diff --git a/experiments/k3/attnres_forward/README.md b/experiments/k3/attnres_forward/README.md new file mode 100644 index 0000000..0e0f983 --- /dev/null +++ b/experiments/k3/attnres_forward/README.md @@ -0,0 +1,58 @@ +# Attention Residuals train-time forward intervention + +This directory implements preregistered protocol +`llm-atlas-k3-attnres-forward-training-v1`: + +- `research/K3_ATTNRES_FORWARD_TRAINING_SCOPING.md` +- `research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md` +- `research/K3_ATTNRES_FORWARD_TRAINING_GROK_REVIEW.md` + +It is a depth-32 reduced Block AttnRes architecture ablation. It is not a +Kimi-K3 checkpoint forward pass and does not claim to recover unpublished +Figure 5 telemetry. + +## Frozen environment + +```text +Python /home/wuyang/.pyenv/versions/3.10.14/envs/navi-router-cu128/bin/python +PyTorch 2.11.0+cu128 +GPU NVIDIA GeForce RTX 5090 +CUBLAS_WORKSPACE_CONFIG=:4096:8 +maximum concurrency 2 +``` + +## Pre-result gates + +The checked-in gate artifacts must pass before formal output: + +```bash +CUBLAS_WORKSPACE_CONFIG=:4096:8 \ +/home/wuyang/.pyenv/versions/3.10.14/envs/navi-router-cu128/bin/python \ + experiments/k3/attnres_forward/verify.py step-zero \ + --cache-dir /home/wuyang/.cache/llm-atlas/k3-attnres-gradient-scale-v1 \ + --parent-manifest experiments/k3/attnres_gradient/manifest.json \ + --study-manifest experiments/k3/attnres_forward/manifest.json \ + --output experiments/k3/attnres_forward/results/gates/step-zero.json +``` + +`learned_reference` is smoke-only. Its 20-step result is compared with a +fresh parent Round 05 smoke using `verify.py smoke-compare`. + +## Formal matrix + +```bash +/home/wuyang/.pyenv/versions/3.10.14/envs/navi-router-cu128/bin/python \ + experiments/k3/attnres_forward/run_matrix.py \ + --python /home/wuyang/.pyenv/versions/3.10.14/envs/navi-router-cu128/bin/python \ + --cache-dir /home/wuyang/.cache/llm-atlas/k3-attnres-gradient-scale-v1 \ + --parent-manifest experiments/k3/attnres_gradient/manifest.json \ + --study-manifest experiments/k3/attnres_forward/manifest.json \ + --output-dir experiments/k3/attnres_forward/results/raw \ + --phase all \ + --concurrency 2 +``` + +This runs 12 formal cells and one full replay. The analyzer reads all cells, +the frozen historical paired references, and generates the only authoritative +status, interaction map, and website compact artifact. + diff --git a/experiments/k3/attnres_forward/analyze.py b/experiments/k3/attnres_forward/analyze.py new file mode 100644 index 0000000..7bbfeb7 --- /dev/null +++ b/experiments/k3/attnres_forward/analyze.py @@ -0,0 +1,684 @@ +#!/usr/bin/env python3 +"""Aggregate and gate preregistered Round 08 forward-training results.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import statistics +from pathlib import Path +from typing import Any, Iterable + + +PROTOCOL_ID = "llm-atlas-k3-attnres-forward-training-v1" +PARENT_PROTOCOL_ID = "llm-atlas-k3-attnres-gradient-scale-v1" +METRICS = ("spike_contrast", "peak_normalized") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--formal", type=Path, action="append", required=True) + parser.add_argument("--replay", type=Path, required=True) + parser.add_argument("--reference-dir", type=Path, required=True) + parser.add_argument("--aggregate-output", type=Path, required=True) + parser.add_argument("--compact-output", type=Path, required=True) + parser.add_argument("--reproduction-output", type=Path, required=True) + return parser.parse_args() + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def mean(values: Iterable[float]) -> float: + return statistics.fmean(values) + + +def read_result(path: Path, expected_protocol: str) -> dict[str, Any]: + value = json.loads(path.read_text()) + if value.get("protocol_id") != expected_protocol: + raise RuntimeError(f"protocol mismatch: {path}") + expected = value.get("canonical_sha256_without_self") + payload = { + key: item + for key, item in value.items() + if key != "canonical_sha256_without_self" + } + if not isinstance(expected, str) or canonical_sha256(payload) != expected: + raise RuntimeError(f"canonical self-hash mismatch: {path}") + return value + + +def exactly_one(values: list[dict[str, Any]], step: int) -> dict[str, Any]: + matches = [value for value in values if value["step"] == step] + if len(matches) != 1: + raise RuntimeError(f"step {step} missing or duplicated") + return matches[0] + + +def spectrum_metrics( + diagnostic: dict[str, Any], + spike_layers: tuple[int, ...], + epsilon: float, +) -> dict[str, Any]: + values = [ + float(value) + for value in diagnostic["activation_grad_rms_by_block"] + ] + if len(values) != 32: + raise RuntimeError("activation-gradient spectrum must have 32 layers") + if any(not math.isfinite(value) or value <= epsilon for value in values): + raise RuntimeError("activation-gradient spectrum is non-finite/non-positive") + spike_indices = {layer - 1 for layer in spike_layers} + spike_values = [ + value for index, value in enumerate(values) if index in spike_indices + ] + reference_values = [ + value for index, value in enumerate(values) if index not in spike_indices + ] + spike_mean = mean(spike_values) + reference_mean = mean(reference_values) + global_mean = mean(values) + contrast = spike_mean / reference_mean + peak = max(values) / global_mean + if any( + not math.isfinite(value) or value <= epsilon + for value in (spike_mean, reference_mean, contrast, peak) + ): + raise RuntimeError("derived spike metric is non-finite/non-positive") + ordered = sorted(range(32), key=lambda index: (-values[index], index)) + return { + "values": values, + "normalized": [value / global_mean for value in values], + "spike_mean": spike_mean, + "reference_mean": reference_mean, + "global_mean": global_mean, + "spike_contrast": contrast, + "peak_normalized": peak, + "peak_layer_1based": ordered[0] + 1, + "top_five_layers_1based": [index + 1 for index in ordered[:5]], + } + + +def final_bpc(value: dict[str, Any], step: int) -> float: + result = float(exactly_one(value["evaluations"], step)["bits_per_byte"]) + if not math.isfinite(result): + raise RuntimeError("final validation BPC is non-finite") + return result + + +def stable_environment(value: dict[str, Any]) -> dict[str, Any]: + keys = ( + "cublas_workspace_config", + "deterministic_algorithms", + "autocast", + "compile", + ) + return {key: value["environment"][key] for key in keys} + + +def pairing_checks( + run: dict[str, Any], reference: dict[str, Any] +) -> dict[str, bool]: + manifest_fields = ( + "formal_schedule_sha256", + "validation_tensor_sha256", + "diagnostic_tensor_sha256", + "input_gate_tensor_hashes", + ) + checks = { + "seed": run["seed"] == reference["seed"], + "architecture": ( + run["architecture"] == reference["architecture"] == "block" + ), + "depth": run["depth"] == reference["depth"] == 32, + "steps": run["steps"] == reference["steps"] == 8000, + "batch_size": run["batch_size"] == reference["batch_size"] == 32, + "initial_public_parameters": ( + run["hashes"]["initial_public_parameters"] + == reference["hashes"]["initial_public_parameters"] + ), + "initial_mixer_parameters": ( + run["hashes"]["initial_mixer_parameters"] + == reference["hashes"]["initial_mixer_parameters"] + ), + "model_topology": run["model"] == reference["model"], + "optimizer_hyperparameters": ( + run["optimizer"] == reference["optimizer"] + ), + "scientific_environment": ( + stable_environment(run) == stable_environment(reference) + ), + } + for field in manifest_fields: + checks[f"manifest.{field}"] = ( + run["manifest"][field] == reference["manifest"][field] + ) + return checks + + +def scientific_replay_payload(value: dict[str, Any]) -> dict[str, Any]: + payload = copy.deepcopy(value) + for key in ( + "run_kind", + "timing", + "canonical_sha256_without_self", + "parent_runner_canonical_sha256", + ): + payload.pop(key, None) + payload["manifest"].pop("path", None) + payload["study_manifest"].pop("path", None) + payload["environment"] = stable_environment(value) + return payload + + +def quality_gate( + variant_runs: dict[int, dict[str, Any]], + references: dict[int, dict[str, Any]], + *, + step: int, + per_seed_maximum: float, + mean_maximum: float, +) -> dict[str, Any]: + per_seed = {} + for seed, run in sorted(variant_runs.items()): + variant_bpc = final_bpc(run, step) + reference_bpc = final_bpc(references[seed], step) + delta = variant_bpc - reference_bpc + per_seed[str(seed)] = { + "variant_bpc": variant_bpc, + "reference_bpc": reference_bpc, + "delta_bpc": delta, + "passed": delta <= per_seed_maximum, + } + mean_delta = mean(item["delta_bpc"] for item in per_seed.values()) + per_seed_passed = all(item["passed"] for item in per_seed.values()) + mean_passed = mean_delta <= mean_maximum + return { + "passed": per_seed_passed and mean_passed, + "passed_checks": ( + sum(item["passed"] for item in per_seed.values()) + + int(mean_passed) + ), + "required_checks": 4, + "per_seed_maximum": per_seed_maximum, + "mean_maximum": mean_maximum, + "mean_delta_bpc": mean_delta, + "mean_passed": mean_passed, + "per_seed": per_seed, + } + + +def variant_effect( + variant: str, + runs: dict[int, dict[str, Any]], + references: dict[int, dict[str, Any]], + metrics_by_cell: dict[tuple[str, int, int], dict[str, Any]], + *, + step: int, + threshold: float, + quality: dict[str, Any], +) -> dict[str, Any]: + cells = [] + for seed in sorted(runs): + candidate = metrics_by_cell[(variant, seed, step)] + reference = metrics_by_cell[("learned_reference", seed, step)] + for metric in METRICS: + reference_value = reference[metric] + candidate_value = candidate[metric] + relative_drop = ( + reference_value - candidate_value + ) / reference_value + cells.append( + { + "seed": seed, + "metric": metric, + "reference": reference_value, + "variant": candidate_value, + "relative_drop": relative_drop, + "passed": relative_drop >= threshold, + } + ) + attenuation_passed = all(cell["passed"] for cell in cells) + return { + "variant": variant, + "threshold": threshold, + "passed_cells": sum(cell["passed"] for cell in cells), + "required_cells": len(cells), + "attenuation_passed": attenuation_passed, + "quality": quality, + "material_response_passed": ( + attenuation_passed and quality["passed"] + ), + "cells": cells, + } + + +def interaction_map( + metrics_by_cell: dict[tuple[str, int, int], dict[str, Any]], + seeds: tuple[int, ...], + steps: tuple[int, ...], +) -> dict[str, Any]: + cells = [] + for step in steps: + for seed in seeds: + reference = metrics_by_cell[ + ("learned_reference", seed, step) + ] + group6 = metrics_by_cell[ + ("uniform_group_6_forward", seed, step) + ] + group7 = metrics_by_cell[ + ("uniform_group_7_forward", seed, step) + ] + joint = metrics_by_cell[ + ("uniform_groups_6_7_forward", seed, step) + ] + for metric in METRICS: + ref = reference[metric] + effects = { + "group6": math.log(ref / group6[metric]), + "group7": math.log(ref / group7[metric]), + "groups6_7": math.log(ref / joint[metric]), + } + residual = ( + effects["groups6_7"] + - effects["group6"] + - effects["group7"] + ) + cells.append( + { + "step": step, + "seed": seed, + "metric": metric, + "log_effects": effects, + "interaction_residual": residual, + "relative_drops": { + "group6": (ref - group6[metric]) / ref, + "group7": (ref - group7[metric]) / ref, + "groups6_7": (ref - joint[metric]) / ref, + }, + } + ) + summaries = [] + for step in steps: + for metric in METRICS: + selected = [ + cell + for cell in cells + if cell["step"] == step and cell["metric"] == metric + ] + residuals = [ + cell["interaction_residual"] for cell in selected + ] + summaries.append( + { + "step": step, + "metric": metric, + "mean_interaction_residual": mean(residuals), + "minimum": min(residuals), + "maximum": max(residuals), + } + ) + return { + "definition": "I67=ln(Xref/X67)-ln(Xref/X6)-ln(Xref/X7)", + "interpretation": ( + "descriptive cross-run log-attenuation residual from three " + "independently trained variants; not a causal interaction" + ), + "cells": cells, + "summaries": summaries, + } + + +def environment_metadata(value: dict[str, Any]) -> dict[str, Any]: + return { + key: value["environment"].get(key) + for key in ("gpu", "torch", "cuda", "compute_capability") + } + + +def write_hashed(path: Path, value: dict[str, Any]) -> None: + value["canonical_sha256_without_self"] = canonical_sha256(value) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + + +def main() -> None: + args = parse_args() + manifest = json.loads(args.manifest.read_text()) + if ( + manifest["protocol_id"] != PROTOCOL_ID + or manifest["status"] != "frozen-before-model-output" + ): + raise RuntimeError("manifest is not the frozen Round 08 contract") + variants = tuple(manifest["variants"].keys()) + seeds = tuple(manifest["formal_seeds"]) + steps = tuple(manifest["diagnostic_steps"]) + primary_step = manifest["primary_step"] + epsilon = manifest["thresholds"]["positive_denominator_epsilon"] + spike_layers = tuple(manifest["spike_layers_1based"]) + expected_cells = {(variant, seed) for variant in variants for seed in seeds} + if len(args.formal) != len(expected_cells): + raise RuntimeError("formal path count does not match the 4×3 matrix") + + runs: dict[tuple[str, int], dict[str, Any]] = {} + run_paths: dict[tuple[str, int], Path] = {} + pairing: dict[str, Any] = {} + references: dict[int, dict[str, Any]] = {} + reference_paths: dict[int, Path] = {} + for seed in seeds: + path = args.reference_dir / ( + f"formal-depth-32-block-seed-{seed}.json" + ) + references[seed] = read_result(path, PARENT_PROTOCOL_ID) + reference_paths[seed] = path + + for path in args.formal: + value = read_result(path, PROTOCOL_ID) + identity = (value["variant"], value["seed"]) + if identity in runs: + raise RuntimeError(f"duplicate formal cell: {identity}") + if ( + value["run_kind"] != "formal" + or value["steps"] != manifest["formal_steps"] + or not value["forward_intervention"]["passed"] + ): + raise RuntimeError(f"invalid formal cell: {path}") + runs[identity] = value + run_paths[identity] = path + if set(runs) != expected_cells: + raise RuntimeError("formal matrix identities do not match manifest") + + for (variant, seed), value in sorted(runs.items()): + checks = pairing_checks(value, references[seed]) + if not all(checks.values()): + raise RuntimeError( + f"historical reference pairing failed: " + f"{variant}/{seed}: {checks}" + ) + pairing[f"{variant}:{seed}"] = { + "passed": True, + "checks": checks, + "run_environment": environment_metadata(value), + "reference_environment": environment_metadata(references[seed]), + "metadata_equal": ( + environment_metadata(value) + == environment_metadata(references[seed]) + ), + } + + replay = read_result(args.replay, PROTOCOL_ID) + replay_contract = manifest["replay"] + if ( + replay["run_kind"] != "replay" + or replay["variant"] != replay_contract["variant"] + or replay["seed"] != replay_contract["seed"] + or replay["steps"] != manifest["formal_steps"] + or not replay["forward_intervention"]["passed"] + ): + raise RuntimeError("invalid replay identity/audit") + formal_primary = runs[ + (replay_contract["variant"], replay_contract["seed"]) + ] + formal_payload = scientific_replay_payload(formal_primary) + replay_payload = scientific_replay_payload(replay) + replay_exact = formal_payload == replay_payload + if not replay_exact: + raise RuntimeError("primary formal/replay scientific payload mismatch") + + metrics_by_cell: dict[tuple[str, int, int], dict[str, Any]] = {} + for seed, reference in references.items(): + for step in steps: + metrics_by_cell[("learned_reference", seed, step)] = ( + spectrum_metrics( + exactly_one(reference["diagnostics"], step), + spike_layers, + epsilon, + ) + ) + for (variant, seed), value in runs.items(): + if tuple(item["step"] for item in value["diagnostics"]) != steps: + raise RuntimeError(f"diagnostic schedule drift: {variant}/{seed}") + for step in steps: + metrics_by_cell[(variant, seed, step)] = spectrum_metrics( + exactly_one(value["diagnostics"], step), + spike_layers, + epsilon, + ) + + runs_by_variant = { + variant: {seed: runs[(variant, seed)] for seed in seeds} + for variant in variants + } + qualities = { + variant: quality_gate( + variant_runs, + references, + step=primary_step, + per_seed_maximum=manifest["thresholds"][ + "final_bpc_delta_per_seed_maximum" + ], + mean_maximum=manifest["thresholds"][ + "final_bpc_delta_mean_maximum" + ], + ) + for variant, variant_runs in runs_by_variant.items() + } + effects = { + variant: variant_effect( + variant, + variant_runs, + references, + metrics_by_cell, + step=primary_step, + threshold=manifest["thresholds"]["material_relative_drop"], + quality=qualities[variant], + ) + for variant, variant_runs in runs_by_variant.items() + } + primary = effects[manifest["primary_variant"]] + if primary["attenuation_passed"] and primary["quality"]["passed"]: + status = ( + "forward_training_attenuation_established_within_reduced_protocol" + ) + elif primary["attenuation_passed"]: + status = "quality_guard_failed" + elif primary["quality"]["passed"]: + status = "attenuation_not_established" + else: + status = "attenuation_and_quality_failed" + secondary = { + variant: ( + "secondary_material_response" + if effect["material_response_passed"] + else "secondary_response_not_established" + ) + for variant, effect in effects.items() + if variant != manifest["primary_variant"] + } + interaction = interaction_map(metrics_by_cell, seeds, steps) + + trajectories = [] + final_spectra = [] + for variant in ("learned_reference",) + variants: + for seed in seeds: + for step in steps: + record = metrics_by_cell[(variant, seed, step)] + reference = metrics_by_cell[ + ("learned_reference", seed, step) + ] + trajectories.append( + { + "variant": variant, + "seed": seed, + "step": step, + "spike_mean": record["spike_mean"], + "reference_mean": record["reference_mean"], + "spike_contrast": record["spike_contrast"], + "peak_normalized": record["peak_normalized"], + "relative_drop": { + metric: ( + reference[metric] - record[metric] + ) + / reference[metric] + for metric in METRICS + }, + } + ) + final = metrics_by_cell[(variant, seed, primary_step)] + final_spectra.append( + { + "variant": variant, + "seed": seed, + **final, + } + ) + + input_files = { + "manifest": { + "path": str(args.manifest), + "sha256": file_sha256(args.manifest), + }, + "formal": [ + { + "variant": variant, + "seed": seed, + "path": str(run_paths[(variant, seed)]), + "sha256": file_sha256(run_paths[(variant, seed)]), + } + for variant, seed in sorted(runs) + ], + "references": [ + { + "seed": seed, + "path": str(reference_paths[seed]), + "sha256": file_sha256(reference_paths[seed]), + } + for seed in seeds + ], + "replay": { + "path": str(args.replay), + "sha256": file_sha256(args.replay), + }, + } + aggregate = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "status": status, + "scope": ( + "depth-32 reduced Block AttnRes train-time architecture " + "ablation; not a real Kimi-K3 checkpoint result" + ), + "primary_step": primary_step, + "spike_layers_1based": list(spike_layers), + "thresholds": manifest["thresholds"], + "input_files": input_files, + "historical_pairing": pairing, + "replay": { + "passed": replay_exact, + "scientific_payload_sha256": canonical_sha256(formal_payload), + "excluded": [ + "run_kind", + "timing", + "self hashes", + "manifest path strings", + "GPU/version metadata", + ], + }, + "primary": primary, + "secondary_status": secondary, + "effects": effects, + "interaction": interaction, + "trajectories": trajectories, + "final_spectra": final_spectra, + "processed_target_bytes": manifest["new_target_bytes"], + "historical_reference_target_bytes": ( + manifest["historical_reference_target_bytes"] + ), + "reporting_boundary": ( + "C can change through spike-window numerator and the 27-layer " + "reference denominator; layers 26-28 are intervened but belong " + "to the denominator." + ), + } + write_hashed(args.aggregate_output, aggregate) + + compact = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "status": status, + "primary_step": primary_step, + "spike_layers_1based": list(spike_layers), + "thresholds": manifest["thresholds"], + "primary": primary, + "secondary_status": secondary, + "effects": effects, + "interaction": interaction, + "trajectories": trajectories, + "final_spectra": final_spectra, + "replay": aggregate["replay"], + "processed_target_bytes": manifest["new_target_bytes"], + "reporting_boundary": aggregate["reporting_boundary"], + "aggregate_sha256": aggregate["canonical_sha256_without_self"], + } + write_hashed(args.compact_output, compact) + + reproduction = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "passed": replay_exact, + "formal_variant": replay_contract["variant"], + "seed": replay_contract["seed"], + "formal_file_sha256": file_sha256( + run_paths[ + (replay_contract["variant"], replay_contract["seed"]) + ] + ), + "replay_file_sha256": file_sha256(args.replay), + "scientific_payload_sha256": canonical_sha256(formal_payload), + "excluded_fields": aggregate["replay"]["excluded"], + } + write_hashed(args.reproduction_output, reproduction) + print( + json.dumps( + { + "status": status, + "primary_attenuation": { + "passed_cells": primary["passed_cells"], + "required_cells": primary["required_cells"], + }, + "primary_quality": { + "passed_checks": primary["quality"]["passed_checks"], + "required_checks": primary["quality"]["required_checks"], + }, + "replay_exact": replay_exact, + "aggregate": str(args.aggregate_output), + "compact": str(args.compact_output), + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/k3/attnres_forward/results/gates/parent-equivalence.json b/experiments/k3/attnres_forward/results/gates/parent-equivalence.json new file mode 100644 index 0000000..91fa1ce --- /dev/null +++ b/experiments/k3/attnres_forward/results/gates/parent-equivalence.json @@ -0,0 +1,39 @@ +{ + "canonical_sha256_without_self": "99aeefc0ba35c199725ed7af377450ecbb5cc5f1aabdda9ac2345a30f03b444f", + "excluded_fields": [ + "protocol wrapper fields", + "timing", + "self hash", + "parent runner self hash", + "study manifest" + ], + "field_checks": { + "architecture": true, + "batch_size": true, + "depth": true, + "diagnostics": true, + "environment": true, + "evaluations": true, + "gradient_gate": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "seed": true, + "steps": true, + "target_bytes_seen": true, + "training_history": true + }, + "gate": "empty-selector-parent-equivalence", + "parent_file": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/results/gates/parent-smoke.json", + "passed": true, + "protocol_id": "llm-atlas-k3-attnres-forward-training-v1", + "schema_version": 1, + "wrapper_file": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/results/gates/wrapper-smoke.json", + "wrapper_identity": { + "forward_audit": true, + "parent_protocol": true, + "protocol": true, + "variant": true + } +} diff --git a/experiments/k3/attnres_forward/results/gates/parent-smoke.json b/experiments/k3/attnres_forward/results/gates/parent-smoke.json new file mode 100644 index 0000000..ff4c261 --- /dev/null +++ b/experiments/k3/attnres_forward/results/gates/parent-smoke.json @@ -0,0 +1,2618 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "fc6cf8d746ae7004851f9d52a5fc29a99061d0d6a5ec28f2f4c70a6313eb712a", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.9337450289167464e-05, + 3.887022103299387e-05, + 3.8419901102315634e-05, + 3.799897967837751e-05, + 3.775028380914591e-05, + 3.7539852201007307e-05, + 3.739175735972822e-05, + 3.724253110704012e-05, + 3.720997119671665e-05, + 3.713165278895758e-05, + 3.7142541259527206e-05, + 3.707727591972798e-05, + 3.688051219796762e-05, + 3.692117388709448e-05, + 3.689077857416123e-05, + 3.6904009903082624e-05, + 3.717117215273902e-05, + 3.7258152588037774e-05, + 3.729091622517444e-05, + 3.7364865420386195e-05, + 3.74124720110558e-05, + 3.742674380191602e-05, + 3.742256740224548e-05, + 3.74479714082554e-05, + 3.765593282878399e-05, + 3.763797212741338e-05, + 3.7667807191610336e-05, + 3.764635039260611e-05, + 3.751127951545641e-05, + 3.7523430364672095e-05, + 3.7504440115299076e-05, + 3.756426303880289e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.8068872072472004e-05, + "first_to_last_ratio": 1.0127680561501509, + "imbalance_abs_log_ratio": 0.01268723177426717, + "last_quartile_mean": 3.7588934446830535e-05, + "mean": 3.750672590285831e-05, + "normalized": [ + 1.0488105624322075, + 1.0363533498942825, + 1.0243469718423952, + 1.013124413386392, + 1.0064937128054956, + 1.0008832095404647, + 0.9969347219635258, + 0.9929560688260969, + 0.9920879602525092, + 0.9899998438980743, + 0.9902901510445264, + 0.9885500540824973, + 0.9833039624276305, + 0.9843880796932157, + 0.9835776833655818, + 0.9839304555311837, + 0.9910535046170554, + 0.9933725669506762, + 0.9942461072650594, + 0.9962177321785024, + 0.997487013607996, + 0.9978675264498043, + 0.9977561757634937, + 0.9984334944416348, + 1.0039781378495187, + 1.0034992717011608, + 1.0042947307416068, + 1.0037226520413813, + 1.0001214078938774, + 1.0004453724341882, + 0.9999390565957383, + 1.001534048482227 + ], + "population_cv": 0.013988937740460563 + }, + "activation_output_rms_by_block": [ + 0.0037520017940551043, + 0.005621022544801235, + 0.0076507688499987125, + 0.009552493691444397, + 0.005728918593376875, + 0.009368892759084702, + 0.013378600589931011, + 0.01761116087436676, + 0.00800272449851036, + 0.013154285028576851, + 0.01923154480755329, + 0.024512210860848427, + 0.011547097004950047, + 0.016813727095723152, + 0.02113443799316883, + 0.025982216000556946, + 0.010269254446029663, + 0.01641162671148777, + 0.02165052480995655, + 0.026260115206241608, + 0.011169232428073883, + 0.016121258959174156, + 0.019987935200333595, + 0.02486182004213333, + 0.01274633314460516, + 0.016767995432019234, + 0.022238746285438538, + 0.02633264660835266, + 0.0102624436840415, + 0.016213588416576385, + 0.020620016381144524, + 0.02528378553688526 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.00908298246213235, + "first_to_last_ratio": 0.4829268696139592, + "imbalance_abs_log_ratio": 0.7278900454597648, + "last_quartile_mean": 0.018808194436132908, + "mean": 0.015944982071232516, + "normalized": [ + 0.2353092513552812, + 0.35252611258450545, + 0.47982298229121323, + 0.5990909020009959, + 0.3592928839796457, + 0.5875762492068142, + 0.8390477035448226, + 1.1044954955540724, + 0.5018961114386356, + 0.8249796061112817, + 1.2061189358281121, + 1.537299367998203, + 0.7241837559531027, + 1.0544839135352808, + 1.3254601290081713, + 1.62949170368986, + 0.6440430224476176, + 1.0292659244250366, + 1.3578268519359336, + 1.6469203344931544, + 0.7004857313841133, + 1.0110553205487558, + 1.2535564550050677, + 1.5592253369157636, + 0.7993946363603464, + 1.0516158222762493, + 1.394717547256516, + 1.6514691889093607, + 0.6436158810461574, + 1.016845822977028, + 1.2931978404884237, + 1.5856891794504773 + ], + "population_cv": 0.416991705473353 + }, + "bits_per_byte": 6.71703146032544, + "branch_output_rms_by_sublayer": [ + 0.002443866338580847, + 0.0027545092161744833, + 0.0026208641938865185, + 0.002763434313237667, + 0.0034182921517640352, + 0.002741520293056965, + 0.0035785960499197245, + 0.002719961805269122, + 0.0047746808268129826, + 0.0028042772319167852, + 0.005376013461500406, + 0.0027222761418670416, + 0.006767150945961475, + 0.0027788877487182617, + 0.007074786350131035, + 0.002833561971783638, + 0.006997238378971815, + 0.0029156480450183153, + 0.008518846705555916, + 0.0027741468511521816, + 0.009081361815333366, + 0.0028916175942867994, + 0.00911131501197815, + 0.0032094581983983517, + 0.010676512494683266, + 0.002878024475648999, + 0.009562043473124504, + 0.0026487084105610847, + 0.008648304268717766, + 0.003162843408063054, + 0.009535389021039009, + 0.0030166516080498695, + 0.009431489743292332, + 0.002794384490698576, + 0.010901342146098614, + 0.00310111534781754, + 0.010227455757558346, + 0.0028713408391922712, + 0.008522198535501957, + 0.0031550333369523287, + 0.010077033191919327, + 0.0032391855493187904, + 0.009554130956530571, + 0.00291044800542295, + 0.010129180736839771, + 0.0026873303577303886, + 0.00920273270457983, + 0.002927709138020873, + 0.011994888074696064, + 0.0029113248456269503, + 0.008809582330286503, + 0.0027162786573171616, + 0.01077704131603241, + 0.0031467911321669817, + 0.010385709814727306, + 0.003254092764109373, + 0.009710066020488739, + 0.0025947801768779755, + 0.009926802478730679, + 0.0031805087346583605, + 0.0101145189255476, + 0.002813765313476324, + 0.009518872946500778, + 0.0030965779442340136 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00016739224645496468, + 0.0001695381430192955, + 0.0001941383401676919, + 0.00020674849822850373, + 0.00024134787186321302, + 0.00026490993231531454, + 0.000322331199796065, + 0.0003097008723390916, + 0.00031886187327784883, + 0.00034582398876540076, + 0.00034801623779775657, + 0.00036856862737498307, + 0.00040270040252095, + 0.0003782002933862871, + 0.0003548760100734237, + 0.0003670443346435906, + 0.0003676146253339179, + 0.0003749331083729355, + 0.00038033249912268637, + 0.0003562102847256418, + 0.00036931756079357426, + 0.00035104688530666954, + 0.0003583944935512815, + 0.0003605180195562536, + 0.0004079256274935599, + 0.0003614699495467052, + 0.00038354947558429407, + 0.0003579111697278514, + 0.0003529832358531257, + 0.00036980562702627114, + 0.00036596233134640236, + 0.0003819940876588112 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002345133880230175, + "first_to_last_ratio": 0.6292279841951004, + "imbalance_abs_log_ratio": 0.46326163295248246, + "last_quartile_mean": 0.00037270018802962763, + "mean": 0.0003331302454070113, + "normalized": [ + 0.502482883985657, + 0.5089244983209419, + 0.582770081205019, + 0.6206236181764371, + 0.7244850180695525, + 0.7952142922107038, + 0.9675831127319218, + 0.9296690306840971, + 0.9571687881064967, + 1.0381044457337716, + 1.044685202247421, + 1.1063799593599613, + 1.2088377085933444, + 1.1352925728019987, + 1.0652770649505088, + 1.1018042933782364, + 1.1035162084570687, + 1.1254850423889022, + 1.141693089614257, + 1.069282329169577, + 1.1086281293442752, + 1.0537826875424299, + 1.0758389503582988, + 1.0822134121018658, + 1.224522939954507, + 1.0850709430633327, + 1.1513499025454192, + 1.07438809493435, + 1.0595952805842004, + 1.1100932206694432, + 1.0985563046047577, + 1.146680894111248 + ], + "population_cv": 0.19646899587719932 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931430697441101, + "mean_weights": [ + 0.49865636229515076, + 0.5013436079025269 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931370496749878, + "mean_weights": [ + 0.49779537320137024, + 0.5022045969963074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931354999542236, + "mean_weights": [ + 0.49762701988220215, + 0.5023729801177979 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931448578834534, + "mean_weights": [ + 0.49909353256225586, + 0.5009064674377441 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931440234184265, + "mean_weights": [ + 0.49918538331985474, + 0.5008145570755005 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931461691856384, + "mean_weights": [ + 0.49945560097694397, + 0.5005444288253784 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931438446044922, + "mean_weights": [ + 0.49891161918640137, + 0.5010883808135986 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931411623954773, + "mean_weights": [ + 0.49836465716362, + 0.5016353130340576 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986062288284302, + "mean_weights": [ + 0.3321005403995514, + 0.3337631821632385, + 0.3341362476348877 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985990762710571, + "mean_weights": [ + 0.33337411284446716, + 0.33537548780441284, + 0.3312504291534424 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986008644104004, + "mean_weights": [ + 0.3350442051887512, + 0.33362624049186707, + 0.3313295841217041 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986016988754272, + "mean_weights": [ + 0.3313179612159729, + 0.3338589072227478, + 0.3348231613636017 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986084938049316, + "mean_weights": [ + 0.33454394340515137, + 0.3326125741004944, + 0.33284348249435425 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985959768295288, + "mean_weights": [ + 0.3312351107597351, + 0.332991361618042, + 0.3357735574245453 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986058712005615, + "mean_weights": [ + 0.33314070105552673, + 0.3320331275463104, + 0.33482617139816284 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.33257031440734863, + 0.3324887454509735, + 0.33494094014167786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862926959991455, + "mean_weights": [ + 0.24962691962718964, + 0.2496621459722519, + 0.25036555528640747, + 0.2503453493118286 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862905502319336, + "mean_weights": [ + 0.24916216731071472, + 0.24970784783363342, + 0.25086167454719543, + 0.2502683103084564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386286973953247, + "mean_weights": [ + 0.25067827105522156, + 0.24852371215820312, + 0.24991214275360107, + 0.25088587403297424 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862855434417725, + "mean_weights": [ + 0.24860653281211853, + 0.24958111345767975, + 0.2513706088066101, + 0.2504417300224304 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386291265487671, + "mean_weights": [ + 0.25055181980133057, + 0.24973659217357635, + 0.2504120469093323, + 0.2492995262145996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862907886505127, + "mean_weights": [ + 0.24893252551555634, + 0.25016576051712036, + 0.2503091096878052, + 0.25059258937835693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862874507904053, + "mean_weights": [ + 0.24930882453918457, + 0.24942341446876526, + 0.2499566376209259, + 0.25131112337112427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862828016281128, + "mean_weights": [ + 0.24865929782390594, + 0.24900496006011963, + 0.25141197443008423, + 0.2509238123893738 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094212532043457, + "mean_weights": [ + 0.19921688735485077, + 0.1992446780204773, + 0.19889645278453827, + 0.20075249671936035, + 0.20188948512077332 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609428882598877, + "mean_weights": [ + 0.19964422285556793, + 0.19919568300247192, + 0.19919705390930176, + 0.20121362805366516, + 0.20074939727783203 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094245910644531, + "mean_weights": [ + 0.20065900683403015, + 0.19955331087112427, + 0.2014925479888916, + 0.19979336857795715, + 0.19850176572799683 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094355583190918, + "mean_weights": [ + 0.19937855005264282, + 0.20006650686264038, + 0.20067161321640015, + 0.1999610960483551, + 0.19992223381996155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094313859939575, + "mean_weights": [ + 0.1997283697128296, + 0.19996783137321472, + 0.19928213953971863, + 0.1997123807668686, + 0.20130927860736847 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094284057617188, + "mean_weights": [ + 0.19933226704597473, + 0.19993427395820618, + 0.19907569885253906, + 0.2001166194677353, + 0.20154115557670593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094276905059814, + "mean_weights": [ + 0.19932052493095398, + 0.19971224665641785, + 0.2011900395154953, + 0.20086491107940674, + 0.19891227781772614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094303131103516, + "mean_weights": [ + 0.19955268502235413, + 0.19991174340248108, + 0.20052853226661682, + 0.20107966661453247, + 0.1989273726940155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917479276657104, + "mean_weights": [ + 0.16630326211452484, + 0.1665954291820526, + 0.1656438112258911, + 0.16612562537193298, + 0.16804365813732147, + 0.16728821396827698 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917546033859253, + "mean_weights": [ + 0.16606619954109192, + 0.16676779091358185, + 0.16676853597164154, + 0.16595947742462158, + 0.16733971238136292, + 0.1670982539653778 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791755199432373, + "mean_weights": [ + 0.16713324189186096, + 0.16643813252449036, + 0.16707737743854523, + 0.16582445800304413, + 0.16707313060760498, + 0.16645365953445435 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791752815246582, + "mean_weights": [ + 0.16595935821533203, + 0.16658073663711548, + 0.16717244684696198, + 0.1671261042356491, + 0.16733530163764954, + 0.16582605242729187 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917518615722656, + "mean_weights": [ + 0.1666388213634491, + 0.16675034165382385, + 0.16638417541980743, + 0.16550619900226593, + 0.16747808456420898, + 0.1672423779964447 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917499542236328, + "mean_weights": [ + 0.16634854674339294, + 0.16704240441322327, + 0.16620638966560364, + 0.16553011536598206, + 0.16771462559700012, + 0.16715793311595917 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917507886886597, + "mean_weights": [ + 0.16621501743793488, + 0.1665443778038025, + 0.16585618257522583, + 0.16727086901664734, + 0.16779379546642303, + 0.16631974279880524 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917485237121582, + "mean_weights": [ + 0.16604092717170715, + 0.1667710244655609, + 0.16534343361854553, + 0.1669321358203888, + 0.16766314208507538, + 0.16724932193756104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9458963871002197, + "mean_weights": [ + 0.14258044958114624, + 0.14276695251464844, + 0.14234250783920288, + 0.14281943440437317, + 0.14429350197315216, + 0.14340519905090332, + 0.14179196953773499 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458930492401123, + "mean_weights": [ + 0.14285841584205627, + 0.14281482994556427, + 0.14142268896102905, + 0.14248794317245483, + 0.1441671997308731, + 0.14373424649238586, + 0.1425146758556366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459006786346436, + "mean_weights": [ + 0.1428324282169342, + 0.14242856204509735, + 0.14238804578781128, + 0.14283627271652222, + 0.14203999936580658, + 0.14380358159542084, + 0.14367109537124634 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.14253869652748108, + 0.14282220602035522, + 0.1414484828710556, + 0.14307048916816711, + 0.1438344419002533, + 0.14343075454235077, + 0.1428549587726593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14281800389289856, + 0.1431872844696045, + 0.1432121843099594, + 0.14210283756256104, + 0.14336752891540527, + 0.14217621088027954, + 0.14313597977161407 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458997249603271, + "mean_weights": [ + 0.14271901547908783, + 0.14267435669898987, + 0.14293994009494781, + 0.14278604090213776, + 0.14186438918113708, + 0.1428055316209793, + 0.14421072602272034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459013938903809, + "mean_weights": [ + 0.14281892776489258, + 0.1429084837436676, + 0.14260512590408325, + 0.14224481582641602, + 0.14402621984481812, + 0.14324231445789337, + 0.14215409755706787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459059238433838, + "mean_weights": [ + 0.14244785904884338, + 0.14288531243801117, + 0.14282266795635223, + 0.14290127158164978, + 0.1435883343219757, + 0.14224958419799805, + 0.14310497045516968 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432487487793, + "mean_weights": [ + 0.12496939301490784, + 0.12501314282417297, + 0.12449057400226593, + 0.12434226274490356, + 0.12589572370052338, + 0.12463213503360748, + 0.12490519136190414, + 0.1257515549659729 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794320106506348, + "mean_weights": [ + 0.12487496435642242, + 0.12489178031682968, + 0.12453664839267731, + 0.12434129416942596, + 0.1249510794878006, + 0.12465334683656693, + 0.12601137161254883, + 0.12573951482772827 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794363021850586, + "mean_weights": [ + 0.12491423636674881, + 0.12523731589317322, + 0.12523618340492249, + 0.12467047572135925, + 0.12580403685569763, + 0.12441615015268326, + 0.12498776614665985, + 0.1247338205575943 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079427480697632, + "mean_weights": [ + 0.12474467605352402, + 0.1245410293340683, + 0.12462934106588364, + 0.1246577650308609, + 0.12418574839830399, + 0.12514574825763702, + 0.12619128823280334, + 0.1259044110774994 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794358253479004, + "mean_weights": [ + 0.12475918978452682, + 0.12487104535102844, + 0.1250094473361969, + 0.1255180537700653, + 0.12490293383598328, + 0.12545335292816162, + 0.12531054019927979, + 0.12417542934417725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079436779022217, + "mean_weights": [ + 0.12494880706071854, + 0.12516388297080994, + 0.12451333552598953, + 0.124576136469841, + 0.12569281458854675, + 0.12472400069236755, + 0.1254362016916275, + 0.12494482100009918 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079434871673584, + "mean_weights": [ + 0.12489444017410278, + 0.12512432038784027, + 0.12540367245674133, + 0.1253792643547058, + 0.1248319000005722, + 0.12483960390090942, + 0.125506192445755, + 0.12402061372995377 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079428195953369, + "mean_weights": [ + 0.12488104403018951, + 0.12459895014762878, + 0.12424551695585251, + 0.12444087862968445, + 0.12461263686418533, + 0.12524595856666565, + 0.12623095512390137, + 0.1257440745830536 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11096074432134628, + 0.11101590096950531, + 0.1106940507888794, + 0.11131151020526886, + 0.11191274225711823, + 0.11117846518754959, + 0.11035685986280441, + 0.11126935482025146, + 0.11130036413669586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216749191284, + "mean_weights": [ + 0.11095131188631058, + 0.11080048978328705, + 0.11054704338312149, + 0.11075608432292938, + 0.1108987033367157, + 0.11123715341091156, + 0.11191022396087646, + 0.11176811158657074, + 0.11113087832927704 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216510772705, + "mean_weights": [ + 0.1109621673822403, + 0.1111164540052414, + 0.11043901741504669, + 0.11095462739467621, + 0.11190879344940186, + 0.11175167560577393, + 0.11079787462949753, + 0.1112256720662117, + 0.1108437031507492 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197213649749756, + "mean_weights": [ + 0.11101574450731277, + 0.11100780963897705, + 0.11082783341407776, + 0.11142586171627045, + 0.11015472561120987, + 0.11108498275279999, + 0.11191210150718689, + 0.11077474802732468, + 0.11179618537425995 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197218894958496, + "mean_weights": [ + 0.11101824045181274, + 0.11099033057689667, + 0.11046893894672394, + 0.11147398501634598, + 0.11092066764831543, + 0.11164902150630951, + 0.11152558028697968, + 0.11112039536237717, + 0.1108328253030777 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216033935547, + "mean_weights": [ + 0.11115090548992157, + 0.11169334501028061, + 0.1117531955242157, + 0.1108616292476654, + 0.11132021248340607, + 0.11068520694971085, + 0.1114465519785881, + 0.11033187806606293, + 0.11075706779956818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972179412841797, + "mean_weights": [ + 0.11092537641525269, + 0.11108530312776566, + 0.11051007360219955, + 0.11084415018558502, + 0.11164555698633194, + 0.11119014024734497, + 0.11111340671777725, + 0.11183696240186691, + 0.1108490377664566 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802533470094204, + 0.014066163450479507, + 0.014108670875430107, + 0.014229752123355865, + 0.01430919673293829, + 0.014553130604326725, + 0.01460581086575985, + 0.014812584035098553, + 0.014851128682494164, + 0.010233328677713871, + 0.01034309808164835, + 0.010804922319948673, + 0.010769715532660484, + 0.011443440802395344, + 0.011492080986499786, + 0.01227729581296444, + 0.012374433688819408, + 0.009845216758549213, + 0.00994168221950531, + 0.010567485354840755, + 0.01064346730709076, + 0.01164715364575386, + 0.01175782922655344, + 0.012674410827457905, + 0.012870704755187035, + 0.011088024824857712, + 0.011239404790103436, + 0.011724143289029598, + 0.011804761365056038, + 0.012471788562834263, + 0.012632876634597778, + 0.01310542318969965, + 0.013265094719827175, + 0.011508332565426826, + 0.011626233346760273, + 0.012281009927392006, + 0.012384294532239437, + 0.012981574982404709, + 0.013078294694423676, + 0.013564560562372208, + 0.01372351124882698, + 0.012140991166234016, + 0.012265652418136597, + 0.012530013918876648, + 0.012632926926016808, + 0.013094775378704071, + 0.01315103005617857, + 0.01351068913936615, + 0.013631501235067844, + 0.01232061255723238, + 0.012407775968313217, + 0.012646659277379513, + 0.012729056179523468, + 0.01315284613519907, + 0.013267731294035912, + 0.013451422564685345, + 0.01357149239629507, + 0.012415111064910889, + 0.012491659261286259, + 0.01274929754436016, + 0.012818088755011559, + 0.013031110167503357, + 0.013103967532515526, + 0.013315660879015923 + ], + "loss_nats": 4.655891418457031, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.197218418121338, + "mean_weights": [ + 0.11057738959789276, + 0.11068764328956604, + 0.11082598567008972, + 0.11127202212810516, + 0.11091771721839905, + 0.11130063980817795, + 0.11115016043186188, + 0.11173908412456512, + 0.11152935773134232 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.002443866338580847, + 0.0037520017940551043, + 0.004732752684503794, + 0.005621022544801235, + 0.007002863567322493, + 0.0076507688499987125, + 0.008998939767479897, + 0.009552493691444397, + 0.0047746808268129826, + 0.005728918593376875, + 0.008738124743103981, + 0.009368892759084702, + 0.012762236408889294, + 0.013378600589931011, + 0.01696598529815674, + 0.01761116087436676, + 0.006997238378971815, + 0.00800272449851036, + 0.012584065087139606, + 0.013154285028576851, + 0.018650829792022705, + 0.01923154480755329, + 0.023473192006349564, + 0.024512210860848427, + 0.010676512494683266, + 0.011547097004950047, + 0.01631898619234562, + 0.016813727095723152, + 0.02041666954755783, + 0.02113443799316883, + 0.02490287274122238, + 0.025982216000556946, + 0.009431489743292332, + 0.010269254446029663, + 0.015727587044239044, + 0.01641162671148777, + 0.02073165774345398, + 0.02165052480995655, + 0.02527376264333725, + 0.026260115206241608, + 0.010077033191919327, + 0.011169232428073883, + 0.015674244612455368, + 0.016121258959174156, + 0.019633492454886436, + 0.019987935200333595, + 0.02385960891842842, + 0.02486182004213333, + 0.011994888074696064, + 0.01274633314460516, + 0.015906881541013718, + 0.016767995432019234, + 0.02143402211368084, + 0.022238746285438538, + 0.02530583366751671, + 0.02633264660835266, + 0.009710066020488739, + 0.0102624436840415, + 0.015399535186588764, + 0.016213588416576385, + 0.01968187838792801, + 0.020620016381144524, + 0.024621067568659782, + 0.02528378553688526 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 6.720464749984615, + "cross_entropy_nats": 4.6582711935043335, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "0f212e3b0862c8a0fa1714765ff7cc8edeb23e1470b2107592e11f4eeae72674", + "final_model_state": "ac8be80504f01ca19f023c383b7139873a7635445148dec83aff2b39b674dd20", + "final_optimizer_state": "de50114df8c8cb4b59e160333f861411e6676277c7a3a849bd21ada1ea6abc54", + "final_public_parameters": "063f8f52a2b325c61dff3a9a7bdd1302f383a71c59990ca1765419e2e5dcbb96", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 6751356416, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + } + ] +} diff --git a/experiments/k3/attnres_forward/results/gates/primary-smoke.json b/experiments/k3/attnres_forward/results/gates/primary-smoke.json new file mode 100644 index 0000000..0698ebe --- /dev/null +++ b/experiments/k3/attnres_forward/results/gates/primary-smoke.json @@ -0,0 +1,3623 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "f0b86fbc1e9319eaa88f686bf658788c2aaba0dda022580df2fd5e54d3202097", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.934350024792366e-05, + 3.887611092068255e-05, + 3.842553996946663e-05, + 3.800440754275769e-05, + 3.7755999073851854e-05, + 3.7545447412412614e-05, + 3.739710518857464e-05, + 3.724786438397132e-05, + 3.721602843143046e-05, + 3.7137662729946896e-05, + 3.714848207891919e-05, + 3.708315853145905e-05, + 3.6884568544337526e-05, + 3.692537211463787e-05, + 3.689494769787416e-05, + 3.690824451041408e-05, + 3.717643267009407e-05, + 3.726343493326567e-05, + 3.729622403625399e-05, + 3.737024235306308e-05, + 3.7416841223603114e-05, + 3.743126580957323e-05, + 3.7427271308843046e-05, + 3.7452777178259566e-05, + 3.766144436667673e-05, + 3.764339999179356e-05, + 3.7673275073757395e-05, + 3.7651498132618144e-05, + 3.7516889278776944e-05, + 3.752906195586547e-05, + 3.751005351659842e-05, + 3.7569901905953884e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.807449684245512e-05, + "first_to_last_ratio": 1.0127693432316311, + "imbalance_abs_log_ratio": 0.012688502628590252, + "last_quartile_mean": 3.759444052775507e-05, + "mean": 3.7512014159801765e-05, + "normalized": [ + 1.0488239869050948, + 1.036364263328269, + 1.0243528861386442, + 1.0131262848445919, + 1.006504180580939, + 1.0008912678607025, + 0.9969367421664534, + 0.99295826199294, + 0.9921095751587637, + 0.9900204924145068, + 0.9903089159826522, + 0.9885675126236682, + 0.9832734757245689, + 0.9843612224429007, + 0.9835501645073257, + 0.9839046326114184, + 0.9910540263639774, + 0.9933733436579241, + 0.994247439696826, + 0.9962206293126586, + 0.9974628678749903, + 0.9978474003052851, + 0.9977409144015111, + 0.9984208530821659, + 1.0039835292831356, + 1.0035025000639017, + 1.0042989137631655, + 1.0037183813223725, + 1.0001299615359072, + 1.0004544622954952, + 0.9999477329264433, + 1.0015431788308011 + ], + "population_cv": 0.013994511621676028 + }, + "activation_output_rms_by_block": [ + 0.003752076532691717, + 0.005621279589831829, + 0.007651050575077534, + 0.009552945382893085, + 0.005728233605623245, + 0.00936705432832241, + 0.013376436196267605, + 0.017608575522899628, + 0.008001524023711681, + 0.01315119955688715, + 0.019228288903832436, + 0.02450871281325817, + 0.011544129811227322, + 0.016811303794384003, + 0.021131416782736778, + 0.02597878687083721, + 0.010270251892507076, + 0.016414040699601173, + 0.021650852635502815, + 0.026260782033205032, + 0.011164487339556217, + 0.016113370656967163, + 0.019974395632743835, + 0.024847183376550674, + 0.012743011116981506, + 0.016761260107159615, + 0.022224903106689453, + 0.026317164301872253, + 0.010263347066938877, + 0.016214758157730103, + 0.020621437579393387, + 0.025286227464675903 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.009082206466700882, + "first_to_last_ratio": 0.48299297446670986, + "imbalance_abs_log_ratio": 0.7277531710529308, + "last_quartile_mean": 0.018804013612680137, + "mean": 0.015941890233079903, + "normalized": [ + 0.23535957642626626, + 0.3526106068756831, + 0.4799337132055629, + 0.5992354258637684, + 0.3593195989856327, + 0.5875748854979249, + 0.8390746643400603, + 1.1045475326609202, + 0.5019181481445831, + 0.8249460612643045, + 1.2061486199380018, + 1.5373780934962062, + 0.7241380816481162, + 1.0545364162337563, + 1.325527680455888, + 1.6295926324301522, + 0.6442304985387488, + 1.0296169688548942, + 1.358110758445485, + 1.647281573844557, + 0.7003239375208825, + 1.010756592937231, + 1.2529502675470918, + 1.5586096136198466, + 0.7993412908175327, + 1.0513972848953317, + 1.3941196923167938, + 1.650818310570433, + 0.6437973738924712, + 1.01711640970178, + 1.2935377974566205, + 1.5861498915734735 + ], + "population_cv": 0.416946448324543 + }, + "bits_per_byte": 6.717052098243545, + "branch_output_rms_by_sublayer": [ + 0.002443928038701415, + 0.0027545003686100245, + 0.002620922401547432, + 0.002763453871011734, + 0.003418463747948408, + 0.0027414956130087376, + 0.0035784339997917414, + 0.0027198803145438433, + 0.004773888736963272, + 0.0028043061029165983, + 0.005374820902943611, + 0.002722355304285884, + 0.00676629226654768, + 0.0027789603918790817, + 0.007075758185237646, + 0.002833461854606867, + 0.006995974574238062, + 0.0029154156800359488, + 0.008517388254404068, + 0.002774328924715519, + 0.009080596268177032, + 0.0028916397131979465, + 0.009109988808631897, + 0.0032102405093610287, + 0.010673855431377888, + 0.002877794438973069, + 0.009562615305185318, + 0.0026486723218113184, + 0.008646698668599129, + 0.0031627838034182787, + 0.009536299854516983, + 0.0030164923518896103, + 0.009432275779545307, + 0.0027947286143898964, + 0.01090320572257042, + 0.0031009595841169357, + 0.010227161459624767, + 0.002871215809136629, + 0.008522190153598785, + 0.003154836595058441, + 0.010072597302496433, + 0.0032395559828728437, + 0.009548105299472809, + 0.0029102955013513565, + 0.010121389292180538, + 0.0026868272107094526, + 0.009201278910040855, + 0.0029280087910592556, + 0.01199205219745636, + 0.0029097299557179213, + 0.008808780461549759, + 0.0027156553696841, + 0.010769994929432869, + 0.0031467543449252844, + 0.010386301204562187, + 0.003253829199820757, + 0.009710980579257011, + 0.0025951166171580553, + 0.009926634840667248, + 0.003180553438141942, + 0.010113959200680256, + 0.0028141052462160587, + 0.009518159553408623, + 0.0030968233477324247 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00016737460466537552, + 0.00016955889217530398, + 0.0001941533654368659, + 0.00020677099590829626, + 0.000241377514962491, + 0.00026501239176498695, + 0.00032262838018369365, + 0.00031000316290161927, + 0.0003190796766484002, + 0.00034607256602154805, + 0.0003482012935130775, + 0.00036880192147718024, + 0.00040270042689994603, + 0.0003782755449508064, + 0.00035486502238655745, + 0.0003672093084735214, + 0.00036789155274553697, + 0.0003751571652154002, + 0.0003805970858161285, + 0.00035647119760667416, + 0.00036940450439703476, + 0.00035111776542448423, + 0.00035831874545173094, + 0.000360763035713257, + 0.0004080638981985186, + 0.0003616992623202981, + 0.00038377302017183443, + 0.00035823205404444226, + 0.0003532695727353617, + 0.0003701571311252852, + 0.0003662503537512869, + 0.0003823335970855767 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023460991349982907, + "first_to_last_ratio": 0.6290276114781215, + "imbalance_abs_log_ratio": 0.46358012582366337, + "last_quartile_mean": 0.0003729723611790755, + "mean": 0.0003332995315678913, + "normalized": [ + 0.5021747371741573, + 0.5087282642662994, + 0.5825191668393267, + 0.6203758971265705, + 0.7242059832098016, + 0.7951178044515355, + 0.9679832991843735, + 0.9301038061569352, + 0.9573361088970069, + 1.0383229895150783, + 1.0447098196480689, + 1.1065179712143018, + 1.208223800992406, + 1.1349417239542496, + 1.0647030336863026, + 1.101739647656009, + 1.1037865880426523, + 1.125585635991159, + 1.1419070528714588, + 1.0695220480202294, + 1.1083259033077553, + 1.0534601227093638, + 1.0750652536658107, + 1.0823988681177357, + 1.224315846706788, + 1.085207832782753, + 1.1514358222062546, + 1.0748051530683667, + 1.0599161993223583, + 1.1105840124767359, + 1.0988624917304564, + 1.1471171150076982 + ], + "population_cv": 0.19654387017806715 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931430101394653, + "mean_weights": [ + 0.4986528158187866, + 0.5013471841812134 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931370496749878, + "mean_weights": [ + 0.4977952837944031, + 0.5022047162055969 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931354999542236, + "mean_weights": [ + 0.497627854347229, + 0.502372145652771 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931448578834534, + "mean_weights": [ + 0.49909159541130066, + 0.500908374786377 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931440234184265, + "mean_weights": [ + 0.49918830394744873, + 0.5008116364479065 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931461691856384, + "mean_weights": [ + 0.49945223331451416, + 0.5005477666854858 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931438446044922, + "mean_weights": [ + 0.4989117980003357, + 0.5010882019996643 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931412220001221, + "mean_weights": [ + 0.498369038105011, + 0.501630961894989 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.33209967613220215, + 0.33376386761665344, + 0.334136426448822 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985990762710571, + "mean_weights": [ + 0.33337461948394775, + 0.3353745937347412, + 0.33125078678131104 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986008644104004, + "mean_weights": [ + 0.33504414558410645, + 0.33362600207328796, + 0.3313298523426056 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986016988754272, + "mean_weights": [ + 0.33131715655326843, + 0.33385908603668213, + 0.33482375741004944 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986084938049316, + "mean_weights": [ + 0.3345443904399872, + 0.3326132297515869, + 0.3328423798084259 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985959768295288, + "mean_weights": [ + 0.331234872341156, + 0.33299222588539124, + 0.33577287197113037 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986058712005615, + "mean_weights": [ + 0.333141565322876, + 0.3320317268371582, + 0.3348267078399658 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.3325701653957367, + 0.33248791098594666, + 0.33494192361831665 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862926959991455, + "mean_weights": [ + 0.24962776899337769, + 0.2496623694896698, + 0.25036606192588806, + 0.25034379959106445 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862905502319336, + "mean_weights": [ + 0.2491619884967804, + 0.24970892071723938, + 0.25086259841918945, + 0.25026649236679077 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862870931625366, + "mean_weights": [ + 0.25067847967147827, + 0.24852469563484192, + 0.24991153180599213, + 0.2508852779865265 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862855434417725, + "mean_weights": [ + 0.24860671162605286, + 0.24958062171936035, + 0.2513701319694519, + 0.2504425346851349 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386291265487671, + "mean_weights": [ + 0.25055184960365295, + 0.24973732233047485, + 0.2504110336303711, + 0.2492997944355011 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862907886505127, + "mean_weights": [ + 0.24893227219581604, + 0.25016486644744873, + 0.25030890107154846, + 0.25059396028518677 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862874507904053, + "mean_weights": [ + 0.24930796027183533, + 0.24942311644554138, + 0.24996015429496765, + 0.25130876898765564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862826824188232, + "mean_weights": [ + 0.24865901470184326, + 0.24900351464748383, + 0.25141385197639465, + 0.25092363357543945 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094211339950562, + "mean_weights": [ + 0.19921651482582092, + 0.19924485683441162, + 0.1988956332206726, + 0.2007521539926529, + 0.20189084112644196 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609428882598877, + "mean_weights": [ + 0.19964434206485748, + 0.19919463992118835, + 0.19919714331626892, + 0.20121338963508606, + 0.200750470161438 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094247102737427, + "mean_weights": [ + 0.20065948367118835, + 0.19955207407474518, + 0.20149102807044983, + 0.19979192316532135, + 0.19850550591945648 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094355583190918, + "mean_weights": [ + 0.19937850534915924, + 0.20006614923477173, + 0.20067325234413147, + 0.19996050000190735, + 0.19992157816886902 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094313859939575, + "mean_weights": [ + 0.1997273862361908, + 0.1999678611755371, + 0.1992817223072052, + 0.19971339404582977, + 0.20130965113639832 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094284057617188, + "mean_weights": [ + 0.19933168590068817, + 0.19993484020233154, + 0.19907662272453308, + 0.20011654496192932, + 0.2015402913093567 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094276905059814, + "mean_weights": [ + 0.19931891560554504, + 0.19971159100532532, + 0.20119035243988037, + 0.20086422562599182, + 0.19891491532325745 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094303131103516, + "mean_weights": [ + 0.19955140352249146, + 0.19991253316402435, + 0.20052893459796906, + 0.20107696950435638, + 0.19893015921115875 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791748046875, + "mean_weights": [ + 0.1663031131029129, + 0.1665959358215332, + 0.1656446009874344, + 0.16612613201141357, + 0.16804239153862, + 0.16728781163692474 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917547225952148, + "mean_weights": [ + 0.16606608033180237, + 0.16676852107048035, + 0.1667691171169281, + 0.165959894657135, + 0.16733849048614502, + 0.16709789633750916 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791755199432373, + "mean_weights": [ + 0.1671326905488968, + 0.166437566280365, + 0.16707676649093628, + 0.16582348942756653, + 0.1670762449502945, + 0.1664532721042633 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791752815246582, + "mean_weights": [ + 0.16595980525016785, + 0.1665806621313095, + 0.16717451810836792, + 0.16712555289268494, + 0.16733399033546448, + 0.1658255010843277 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917518615722656, + "mean_weights": [ + 0.16663867235183716, + 0.16675058007240295, + 0.16638454794883728, + 0.16550561785697937, + 0.1674773097038269, + 0.16724327206611633 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917499542236328, + "mean_weights": [ + 0.16634856164455414, + 0.16704332828521729, + 0.16620689630508423, + 0.1655295491218567, + 0.16771358251571655, + 0.1671580672264099 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917507886886597, + "mean_weights": [ + 0.16621428728103638, + 0.16654521226882935, + 0.16585756838321686, + 0.1672719269990921, + 0.1677929311990738, + 0.16631805896759033 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07942795753479, + "mean_weights": [ + 0.12488093227148056, + 0.12459862232208252, + 0.12424520403146744, + 0.12444067001342773, + 0.12461253255605698, + 0.12524709105491638, + 0.12623067200183868, + 0.1257442682981491 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11096031218767166, + 0.11101566255092621, + 0.11069408804178238, + 0.11131208389997482, + 0.11191314458847046, + 0.11117817461490631, + 0.11035752296447754, + 0.11126880347728729, + 0.11130021512508392 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216749191284, + "mean_weights": [ + 0.11095122992992401, + 0.11080076545476913, + 0.11054693907499313, + 0.11075648665428162, + 0.11089976876974106, + 0.11123757809400558, + 0.11190937459468842, + 0.1117669939994812, + 0.11113084852695465 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216510772705, + "mean_weights": [ + 0.11096195876598358, + 0.11111634969711304, + 0.11043959856033325, + 0.11095423996448517, + 0.11190803349018097, + 0.11175134778022766, + 0.11079943180084229, + 0.1112256795167923, + 0.11084336042404175 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197213649749756, + "mean_weights": [ + 0.11101563274860382, + 0.1110072135925293, + 0.11082680523395538, + 0.11142624914646149, + 0.1101548969745636, + 0.11108594387769699, + 0.11191216111183167, + 0.11077424138784409, + 0.11179685592651367 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197218894958496, + "mean_weights": [ + 0.11101803183555603, + 0.11099022626876831, + 0.11046922951936722, + 0.11147402226924896, + 0.11091943085193634, + 0.11164889484643936, + 0.11152636259794235, + 0.11112086474895477, + 0.11083294451236725 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216033935547, + "mean_weights": [ + 0.1111505776643753, + 0.11169322580099106, + 0.11175311356782913, + 0.11086191982030869, + 0.11131928861141205, + 0.11068505048751831, + 0.11144685000181198, + 0.11033272743225098, + 0.1107572466135025 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972179412841797, + "mean_weights": [ + 0.11092513799667358, + 0.11108554899692535, + 0.11051031202077866, + 0.11084482073783875, + 0.11164578050374985, + 0.11118975281715393, + 0.11111322790384293, + 0.11183710396289825, + 0.1108483225107193 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028025340288877487, + 0.014066023752093315, + 0.014108708128333092, + 0.014229794964194298, + 0.014309127815067768, + 0.01455320231616497, + 0.014605770818889141, + 0.01481270045042038, + 0.014851206913590431, + 0.010233239270746708, + 0.010343058034777641, + 0.010804706253111362, + 0.010769513435661793, + 0.01144309900701046, + 0.011491727083921432, + 0.012276813387870789, + 0.012374046258628368, + 0.009844657965004444, + 0.009941100142896175, + 0.010566700249910355, + 0.010642717592418194, + 0.011646243743598461, + 0.011756961233913898, + 0.012673276476562023, + 0.012869661673903465, + 0.011087263002991676, + 0.01123880036175251, + 0.011723686009645462, + 0.0118045499548316, + 0.012471769005060196, + 0.012632780708372593, + 0.013105461373925209, + 0.013265169225633144, + 0.011508514173328876, + 0.0116264121606946, + 0.012281306087970734, + 0.012384527362883091, + 0.012981628999114037, + 0.013078375719487667, + 0.013564801774919033, + 0.013736819848418236, + 0.012090411968529224, + 0.01221960224211216, + 0.012518657371401787, + 0.012597586959600449, + 0.013060149736702442, + 0.013144387863576412, + 0.013466374948620796, + 0.013602014631032944, + 0.012321850284934044, + 0.012410826981067657, + 0.012648497708141804, + 0.012738197110593319, + 0.013148728758096695, + 0.013268771581351757, + 0.013453229330480099, + 0.013569011352956295, + 0.01241307333111763, + 0.012489519082009792, + 0.012747267261147499, + 0.012816172093153, + 0.01302917767316103, + 0.013102042488753796, + 0.013313685543835163 + ], + "loss_nats": 4.655905723571777, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.197218418121338, + "mean_weights": [ + 0.11057741940021515, + 0.1106872707605362, + 0.11082567274570465, + 0.11127195507287979, + 0.11091846227645874, + 0.11130088567733765, + 0.11114984005689621, + 0.11173895001411438, + 0.11152952909469604 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.002443928038701415, + 0.003752076532691717, + 0.004733026959002018, + 0.005621279589831829, + 0.007003156002610922, + 0.007651050575077534, + 0.008999467827379704, + 0.009552945382893085, + 0.004773888736963272, + 0.005728233605623245, + 0.008736123330891132, + 0.00936705432832241, + 0.012760105542838573, + 0.013376436196267605, + 0.016963720321655273, + 0.017608575522899628, + 0.006995974574238062, + 0.008001524023711681, + 0.012580735608935356, + 0.01315119955688715, + 0.0186475720256567, + 0.019228288903832436, + 0.02346930466592312, + 0.02450871281325817, + 0.010673855431377888, + 0.011544129811227322, + 0.016316605731844902, + 0.016811303794384003, + 0.020413896068930626, + 0.021131416782736778, + 0.024899384006857872, + 0.02597878687083721, + 0.009432275779545307, + 0.010270251892507076, + 0.015730082988739014, + 0.016414040699601173, + 0.020732076838612556, + 0.021650852635502815, + 0.025274164974689484, + 0.026260782033205032, + 0.010072597302496433, + 0.011164487339556217, + 0.015665393322706223, + 0.016113370656967163, + 0.019619770348072052, + 0.019974395632743835, + 0.02384515292942524, + 0.024847183376550674, + 0.01199205219745636, + 0.012743011116981506, + 0.0159013532102108, + 0.016761260107159615, + 0.021420039236545563, + 0.022224903106689453, + 0.025291141122579575, + 0.026317164301872253, + 0.009710980579257011, + 0.010263347066938877, + 0.01540068257600069, + 0.016214758157730103, + 0.019683225080370903, + 0.020621437579393387, + 0.024622956290841103, + 0.025286227464675903 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 6.720469307524863, + "cross_entropy_nats": 4.658274352550507, + "step": 20 + } + ], + "forward_intervention": { + "depth_visit_counts": [ + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39 + ], + "forward_calls": 39, + "output_mixer_selected": false, + "output_visit_count": 39, + "passed": true, + "selected_depth_indices": [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ], + "selected_parameter_reachability_gate": true, + "selected_parameters": { + "mixers.40.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.40.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.41.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.41.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.42.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.42.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.43.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.43.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.44.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.44.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.45.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.45.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.46.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.46.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.47.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.47.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.48.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.48.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.49.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.49.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.50.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.50.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.51.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.51.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.52.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.52.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.53.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.53.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.54.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.54.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.55.key_norm.weight": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + }, + "mixers.55.query": { + "final_equals_initial": true, + "gradient_hook_calls": 0, + "in_optimizer_param_group": true, + "optimizer_state_present": false + } + }, + "selected_source_count_checks": { + "40": true, + "41": true, + "42": true, + "43": true, + "44": true, + "45": true, + "46": true, + "47": true, + "48": true, + "49": true, + "50": true, + "51": true, + "52": true, + "53": true, + "54": true, + "55": true + }, + "selector_gate": true, + "semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel", + "source_counts_by_depth_index": { + "0": [ + 1 + ], + "1": [ + 2 + ], + "10": [ + 3 + ], + "11": [ + 3 + ], + "12": [ + 3 + ], + "13": [ + 3 + ], + "14": [ + 3 + ], + "15": [ + 3 + ], + "16": [ + 3 + ], + "17": [ + 4 + ], + "18": [ + 4 + ], + "19": [ + 4 + ], + "2": [ + 2 + ], + "20": [ + 4 + ], + "21": [ + 4 + ], + "22": [ + 4 + ], + "23": [ + 4 + ], + "24": [ + 4 + ], + "25": [ + 5 + ], + "26": [ + 5 + ], + "27": [ + 5 + ], + "28": [ + 5 + ], + "29": [ + 5 + ], + "3": [ + 2 + ], + "30": [ + 5 + ], + "31": [ + 5 + ], + "32": [ + 5 + ], + "33": [ + 6 + ], + "34": [ + 6 + ], + "35": [ + 6 + ], + "36": [ + 6 + ], + "37": [ + 6 + ], + "38": [ + 6 + ], + "39": [ + 6 + ], + "4": [ + 2 + ], + "40": [ + 6 + ], + "41": [ + 7 + ], + "42": [ + 7 + ], + "43": [ + 7 + ], + "44": [ + 7 + ], + "45": [ + 7 + ], + "46": [ + 7 + ], + "47": [ + 7 + ], + "48": [ + 7 + ], + "49": [ + 8 + ], + "5": [ + 2 + ], + "50": [ + 8 + ], + "51": [ + 8 + ], + "52": [ + 8 + ], + "53": [ + 8 + ], + "54": [ + 8 + ], + "55": [ + 8 + ], + "56": [ + 8 + ], + "57": [ + 9 + ], + "58": [ + 9 + ], + "59": [ + 9 + ], + "6": [ + 2 + ], + "60": [ + 9 + ], + "61": [ + 9 + ], + "62": [ + 9 + ], + "63": [ + 9 + ], + "7": [ + 2 + ], + "8": [ + 2 + ], + "9": [ + 3 + ] + }, + "uniform_weight_gate": true, + "uniform_weight_max_abs_error": 0.0, + "uniform_weight_threshold": 1e-12, + "unselected_parameter_reachability_gate": true, + "unselected_parameters": { + "mixers.0.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.0.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.1.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.1.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.10.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.10.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.11.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.11.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.12.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.12.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.13.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.13.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.14.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.14.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.15.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.15.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.16.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.16.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.17.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.17.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.18.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.18.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.19.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.19.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.2.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.2.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.20.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.20.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.21.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.21.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.22.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.22.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.23.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.23.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.24.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.24.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.25.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.25.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.26.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.26.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.27.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.27.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.28.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.28.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.29.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.29.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.3.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.3.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.30.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.30.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.31.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.31.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.32.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.32.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.33.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.33.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.34.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.34.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.35.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.35.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.36.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.36.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.37.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.37.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.38.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.38.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.39.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.39.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.4.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.4.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.5.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.5.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.56.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.56.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.57.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.57.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.58.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.58.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.59.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.59.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.6.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.6.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.60.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.60.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.61.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.61.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.62.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.62.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.63.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.63.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.7.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.7.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.8.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.8.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.9.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.9.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "output_mixer.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "output_mixer.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + } + }, + "variant": "uniform_groups_6_7_forward", + "visit_gate": true + }, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "30089f6da7b0a0bb5e6cdbe490b747650c45dffe323fff195a5826f29e2bda97", + "final_model_state": "71293f52d25e18a11cac9db25eecf5c920366a19345d3d559938ba4cc52d1247", + "final_optimizer_state": "7e585c88cf40b3eb2c7b31477ccbe63c4ed526936da5da54efb9d7514f9d2e7a", + "final_public_parameters": "7688156eb65ae21c33116d50a5e129a6cb020cb4391543dccf282e44f8ab6905", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "parent_runner_canonical_sha256": "15cea1697eaa21a7245460ad114a4b39e8f1a153d60fda0b6828a1ecaa98232a", + "protocol_id": "llm-atlas-k3-attnres-forward-training-v1", + "run_kind": "smoke", + "schema_version": 2, + "seed": 2026073001, + "steps": 20, + "study_manifest": { + "file_sha256": "49546ed5baf36bcb30885062b7c671fafe4ccff2e606624cd7dbe23717f9a712", + "path": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/manifest.json", + "status": "frozen-before-model-output" + }, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 5610759680, + "peak_reserved_bytes": 11068768256, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.321983337402344 + }, + { + "bits_per_byte": 7.24805334829669, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.023967742919922, + "step": 10, + "unclipped_grad_norm": 11.155019760131836 + }, + { + "bits_per_byte": 6.733684884373789, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6674346923828125, + "step": 20, + "unclipped_grad_norm": 3.4893624782562256 + } + ], + "variant": "uniform_groups_6_7_forward" +} diff --git a/experiments/k3/attnres_forward/results/gates/step-zero.json b/experiments/k3/attnres_forward/results/gates/step-zero.json new file mode 100644 index 0000000..7bddaac --- /dev/null +++ b/experiments/k3/attnres_forward/results/gates/step-zero.json @@ -0,0 +1,6468 @@ +{ + "canonical_sha256_without_self": "00db0569d2fd00599383ecdc50578c52df784a937a96dd69576f1366e88b0dcb", + "environment": { + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "gpu": "NVIDIA GeForce RTX 5090", + "torch": "2.11.0+cu128" + }, + "gate": "step-zero-cross-variant-byte-exact", + "parent_manifest_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "passed": true, + "protocol_id": "llm-atlas-k3-attnres-forward-training-v1", + "schema_version": 1, + "seed": 2026073001, + "study_manifest_sha256": "49546ed5baf36bcb30885062b7c671fafe4ccff2e606624cd7dbe23717f9a712", + "variants": { + "learned_reference": { + "exact_vs_learned_reference": { + "diagnostic": true, + "evaluation": true, + "initial_mixer_hash": true, + "initial_public_hash": true, + "logits_sha256": true, + "loss_nats": true + }, + "passed": true, + "payload": { + "diagnostic": { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + "evaluation": { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079 + }, + "initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58", + "logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939", + "loss_nats": 5.516995429992676 + }, + "pre_reduction_uniform_weight_gate": true, + "pre_reduction_uniform_weight_max_abs_error": 0.0, + "selected_capture_checks": {} + }, + "uniform_group_6_forward": { + "exact_vs_learned_reference": { + "diagnostic": true, + "evaluation": true, + "initial_mixer_hash": true, + "initial_public_hash": true, + "logits_sha256": true, + "loss_nats": true + }, + "passed": true, + "payload": { + "diagnostic": { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + "evaluation": { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079 + }, + "initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58", + "logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939", + "loss_nats": 5.516995429992676 + }, + "pre_reduction_uniform_weight_gate": true, + "pre_reduction_uniform_weight_max_abs_error": 0.0, + "selected_capture_checks": { + "40": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 6, + "passed": true, + "source_count": 6 + }, + "41": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "42": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "43": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "44": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "45": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "46": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "47": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + } + } + }, + "uniform_group_7_forward": { + "exact_vs_learned_reference": { + "diagnostic": true, + "evaluation": true, + "initial_mixer_hash": true, + "initial_public_hash": true, + "logits_sha256": true, + "loss_nats": true + }, + "passed": true, + "payload": { + "diagnostic": { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + "evaluation": { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079 + }, + "initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58", + "logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939", + "loss_nats": 5.516995429992676 + }, + "pre_reduction_uniform_weight_gate": true, + "pre_reduction_uniform_weight_max_abs_error": 0.0, + "selected_capture_checks": { + "48": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "49": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "50": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "51": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "52": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "53": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "54": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "55": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + } + } + }, + "uniform_group_7_mlp_forward": { + "exact_vs_learned_reference": { + "diagnostic": true, + "evaluation": true, + "initial_mixer_hash": true, + "initial_public_hash": true, + "logits_sha256": true, + "loss_nats": true + }, + "passed": true, + "payload": { + "diagnostic": { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + "evaluation": { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079 + }, + "initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58", + "logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939", + "loss_nats": 5.516995429992676 + }, + "pre_reduction_uniform_weight_gate": true, + "pre_reduction_uniform_weight_max_abs_error": 0.0, + "selected_capture_checks": { + "49": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "51": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "53": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "55": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + } + } + }, + "uniform_groups_6_7_forward": { + "exact_vs_learned_reference": { + "diagnostic": true, + "evaluation": true, + "initial_mixer_hash": true, + "initial_public_hash": true, + "logits_sha256": true, + "loss_nats": true + }, + "passed": true, + "payload": { + "diagnostic": { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + "evaluation": { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079 + }, + "initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58", + "logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939", + "loss_nats": 5.516995429992676 + }, + "pre_reduction_uniform_weight_gate": true, + "pre_reduction_uniform_weight_max_abs_error": 0.0, + "selected_capture_checks": { + "40": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 6, + "passed": true, + "source_count": 6 + }, + "41": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "42": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "43": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "44": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "45": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "46": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "47": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "48": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 7, + "passed": true, + "source_count": 7 + }, + "49": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "50": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "51": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "52": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "53": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "54": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + }, + "55": { + "capture_summary_exact_vs_learned": true, + "expected_source_count": 8, + "passed": true, + "source_count": 8 + } + } + } + } +} diff --git a/experiments/k3/attnres_forward/results/gates/wrapper-smoke.json b/experiments/k3/attnres_forward/results/gates/wrapper-smoke.json new file mode 100644 index 0000000..c26d872 --- /dev/null +++ b/experiments/k3/attnres_forward/results/gates/wrapper-smoke.json @@ -0,0 +1,3428 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "a467cc1c05e1b860abbf44086273071bcd2a891db4354cc6e79b319e44d1cce5", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.9337450289167464e-05, + 3.887022103299387e-05, + 3.8419901102315634e-05, + 3.799897967837751e-05, + 3.775028380914591e-05, + 3.7539852201007307e-05, + 3.739175735972822e-05, + 3.724253110704012e-05, + 3.720997119671665e-05, + 3.713165278895758e-05, + 3.7142541259527206e-05, + 3.707727591972798e-05, + 3.688051219796762e-05, + 3.692117388709448e-05, + 3.689077857416123e-05, + 3.6904009903082624e-05, + 3.717117215273902e-05, + 3.7258152588037774e-05, + 3.729091622517444e-05, + 3.7364865420386195e-05, + 3.74124720110558e-05, + 3.742674380191602e-05, + 3.742256740224548e-05, + 3.74479714082554e-05, + 3.765593282878399e-05, + 3.763797212741338e-05, + 3.7667807191610336e-05, + 3.764635039260611e-05, + 3.751127951545641e-05, + 3.7523430364672095e-05, + 3.7504440115299076e-05, + 3.756426303880289e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.8068872072472004e-05, + "first_to_last_ratio": 1.0127680561501509, + "imbalance_abs_log_ratio": 0.01268723177426717, + "last_quartile_mean": 3.7588934446830535e-05, + "mean": 3.750672590285831e-05, + "normalized": [ + 1.0488105624322075, + 1.0363533498942825, + 1.0243469718423952, + 1.013124413386392, + 1.0064937128054956, + 1.0008832095404647, + 0.9969347219635258, + 0.9929560688260969, + 0.9920879602525092, + 0.9899998438980743, + 0.9902901510445264, + 0.9885500540824973, + 0.9833039624276305, + 0.9843880796932157, + 0.9835776833655818, + 0.9839304555311837, + 0.9910535046170554, + 0.9933725669506762, + 0.9942461072650594, + 0.9962177321785024, + 0.997487013607996, + 0.9978675264498043, + 0.9977561757634937, + 0.9984334944416348, + 1.0039781378495187, + 1.0034992717011608, + 1.0042947307416068, + 1.0037226520413813, + 1.0001214078938774, + 1.0004453724341882, + 0.9999390565957383, + 1.001534048482227 + ], + "population_cv": 0.013988937740460563 + }, + "activation_output_rms_by_block": [ + 0.0037520017940551043, + 0.005621022544801235, + 0.0076507688499987125, + 0.009552493691444397, + 0.005728918593376875, + 0.009368892759084702, + 0.013378600589931011, + 0.01761116087436676, + 0.00800272449851036, + 0.013154285028576851, + 0.01923154480755329, + 0.024512210860848427, + 0.011547097004950047, + 0.016813727095723152, + 0.02113443799316883, + 0.025982216000556946, + 0.010269254446029663, + 0.01641162671148777, + 0.02165052480995655, + 0.026260115206241608, + 0.011169232428073883, + 0.016121258959174156, + 0.019987935200333595, + 0.02486182004213333, + 0.01274633314460516, + 0.016767995432019234, + 0.022238746285438538, + 0.02633264660835266, + 0.0102624436840415, + 0.016213588416576385, + 0.020620016381144524, + 0.02528378553688526 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.00908298246213235, + "first_to_last_ratio": 0.4829268696139592, + "imbalance_abs_log_ratio": 0.7278900454597648, + "last_quartile_mean": 0.018808194436132908, + "mean": 0.015944982071232516, + "normalized": [ + 0.2353092513552812, + 0.35252611258450545, + 0.47982298229121323, + 0.5990909020009959, + 0.3592928839796457, + 0.5875762492068142, + 0.8390477035448226, + 1.1044954955540724, + 0.5018961114386356, + 0.8249796061112817, + 1.2061189358281121, + 1.537299367998203, + 0.7241837559531027, + 1.0544839135352808, + 1.3254601290081713, + 1.62949170368986, + 0.6440430224476176, + 1.0292659244250366, + 1.3578268519359336, + 1.6469203344931544, + 0.7004857313841133, + 1.0110553205487558, + 1.2535564550050677, + 1.5592253369157636, + 0.7993946363603464, + 1.0516158222762493, + 1.394717547256516, + 1.6514691889093607, + 0.6436158810461574, + 1.016845822977028, + 1.2931978404884237, + 1.5856891794504773 + ], + "population_cv": 0.416991705473353 + }, + "bits_per_byte": 6.71703146032544, + "branch_output_rms_by_sublayer": [ + 0.002443866338580847, + 0.0027545092161744833, + 0.0026208641938865185, + 0.002763434313237667, + 0.0034182921517640352, + 0.002741520293056965, + 0.0035785960499197245, + 0.002719961805269122, + 0.0047746808268129826, + 0.0028042772319167852, + 0.005376013461500406, + 0.0027222761418670416, + 0.006767150945961475, + 0.0027788877487182617, + 0.007074786350131035, + 0.002833561971783638, + 0.006997238378971815, + 0.0029156480450183153, + 0.008518846705555916, + 0.0027741468511521816, + 0.009081361815333366, + 0.0028916175942867994, + 0.00911131501197815, + 0.0032094581983983517, + 0.010676512494683266, + 0.002878024475648999, + 0.009562043473124504, + 0.0026487084105610847, + 0.008648304268717766, + 0.003162843408063054, + 0.009535389021039009, + 0.0030166516080498695, + 0.009431489743292332, + 0.002794384490698576, + 0.010901342146098614, + 0.00310111534781754, + 0.010227455757558346, + 0.0028713408391922712, + 0.008522198535501957, + 0.0031550333369523287, + 0.010077033191919327, + 0.0032391855493187904, + 0.009554130956530571, + 0.00291044800542295, + 0.010129180736839771, + 0.0026873303577303886, + 0.00920273270457983, + 0.002927709138020873, + 0.011994888074696064, + 0.0029113248456269503, + 0.008809582330286503, + 0.0027162786573171616, + 0.01077704131603241, + 0.0031467911321669817, + 0.010385709814727306, + 0.003254092764109373, + 0.009710066020488739, + 0.0025947801768779755, + 0.009926802478730679, + 0.0031805087346583605, + 0.0101145189255476, + 0.002813765313476324, + 0.009518872946500778, + 0.0030965779442340136 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00016739224645496468, + 0.0001695381430192955, + 0.0001941383401676919, + 0.00020674849822850373, + 0.00024134787186321302, + 0.00026490993231531454, + 0.000322331199796065, + 0.0003097008723390916, + 0.00031886187327784883, + 0.00034582398876540076, + 0.00034801623779775657, + 0.00036856862737498307, + 0.00040270040252095, + 0.0003782002933862871, + 0.0003548760100734237, + 0.0003670443346435906, + 0.0003676146253339179, + 0.0003749331083729355, + 0.00038033249912268637, + 0.0003562102847256418, + 0.00036931756079357426, + 0.00035104688530666954, + 0.0003583944935512815, + 0.0003605180195562536, + 0.0004079256274935599, + 0.0003614699495467052, + 0.00038354947558429407, + 0.0003579111697278514, + 0.0003529832358531257, + 0.00036980562702627114, + 0.00036596233134640236, + 0.0003819940876588112 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002345133880230175, + "first_to_last_ratio": 0.6292279841951004, + "imbalance_abs_log_ratio": 0.46326163295248246, + "last_quartile_mean": 0.00037270018802962763, + "mean": 0.0003331302454070113, + "normalized": [ + 0.502482883985657, + 0.5089244983209419, + 0.582770081205019, + 0.6206236181764371, + 0.7244850180695525, + 0.7952142922107038, + 0.9675831127319218, + 0.9296690306840971, + 0.9571687881064967, + 1.0381044457337716, + 1.044685202247421, + 1.1063799593599613, + 1.2088377085933444, + 1.1352925728019987, + 1.0652770649505088, + 1.1018042933782364, + 1.1035162084570687, + 1.1254850423889022, + 1.141693089614257, + 1.069282329169577, + 1.1086281293442752, + 1.0537826875424299, + 1.0758389503582988, + 1.0822134121018658, + 1.224522939954507, + 1.0850709430633327, + 1.1513499025454192, + 1.07438809493435, + 1.0595952805842004, + 1.1100932206694432, + 1.0985563046047577, + 1.146680894111248 + ], + "population_cv": 0.19646899587719932 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931430697441101, + "mean_weights": [ + 0.49865636229515076, + 0.5013436079025269 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931370496749878, + "mean_weights": [ + 0.49779537320137024, + 0.5022045969963074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931354999542236, + "mean_weights": [ + 0.49762701988220215, + 0.5023729801177979 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931448578834534, + "mean_weights": [ + 0.49909353256225586, + 0.5009064674377441 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931440234184265, + "mean_weights": [ + 0.49918538331985474, + 0.5008145570755005 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931461691856384, + "mean_weights": [ + 0.49945560097694397, + 0.5005444288253784 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931438446044922, + "mean_weights": [ + 0.49891161918640137, + 0.5010883808135986 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931411623954773, + "mean_weights": [ + 0.49836465716362, + 0.5016353130340576 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986062288284302, + "mean_weights": [ + 0.3321005403995514, + 0.3337631821632385, + 0.3341362476348877 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985990762710571, + "mean_weights": [ + 0.33337411284446716, + 0.33537548780441284, + 0.3312504291534424 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986008644104004, + "mean_weights": [ + 0.3350442051887512, + 0.33362624049186707, + 0.3313295841217041 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986016988754272, + "mean_weights": [ + 0.3313179612159729, + 0.3338589072227478, + 0.3348231613636017 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986084938049316, + "mean_weights": [ + 0.33454394340515137, + 0.3326125741004944, + 0.33284348249435425 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985959768295288, + "mean_weights": [ + 0.3312351107597351, + 0.332991361618042, + 0.3357735574245453 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986058712005615, + "mean_weights": [ + 0.33314070105552673, + 0.3320331275463104, + 0.33482617139816284 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.33257031440734863, + 0.3324887454509735, + 0.33494094014167786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862926959991455, + "mean_weights": [ + 0.24962691962718964, + 0.2496621459722519, + 0.25036555528640747, + 0.2503453493118286 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862905502319336, + "mean_weights": [ + 0.24916216731071472, + 0.24970784783363342, + 0.25086167454719543, + 0.2502683103084564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386286973953247, + "mean_weights": [ + 0.25067827105522156, + 0.24852371215820312, + 0.24991214275360107, + 0.25088587403297424 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862855434417725, + "mean_weights": [ + 0.24860653281211853, + 0.24958111345767975, + 0.2513706088066101, + 0.2504417300224304 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386291265487671, + "mean_weights": [ + 0.25055181980133057, + 0.24973659217357635, + 0.2504120469093323, + 0.2492995262145996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862907886505127, + "mean_weights": [ + 0.24893252551555634, + 0.25016576051712036, + 0.2503091096878052, + 0.25059258937835693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862874507904053, + "mean_weights": [ + 0.24930882453918457, + 0.24942341446876526, + 0.2499566376209259, + 0.25131112337112427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862828016281128, + "mean_weights": [ + 0.24865929782390594, + 0.24900496006011963, + 0.25141197443008423, + 0.2509238123893738 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094212532043457, + "mean_weights": [ + 0.19921688735485077, + 0.1992446780204773, + 0.19889645278453827, + 0.20075249671936035, + 0.20188948512077332 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609428882598877, + "mean_weights": [ + 0.19964422285556793, + 0.19919568300247192, + 0.19919705390930176, + 0.20121362805366516, + 0.20074939727783203 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094245910644531, + "mean_weights": [ + 0.20065900683403015, + 0.19955331087112427, + 0.2014925479888916, + 0.19979336857795715, + 0.19850176572799683 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094355583190918, + "mean_weights": [ + 0.19937855005264282, + 0.20006650686264038, + 0.20067161321640015, + 0.1999610960483551, + 0.19992223381996155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094313859939575, + "mean_weights": [ + 0.1997283697128296, + 0.19996783137321472, + 0.19928213953971863, + 0.1997123807668686, + 0.20130927860736847 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094284057617188, + "mean_weights": [ + 0.19933226704597473, + 0.19993427395820618, + 0.19907569885253906, + 0.2001166194677353, + 0.20154115557670593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094276905059814, + "mean_weights": [ + 0.19932052493095398, + 0.19971224665641785, + 0.2011900395154953, + 0.20086491107940674, + 0.19891227781772614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094303131103516, + "mean_weights": [ + 0.19955268502235413, + 0.19991174340248108, + 0.20052853226661682, + 0.20107966661453247, + 0.1989273726940155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917479276657104, + "mean_weights": [ + 0.16630326211452484, + 0.1665954291820526, + 0.1656438112258911, + 0.16612562537193298, + 0.16804365813732147, + 0.16728821396827698 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917546033859253, + "mean_weights": [ + 0.16606619954109192, + 0.16676779091358185, + 0.16676853597164154, + 0.16595947742462158, + 0.16733971238136292, + 0.1670982539653778 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791755199432373, + "mean_weights": [ + 0.16713324189186096, + 0.16643813252449036, + 0.16707737743854523, + 0.16582445800304413, + 0.16707313060760498, + 0.16645365953445435 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791752815246582, + "mean_weights": [ + 0.16595935821533203, + 0.16658073663711548, + 0.16717244684696198, + 0.1671261042356491, + 0.16733530163764954, + 0.16582605242729187 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917518615722656, + "mean_weights": [ + 0.1666388213634491, + 0.16675034165382385, + 0.16638417541980743, + 0.16550619900226593, + 0.16747808456420898, + 0.1672423779964447 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917499542236328, + "mean_weights": [ + 0.16634854674339294, + 0.16704240441322327, + 0.16620638966560364, + 0.16553011536598206, + 0.16771462559700012, + 0.16715793311595917 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917507886886597, + "mean_weights": [ + 0.16621501743793488, + 0.1665443778038025, + 0.16585618257522583, + 0.16727086901664734, + 0.16779379546642303, + 0.16631974279880524 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917485237121582, + "mean_weights": [ + 0.16604092717170715, + 0.1667710244655609, + 0.16534343361854553, + 0.1669321358203888, + 0.16766314208507538, + 0.16724932193756104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9458963871002197, + "mean_weights": [ + 0.14258044958114624, + 0.14276695251464844, + 0.14234250783920288, + 0.14281943440437317, + 0.14429350197315216, + 0.14340519905090332, + 0.14179196953773499 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458930492401123, + "mean_weights": [ + 0.14285841584205627, + 0.14281482994556427, + 0.14142268896102905, + 0.14248794317245483, + 0.1441671997308731, + 0.14373424649238586, + 0.1425146758556366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459006786346436, + "mean_weights": [ + 0.1428324282169342, + 0.14242856204509735, + 0.14238804578781128, + 0.14283627271652222, + 0.14203999936580658, + 0.14380358159542084, + 0.14367109537124634 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.14253869652748108, + 0.14282220602035522, + 0.1414484828710556, + 0.14307048916816711, + 0.1438344419002533, + 0.14343075454235077, + 0.1428549587726593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14281800389289856, + 0.1431872844696045, + 0.1432121843099594, + 0.14210283756256104, + 0.14336752891540527, + 0.14217621088027954, + 0.14313597977161407 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458997249603271, + "mean_weights": [ + 0.14271901547908783, + 0.14267435669898987, + 0.14293994009494781, + 0.14278604090213776, + 0.14186438918113708, + 0.1428055316209793, + 0.14421072602272034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459013938903809, + "mean_weights": [ + 0.14281892776489258, + 0.1429084837436676, + 0.14260512590408325, + 0.14224481582641602, + 0.14402621984481812, + 0.14324231445789337, + 0.14215409755706787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459059238433838, + "mean_weights": [ + 0.14244785904884338, + 0.14288531243801117, + 0.14282266795635223, + 0.14290127158164978, + 0.1435883343219757, + 0.14224958419799805, + 0.14310497045516968 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432487487793, + "mean_weights": [ + 0.12496939301490784, + 0.12501314282417297, + 0.12449057400226593, + 0.12434226274490356, + 0.12589572370052338, + 0.12463213503360748, + 0.12490519136190414, + 0.1257515549659729 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794320106506348, + "mean_weights": [ + 0.12487496435642242, + 0.12489178031682968, + 0.12453664839267731, + 0.12434129416942596, + 0.1249510794878006, + 0.12465334683656693, + 0.12601137161254883, + 0.12573951482772827 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794363021850586, + "mean_weights": [ + 0.12491423636674881, + 0.12523731589317322, + 0.12523618340492249, + 0.12467047572135925, + 0.12580403685569763, + 0.12441615015268326, + 0.12498776614665985, + 0.1247338205575943 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079427480697632, + "mean_weights": [ + 0.12474467605352402, + 0.1245410293340683, + 0.12462934106588364, + 0.1246577650308609, + 0.12418574839830399, + 0.12514574825763702, + 0.12619128823280334, + 0.1259044110774994 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794358253479004, + "mean_weights": [ + 0.12475918978452682, + 0.12487104535102844, + 0.1250094473361969, + 0.1255180537700653, + 0.12490293383598328, + 0.12545335292816162, + 0.12531054019927979, + 0.12417542934417725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079436779022217, + "mean_weights": [ + 0.12494880706071854, + 0.12516388297080994, + 0.12451333552598953, + 0.124576136469841, + 0.12569281458854675, + 0.12472400069236755, + 0.1254362016916275, + 0.12494482100009918 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079434871673584, + "mean_weights": [ + 0.12489444017410278, + 0.12512432038784027, + 0.12540367245674133, + 0.1253792643547058, + 0.1248319000005722, + 0.12483960390090942, + 0.125506192445755, + 0.12402061372995377 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079428195953369, + "mean_weights": [ + 0.12488104403018951, + 0.12459895014762878, + 0.12424551695585251, + 0.12444087862968445, + 0.12461263686418533, + 0.12524595856666565, + 0.12623095512390137, + 0.1257440745830536 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11096074432134628, + 0.11101590096950531, + 0.1106940507888794, + 0.11131151020526886, + 0.11191274225711823, + 0.11117846518754959, + 0.11035685986280441, + 0.11126935482025146, + 0.11130036413669586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216749191284, + "mean_weights": [ + 0.11095131188631058, + 0.11080048978328705, + 0.11054704338312149, + 0.11075608432292938, + 0.1108987033367157, + 0.11123715341091156, + 0.11191022396087646, + 0.11176811158657074, + 0.11113087832927704 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216510772705, + "mean_weights": [ + 0.1109621673822403, + 0.1111164540052414, + 0.11043901741504669, + 0.11095462739467621, + 0.11190879344940186, + 0.11175167560577393, + 0.11079787462949753, + 0.1112256720662117, + 0.1108437031507492 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197213649749756, + "mean_weights": [ + 0.11101574450731277, + 0.11100780963897705, + 0.11082783341407776, + 0.11142586171627045, + 0.11015472561120987, + 0.11108498275279999, + 0.11191210150718689, + 0.11077474802732468, + 0.11179618537425995 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197218894958496, + "mean_weights": [ + 0.11101824045181274, + 0.11099033057689667, + 0.11046893894672394, + 0.11147398501634598, + 0.11092066764831543, + 0.11164902150630951, + 0.11152558028697968, + 0.11112039536237717, + 0.1108328253030777 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216033935547, + "mean_weights": [ + 0.11115090548992157, + 0.11169334501028061, + 0.1117531955242157, + 0.1108616292476654, + 0.11132021248340607, + 0.11068520694971085, + 0.1114465519785881, + 0.11033187806606293, + 0.11075706779956818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972179412841797, + "mean_weights": [ + 0.11092537641525269, + 0.11108530312776566, + 0.11051007360219955, + 0.11084415018558502, + 0.11164555698633194, + 0.11119014024734497, + 0.11111340671777725, + 0.11183696240186691, + 0.1108490377664566 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802533470094204, + 0.014066163450479507, + 0.014108670875430107, + 0.014229752123355865, + 0.01430919673293829, + 0.014553130604326725, + 0.01460581086575985, + 0.014812584035098553, + 0.014851128682494164, + 0.010233328677713871, + 0.01034309808164835, + 0.010804922319948673, + 0.010769715532660484, + 0.011443440802395344, + 0.011492080986499786, + 0.01227729581296444, + 0.012374433688819408, + 0.009845216758549213, + 0.00994168221950531, + 0.010567485354840755, + 0.01064346730709076, + 0.01164715364575386, + 0.01175782922655344, + 0.012674410827457905, + 0.012870704755187035, + 0.011088024824857712, + 0.011239404790103436, + 0.011724143289029598, + 0.011804761365056038, + 0.012471788562834263, + 0.012632876634597778, + 0.01310542318969965, + 0.013265094719827175, + 0.011508332565426826, + 0.011626233346760273, + 0.012281009927392006, + 0.012384294532239437, + 0.012981574982404709, + 0.013078294694423676, + 0.013564560562372208, + 0.01372351124882698, + 0.012140991166234016, + 0.012265652418136597, + 0.012530013918876648, + 0.012632926926016808, + 0.013094775378704071, + 0.01315103005617857, + 0.01351068913936615, + 0.013631501235067844, + 0.01232061255723238, + 0.012407775968313217, + 0.012646659277379513, + 0.012729056179523468, + 0.01315284613519907, + 0.013267731294035912, + 0.013451422564685345, + 0.01357149239629507, + 0.012415111064910889, + 0.012491659261286259, + 0.01274929754436016, + 0.012818088755011559, + 0.013031110167503357, + 0.013103967532515526, + 0.013315660879015923 + ], + "loss_nats": 4.655891418457031, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.197218418121338, + "mean_weights": [ + 0.11057738959789276, + 0.11068764328956604, + 0.11082598567008972, + 0.11127202212810516, + 0.11091771721839905, + 0.11130063980817795, + 0.11115016043186188, + 0.11173908412456512, + 0.11152935773134232 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.002443866338580847, + 0.0037520017940551043, + 0.004732752684503794, + 0.005621022544801235, + 0.007002863567322493, + 0.0076507688499987125, + 0.008998939767479897, + 0.009552493691444397, + 0.0047746808268129826, + 0.005728918593376875, + 0.008738124743103981, + 0.009368892759084702, + 0.012762236408889294, + 0.013378600589931011, + 0.01696598529815674, + 0.01761116087436676, + 0.006997238378971815, + 0.00800272449851036, + 0.012584065087139606, + 0.013154285028576851, + 0.018650829792022705, + 0.01923154480755329, + 0.023473192006349564, + 0.024512210860848427, + 0.010676512494683266, + 0.011547097004950047, + 0.01631898619234562, + 0.016813727095723152, + 0.02041666954755783, + 0.02113443799316883, + 0.02490287274122238, + 0.025982216000556946, + 0.009431489743292332, + 0.010269254446029663, + 0.015727587044239044, + 0.01641162671148777, + 0.02073165774345398, + 0.02165052480995655, + 0.02527376264333725, + 0.026260115206241608, + 0.010077033191919327, + 0.011169232428073883, + 0.015674244612455368, + 0.016121258959174156, + 0.019633492454886436, + 0.019987935200333595, + 0.02385960891842842, + 0.02486182004213333, + 0.011994888074696064, + 0.01274633314460516, + 0.015906881541013718, + 0.016767995432019234, + 0.02143402211368084, + 0.022238746285438538, + 0.02530583366751671, + 0.02633264660835266, + 0.009710066020488739, + 0.0102624436840415, + 0.015399535186588764, + 0.016213588416576385, + 0.01968187838792801, + 0.020620016381144524, + 0.024621067568659782, + 0.02528378553688526 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 6.720464749984615, + "cross_entropy_nats": 4.6582711935043335, + "step": 20 + } + ], + "forward_intervention": { + "depth_visit_counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "forward_calls": 0, + "output_mixer_selected": false, + "output_visit_count": 0, + "passed": true, + "selected_depth_indices": [], + "selected_parameter_reachability_gate": true, + "selected_parameters": {}, + "selected_source_count_checks": {}, + "selector_gate": true, + "semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel", + "source_counts_by_depth_index": { + "0": [], + "1": [], + "10": [], + "11": [], + "12": [], + "13": [], + "14": [], + "15": [], + "16": [], + "17": [], + "18": [], + "19": [], + "2": [], + "20": [], + "21": [], + "22": [], + "23": [], + "24": [], + "25": [], + "26": [], + "27": [], + "28": [], + "29": [], + "3": [], + "30": [], + "31": [], + "32": [], + "33": [], + "34": [], + "35": [], + "36": [], + "37": [], + "38": [], + "39": [], + "4": [], + "40": [], + "41": [], + "42": [], + "43": [], + "44": [], + "45": [], + "46": [], + "47": [], + "48": [], + "49": [], + "5": [], + "50": [], + "51": [], + "52": [], + "53": [], + "54": [], + "55": [], + "56": [], + "57": [], + "58": [], + "59": [], + "6": [], + "60": [], + "61": [], + "62": [], + "63": [], + "7": [], + "8": [], + "9": [] + }, + "uniform_weight_gate": true, + "uniform_weight_max_abs_error": 0.0, + "uniform_weight_threshold": 1e-12, + "unselected_parameter_reachability_gate": true, + "unselected_parameters": { + "mixers.0.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.0.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.1.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.1.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.10.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.10.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.11.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.11.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.12.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.12.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.13.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.13.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.14.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.14.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.15.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.15.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.16.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.16.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.17.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.17.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.18.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.18.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.19.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.19.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.2.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.2.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.20.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.20.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.21.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.21.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.22.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.22.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.23.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.23.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.24.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.24.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.25.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.25.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.26.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.26.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.27.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.27.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.28.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.28.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.29.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.29.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.3.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.3.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.30.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.30.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.31.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.31.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.32.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.32.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.33.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.33.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.34.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.34.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.35.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.35.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.36.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.36.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.37.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.37.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.38.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.38.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.39.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.39.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.4.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.4.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.40.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.40.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.41.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.41.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.42.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.42.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.43.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.43.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.44.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.44.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.45.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.45.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.46.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.46.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.47.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.47.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.48.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.48.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.49.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.49.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.5.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.5.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.50.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.50.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.51.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.51.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.52.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.52.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.53.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.53.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.54.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.54.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.55.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.55.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.56.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.56.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.57.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.57.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.58.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.58.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.59.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.59.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.6.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.6.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.60.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.60.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.61.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.61.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.62.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.62.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.63.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.63.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.7.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.7.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.8.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.8.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.9.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "mixers.9.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "output_mixer.key_norm.weight": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + }, + "output_mixer.query": { + "gradient_hook_calls": 23, + "in_optimizer_param_group": true, + "optimizer_state_present": true + } + }, + "variant": "learned_reference", + "visit_gate": true + }, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "0f212e3b0862c8a0fa1714765ff7cc8edeb23e1470b2107592e11f4eeae72674", + "final_model_state": "ac8be80504f01ca19f023c383b7139873a7635445148dec83aff2b39b674dd20", + "final_optimizer_state": "de50114df8c8cb4b59e160333f861411e6676277c7a3a849bd21ada1ea6abc54", + "final_public_parameters": "063f8f52a2b325c61dff3a9a7bdd1302f383a71c59990ca1765419e2e5dcbb96", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "parent_runner_canonical_sha256": "fc6cf8d746ae7004851f9d52a5fc29a99061d0d6a5ec28f2f4c70a6313eb712a", + "protocol_id": "llm-atlas-k3-attnres-forward-training-v1", + "run_kind": "smoke", + "schema_version": 2, + "seed": 2026073001, + "steps": 20, + "study_manifest": { + "file_sha256": "49546ed5baf36bcb30885062b7c671fafe4ccff2e606624cd7dbe23717f9a712", + "path": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/manifest.json", + "status": "frozen-before-model-output" + }, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 6751356416, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + } + ], + "variant": "learned_reference" +} diff --git a/experiments/k3/attnres_forward/run_matrix.py b/experiments/k3/attnres_forward/run_matrix.py new file mode 100644 index 0000000..0106cbc --- /dev/null +++ b/experiments/k3/attnres_forward/run_matrix.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Run the frozen Round 08 matrix with at most two isolated processes.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any + + +VARIANTS = ( + "uniform_group_6_forward", + "uniform_group_7_forward", + "uniform_groups_6_7_forward", + "uniform_group_7_mlp_forward", +) +SEEDS = (2026073001, 2026073002, 2026073003) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--cache-dir", type=Path, required=True) + parser.add_argument("--parent-manifest", type=Path, required=True) + parser.add_argument("--study-manifest", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--phase", choices=("formal", "replay", "all"), default="all" + ) + parser.add_argument("--concurrency", type=int, default=2) + return parser.parse_args() + + +def cell_output( + output_dir: Path, variant: str, seed: int, run_kind: str +) -> Path: + return output_dir / ( + f"{run_kind}-{variant}-seed-{seed}.json" + ) + + +def command_for( + args: argparse.Namespace, variant: str, seed: int, run_kind: str +) -> list[str]: + runner = Path(__file__).resolve().parent / "train.py" + return [ + str(args.python), + str(runner), + "--variant", + variant, + "--study-manifest", + str(args.study_manifest), + "--run-kind", + run_kind, + "--architecture", + "block", + "--depth", + "32", + "--seed", + str(seed), + "--cache-dir", + str(args.cache_dir), + "--manifest", + str(args.parent_manifest), + "--output", + str(cell_output(args.output_dir, variant, seed, run_kind)), + ] + + +def validate_manifest(args: argparse.Namespace) -> None: + manifest = json.loads(args.study_manifest.read_text()) + if ( + manifest["status"] != "frozen-before-model-output" + or tuple(manifest["variants"]) != VARIANTS + or tuple(manifest["formal_seeds"]) != SEEDS + or manifest["concurrency_maximum"] != 2 + ): + raise RuntimeError("study manifest matrix/concurrency drift") + if args.concurrency < 1 or args.concurrency > 2: + raise ValueError("the frozen protocol permits one or two processes") + + +def run_cells( + args: argparse.Namespace, + cells: list[tuple[str, int, str]], +) -> None: + args.output_dir.mkdir(parents=True, exist_ok=True) + for variant, seed, run_kind in cells: + output = cell_output(args.output_dir, variant, seed, run_kind) + if output.exists(): + raise FileExistsError( + f"refusing to overwrite existing result: {output}" + ) + + environment = dict(os.environ) + environment["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + pending = list(cells) + running: list[dict[str, Any]] = [] + completed = 0 + while pending or running: + while pending and len(running) < args.concurrency: + variant, seed, run_kind = pending.pop(0) + command = command_for(args, variant, seed, run_kind) + process = subprocess.Popen(command, env=environment) + running.append( + { + "identity": (variant, seed, run_kind), + "process": process, + "started": time.monotonic(), + } + ) + print( + json.dumps( + { + "event": "cell_started", + "variant": variant, + "seed": seed, + "run_kind": run_kind, + "pid": process.pid, + "active": len(running), + "remaining": len(pending), + }, + sort_keys=True, + ), + flush=True, + ) + time.sleep(1) + survivors = [] + for item in running: + return_code = item["process"].poll() + if return_code is None: + survivors.append(item) + continue + variant, seed, run_kind = item["identity"] + elapsed = time.monotonic() - item["started"] + if return_code != 0: + for survivor in survivors: + survivor["process"].terminate() + for survivor in running: + if survivor is not item and survivor not in survivors: + survivor["process"].terminate() + raise RuntimeError( + f"cell failed: {variant}/{seed}/{run_kind}: {return_code}" + ) + completed += 1 + print( + json.dumps( + { + "event": "cell_completed", + "variant": variant, + "seed": seed, + "run_kind": run_kind, + "elapsed_seconds": elapsed, + "completed": completed, + "total": len(cells), + }, + sort_keys=True, + ), + flush=True, + ) + running = survivors + + +def main() -> None: + args = parse_args() + validate_manifest(args) + formal = [ + (variant, seed, "formal") + for variant in VARIANTS + for seed in SEEDS + ] + replay = [ + ("uniform_groups_6_7_forward", 2026073001, "replay") + ] + cells = ( + formal + if args.phase == "formal" + else replay + if args.phase == "replay" + else formal + replay + ) + run_cells(args, cells) + + +if __name__ == "__main__": + main() diff --git a/experiments/k3/attnres_forward/train.py b/experiments/k3/attnres_forward/train.py new file mode 100644 index 0000000..b43a545 --- /dev/null +++ b/experiments/k3/attnres_forward/train.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Run one preregistered Round 08 train-time uniform-forward cell.""" + +from __future__ import annotations + +import importlib.util +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + +import torch +import torch.nn.functional as F + + +PROTOCOL_ID = "llm-atlas-k3-attnres-forward-training-v1" +PARENT_PROTOCOL_ID = "llm-atlas-k3-attnres-gradient-scale-v1" +VARIANTS = { + "learned_reference": (), + "uniform_group_6_forward": tuple(range(40, 48)), + "uniform_group_7_forward": tuple(range(48, 56)), + "uniform_groups_6_7_forward": tuple(range(40, 56)), + "uniform_group_7_mlp_forward": (49, 51, 53, 55), +} +FORMAL_VARIANTS = tuple(name for name in VARIANTS if name != "learned_reference") +EXPECTED_SOURCE_COUNTS = { + **{40: 6}, + **{index: 7 for index in range(41, 49)}, + **{index: 8 for index in range(49, 56)}, +} + + +def load_parent_module() -> Any: + path = Path(__file__).resolve().parents[1] / "attnres_gradient" / "train.py" + spec = importlib.util.spec_from_file_location("k3_attnres_round05_train", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import Round 05 runner from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +parent = load_parent_module() +ACTIVE_VARIANT = "learned_reference" +LAST_MODEL: ForwardInterventionLanguageModel | None = None +LAST_OPTIMIZER: torch.optim.Optimizer | None = None + + +def extract_wrapper_argument(name: str) -> str: + try: + index = sys.argv.index(name) + except ValueError as error: + raise ValueError(f"missing required wrapper argument: {name}") from error + if index + 1 >= len(sys.argv): + raise ValueError(f"missing value for wrapper argument: {name}") + value = sys.argv[index + 1] + del sys.argv[index : index + 2] + return value + + +def argument_value(name: str, default: str | None = None) -> str | None: + try: + index = sys.argv.index(name) + except ValueError: + return default + if index + 1 >= len(sys.argv): + raise ValueError(f"missing value for argument: {name}") + return sys.argv[index + 1] + + +def parameter_names_for_indices(indices: tuple[int, ...]) -> tuple[str, ...]: + names = [] + for index in indices: + names.extend( + ( + f"mixers.{index}.query", + f"mixers.{index}.key_norm.weight", + ) + ) + return tuple(names) + + +class ForwardInterventionLanguageModel(parent.GradientLanguageModel): + """Round 05 model with one frozen selector and parameter-free uniform mixers.""" + + def __init__(self, architecture: str): + super().__init__(architecture) + global LAST_MODEL + if architecture != "block": + raise ValueError("Round 08 only permits the block architecture") + if ACTIVE_VARIANT not in VARIANTS: + raise ValueError(f"unknown Round 08 variant: {ACTIVE_VARIANT}") + self.forward_variant = ACTIVE_VARIANT + self.selected_indices = tuple(VARIANTS[ACTIVE_VARIANT]) + self.selected_set = frozenset(self.selected_indices) + self.forward_calls = 0 + self.depth_visits = [0] * len(self.mixers) + self.output_visits = 0 + self.source_counts: dict[int, set[int]] = { + index: set() for index in range(len(self.mixers)) + } + self.uniform_weight_max_abs_error = 0.0 + selected_names = parameter_names_for_indices(self.selected_indices) + named_parameters = dict(self.named_parameters()) + self.selected_initial_tensors = { + name: named_parameters[name].detach().cpu().clone() + for name in selected_names + } + self.gradient_hook_calls = { + name: 0 + for name in named_parameters + if name.startswith("mixers.") or name.startswith("output_mixer.") + } + self._gradient_hooks = [] + for name, parameter in named_parameters.items(): + if name not in self.gradient_hook_calls: + continue + + def count_hook( + gradient: torch.Tensor, *, parameter_name: str = name + ) -> torch.Tensor: + self.gradient_hook_calls[parameter_name] += 1 + return gradient + + self._gradient_hooks.append(parameter.register_hook(count_hook)) + LAST_MODEL = self + + def mix( + self, + mixer_index: int, + sources: list[torch.Tensor], + capture: bool, + ) -> tuple[torch.Tensor, dict[str, Any] | None]: + self.depth_visits[mixer_index] += 1 + self.source_counts[mixer_index].add(len(sources)) + if mixer_index not in self.selected_set: + return self.mixers[mixer_index](sources, capture) + + values = torch.stack(sources, dim=0) + logits = torch.zeros( + values.shape[0], + values.shape[1], + values.shape[2], + dtype=torch.float32, + device=values.device, + ) + weights = torch.softmax(logits, dim=0) + expected = torch.tensor( + 1.0 / len(sources), dtype=weights.dtype, device=weights.device + ) + error = (weights - expected).abs().max().detach().cpu().item() + self.uniform_weight_max_abs_error = max( + self.uniform_weight_max_abs_error, error + ) + output = torch.einsum( + "nbt,nbtd->btd", weights, values.float() + ).to(values.dtype) + if not capture: + return output, None + entropy = -(weights * torch.log(weights.clamp_min(1e-30))).sum(dim=0) + return output, { + "mean_weights": weights.mean(dim=(1, 2)).detach().cpu().tolist(), + "entropy_mean": entropy.mean().detach().cpu().item(), + "sources": len(sources), + } + + def forward( + self, input_ids: torch.Tensor, capture: bool = False + ) -> tuple[torch.Tensor, parent.ActivationTrace | None]: + if not self.selected_indices: + return super().forward(input_ids, capture) + + self.forward_calls += 1 + embedded = self.embed(input_ids) + trace = parent.ActivationTrace([], [], [], [], []) if capture else None + completed = [embedded] + partial: torch.Tensor | None = None + mixer_index = 0 + for block in self.blocks: + for branch_index in range(2): + sources = completed + ([] if partial is None else [partial]) + branch_input, weights = self.mix( + mixer_index, sources, capture + ) + mixer_index += 1 + if branch_index == 0: + branch_output = block.attention( + block.attention_norm(branch_input) + ) + else: + branch_output = block.mlp(block.mlp_norm(branch_input)) + branch_for_residual = branch_output.float() + partial = ( + branch_for_residual + if partial is None + else partial + branch_for_residual + ) + if trace is not None: + trace.layer_input_rms.append(parent.rms(branch_input)) + trace.branch_output_rms.append(parent.rms(branch_output)) + trace.stream_state_rms.append(parent.rms(partial)) + trace.depth_weights.append(weights or {}) + if branch_index == 1: + partial.retain_grad() + trace.block_outputs.append(partial) + if mixer_index % parent.round04.SUBLAYERS_PER_BLOCK == 0: + completed.append(partial) + partial = None + if partial is not None or len(completed) != parent.BLOCK_GROUPS + 1: + raise RuntimeError("Round 08 Block AttnRes aggregation failed") + if self.output_mixer is None: + raise RuntimeError("Round 08 output mixer missing") + self.output_visits += 1 + hidden, output_weights = self.output_mixer(completed, capture) + if trace is not None: + trace.output_weights = output_weights + normalized = self.final_norm(hidden) + logits = F.linear(normalized, self.token_embedding.weight) + return logits, trace + + +def tensor_exact(left: torch.Tensor, right: torch.Tensor) -> bool: + return ( + left.dtype == right.dtype + and tuple(left.shape) == tuple(right.shape) + and torch.equal(left.detach().cpu(), right.detach().cpu()) + ) + + +def build_intervention_audit( + model: ForwardInterventionLanguageModel, + optimizer: torch.optim.Optimizer, + study_manifest: dict[str, Any], +) -> dict[str, Any]: + selected = tuple(model.selected_indices) + selected_names = set(parameter_names_for_indices(selected)) + mixer_parameters = { + name: parameter + for name, parameter in model.named_parameters() + if name.startswith("mixers.") or name.startswith("output_mixer.") + } + optimizer_parameters = { + parameter + for group in optimizer.param_groups + for parameter in group["params"] + } + selected_parameter_checks = {} + for name in sorted(selected_names): + parameter = mixer_parameters[name] + selected_parameter_checks[name] = { + "gradient_hook_calls": model.gradient_hook_calls[name], + "in_optimizer_param_group": parameter in optimizer_parameters, + "optimizer_state_present": parameter in optimizer.state, + "final_equals_initial": tensor_exact( + parameter, model.selected_initial_tensors[name] + ), + } + unselected_parameter_checks = {} + for name, parameter in sorted(mixer_parameters.items()): + if name in selected_names: + continue + unselected_parameter_checks[name] = { + "gradient_hook_calls": model.gradient_hook_calls[name], + "in_optimizer_param_group": parameter in optimizer_parameters, + "optimizer_state_present": parameter in optimizer.state, + } + + source_counts = { + str(index): sorted(values) + for index, values in model.source_counts.items() + } + selected_source_gate = { + str(index): ( + source_counts[str(index)] + == [study_manifest["selected_source_counts"][str(index)]] + ) + for index in selected + } + visit_gate = ( + all(value == model.forward_calls for value in model.depth_visits) + and model.output_visits == model.forward_calls + ) + selected_parameter_gate = all( + check["gradient_hook_calls"] == 0 + and check["in_optimizer_param_group"] + and not check["optimizer_state_present"] + and check["final_equals_initial"] + for check in selected_parameter_checks.values() + ) + unselected_parameter_gate = all( + check["gradient_hook_calls"] > 0 + and check["in_optimizer_param_group"] + and check["optimizer_state_present"] + for check in unselected_parameter_checks.values() + ) + expected_selected = tuple( + study_manifest["variants"] + .get(model.forward_variant, {"selected_depth_indices": []})[ + "selected_depth_indices" + ] + ) + selector_gate = ( + selected == expected_selected + and 64 not in selected + and selected_source_gate == { + str(index): True for index in selected + } + ) + threshold = study_manifest["thresholds"][ + "uniform_weight_max_abs_error" + ] + uniform_gate = model.uniform_weight_max_abs_error <= threshold + passed = ( + visit_gate + and selector_gate + and selected_parameter_gate + and unselected_parameter_gate + and uniform_gate + ) + return { + "passed": passed, + "variant": model.forward_variant, + "selected_depth_indices": list(selected), + "output_mixer_selected": False, + "forward_calls": model.forward_calls, + "depth_visit_counts": model.depth_visits, + "output_visit_count": model.output_visits, + "visit_gate": visit_gate, + "source_counts_by_depth_index": source_counts, + "selected_source_count_checks": selected_source_gate, + "selector_gate": selector_gate, + "uniform_weight_max_abs_error": model.uniform_weight_max_abs_error, + "uniform_weight_threshold": threshold, + "uniform_weight_gate": uniform_gate, + "selected_parameters": selected_parameter_checks, + "selected_parameter_reachability_gate": selected_parameter_gate, + "unselected_parameters": unselected_parameter_checks, + "unselected_parameter_reachability_gate": unselected_parameter_gate, + "semantics": ( + "selected depth mixers use parameter-free constant-zero logits " + "with the parent softmax+einsum arithmetic kernel" + ), + } + + +def rewrite_result( + output_path: Path, + study_manifest_path: Path, + study_manifest: dict[str, Any], +) -> None: + if LAST_MODEL is None or LAST_OPTIMIZER is None: + raise RuntimeError("runner capture state missing") + result = json.loads(output_path.read_text()) + parent_self_hash = result.pop("canonical_sha256_without_self") + if result["protocol_id"] != PARENT_PROTOCOL_ID: + raise RuntimeError("parent runner protocol drift") + result["schema_version"] = 2 + result["protocol_id"] = PROTOCOL_ID + result["parent_protocol_id"] = PARENT_PROTOCOL_ID + result["variant"] = ACTIVE_VARIANT + result["parent_runner_canonical_sha256"] = parent_self_hash + result["study_manifest"] = { + "path": str(study_manifest_path), + "file_sha256": parent.file_sha256(study_manifest_path), + "status": study_manifest["status"], + } + result["forward_intervention"] = build_intervention_audit( + LAST_MODEL, LAST_OPTIMIZER, study_manifest + ) + result["canonical_sha256_without_self"] = parent.canonical_sha256(result) + temporary = output_path.with_suffix(output_path.suffix + ".round08.tmp") + temporary.write_text( + json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + os.replace(temporary, output_path) + if not result["forward_intervention"]["passed"]: + raise RuntimeError( + f"forward intervention audit failed: " + f"{result['forward_intervention']}" + ) + + +def main() -> None: + global ACTIVE_VARIANT, LAST_OPTIMIZER + variant = extract_wrapper_argument("--variant") + study_manifest_path = Path( + extract_wrapper_argument("--study-manifest") + ).resolve() + if variant not in VARIANTS: + raise ValueError(f"unknown variant: {variant}") + run_kind = argument_value("--run-kind", "formal") + if run_kind in ("formal", "replay") and variant not in FORMAL_VARIANTS: + raise ValueError("learned_reference is smoke-only") + if argument_value("--architecture") != "block": + raise ValueError("Round 08 requires --architecture block") + if argument_value("--depth") != "32": + raise ValueError("Round 08 requires --depth 32") + if run_kind == "replay" and variant != "uniform_groups_6_7_forward": + raise ValueError("the frozen replay uses the primary joint variant") + + study_manifest = json.loads(study_manifest_path.read_text()) + if ( + study_manifest["protocol_id"] != PROTOCOL_ID + or study_manifest["status"] != "frozen-before-model-output" + ): + raise ValueError("study manifest is not the frozen Round 08 contract") + expected = tuple( + study_manifest["variants"] + .get(variant, {"selected_depth_indices": []})[ + "selected_depth_indices" + ] + ) + if expected != VARIANTS[variant]: + raise ValueError("study manifest selector drift") + + output_value = argument_value("--output") + if output_value is None: + raise ValueError("--output is required") + output_path = Path(output_value).resolve() + ACTIVE_VARIANT = variant + parent.GradientLanguageModel = ForwardInterventionLanguageModel + + original_adamw = torch.optim.AdamW + + def capture_adamw(*args: Any, **kwargs: Any) -> torch.optim.Optimizer: + global LAST_OPTIMIZER + LAST_OPTIMIZER = original_adamw(*args, **kwargs) + return LAST_OPTIMIZER + + torch.optim.AdamW = capture_adamw # type: ignore[assignment] + try: + parent.main() + finally: + torch.optim.AdamW = original_adamw # type: ignore[assignment] + rewrite_result(output_path, study_manifest_path, study_manifest) + + +if __name__ == "__main__": + main() diff --git a/experiments/k3/attnres_forward/verify.py b/experiments/k3/attnres_forward/verify.py new file mode 100644 index 0000000..c9efb02 --- /dev/null +++ b/experiments/k3/attnres_forward/verify.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Run pre-result Round 08 identity gates.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +from pathlib import Path +from typing import Any + +import torch + + +def load_runner() -> Any: + path = Path(__file__).resolve().parent / "train.py" + spec = importlib.util.spec_from_file_location("k3_attnres_round08_train", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import Round 08 runner from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +runner = load_runner() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + step_zero = subparsers.add_parser("step-zero") + step_zero.add_argument("--cache-dir", type=Path, required=True) + step_zero.add_argument("--parent-manifest", type=Path, required=True) + step_zero.add_argument("--study-manifest", type=Path, required=True) + step_zero.add_argument("--output", type=Path, required=True) + step_zero.add_argument("--seed", type=int, default=2026073001) + + smoke = subparsers.add_parser("smoke-compare") + smoke.add_argument("--parent", type=Path, required=True) + smoke.add_argument("--wrapper", type=Path, required=True) + smoke.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + + +def tensor_sha256(value: torch.Tensor) -> str: + return hashlib.sha256(runner.parent.tensor_bytes(value)).hexdigest() + + +def read_and_verify(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text()) + expected = value["canonical_sha256_without_self"] + payload = { + key: item + for key, item in value.items() + if key != "canonical_sha256_without_self" + } + if canonical_sha256(payload) != expected: + raise RuntimeError(f"canonical self-hash failed: {path}") + return value + + +def smoke_compare(args: argparse.Namespace) -> None: + parent_result = read_and_verify(args.parent) + wrapper_result = read_and_verify(args.wrapper) + fields = ( + "architecture", + "depth", + "seed", + "steps", + "batch_size", + "target_bytes_seen", + "manifest", + "model", + "optimizer", + "hashes", + "evaluations", + "diagnostics", + "training_history", + "gradient_gate", + "environment", + ) + checks = { + field: parent_result[field] == wrapper_result[field] + for field in fields + } + wrapper_identity = { + "protocol": wrapper_result["protocol_id"] == runner.PROTOCOL_ID, + "parent_protocol": ( + wrapper_result["parent_protocol_id"] + == runner.PARENT_PROTOCOL_ID + ), + "variant": wrapper_result["variant"] == "learned_reference", + "forward_audit": wrapper_result["forward_intervention"]["passed"], + } + passed = all(checks.values()) and all(wrapper_identity.values()) + result = { + "schema_version": 1, + "protocol_id": runner.PROTOCOL_ID, + "gate": "empty-selector-parent-equivalence", + "passed": passed, + "field_checks": checks, + "wrapper_identity": wrapper_identity, + "excluded_fields": [ + "protocol wrapper fields", + "timing", + "self hash", + "parent runner self hash", + "study manifest", + ], + "parent_file": str(args.parent.resolve()), + "wrapper_file": str(args.wrapper.resolve()), + } + result["canonical_sha256_without_self"] = canonical_sha256(result) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + if not passed: + raise RuntimeError(f"empty-selector parent equivalence failed: {checks}") + + +def exact_structure(value: Any) -> Any: + return json.loads( + json.dumps(value, ensure_ascii=False, sort_keys=True) + ) + + +def step_zero(args: argparse.Namespace) -> None: + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required by the frozen step-zero gate") + study_manifest = json.loads(args.study_manifest.read_text()) + if study_manifest["protocol_id"] != runner.PROTOCOL_ID: + raise RuntimeError("study manifest mismatch") + parent_manifest = json.loads(args.parent_manifest.read_text()) + if parent_manifest["protocol_id"] != runner.PARENT_PROTOCOL_ID: + raise RuntimeError("parent manifest mismatch") + + parent = runner.parent + parent.configure_round04_globals(32) + device = torch.device("cuda") + corpus = parent.round04.ByteCorpus( + args.cache_dir, parent_manifest, device + ) + inputs, targets = corpus.fixed_batch( + corpus.diagnostic_starts, 0, 16 + ) + + variants = ("learned_reference",) + tuple( + study_manifest["variants"].keys() + ) + observations: dict[str, Any] = {} + reference_payload: dict[str, Any] | None = None + for variant in variants: + parent.configure_determinism(args.seed) + runner.ACTIVE_VARIANT = variant + model = runner.ForwardInterventionLanguageModel("block").to(device) + initial_public = parent.named_state_hash( + model, include_mixers=False + ) + initial_mixer = parent.named_state_hash( + model, include_mixers=True + ) + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + logits, trace = model(inputs, capture=True) + loss = parent.cross_entropy(logits, targets) + if trace is None: + raise RuntimeError("step-zero trace missing") + evaluation = parent.evaluate(model, corpus, 64, 8) + diagnostic = parent.diagnostic(model, corpus, 16) + payload = { + "initial_public_hash": initial_public, + "initial_mixer_hash": initial_mixer, + "logits_sha256": tensor_sha256(logits), + "loss_nats": loss.detach().cpu().item(), + "evaluation": exact_structure(evaluation), + "diagnostic": exact_structure(diagnostic), + } + if reference_payload is None: + reference_payload = payload + exact_checks = { + key: payload[key] == reference_payload[key] + for key in payload + } + selected = tuple(runner.VARIANTS[variant]) + capture_checks = {} + for index in selected: + summary = trace.depth_weights[index] + source_count = summary["sources"] + capture_checks[str(index)] = { + "source_count": source_count, + "expected_source_count": study_manifest[ + "selected_source_counts" + ][str(index)], + "capture_summary_exact_vs_learned": ( + payload["diagnostic"]["depth_weights"][index] + == reference_payload["diagnostic"]["depth_weights"][index] + ), + "passed": ( + source_count + == study_manifest["selected_source_counts"][str(index)] + and payload["diagnostic"]["depth_weights"][index] + == reference_payload["diagnostic"]["depth_weights"][index] + ), + } + runtime_uniform_gate = ( + model.uniform_weight_max_abs_error + <= study_manifest["thresholds"][ + "uniform_weight_max_abs_error" + ] + ) + observations[variant] = { + "payload": payload, + "exact_vs_learned_reference": exact_checks, + "selected_capture_checks": capture_checks, + "pre_reduction_uniform_weight_max_abs_error": ( + model.uniform_weight_max_abs_error + ), + "pre_reduction_uniform_weight_gate": runtime_uniform_gate, + "passed": ( + all(exact_checks.values()) + and all( + item["passed"] for item in capture_checks.values() + ) + and runtime_uniform_gate + ), + } + del model, logits, loss, trace + torch.cuda.empty_cache() + + passed = all(item["passed"] for item in observations.values()) + result = { + "schema_version": 1, + "protocol_id": runner.PROTOCOL_ID, + "gate": "step-zero-cross-variant-byte-exact", + "seed": args.seed, + "passed": passed, + "variants": observations, + "parent_manifest_sha256": runner.parent.file_sha256( + args.parent_manifest + ), + "study_manifest_sha256": runner.parent.file_sha256( + args.study_manifest + ), + "environment": { + "gpu": torch.cuda.get_device_name(0), + "torch": torch.__version__, + "cuda": torch.version.cuda, + "cublas_workspace_config": os.environ.get( + "CUBLAS_WORKSPACE_CONFIG" + ), + }, + } + result["canonical_sha256_without_self"] = canonical_sha256(result) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + if not passed: + failed = [ + name + for name, value in observations.items() + if not value["passed"] + ] + raise RuntimeError(f"step-zero exactness failed: {failed}") + + +def main() -> None: + args = parse_args() + if args.command == "step-zero": + step_zero(args) + else: + smoke_compare(args) + + +if __name__ == "__main__": + main() diff --git a/research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md b/research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md index 61564e0..c110d6d 100644 --- a/research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md +++ b/research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md @@ -169,7 +169,9 @@ uniform。选中分支用同 dtype 的 constant-zero logits 和同一个 - CE byte-exact； - activation-gradient spectrum byte-exact； - validation metrics byte-exact； -- selected capture weights 等于 `1/N`，max absolute error `≤ 1e-12`； +- selected 的**归约前 weight tensor** 等于 FP32 `1/N`，max absolute error + `≤ 1e-12`；capture 的 `mean_weights` 因 FP32 大规模 mean 可有约 `1e-8` 的归约舍入， + 但必须与父 learned capture summary byte-exact； - 若任何跨 variant byte-exact 比较失败，hard-fail；不得在结果后改成容差 gate。 这个负控制只约束初始化；训练开始后 forward 必须允许分化。