From 26fc8244092f6f61b74c76e8b88fda0fa6d29145 Mon Sep 17 00:00:00 2001 From: wuyang <5700876+banisherwy@user.noreply.gitee.com> Date: Thu, 30 Jul 2026 09:49:53 +0800 Subject: [PATCH] research: audit AttnRes gradient scale study --- experiments/k3/attnres_gradient/analyze.py | 685 + .../k3/attnres_gradient/reproduction.json | 217 + ...mal-depth-16-baseline-seed-2026073001.json | 7366 + ...mal-depth-16-baseline-seed-2026073002.json | 7366 + ...mal-depth-16-baseline-seed-2026073003.json | 7366 + ...formal-depth-16-block-seed-2026073001.json | 9616 ++ ...formal-depth-16-block-seed-2026073002.json | 9616 ++ ...formal-depth-16-block-seed-2026073003.json | 9616 ++ ...mal-depth-32-baseline-seed-2026073001.json | 8614 ++ ...mal-depth-32-baseline-seed-2026073002.json | 8614 ++ ...mal-depth-32-baseline-seed-2026073003.json | 8614 ++ ...formal-depth-32-block-seed-2026073001.json | 13072 ++ ...formal-depth-32-block-seed-2026073002.json | 13072 ++ ...formal-depth-32-block-seed-2026073003.json | 13072 ++ ...replay-depth-32-block-seed-2026073001.json | 13072 ++ .../raw/smoke-a-depth-16-baseline.json | 700 + .../results/raw/smoke-a-depth-16-block.json | 1450 + .../raw/smoke-a-depth-32-baseline.json | 1132 + .../results/raw/smoke-a-depth-32-block.json | 2618 + .../raw/smoke-b-depth-16-baseline.json | 700 + .../results/raw/smoke-b-depth-16-block.json | 1450 + .../raw/smoke-b-depth-32-baseline.json | 1132 + .../results/raw/smoke-b-depth-32-block.json | 2618 + research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md | 545 + src/data/k3-attnres-gradient-compact.json | 34530 +++++ src/data/k3-attnres-gradient.json | 116733 +++++++++++++++ 26 files changed, 293586 insertions(+) create mode 100644 experiments/k3/attnres_gradient/analyze.py create mode 100644 experiments/k3/attnres_gradient/reproduction.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073001.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073002.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073003.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073001.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073002.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073003.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073001.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073002.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073003.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073001.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073002.json create mode 100644 experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073003.json create mode 100644 experiments/k3/attnres_gradient/results/raw/replay-depth-32-block-seed-2026073001.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-baseline.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-block.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-baseline.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-block.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-baseline.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-block.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-baseline.json create mode 100644 experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-block.json create mode 100644 research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md create mode 100644 src/data/k3-attnres-gradient-compact.json create mode 100644 src/data/k3-attnres-gradient.json diff --git a/experiments/k3/attnres_gradient/analyze.py b/experiments/k3/attnres_gradient/analyze.py new file mode 100644 index 0000000..7abf66e --- /dev/null +++ b/experiments/k3/attnres_gradient/analyze.py @@ -0,0 +1,685 @@ +#!/usr/bin/env python3 +"""Validate, aggregate, and publish K3 AttnRes Round 05 experiment data.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import shutil +import statistics +from pathlib import Path +from typing import Any, Iterable + + +PROTOCOL_ID = "llm-atlas-k3-attnres-gradient-scale-v1" +ARCHITECTURES = ("baseline", "block") +DEPTHS = (16, 32) +SEEDS = (2026073001, 2026073002, 2026073003) +STEPS = (0, 100, 500, 2000, 4000, 8000) +FORMAL_STEPS = 8000 +FORMAL_BATCH = 32 +TARGET_BYTES_PER_RUN = 65_536_000 +EXPECTED_TOTAL_TARGET_BYTES = 786_432_000 +SMOKE_COMPARE_FIELDS = ( + "protocol_id", + "run_kind", + "architecture", + "depth", + "seed", + "steps", + "batch_size", + "target_bytes_seen", + "manifest", + "model", + "optimizer", + "hashes", + "evaluations", + "diagnostics", + "training_history", + "gradient_gate", + "environment", +) +REPLAY_COMPARE_FIELDS = tuple( + field for field in SMOKE_COMPARE_FIELDS if field != "run_kind" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--formal-dir", type=Path, required=True) + parser.add_argument("--smoke-a-dir", type=Path, required=True) + parser.add_argument("--smoke-b-dir", type=Path, required=True) + parser.add_argument("--replay", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--raw-output-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--compact-output", type=Path, required=True) + parser.add_argument("--reproduction-output", type=Path, required=True) + return parser.parse_args() + + +def read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + os.replace(temporary, path) + + +def mean(values: Iterable[float]) -> float: + return statistics.fmean(values) + + +def require_finite(value: Any, path: str = "root") -> None: + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"non-finite float at {path}") + elif isinstance(value, dict): + for key, child in value.items(): + require_finite(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + require_finite(child, f"{path}[{index}]") + + +def selected(run: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]: + return {field: run[field] for field in fields} + + +def final_diagnostic(run: dict[str, Any]) -> dict[str, Any]: + diagnostic = run["diagnostics"][-1] + if diagnostic["step"] != FORMAL_STEPS: + raise ValueError("final diagnostic is not step 8000") + return diagnostic + + +def validate_run( + run: dict[str, Any], + *, + path: Path, + depth: int, + architecture: str, + seed: int, + manifest: dict[str, Any], + manifest_hash: str, +) -> None: + if run["protocol_id"] != PROTOCOL_ID or run["run_kind"] != "formal": + raise ValueError(f"formal protocol/kind mismatch: {path}") + if ( + run["depth"] != depth + or run["architecture"] != architecture + or run["seed"] != seed + ): + raise ValueError(f"formal identity mismatch: {path}") + if ( + run["steps"] != FORMAL_STEPS + or run["batch_size"] != FORMAL_BATCH + or run["target_bytes_seen"] != TARGET_BYTES_PER_RUN + ): + raise ValueError(f"formal budget mismatch: {path}") + if run["manifest"]["file_sha256"] != manifest_hash: + raise ValueError(f"manifest file hash mismatch: {path}") + for key in ( + "formal_schedule_sha256", + "validation_tensor_sha256", + "diagnostic_tensor_sha256", + ): + if run["manifest"][key] != manifest["windows"][key]: + raise ValueError(f"manifest {key} mismatch: {path}") + if [row["step"] for row in run["evaluations"]] != list(STEPS): + raise ValueError(f"evaluation steps mismatch: {path}") + if [row["step"] for row in run["diagnostics"]] != list(STEPS): + raise ValueError(f"diagnostic steps mismatch: {path}") + if run["model"]["layers"] != depth: + raise ValueError(f"model depth mismatch: {path}") + + for diagnostic in run["diagnostics"]: + capture = diagnostic["capture"] + if ( + capture["count"] != depth + or capture["shape"] != [16, 256, 192] + or set(capture["dtypes"]) != {"torch.float32"} + or not capture["all_gradients_finite"] + or not capture["all_gradients_present"] + or not capture["storage_unique"] + ): + raise ValueError(f"activation capture gate mismatch: {path}") + for field in ( + "activation_grad_rms_by_block", + "activation_output_rms_by_block", + "core_parameter_grad_rms_by_block", + ): + if len(diagnostic[field]) != depth: + raise ValueError(f"{field} length mismatch: {path}") + for field in ( + "layer_input_rms_by_sublayer", + "branch_output_rms_by_sublayer", + "stream_state_rms_by_sublayer", + ): + if len(diagnostic[field]) != depth * 2: + raise ValueError(f"{field} length mismatch: {path}") + if architecture == "baseline": + if diagnostic["depth_weights"] or diagnostic["output_weights"] is not None: + raise ValueError(f"unexpected Baseline mixer trace: {path}") + else: + if ( + len(diagnostic["depth_weights"]) != depth * 2 + or diagnostic["output_weights"]["sources"] != 9 + ): + raise ValueError(f"Block mixer trace mismatch: {path}") + require_finite(run, path.name) + + +def relative_reduction(baseline: float, block: float) -> float: + return (baseline - block) / baseline + + +def depth_verdict(rows: list[dict[str, Any]]) -> dict[str, Any]: + cv_reductions = [row["relative_cv_reduction"] for row in rows] + imbalance_reductions = [ + row["relative_imbalance_reduction"] for row in rows + ] + mean_cv = mean(cv_reductions) + mean_imbalance = mean(imbalance_reductions) + support = ( + all(value > 0 for value in cv_reductions) + and mean_cv >= 0.20 + and all(value > 0 for value in imbalance_reductions) + and mean_imbalance >= 0.20 + ) + concern = ( + all(value < 0 for value in cv_reductions) + and -mean_cv >= 0.20 + and all(value < 0 for value in imbalance_reductions) + and -mean_imbalance >= 0.20 + ) + if support: + label = "joint directional support at this depth" + elif concern: + label = "joint directional concern at this depth" + else: + label = "mixed / inconclusive at this depth" + return { + "label": label, + "threshold_relative": 0.20, + "cv_reductions": cv_reductions, + "mean_cv_reduction": mean_cv, + "imbalance_reductions": imbalance_reductions, + "mean_imbalance_reduction": mean_imbalance, + "all_cv_improve": all(value > 0 for value in cv_reductions), + "all_imbalance_improve": all( + value > 0 for value in imbalance_reductions + ), + } + + +def summarize_depth( + depth: int, runs: dict[tuple[int, str, int], dict[str, Any]] +) -> dict[str, Any]: + rows = [] + for seed in SEEDS: + baseline = runs[(depth, "baseline", seed)] + block = runs[(depth, "block", seed)] + baseline_diagnostic = final_diagnostic(baseline) + block_diagnostic = final_diagnostic(block) + baseline_activation = baseline_diagnostic[ + "activation_grad_statistics" + ] + block_activation = block_diagnostic["activation_grad_statistics"] + baseline_bpc = baseline["evaluations"][-1]["bits_per_byte"] + block_bpc = block["evaluations"][-1]["bits_per_byte"] + rows.append( + { + "seed": seed, + "baseline_bpc": baseline_bpc, + "block_bpc": block_bpc, + "block_minus_baseline_bpc": block_bpc - baseline_bpc, + "baseline_activation_grad_mean": baseline_activation["mean"], + "block_activation_grad_mean": block_activation["mean"], + "block_to_baseline_activation_grad_mean": ( + block_activation["mean"] / baseline_activation["mean"] + ), + "baseline_cv": baseline_activation["population_cv"], + "block_cv": block_activation["population_cv"], + "relative_cv_reduction": relative_reduction( + baseline_activation["population_cv"], + block_activation["population_cv"], + ), + "baseline_first_to_last_ratio": baseline_activation[ + "first_to_last_ratio" + ], + "block_first_to_last_ratio": block_activation[ + "first_to_last_ratio" + ], + "baseline_imbalance": baseline_activation[ + "imbalance_abs_log_ratio" + ], + "block_imbalance": block_activation[ + "imbalance_abs_log_ratio" + ], + "relative_imbalance_reduction": relative_reduction( + baseline_activation["imbalance_abs_log_ratio"], + block_activation["imbalance_abs_log_ratio"], + ), + "baseline_parameter_grad_cv": baseline_diagnostic[ + "core_parameter_grad_statistics" + ]["population_cv"], + "block_parameter_grad_cv": block_diagnostic[ + "core_parameter_grad_statistics" + ]["population_cv"], + } + ) + verdict = depth_verdict(rows) + return { + "depth": depth, + "by_seed": rows, + "means": { + "baseline_bpc": mean(row["baseline_bpc"] for row in rows), + "block_bpc": mean(row["block_bpc"] for row in rows), + "block_minus_baseline_bpc": mean( + row["block_minus_baseline_bpc"] for row in rows + ), + "baseline_cv": mean(row["baseline_cv"] for row in rows), + "block_cv": mean(row["block_cv"] for row in rows), + "relative_cv_reduction": mean( + row["relative_cv_reduction"] for row in rows + ), + "baseline_imbalance": mean( + row["baseline_imbalance"] for row in rows + ), + "block_imbalance": mean( + row["block_imbalance"] for row in rows + ), + "relative_imbalance_reduction": mean( + row["relative_imbalance_reduction"] for row in rows + ), + "block_to_baseline_activation_grad_mean": mean( + row["block_to_baseline_activation_grad_mean"] for row in rows + ), + "baseline_parameter_grad_cv": mean( + row["baseline_parameter_grad_cv"] for row in rows + ), + "block_parameter_grad_cv": mean( + row["block_parameter_grad_cv"] for row in rows + ), + }, + "verdict": verdict, + } + + +def compact_cell(run: dict[str, Any]) -> dict[str, Any]: + diagnostics = [] + for row in run["diagnostics"]: + compact = { + "step": row["step"], + "loss_nats": row["loss_nats"], + "bits_per_byte": row["bits_per_byte"], + "activation_grad_rms_by_block": row[ + "activation_grad_rms_by_block" + ], + "activation_grad_statistics": row[ + "activation_grad_statistics" + ], + "activation_output_rms_by_block": row[ + "activation_output_rms_by_block" + ], + "activation_output_statistics": row[ + "activation_output_statistics" + ], + "core_parameter_grad_rms_by_block": row[ + "core_parameter_grad_rms_by_block" + ], + "core_parameter_grad_statistics": row[ + "core_parameter_grad_statistics" + ], + } + if row["depth_weights"]: + compact["depth_weights"] = row["depth_weights"] + compact["output_weights"] = row["output_weights"] + diagnostics.append(compact) + return { + "architecture": run["architecture"], + "depth": run["depth"], + "seed": run["seed"], + "evaluations": run["evaluations"], + "diagnostics": diagnostics, + "timing": run["timing"], + "parameters": run["model"]["parameters"], + "hashes": run["hashes"], + } + + +def main() -> None: + args = parse_args() + manifest = read_json(args.manifest) + manifest_hash = file_sha256(args.manifest) + if manifest["protocol_id"] != PROTOCOL_ID: + raise ValueError("manifest protocol mismatch") + if ( + manifest["windows"]["formal_schedule_cells"] != 768_000 + or manifest["windows"]["formal_steps"] != FORMAL_STEPS + or manifest["windows"]["formal_batch"] != FORMAL_BATCH + ): + raise ValueError("manifest schedule budget mismatch") + + runs: dict[tuple[int, str, int], dict[str, Any]] = {} + source_paths: dict[str, Path] = {} + formal_hashes: dict[str, str] = {} + for depth in DEPTHS: + for architecture in ARCHITECTURES: + for seed in SEEDS: + name = f"depth-{depth}-{architecture}-seed-{seed}.json" + path = args.formal_dir / name + run = read_json(path) + validate_run( + run, + path=path, + depth=depth, + architecture=architecture, + seed=seed, + manifest=manifest, + manifest_hash=manifest_hash, + ) + runs[(depth, architecture, seed)] = run + public_name = f"formal-{name}" + source_paths[public_name] = path + formal_hashes[public_name] = file_sha256(path) + + if sum(run["target_bytes_seen"] for run in runs.values()) != ( + EXPECTED_TOTAL_TARGET_BYTES + ): + raise ValueError("formal total target-byte budget mismatch") + + common_initial_exact: dict[str, Any] = {} + input_gate_exact: dict[str, Any] = {} + for depth in DEPTHS: + for seed in SEEDS: + baseline = runs[(depth, "baseline", seed)] + block = runs[(depth, "block", seed)] + public_fields = ( + "initial_public_parameter_structure", + "initial_public_parameter_tensors", + "initial_public_parameter_elements", + "initial_public_parameters", + ) + exact = all( + baseline["hashes"][field] == block["hashes"][field] + for field in public_fields + ) + gate_exact = ( + baseline["manifest"]["input_gate_tensor_hashes"] + == block["manifest"]["input_gate_tensor_hashes"] + ) + key = f"depth-{depth}-seed-{seed}" + common_initial_exact[key] = { + "exact": exact, + "baseline": { + field: baseline["hashes"][field] for field in public_fields + }, + "block": { + field: block["hashes"][field] for field in public_fields + }, + } + input_gate_exact[key] = { + "exact": gate_exact, + "hashes": baseline["manifest"]["input_gate_tensor_hashes"], + } + if not exact or not gate_exact: + raise ValueError(f"paired equality gate failed: {key}") + + smoke_exact: dict[str, Any] = {} + smoke_hashes: dict[str, str] = {} + for depth in DEPTHS: + for architecture in ARCHITECTURES: + name = f"depth-{depth}-{architecture}.json" + left_path = args.smoke_a_dir / name + right_path = args.smoke_b_dir / name + left = read_json(left_path) + right = read_json(right_path) + left_selected = selected(left, SMOKE_COMPARE_FIELDS) + right_selected = selected(right, SMOKE_COMPARE_FIELDS) + exact = left_selected == right_selected + if ( + not exact + or left["run_kind"] != "smoke" + or left["steps"] != 20 + or not left["gradient_gate"]["passed"] + ): + raise ValueError(f"smoke gate failed: {name}") + key = f"depth-{depth}-{architecture}" + smoke_exact[key] = { + "exact": exact, + "compare_sha256": canonical_sha256(left_selected), + "gradient_gate": left["gradient_gate"], + } + for label, path in (("a", left_path), ("b", right_path)): + public_name = f"smoke-{label}-{name}" + source_paths[public_name] = path + smoke_hashes[public_name] = file_sha256(path) + + replay = read_json(args.replay) + replay_formal = runs[(32, "block", 2026073001)] + replay_left = selected(replay_formal, REPLAY_COMPARE_FIELDS) + replay_right = selected(replay, REPLAY_COMPARE_FIELDS) + replay_exact = replay_left == replay_right + if ( + replay["run_kind"] != "replay" + or replay["depth"] != 32 + or replay["architecture"] != "block" + or replay["seed"] != 2026073001 + or not replay_exact + ): + raise ValueError("formal replay gate failed") + replay_public_name = "replay-depth-32-block-seed-2026073001.json" + source_paths[replay_public_name] = args.replay + + depth_summaries = { + str(depth): summarize_depth(depth, runs) for depth in DEPTHS + } + depth_labels = [ + depth_summaries[str(depth)]["verdict"]["label"] for depth in DEPTHS + ] + if all( + label == "joint directional support at this depth" + for label in depth_labels + ): + overall_verdict = ( + "scale-consistent directional support in this operationalization" + ) + elif all( + label == "joint directional concern at this depth" + for label in depth_labels + ): + overall_verdict = ( + "scale-consistent directional concern in this operationalization" + ) + else: + overall_verdict = "depth-dependent or inconclusive" + + full = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "manifest": manifest, + "study": { + "architectures": list(ARCHITECTURES), + "depths": list(DEPTHS), + "seeds": list(SEEDS), + "diagnostic_steps": list(STEPS), + "formal_runs": len(runs), + "formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES, + "replay_target_bytes": TARGET_BYTES_PER_RUN, + "gradient_object": ( + "RMS of d(mean token CE)/d(post-MLP Transformer-block output) " + "over batch×time×channel" + ), + }, + "depth_summaries": depth_summaries, + "overall_verdict": overall_verdict, + "gates": { + "common_initial_parameters": common_initial_exact, + "paired_input_tensors": input_gate_exact, + "smoke_exact": smoke_exact, + "replay": { + "exact": replay_exact, + "compare_fields": list(REPLAY_COMPARE_FIELDS), + "formal_compare_sha256": canonical_sha256(replay_left), + "replay_compare_sha256": canonical_sha256(replay_right), + "formal_final_model_state": replay_formal["hashes"][ + "final_model_state" + ], + "replay_final_model_state": replay["hashes"][ + "final_model_state" + ], + "formal_final_optimizer_state": replay_formal["hashes"][ + "final_optimizer_state" + ], + "replay_final_optimizer_state": replay["hashes"][ + "final_optimizer_state" + ], + }, + }, + "runs": { + f"depth-{depth}-{architecture}-seed-{seed}": run + for (depth, architecture, seed), run in sorted(runs.items()) + }, + } + full["canonical_sha256_without_self"] = canonical_sha256(full) + + compact = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "study": full["study"], + "manifest_summary": { + "file_sha256": manifest_hash, + "dataset_revision": manifest["dataset"]["revision"], + "train_bytes_sha256": manifest["dataset"]["splits"]["train"][ + "concatenated_sha256" + ], + "formal_schedule_sha256": manifest["windows"][ + "formal_schedule_sha256" + ], + "validation_tensor_sha256": manifest["windows"][ + "validation_tensor_sha256" + ], + "diagnostic_tensor_sha256": manifest["windows"][ + "diagnostic_tensor_sha256" + ], + }, + "depth_summaries": depth_summaries, + "overall_verdict": overall_verdict, + "replay_exact": replay_exact, + "cells": [ + compact_cell(runs[(depth, architecture, seed)]) + for depth in DEPTHS + for architecture in ARCHITECTURES + for seed in SEEDS + ], + } + compact["canonical_sha256_without_self"] = canonical_sha256(compact) + + args.raw_output_dir.mkdir(parents=True, exist_ok=True) + for public_name, source_path in sorted(source_paths.items()): + target = args.raw_output_dir / public_name + temporary = target.with_suffix(target.suffix + ".tmp") + shutil.copyfile(source_path, temporary) + os.replace(temporary, target) + + reproduction = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "manifest": { + "path": str(args.manifest), + "sha256": manifest_hash, + }, + "protocol_sha256": file_sha256( + Path("research/K3_ATTNRES_GRADIENT_SCALE_PROTOCOL.md") + ), + "definition_audit_sha256": file_sha256( + Path("research/K3_ATTNRES_GRADIENT_DEFINITION_AUDIT.md") + ), + "runner_sha256": file_sha256( + Path("experiments/k3/attnres_gradient/train.py") + ), + "analyzer_sha256": file_sha256(Path(__file__)), + "formal_raw_sha256": formal_hashes, + "smoke_raw_sha256": smoke_hashes, + "replay_raw_sha256": { + replay_public_name: file_sha256(args.replay) + }, + "formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES, + "replay_target_bytes": TARGET_BYTES_PER_RUN, + "smoke_exact": smoke_exact, + "replay_exact": { + "exact": replay_exact, + "compare_sha256": canonical_sha256(replay_left), + "final_model_state": replay["hashes"]["final_model_state"], + "final_optimizer_state": replay["hashes"][ + "final_optimizer_state" + ], + }, + "aggregate_sha256": full["canonical_sha256_without_self"], + "compact_sha256": compact["canonical_sha256_without_self"], + "overall_verdict": overall_verdict, + } + reproduction["canonical_sha256_without_self"] = canonical_sha256( + reproduction + ) + + atomic_json(args.output, full) + atomic_json(args.compact_output, compact) + atomic_json(args.reproduction_output, reproduction) + print( + json.dumps( + { + "formal_runs": len(runs), + "formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES, + "smoke_exact": all( + row["exact"] for row in smoke_exact.values() + ), + "replay_exact": replay_exact, + "depth_verdicts": { + depth: depth_summaries[str(depth)]["verdict"]["label"] + for depth in DEPTHS + }, + "overall_verdict": overall_verdict, + "aggregate_sha256": full[ + "canonical_sha256_without_self" + ], + "compact_sha256": compact[ + "canonical_sha256_without_self" + ], + "reproduction_sha256": reproduction[ + "canonical_sha256_without_self" + ], + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/k3/attnres_gradient/reproduction.json b/experiments/k3/attnres_gradient/reproduction.json new file mode 100644 index 0000000..599dfc9 --- /dev/null +++ b/experiments/k3/attnres_gradient/reproduction.json @@ -0,0 +1,217 @@ +{ + "aggregate_sha256": "69be133c8f5b11f8a58e831bc03f5432e737ed21efe89bf186784ead7c7f7b51", + "analyzer_sha256": "017d38d92bfd5bc26e10028015f9bf85d28516fb5c5bae06d7bf315b5eca2fc8", + "canonical_sha256_without_self": "addb2e5990af4bbec4b88b4654fc72464f4df5b57fd00e46cad1febbb042f68a", + "compact_sha256": "8cdb71808a429e5f9513c1c6c9426bdc93e30f237001a1b47a2a2fba574a044f", + "definition_audit_sha256": "79221c5648ba280d9177386d2fcfb2fa653b3dd14405185b5997754db2961cc6", + "formal_raw_sha256": { + "formal-depth-16-baseline-seed-2026073001.json": "c9144ddb458010868668ab0bb1272d4e5051300936d9b21dccee75c32a9e46de", + "formal-depth-16-baseline-seed-2026073002.json": "939cad5054960ba8a72b50b4b250b94c3d22a6b46796f669c6e4f033d59fb9de", + "formal-depth-16-baseline-seed-2026073003.json": "afd9a60ff323e97d21b24b413d9b9fa97b8d5f960bfc88d2040502aa5ad96a1e", + "formal-depth-16-block-seed-2026073001.json": "23e3c68e54ec55d981066ad5730316db9084a6637a8064763952ba109ea1f715", + "formal-depth-16-block-seed-2026073002.json": "027f8786f530d6ad5d910f0aad3d0f287887d6457c670bc8b50af7248be1487d", + "formal-depth-16-block-seed-2026073003.json": "254b637f403a4c61e8e8bb2dd083349406b6baa7a3d8a616e37a9ab6f8ebaeb9", + "formal-depth-32-baseline-seed-2026073001.json": "232ed3dfbbbc898c42622c4a9aee8400f76cb773c576bf1c522cdbb19ead998c", + "formal-depth-32-baseline-seed-2026073002.json": "f4fb5fca608a6623ce4c9707714a8a9ebd2c10f6e63f54e243ec3c06d18b1fff", + "formal-depth-32-baseline-seed-2026073003.json": "1b0bd279b165c3a2431685d8c1e6bc36ff72eb55333d5de689dca4f89506bb5e", + "formal-depth-32-block-seed-2026073001.json": "29e1d638b481619c7b32de402122523db8b17cc1fc67a8881fa1ba132a1d5c38", + "formal-depth-32-block-seed-2026073002.json": "21199deb2199395061e51220fd8c7afd04a1135a6381e406da9b5795e3ad5032", + "formal-depth-32-block-seed-2026073003.json": "c0d7f1bcfa9fa7f8f3134ca4571bdf23a951182d03b7de9611a6b4b89667d4ac" + }, + "formal_target_bytes": 786432000, + "manifest": { + "path": "experiments/k3/attnres_gradient/manifest.json", + "sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371" + }, + "overall_verdict": "depth-dependent or inconclusive", + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "protocol_sha256": "f772629b3b82975b6756721c3a3bb57cc4171dfa26ba5b1e8043dc91c9dcce22", + "replay_exact": { + "compare_sha256": "46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817", + "exact": true, + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637" + }, + "replay_raw_sha256": { + "replay-depth-32-block-seed-2026073001.json": "5cea76d68a642c2b4b4f189a813b2e4af9361a939ddd31a68dfdb0ab090b4aad" + }, + "replay_target_bytes": 65536000, + "runner_sha256": "04ae69e10c58972c9193c2d31c7e09d924a0d0e834107afa4ba128c64ac5800f", + "schema_version": 1, + "smoke_exact": { + "depth-16-baseline": { + "compare_sha256": "525cdcd9a79ec61096ecf64fedcbf93144d3168b434e2f8fd384c4374cbebd2a", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-16-block": { + "compare_sha256": "e4330a0d878d10b474b9aeb0be58131f801ebe2e5dd9553d4713c3d16b6590cc", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-32-baseline": { + "compare_sha256": "8ee0dc37e88742b6705968f5f2845a3a1df1250f5e0fd77be6b909236ea28e56", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-32-block": { + "compare_sha256": "289073b1940d4747b7e049f8d767f9c22a49867d37403ff6ef7746a55bd1734c", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + } + }, + "smoke_raw_sha256": { + "smoke-a-depth-16-baseline.json": "fe0ff3ed21bf9b86d238d7a22e99b6e07ba99135b58d29eb6b205f2602e64556", + "smoke-a-depth-16-block.json": "faa06a2bd02135858d3a2503e1ecfb34187b4ee3b4ce3428ed473a55b15b6705", + "smoke-a-depth-32-baseline.json": "e327a6f5db87515c1b29b2808205e23b81228b08ce356b98804717e2e55eef1c", + "smoke-a-depth-32-block.json": "2d9a97c2d6272c928d5b5bc01febd142d4b2f2ad3080330e849f2731a8d080c2", + "smoke-b-depth-16-baseline.json": "fe0ff3ed21bf9b86d238d7a22e99b6e07ba99135b58d29eb6b205f2602e64556", + "smoke-b-depth-16-block.json": "faa06a2bd02135858d3a2503e1ecfb34187b4ee3b4ce3428ed473a55b15b6705", + "smoke-b-depth-32-baseline.json": "e327a6f5db87515c1b29b2808205e23b81228b08ce356b98804717e2e55eef1c", + "smoke-b-depth-32-block.json": "2d9a97c2d6272c928d5b5bc01febd142d4b2f2ad3080330e849f2731a8d080c2" + } +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073001.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073001.json new file mode 100644 index 0000000..5c03f75 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073001.json @@ -0,0 +1,7366 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "09d1bf3e3857ce9c1fd57a3596a5acf1e980ac95e31a3ef6ad060395c15d99ae", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002865509013645351, + 0.00024785567075014114, + 0.00021551651298068464, + 0.00019683866412378848, + 0.0001823508064262569, + 0.0001679604029050097, + 0.00015584587526973337, + 0.00014737028686795384, + 0.00014130656199995428, + 0.00013612695329356939, + 0.00013129493163432926, + 0.00012807638267986476, + 0.00012550120300147682, + 0.00012324050476308912, + 0.00012108208466088399, + 0.00011888353037647903 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023669043730478734, + "first_to_last_ratio": 1.9372775995887173, + "imbalance_abs_log_ratio": 0.6612836883477485, + "last_quartile_mean": 0.00012217683070048224, + "mean": 0.00016411257956860936, + "normalized": [ + 1.7460629899168627, + 1.5102783187106137, + 1.313223602646409, + 1.1994124072706904, + 1.1111324123086055, + 1.0234462424910682, + 0.9496278449793059, + 0.8979828801383491, + 0.8610343117596252, + 0.8294729974472175, + 0.8000296624393728, + 0.7804178266926869, + 0.7647262832098098, + 0.7509509940495869, + 0.7377989242455608, + 0.7244023016942358 + ], + "population_cv": 0.29361872380036635 + }, + "activation_output_rms_by_block": [ + 0.02866268903017044, + 0.029099803417921066, + 0.02953805774450302, + 0.030088091269135475, + 0.030644793063402176, + 0.03131929785013199, + 0.03213750571012497, + 0.03286394104361534, + 0.033869802951812744, + 0.03499744459986687, + 0.03588006645441055, + 0.037365153431892395, + 0.03789033368229866, + 0.039445169270038605, + 0.04050002992153168, + 0.04121527820825577 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0293471603654325, + "first_to_last_ratio": 0.7380574840395957, + "imbalance_abs_log_ratio": 0.3037335657624931, + "last_quartile_mean": 0.03976270277053118, + "mean": 0.034094841103069484, + "normalized": [ + 0.8406752488894867, + 0.8534957922212247, + 0.8663497699023965, + 0.8824822259232266, + 0.8988102619619861, + 0.9185934539320196, + 0.942591449919727, + 0.9638977622528551, + 0.9933996421752943, + 1.0264733158329964, + 1.0523605710888722, + 1.0959180985456622, + 1.1113216092650298, + 1.1569248600043318, + 1.1878638706395246, + 1.2088420674453664 + ], + "population_cv": 0.11952987076282337 + }, + "bits_per_byte": 8.096463027059821, + "branch_output_rms_by_sublayer": [ + 0.0033055038657039404, + 0.0037518907338380814, + 0.0033751516602933407, + 0.003911525942385197, + 0.004229962360113859, + 0.0038461871445178986, + 0.004243654198944569, + 0.0038366704247891903, + 0.004372371360659599, + 0.003851204412057996, + 0.005078981164842844, + 0.003818925702944398, + 0.0054668826051056385, + 0.0038749484810978174, + 0.005823117680847645, + 0.0037566579412668943, + 0.0065947300754487514, + 0.003876061411574483, + 0.006763988174498081, + 0.003795720636844635, + 0.006863070651888847, + 0.0038706199266016483, + 0.007856340147554874, + 0.003839300014078617, + 0.00830968376249075, + 0.003927029203623533, + 0.009655521251261234, + 0.0038513424806296825, + 0.009004893712699413, + 0.0039484030567109585, + 0.008608299307525158, + 0.003791053779423237 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00809059897248305, + 0.007313812062277777, + 0.007090710224412361, + 0.0067033738367094624, + 0.006535407867454384, + 0.006785021795736036, + 0.006295993569919972, + 0.005509300326230647, + 0.006231736048910802, + 0.006047390980174007, + 0.005629405359532731, + 0.005784042737956458, + 0.0059591736113607996, + 0.0061983173180850636, + 0.006571690039252182, + 0.005060205437758869 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007299623773970663, + "first_to_last_ratio": 1.227374872012571, + "imbalance_abs_log_ratio": 0.2048776382299483, + "last_quartile_mean": 0.005947346601614228, + "mean": 0.006362886261765913, + "normalized": [ + 1.271529717747562, + 1.1494488132258316, + 1.1143858200043408, + 1.05351149791715, + 1.0271137340180259, + 1.066343404015675, + 0.9894870520870547, + 0.8658492545019393, + 0.9793882512652816, + 0.9504163254515969, + 0.8847251275509024, + 0.909028151691524, + 0.9365519618304368, + 0.9741361173356609, + 1.0328158902888074, + 0.7952688810682109 + ], + "population_cv": 0.11384977604868386 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02823694422841072, + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679 + ], + "loss_nats": 5.6120405197143555, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679, + 0.04121527820825577 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.509291365044191e-05, + 7.616560469614342e-05, + 6.756903894711286e-05, + 5.824641266372055e-05, + 5.0209364417241886e-05, + 4.2265295633114874e-05, + 3.587255923775956e-05, + 3.076014036196284e-05, + 2.70738746621646e-05, + 2.470596882631071e-05, + 2.293860052304808e-05, + 2.172161293856334e-05, + 2.083007166220341e-05, + 2.0244739062036388e-05, + 1.9782984963967465e-05, + 1.9472558051347733e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.176849248935468e-05, + "first_to_last_ratio": 3.5736674444153773, + "imbalance_abs_log_ratio": 1.2735923638392248, + "last_quartile_mean": 2.008258843488875e-05, + "mean": 3.8934483768571226e-05, + "normalized": [ + 2.185541078603713, + 1.9562505348440236, + 1.7354548566444863, + 1.496010914384804, + 1.28958598027623, + 1.0855491467240759, + 0.921357001957136, + 0.7900487533057547, + 0.6953700689366589, + 0.6345523668212576, + 0.5891589743271394, + 0.5579016551928063, + 0.5350031551983206, + 0.519969371685323, + 0.5081095997460412, + 0.5001365413522295 + ], + "population_cv": 0.5464287537448471 + }, + "activation_output_rms_by_block": [ + 0.030502017587423325, + 0.03408988192677498, + 0.038947492837905884, + 0.04481571167707443, + 0.0512830950319767, + 0.05888458341360092, + 0.06758055835962296, + 0.07802816480398178, + 0.08916938304901123, + 0.10193294286727905, + 0.11700897663831711, + 0.13432858884334564, + 0.15294352173805237, + 0.17351281642913818, + 0.19949108362197876, + 0.2260059416294098 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037088776007294655, + "first_to_last_ratio": 0.19729295890734105, + "imbalance_abs_log_ratio": 1.623065553831421, + "last_quartile_mean": 0.18798834085464478, + "mean": 0.09990779752843082, + "normalized": [ + 0.3053016715611547, + 0.3412134264802905, + 0.38983436529889043, + 0.4485707100521478, + 0.5133042295060407, + 0.5893892656060614, + 0.676429268099835, + 0.7810017509571989, + 0.8925167529955432, + 1.0202701429612833, + 1.1711696137132819, + 1.344525574243789, + 1.530846695870051, + 1.7367294717888415, + 1.9967518908142226, + 2.262145170051368 + ], + "population_cv": 0.5980263271119333 + }, + "bits_per_byte": 5.097173307164379, + "branch_output_rms_by_sublayer": [ + 0.004186291713267565, + 0.008577318862080574, + 0.004227571655064821, + 0.009763575159013271, + 0.005347618833184242, + 0.009802278131246567, + 0.006270815618336201, + 0.009791693650186062, + 0.005826776847243309, + 0.010311179794371128, + 0.006844967603683472, + 0.010455452837049961, + 0.00757947750389576, + 0.01094198040664196, + 0.009920897893607616, + 0.011405051685869694, + 0.010475707240402699, + 0.012199299409985542, + 0.012102399952709675, + 0.012131479568779469, + 0.013124288059771061, + 0.013151587918400764, + 0.01721765287220478, + 0.012752228416502476, + 0.016962353140115738, + 0.012886312790215015, + 0.01880411058664322, + 0.012915017083287239, + 0.024245616048574448, + 0.014021328650414944, + 0.02384282648563385, + 0.013684137724339962 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005223696653307753, + 0.0004106838611462913, + 0.0004143731432945227, + 0.00042601318321813184, + 0.0003557105812490162, + 0.00034956609676611394, + 0.00031105173709675404, + 0.00028609388990673116, + 0.0002585596876336936, + 0.00022586454081316873, + 0.00020762781370074637, + 0.0002072362198298983, + 0.00019288470561650055, + 0.00019339001384203696, + 0.00020597582961175812, + 0.0001909336964292271 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004433599632474303, + "first_to_last_ratio": 2.2643967408442993, + "imbalance_abs_log_ratio": 0.8173083838772207, + "last_quartile_mean": 0.0001957960613748807, + "mean": 0.00029739591659283537, + "normalized": [ + 1.7564789433405417, + 1.3809330869480578, + 1.3933383754620974, + 1.4324782535645448, + 1.1960842816010129, + 1.175423323800053, + 1.0459179825344234, + 0.9619966984900542, + 0.8694123665043039, + 0.7594742503556288, + 0.6981528733799311, + 0.696836130785297, + 0.6485788635779385, + 0.6502779730725327, + 0.6925980422715741, + 0.6420185543120094 + ], + "population_cv": 0.3407962002760142 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028665566816926003, + 0.0289339367300272, + 0.030502017587423325, + 0.030956681817770004, + 0.03408988192677498, + 0.03459245339035988, + 0.038947492837905884, + 0.03989224135875702, + 0.04481571167707443, + 0.04556844383478165, + 0.0512830950319767, + 0.052534833550453186, + 0.05888458341360092, + 0.060703132301568985, + 0.06758055835962296, + 0.07060512155294418, + 0.07802816480398178, + 0.08117377012968063, + 0.08916938304901123, + 0.09386278688907623, + 0.10193294286727905, + 0.10801172256469727, + 0.11700897663831711, + 0.12567132711410522, + 0.13432858884334564, + 0.14358282089233398, + 0.15294352173805237, + 0.16399860382080078, + 0.17351281642913818, + 0.18877196311950684, + 0.19949108362197876, + 0.21516621112823486 + ], + "loss_nats": 3.5330913066864014, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.0289339367300272, + 0.030502017587423325, + 0.030956681817770004, + 0.03408988192677498, + 0.03459245339035988, + 0.038947492837905884, + 0.03989224135875702, + 0.04481571167707443, + 0.04556844383478165, + 0.0512830950319767, + 0.052534833550453186, + 0.05888458341360092, + 0.060703132301568985, + 0.06758055835962296, + 0.07060512155294418, + 0.07802816480398178, + 0.08117377012968063, + 0.08916938304901123, + 0.09386278688907623, + 0.10193294286727905, + 0.10801172256469727, + 0.11700897663831711, + 0.12567132711410522, + 0.13432858884334564, + 0.14358282089233398, + 0.15294352173805237, + 0.16399860382080078, + 0.17351281642913818, + 0.18877196311950684, + 0.19949108362197876, + 0.21516621112823486, + 0.2260059416294098 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014349650882650167, + 0.00014052700134925544, + 0.00012983661144971848, + 0.0001149297459051013, + 0.0001033403241308406, + 9.356468945043162e-05, + 8.457523654215038e-05, + 7.733132952125743e-05, + 7.047204417176545e-05, + 6.370909250108525e-05, + 5.659156886395067e-05, + 4.938157871947624e-05, + 4.27361810579896e-05, + 3.8237969420151785e-05, + 3.594611553126015e-05, + 3.5643912269733846e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013219746688264422, + "first_to_last_ratio": 3.4660158989817993, + "imbalance_abs_log_ratio": 1.243005778409783, + "last_quartile_mean": 3.8141044569783844e-05, + "mean": 8.001999435691687e-05, + "normalized": [ + 1.7932581722820122, + 1.7561486035909524, + 1.622552119544051, + 1.4362628594107976, + 1.2914312849099563, + 1.1692663840127346, + 1.0569262997559778, + 0.9664000871623777, + 0.8806804441579406, + 0.7961646712560445, + 0.7072178562214424, + 0.6171154986492164, + 0.5340687837013764, + 0.47785518766218865, + 0.44921417228459215, + 0.44543757539833934 + ], + "population_cv": 0.4531561582313658 + }, + "activation_output_rms_by_block": [ + 0.05229421332478523, + 0.06407254934310913, + 0.07844720035791397, + 0.08364550769329071, + 0.0890495702624321, + 0.09326893091201782, + 0.09662611037492752, + 0.09771094471216202, + 0.09944888204336166, + 0.10062506794929504, + 0.10494095832109451, + 0.11238908767700195, + 0.12405165284872055, + 0.14480136334896088, + 0.17238570749759674, + 0.2138574868440628 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.06961486767977476, + "first_to_last_ratio": 0.4250665264722616, + "imbalance_abs_log_ratio": 0.8555095894318028, + "last_quartile_mean": 0.16377405263483524, + "mean": 0.10797595209442079, + "normalized": [ + 0.48431351898667185, + 0.5933964748658124, + 0.7265247384835739, + 0.7746679336538376, + 0.8247166941816979, + 0.8637935494234656, + 0.8948854675569957, + 0.9049324670619027, + 0.9210280633265217, + 0.9319210990730817, + 0.9718919471006628, + 1.040871467183008, + 1.1488822320385021, + 1.3410519707419453, + 1.596519448579181, + 1.9806029277431396 + ], + "population_cv": 0.361888909970992 + }, + "bits_per_byte": 3.1304997557876018, + "branch_output_rms_by_sublayer": [ + 0.014489351771771908, + 0.02032402716577053, + 0.0192408449947834, + 0.013354538939893246, + 0.02438616193830967, + 0.010533644817769527, + 0.026007508859038353, + 0.009820356033742428, + 0.019688289612531662, + 0.009847180917859077, + 0.015725189819931984, + 0.00970252975821495, + 0.017999378964304924, + 0.012247176840901375, + 0.019808221608400345, + 0.012050271965563297, + 0.019276488572359085, + 0.012983737513422966, + 0.01952224411070347, + 0.015011198818683624, + 0.01836247928440571, + 0.01764002814888954, + 0.020674988627433777, + 0.02150883898139, + 0.01928989216685295, + 0.025364212691783905, + 0.025591954588890076, + 0.031167680397629738, + 0.011463184840977192, + 0.039293259382247925, + 0.019522786140441895, + 0.04468192160129547 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006452065099731355, + 0.00042477980025779674, + 0.0003406169302843733, + 0.00031739507901072504, + 0.0002899614562735046, + 0.0003022750042151422, + 0.0002946052417003999, + 0.0002490739832611916, + 0.00022953214745651343, + 0.00020477988838642542, + 0.00019411347501299613, + 0.00019138854173966402, + 0.00017481673923870362, + 0.00016715970609089626, + 0.0001418296786517743, + 0.00014407196058940572 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00043199957988150763, + "first_to_last_ratio": 2.752123958438998, + "imbalance_abs_log_ratio": 1.0123729620941166, + "last_quartile_mean": 0.00015696952114269496, + "mean": 0.00026947538388391546, + "normalized": [ + 2.394305931301975, + 1.576321347558719, + 1.2640001671956211, + 1.177825872018991, + 1.0760220547581225, + 1.121716573360022, + 1.0932547435475957, + 0.9242921548948887, + 0.8517740809876394, + 0.7599205739498657, + 0.720338430231595, + 0.7102264369427908, + 0.6487299014815064, + 0.6203153092562447, + 0.526317753434843, + 0.5346386690795809 + ], + "population_cv": 0.4584519745606109 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.043568700551986694, + 0.04671650007367134, + 0.05229421332478523, + 0.059547748416662216, + 0.06407254934310913, + 0.07493934780359268, + 0.07844720035791397, + 0.08121047168970108, + 0.08364550769329071, + 0.0867149755358696, + 0.0890495702624321, + 0.09094811230897903, + 0.09326893091201782, + 0.09377976506948471, + 0.09662611037492752, + 0.09500574320554733, + 0.09771094471216202, + 0.0959710031747818, + 0.09944888204336166, + 0.09544596821069717, + 0.10062506794929504, + 0.097707599401474, + 0.10494095832109451, + 0.10151325911283493, + 0.11238908767700195, + 0.10858181118965149, + 0.12405165284872055, + 0.1238408088684082, + 0.14480136334896088, + 0.1446269154548645, + 0.17238570749759674, + 0.18076260387897491 + ], + "loss_nats": 2.1698970794677734, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04671650007367134, + 0.05229421332478523, + 0.059547748416662216, + 0.06407254934310913, + 0.07493934780359268, + 0.07844720035791397, + 0.08121047168970108, + 0.08364550769329071, + 0.0867149755358696, + 0.0890495702624321, + 0.09094811230897903, + 0.09326893091201782, + 0.09377976506948471, + 0.09662611037492752, + 0.09500574320554733, + 0.09771094471216202, + 0.0959710031747818, + 0.09944888204336166, + 0.09544596821069717, + 0.10062506794929504, + 0.097707599401474, + 0.10494095832109451, + 0.10151325911283493, + 0.11238908767700195, + 0.10858181118965149, + 0.12405165284872055, + 0.1238408088684082, + 0.14480136334896088, + 0.1446269154548645, + 0.17238570749759674, + 0.18076260387897491, + 0.2138574868440628 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00016704612062312663, + 0.00018574431305751204, + 0.0002023741544689983, + 0.00018453448137734085, + 0.00017654163821134716, + 0.00017114127695094794, + 0.00016412502736784518, + 0.00015690893633291125, + 0.0001434983714716509, + 0.00012688931019511074, + 0.00010475149611011147, + 9.039731958182529e-05, + 7.724175520706922e-05, + 6.240545917535201e-05, + 4.393732160679065e-05, + 2.913598837039899e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018492476738174446, + "first_to_last_ratio": 3.4773281598182453, + "imbalance_abs_log_ratio": 1.246264228732751, + "last_quartile_mean": 5.3180131089902716e-05, + "mean": 0.00013041706063177116, + "normalized": [ + 1.2808609534206308, + 1.4242332418605552, + 1.551746017650221, + 1.4149566052433022, + 1.3536698140269194, + 1.312261418267664, + 1.2584628619352762, + 1.2031319796106978, + 1.100303677881642, + 0.9729502381086403, + 0.8032039336162798, + 0.6931402927187534, + 0.5922672603790627, + 0.47850686768314793, + 0.33689857288569336, + 0.22340626471151362 + ], + "population_cv": 0.41009229518876916 + }, + "activation_output_rms_by_block": [ + 0.08545628190040588, + 0.1388079822063446, + 0.16856427490711212, + 0.17068639397621155, + 0.17051894962787628, + 0.1691380739212036, + 0.16795755922794342, + 0.15964734554290771, + 0.14745007455348969, + 0.13513104617595673, + 0.13549363613128662, + 0.14635984599590302, + 0.16386321187019348, + 0.19344064593315125, + 0.2505952715873718, + 0.40341031551361084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14087873324751854, + "first_to_last_ratio": 0.5572131614407935, + "imbalance_abs_log_ratio": 0.5848074166792235, + "last_quartile_mean": 0.25282736122608185, + "mean": 0.17540755681693554, + "normalized": [ + 0.4871870029498929, + 0.7913455082850954, + 0.9609863905865502, + 0.9730846097716802, + 0.9721300080921755, + 0.9642576237335369, + 0.9575274992469833, + 0.9101508990831222, + 0.8406141515748735, + 0.77038326414287, + 0.7724503926173194, + 0.8343987491294447, + 0.9341855895137383, + 1.1028067971725755, + 1.4286458128420665, + 2.2998457012580755 + ], + "population_cv": 0.3845661956400748 + }, + "bits_per_byte": 2.092252187349085, + "branch_output_rms_by_sublayer": [ + 0.036218997091054916, + 0.037682514637708664, + 0.07264158874750137, + 0.009124485775828362, + 0.041704896837472916, + 0.003249978879466653, + 0.02913317084312439, + 0.0056674908846616745, + 0.02289137803018093, + 0.008465219289064407, + 0.019867170602083206, + 0.010350172407925129, + 0.024333713576197624, + 0.014746740460395813, + 0.03023042529821396, + 0.01600552536547184, + 0.0432618223130703, + 0.020560188218951225, + 0.04687124863266945, + 0.025769658386707306, + 0.04757922887802124, + 0.04326478764414787, + 0.029664678499102592, + 0.051030874252319336, + 0.02047067880630493, + 0.05886564403772354, + 0.0222918838262558, + 0.07155895233154297, + 0.011497496627271175, + 0.10460206121206284, + 0.015142823569476604, + 0.212371364235878 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040658806478613896, + 0.0003235049151816101, + 0.00024397059654808256, + 0.0002149925859972515, + 0.00020566351114887415, + 0.00024487046739718384, + 0.00031061648932541417, + 0.0003911240721720923, + 0.000499392430316199, + 0.0005175204124242609, + 0.0005752549257371158, + 0.00046811732551685614, + 0.00041118767134459974, + 0.0003618888130405475, + 0.0003117946917671641, + 0.00025214502653225866 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029726404062827076, + "first_to_last_ratio": 0.8893356416516115, + "imbalance_abs_log_ratio": 0.11728056505781807, + "last_quartile_mean": 0.0003342540506711425, + "mean": 0.0003586644999522281, + "normalized": [ + 1.1336166942652368, + 0.9019708257290558, + 0.6802195271084202, + 0.5994253293144072, + 0.5734147404503854, + 0.6827284758591918, + 0.8660363358146301, + 1.0905012127606375, + 1.3923664884110778, + 1.4429094947874448, + 1.6038802998728234, + 1.3051677140592575, + 1.1464409535913576, + 1.0089897748139245, + 0.8693213067049939, + 0.7030108264571566 + ], + "population_cv": 0.30960458504189897 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05539870634675026, + 0.07199405878782272, + 0.08545628190040588, + 0.1365138292312622, + 0.1388079822063446, + 0.16841135919094086, + 0.16856427490711212, + 0.17091451585292816, + 0.17068639397621155, + 0.17101584374904633, + 0.17051894962787628, + 0.16970102488994598, + 0.1691380739212036, + 0.16952496767044067, + 0.16795755922794342, + 0.16297046840190887, + 0.15964734554290771, + 0.14957723021507263, + 0.14745007455348969, + 0.1322014182806015, + 0.13513104617595673, + 0.12475068122148514, + 0.13549363613128662, + 0.1319173276424408, + 0.14635984599590302, + 0.14501839876174927, + 0.16386321187019348, + 0.16489192843437195, + 0.19344064593315125, + 0.19336768984794617, + 0.2505952715873718, + 0.2517329454421997 + ], + "loss_nats": 1.4502387046813965, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.07199405878782272, + 0.08545628190040588, + 0.1365138292312622, + 0.1388079822063446, + 0.16841135919094086, + 0.16856427490711212, + 0.17091451585292816, + 0.17068639397621155, + 0.17101584374904633, + 0.17051894962787628, + 0.16970102488994598, + 0.1691380739212036, + 0.16952496767044067, + 0.16795755922794342, + 0.16297046840190887, + 0.15964734554290771, + 0.14957723021507263, + 0.14745007455348969, + 0.1322014182806015, + 0.13513104617595673, + 0.12475068122148514, + 0.13549363613128662, + 0.1319173276424408, + 0.14635984599590302, + 0.14501839876174927, + 0.16386321187019348, + 0.16489192843437195, + 0.19344064593315125, + 0.19336768984794617, + 0.2505952715873718, + 0.2517329454421997, + 0.40341031551361084 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00017181786824949086, + 0.0001916729233926162, + 0.00020346585370134562, + 0.00018768427253235132, + 0.00018128244846593589, + 0.00017597789701540023, + 0.0001705518807284534, + 0.00016394301201216877, + 0.0001481794024584815, + 0.0001306164776906371, + 0.00010858976020244882, + 9.323586709797382e-05, + 7.920334610389546e-05, + 6.374791701091453e-05, + 4.397464726935141e-05, + 2.655091702763457e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000188660229468951, + "first_to_last_ratio": 3.535001559771659, + "imbalance_abs_log_ratio": 1.2627137405851174, + "last_quartile_mean": 5.336920685294899e-05, + "mean": 0.00013378090568494372, + "normalized": [ + 1.2843228065305883, + 1.4327375226776322, + 1.5208885951221702, + 1.4029227233245896, + 1.3550696755847886, + 1.315418640010041, + 1.2748596659235207, + 1.2254589784154797, + 1.107627442794015, + 0.9763461909746755, + 0.8116984979768304, + 0.6969295552352281, + 0.5920377478264399, + 0.476509832883341, + 0.3287064551118561, + 0.19846566960880369 + ], + "population_cv": 0.41282075642614385 + }, + "activation_output_rms_by_block": [ + 0.10104851424694061, + 0.1665971875190735, + 0.18799307942390442, + 0.18892255425453186, + 0.18902969360351562, + 0.1888171136379242, + 0.18645423650741577, + 0.1778823733329773, + 0.16393223404884338, + 0.15616974234580994, + 0.16218091547489166, + 0.17675508558750153, + 0.19963829219341278, + 0.23325374722480774, + 0.2989467680454254, + 0.5114414095878601 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1611403338611126, + "first_to_last_ratio": 0.5184360907576058, + "imbalance_abs_log_ratio": 0.6569385168504118, + "last_quartile_mean": 0.3108200542628765, + "mean": 0.20556643418967724, + "normalized": [ + 0.49156135166358245, + 0.8104299136957027, + 0.9145125281029207, + 0.9190340582558922, + 0.9195552491273791, + 0.9185211310505179, + 0.907026661441106, + 0.8653279122837938, + 0.797465961284235, + 0.7597044865880744, + 0.7889464839636536, + 0.8598440999585013, + 0.9711619164887857, + 1.1346879082875139, + 1.4542586644743063, + 2.4879616733340346 + ], + "population_cv": 0.4294257518805603 + }, + "bits_per_byte": 1.9625077876075785, + "branch_output_rms_by_sublayer": [ + 0.053652405738830566, + 0.041928116232156754, + 0.08492356538772583, + 0.010198036208748817, + 0.03331374749541283, + 0.00243964372202754, + 0.028503542765975, + 0.005333597306162119, + 0.022150175645947456, + 0.008892077021300793, + 0.02107803337275982, + 0.011626621708273888, + 0.028634268790483475, + 0.017805112525820732, + 0.03554495424032211, + 0.0204806849360466, + 0.0531805157661438, + 0.028403647243976593, + 0.05312826856970787, + 0.03512146323919296, + 0.05336543545126915, + 0.05582646653056145, + 0.03752146288752556, + 0.06727498769760132, + 0.032218869775533676, + 0.07892175018787384, + 0.028152117505669594, + 0.09350594878196716, + 0.016922952607274055, + 0.13253848254680634, + 0.02406182885169983, + 0.29775944352149963 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00044260282991257096, + 0.0003095538604202773, + 0.00019879384700301542, + 0.00020176150294426366, + 0.00022494965081173505, + 0.00026863158839826584, + 0.00037546716364749414, + 0.0004673111939829261, + 0.0006383031641084419, + 0.0006422551721113508, + 0.0007081700332195422, + 0.0005856936655792496, + 0.0005168621280586129, + 0.00045203145589075384, + 0.0003829553066507503, + 0.0003084257693651058 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002881780100700318, + "first_to_last_ratio": 0.6942899678444003, + "imbalance_abs_log_ratio": 0.36486558463682156, + "last_quartile_mean": 0.0004150686649913057, + "mean": 0.00042023552075652225, + "normalized": [ + 1.0532256509772957, + 0.7366199312780793, + 0.47305341215597385, + 0.48011529958497023, + 0.5352942331166415, + 0.6392405570920472, + 0.8934684125976915, + 1.1120221183151215, + 1.5189176844435874, + 1.5283219537347563, + 1.6851741422159423, + 1.3937271759532939, + 1.2299344118463382, + 1.075662178858635, + 0.9112873323067531, + 0.7339355055228726 + ], + "population_cv": 0.3791677394455083 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.058993492275476456, + 0.08599413186311722, + 0.10104851424694061, + 0.16386346518993378, + 0.1665971875190735, + 0.18777909874916077, + 0.18799307942390442, + 0.18934063613414764, + 0.18892255425453186, + 0.18942749500274658, + 0.18902969360351562, + 0.18932336568832397, + 0.1888171136379242, + 0.18826909363269806, + 0.18645423650741577, + 0.18273916840553284, + 0.1778823733329773, + 0.16554005444049835, + 0.16393223404884338, + 0.15082815289497375, + 0.15616974234580994, + 0.1484535187482834, + 0.16218091547489166, + 0.15933844447135925, + 0.17675508558750153, + 0.177049919962883, + 0.19963829219341278, + 0.20120804011821747, + 0.23325374722480774, + 0.23333875834941864, + 0.2989467680454254, + 0.3003007471561432 + ], + "loss_nats": 1.360306739807129, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08599413186311722, + 0.10104851424694061, + 0.16386346518993378, + 0.1665971875190735, + 0.18777909874916077, + 0.18799307942390442, + 0.18934063613414764, + 0.18892255425453186, + 0.18942749500274658, + 0.18902969360351562, + 0.18932336568832397, + 0.1888171136379242, + 0.18826909363269806, + 0.18645423650741577, + 0.18273916840553284, + 0.1778823733329773, + 0.16554005444049835, + 0.16393223404884338, + 0.15082815289497375, + 0.15616974234580994, + 0.1484535187482834, + 0.16218091547489166, + 0.15933844447135925, + 0.17675508558750153, + 0.177049919962883, + 0.19963829219341278, + 0.20120804011821747, + 0.23325374722480774, + 0.23333875834941864, + 0.2989467680454254, + 0.3003007471561432, + 0.5114414095878601 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0002529843186493963, + 0.0002798262285068631, + 0.0002946711319964379, + 0.00027859603869728744, + 0.0002704691141843796, + 0.0002655670396052301, + 0.0002578931744210422, + 0.00024818466044962406, + 0.00022750988136976957, + 0.00020225343178026378, + 0.00016641257388982922, + 0.0001406865194439888, + 0.000117213734483812, + 9.274082549382001e-05, + 6.181560456752777e-05, + 3.130940967821516e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002765194294624962, + "first_to_last_ratio": 3.6494630846841103, + "imbalance_abs_log_ratio": 1.2945800566875763, + "last_quartile_mean": 7.576989355584374e-05, + "mean": 0.00019925835545109294, + "normalized": [ + 1.269629662839861, + 1.4043387434036372, + 1.478839526349316, + 1.3981649003706025, + 1.3573790347314445, + 1.3327774336188996, + 1.2942652961137227, + 1.2455420496057434, + 1.1417833939998108, + 1.0150311203883542, + 0.8351598281190995, + 0.7060507908212645, + 0.5882500345767512, + 0.4654300457507431, + 0.31022841891666686, + 0.15712972039408363 + ], + "population_cv": 0.41719159337744666 + }, + "activation_output_rms_by_block": [ + 0.10425227880477905, + 0.15637855231761932, + 0.1697961539030075, + 0.17037443816661835, + 0.16992402076721191, + 0.16921496391296387, + 0.1664763242006302, + 0.1570158749818802, + 0.1424521654844284, + 0.13391907513141632, + 0.13948261737823486, + 0.15248820185661316, + 0.17248840630054474, + 0.20097237825393677, + 0.2573379576206207, + 0.4688045382499695 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15020035579800606, + "first_to_last_ratio": 0.5463801662721245, + "imbalance_abs_log_ratio": 0.6044402703006951, + "last_quartile_mean": 0.27490082010626793, + "mean": 0.18321112170815468, + "normalized": [ + 0.5690281126647281, + 0.853542900996599, + 0.9267786383267906, + 0.9299350202004414, + 0.927476559189958, + 0.9236063964638243, + 0.9086583972004597, + 0.8570215253198324, + 0.7775301202038757, + 0.7309549435800197, + 0.7613217804562272, + 0.8323086526347376, + 0.9414734470940544, + 1.0969442050252538, + 1.4045979044358783, + 2.5588213962073194 + ], + "population_cv": 0.43808900348348184 + }, + "bits_per_byte": 1.8689156889212923, + "branch_output_rms_by_sublayer": [ + 0.06537462025880814, + 0.033620789647102356, + 0.06649091839790344, + 0.009441502392292023, + 0.024681083858013153, + 0.00226230570115149, + 0.0236189141869545, + 0.004977936390787363, + 0.01721731387078762, + 0.008134713396430016, + 0.01816825568675995, + 0.010567971505224705, + 0.02614794671535492, + 0.015832560136914253, + 0.03134627267718315, + 0.01896953023970127, + 0.04721185564994812, + 0.02692008763551712, + 0.04645893722772598, + 0.03406013175845146, + 0.04632203280925751, + 0.053266141563653946, + 0.032039228826761246, + 0.06378299742937088, + 0.028799353167414665, + 0.07541537284851074, + 0.025642387568950653, + 0.08898185938596725, + 0.017618922516703606, + 0.12298691272735596, + 0.026708398014307022, + 0.2926487922668457 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005010958566464473, + 0.000367257410216879, + 0.0002469638962710481, + 0.00025453790022962814, + 0.00027094925891000524, + 0.0003539210655471446, + 0.0005067460397887177, + 0.0006525825825832493, + 0.0009467684204760593, + 0.0009621097966915796, + 0.0010586988551551607, + 0.0008532929973946807, + 0.0007592856290288629, + 0.0006522457278840908, + 0.0005350698764559804, + 0.0004007556744895193 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034246376584100063, + "first_to_last_ratio": 0.5835734049551724, + "imbalance_abs_log_ratio": 0.5385850340449921, + "last_quartile_mean": 0.0005868392269646134, + "mean": 0.0005826425617355658, + "normalized": [ + 0.8600399104963966, + 0.6303305565643863, + 0.4238686160094385, + 0.43686801642402323, + 0.4650351290899624, + 0.6074411462370523, + 0.8697374223387168, + 1.1200393267518036, + 1.6249558179475279, + 1.6512865002955899, + 1.8170640538197664, + 1.4645222533226991, + 1.3031757013547307, + 1.1194611769198464, + 0.9183501371100032, + 0.6878242353180568 + ], + "population_cv": 0.44619098566843096 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060730163007974625, + 0.09550311416387558, + 0.10425227880477905, + 0.15394315123558044, + 0.15637855231761932, + 0.1696780025959015, + 0.1697961539030075, + 0.17087572813034058, + 0.17037443816661835, + 0.1706146001815796, + 0.16992402076721191, + 0.17012864351272583, + 0.16921496391296387, + 0.16865167021751404, + 0.1664763242006302, + 0.16138805449008942, + 0.1570158749818802, + 0.14425092935562134, + 0.1424521654844284, + 0.1296483427286148, + 0.13391907513141632, + 0.1270018070936203, + 0.13948261737823486, + 0.13754454255104065, + 0.15248820185661316, + 0.1527755856513977, + 0.17248840630054474, + 0.17395897209644318, + 0.20097237825393677, + 0.2011977583169937, + 0.2573379576206207, + 0.25867754220962524 + ], + "loss_nats": 1.2954336404800415, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.09550311416387558, + 0.10425227880477905, + 0.15394315123558044, + 0.15637855231761932, + 0.1696780025959015, + 0.1697961539030075, + 0.17087572813034058, + 0.17037443816661835, + 0.1706146001815796, + 0.16992402076721191, + 0.17012864351272583, + 0.16921496391296387, + 0.16865167021751404, + 0.1664763242006302, + 0.16138805449008942, + 0.1570158749818802, + 0.14425092935562134, + 0.1424521654844284, + 0.1296483427286148, + 0.13391907513141632, + 0.1270018070936203, + 0.13948261737823486, + 0.13754454255104065, + 0.15248820185661316, + 0.1527755856513977, + 0.17248840630054474, + 0.17395897209644318, + 0.20097237825393677, + 0.2011977583169937, + 0.2573379576206207, + 0.25867754220962524, + 0.4688045382499695 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.076786148009306, + "cross_entropy_nats": 5.5984015464782715, + "step": 0 + }, + { + "bits_per_byte": 5.114718375266125, + "cross_entropy_nats": 3.5452526211738586, + "step": 100 + }, + { + "bits_per_byte": 3.1628078576350735, + "cross_entropy_nats": 2.192291349172592, + "step": 500 + }, + { + "bits_per_byte": 1.9765894690862489, + "cross_entropy_nats": 1.3700674176216125, + "step": 2000 + }, + { + "bits_per_byte": 1.840425322451167, + "cross_entropy_nats": 1.2756856232881546, + "step": 4000 + }, + { + "bits_per_byte": 1.7488000388579794, + "cross_entropy_nats": 1.2121758162975311, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "final_optimizer_state": "e9f9db84e99e09e299da28a6a358f3b99d039dde4de0eceec7d22aff7014508e", + "final_public_parameters": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.257796359896474, + "measured_steps": 7980, + "median_ms": 21.201465002377518, + "p95_ms": 21.46604766312521, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.088097790921855, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.6062421798706055, + "step": 1, + "unclipped_grad_norm": 19.475919723510742 + }, + { + "bits_per_byte": 7.474302716882146, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.180791854858398, + "step": 10, + "unclipped_grad_norm": 12.938376426696777 + }, + { + "bits_per_byte": 6.789615018295581, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.706202507019043, + "step": 20, + "unclipped_grad_norm": 4.482712745666504 + }, + { + "bits_per_byte": 6.558995914517002, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.54634952545166, + "step": 30, + "unclipped_grad_norm": 2.908937454223633 + }, + { + "bits_per_byte": 6.417572516920704, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.448322296142578, + "step": 40, + "unclipped_grad_norm": 2.8319714069366455 + }, + { + "bits_per_byte": 6.245190178377291, + "learning_rate": 3.75e-05, + "loss_nats": 4.328835964202881, + "step": 50, + "unclipped_grad_norm": 2.816138982772827 + }, + { + "bits_per_byte": 5.943838049951856, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.119954586029053, + "step": 60, + "unclipped_grad_norm": 2.6310009956359863 + }, + { + "bits_per_byte": 5.699140446373649, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.950343132019043, + "step": 70, + "unclipped_grad_norm": 2.5801262855529785 + }, + { + "bits_per_byte": 5.518637431021795, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.825227975845337, + "step": 80, + "unclipped_grad_norm": 2.382185459136963 + }, + { + "bits_per_byte": 5.304690325417858, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.676931142807007, + "step": 90, + "unclipped_grad_norm": 2.2607486248016357 + }, + { + "bits_per_byte": 5.142740110510481, + "learning_rate": 7.5e-05, + "loss_nats": 3.564675807952881, + "step": 100, + "unclipped_grad_norm": 2.145101547241211 + }, + { + "bits_per_byte": 4.946587017895412, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.428712844848633, + "step": 110, + "unclipped_grad_norm": 1.954435110092163 + }, + { + "bits_per_byte": 4.732811894942338, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2805352210998535, + "step": 120, + "unclipped_grad_norm": 1.7519731521606445 + }, + { + "bits_per_byte": 4.552989930887901, + "learning_rate": 9.75e-05, + "loss_nats": 3.1558921337127686, + "step": 130, + "unclipped_grad_norm": 1.5486056804656982 + }, + { + "bits_per_byte": 4.412413719829753, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0584521293640137, + "step": 140, + "unclipped_grad_norm": 1.3235344886779785 + }, + { + "bits_per_byte": 4.26477418130067, + "learning_rate": 0.0001125, + "loss_nats": 2.956116199493408, + "step": 150, + "unclipped_grad_norm": 1.2421116828918457 + }, + { + "bits_per_byte": 4.169433535003061, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.890031099319458, + "step": 160, + "unclipped_grad_norm": 0.9132174849510193 + }, + { + "bits_per_byte": 4.040492638206386, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.8006560802459717, + "step": 170, + "unclipped_grad_norm": 0.901505708694458 + }, + { + "bits_per_byte": 3.967424777091417, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.750009298324585, + "step": 180, + "unclipped_grad_norm": 0.677867591381073 + }, + { + "bits_per_byte": 3.8797366708266, + "learning_rate": 0.0001425, + "loss_nats": 2.6892285346984863, + "step": 190, + "unclipped_grad_norm": 0.7871851325035095 + }, + { + "bits_per_byte": 3.825796032209989, + "learning_rate": 0.00015, + "loss_nats": 2.6518397331237793, + "step": 200, + "unclipped_grad_norm": 0.6514210104942322 + }, + { + "bits_per_byte": 3.7775081193619364, + "learning_rate": 0.0001575, + "loss_nats": 2.6183691024780273, + "step": 210, + "unclipped_grad_norm": 0.6251815557479858 + }, + { + "bits_per_byte": 3.6875825188093945, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.556037425994873, + "step": 220, + "unclipped_grad_norm": 0.7170947790145874 + }, + { + "bits_per_byte": 3.6778121844138782, + "learning_rate": 0.0001725, + "loss_nats": 2.549265146255493, + "step": 230, + "unclipped_grad_norm": 0.6953222155570984 + }, + { + "bits_per_byte": 3.5779222530356467, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4800267219543457, + "step": 240, + "unclipped_grad_norm": 0.6264667510986328 + }, + { + "bits_per_byte": 3.617201026700902, + "learning_rate": 0.0001875, + "loss_nats": 2.5072526931762695, + "step": 250, + "unclipped_grad_norm": 0.8498449921607971 + }, + { + "bits_per_byte": 3.6087914190390125, + "learning_rate": 0.000195, + "loss_nats": 2.5014235973358154, + "step": 260, + "unclipped_grad_norm": 0.6882923245429993 + }, + { + "bits_per_byte": 3.6099880743237156, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.5022530555725098, + "step": 270, + "unclipped_grad_norm": 0.7451158165931702 + }, + { + "bits_per_byte": 3.535934064688699, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.450922727584839, + "step": 280, + "unclipped_grad_norm": 0.7390120029449463 + }, + { + "bits_per_byte": 3.452133454263985, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.392836570739746, + "step": 290, + "unclipped_grad_norm": 0.601737380027771 + }, + { + "bits_per_byte": 3.4977253670771122, + "learning_rate": 0.000225, + "loss_nats": 2.4244384765625, + "step": 300, + "unclipped_grad_norm": 0.8893438577651978 + }, + { + "bits_per_byte": 3.4549257645833937, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.3947720527648926, + "step": 310, + "unclipped_grad_norm": 1.0071029663085938 + }, + { + "bits_per_byte": 3.428612419001384, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.376533031463623, + "step": 320, + "unclipped_grad_norm": 0.7961780428886414 + }, + { + "bits_per_byte": 3.4202605975101843, + "learning_rate": 0.0002475, + "loss_nats": 2.370743989944458, + "step": 330, + "unclipped_grad_norm": 1.010308027267456 + }, + { + "bits_per_byte": 3.507607834160991, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.431288480758667, + "step": 340, + "unclipped_grad_norm": 0.7371622920036316 + }, + { + "bits_per_byte": 3.4475772898773274, + "learning_rate": 0.0002625, + "loss_nats": 2.389678478240967, + "step": 350, + "unclipped_grad_norm": 0.8701758980751038 + }, + { + "bits_per_byte": 3.4091412312013003, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.363036632537842, + "step": 360, + "unclipped_grad_norm": 0.7921563386917114 + }, + { + "bits_per_byte": 3.4048547356111953, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.360065460205078, + "step": 370, + "unclipped_grad_norm": 0.8983097076416016 + }, + { + "bits_per_byte": 3.4312740225061367, + "learning_rate": 0.000285, + "loss_nats": 2.378377914428711, + "step": 380, + "unclipped_grad_norm": 0.6835634708404541 + }, + { + "bits_per_byte": 3.36810135519121, + "learning_rate": 0.0002925, + "loss_nats": 2.334589958190918, + "step": 390, + "unclipped_grad_norm": 0.7815158367156982 + }, + { + "bits_per_byte": 3.32671476215698, + "learning_rate": 0.0003, + "loss_nats": 2.3059029579162598, + "step": 400, + "unclipped_grad_norm": 0.9281390309333801 + }, + { + "bits_per_byte": 3.3045228088202467, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.290520668029785, + "step": 410, + "unclipped_grad_norm": 0.8773083090782166 + }, + { + "bits_per_byte": 3.346598020388531, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3196849822998047, + "step": 420, + "unclipped_grad_norm": 0.9256104230880737 + }, + { + "bits_per_byte": 3.314825601502834, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2976620197296143, + "step": 430, + "unclipped_grad_norm": 0.7776153087615967 + }, + { + "bits_per_byte": 3.382362156600754, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.3444747924804688, + "step": 440, + "unclipped_grad_norm": 1.105894923210144 + }, + { + "bits_per_byte": 3.28705625079862, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.278413772583008, + "step": 450, + "unclipped_grad_norm": 1.029218077659607 + }, + { + "bits_per_byte": 3.3030692114551545, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.289513111114502, + "step": 460, + "unclipped_grad_norm": 0.986072301864624 + }, + { + "bits_per_byte": 3.2452720338755956, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.249451160430908, + "step": 470, + "unclipped_grad_norm": 0.9585361480712891 + }, + { + "bits_per_byte": 3.221687708997477, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2331037521362305, + "step": 480, + "unclipped_grad_norm": 1.0470387935638428 + }, + { + "bits_per_byte": 3.155786020978659, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1874241828918457, + "step": 490, + "unclipped_grad_norm": 1.4241349697113037 + }, + { + "bits_per_byte": 3.107888164782763, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.154223918914795, + "step": 500, + "unclipped_grad_norm": 1.0626088380813599 + }, + { + "bits_per_byte": 3.1122186879314886, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1572256088256836, + "step": 510, + "unclipped_grad_norm": 0.9446885585784912 + }, + { + "bits_per_byte": 3.0380233085574626, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.105797290802002, + "step": 520, + "unclipped_grad_norm": 1.1227535009384155 + }, + { + "bits_per_byte": 3.076978410873794, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.132798910140991, + "step": 530, + "unclipped_grad_norm": 0.9776996374130249 + }, + { + "bits_per_byte": 3.01776100056341, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.091752529144287, + "step": 540, + "unclipped_grad_norm": 1.1720482110977173 + }, + { + "bits_per_byte": 2.99280700585382, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.074455738067627, + "step": 550, + "unclipped_grad_norm": 1.1986867189407349 + }, + { + "bits_per_byte": 2.977761275625738, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0640268325805664, + "step": 560, + "unclipped_grad_norm": 1.1189862489700317 + }, + { + "bits_per_byte": 2.9082912915668184, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.015873908996582, + "step": 570, + "unclipped_grad_norm": 0.9214995503425598 + }, + { + "bits_per_byte": 2.96765970264466, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0570249557495117, + "step": 580, + "unclipped_grad_norm": 0.9309345483779907 + }, + { + "bits_per_byte": 2.863993376218783, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9851689338684082, + "step": 590, + "unclipped_grad_norm": 0.9733412861824036 + }, + { + "bits_per_byte": 2.9319891249945065, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0322999954223633, + "step": 600, + "unclipped_grad_norm": 1.4851497411727905 + }, + { + "bits_per_byte": 2.842428815592397, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9702215194702148, + "step": 610, + "unclipped_grad_norm": 1.0783556699752808 + }, + { + "bits_per_byte": 2.8677622040297908, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.987781286239624, + "step": 620, + "unclipped_grad_norm": 0.9511895179748535 + }, + { + "bits_per_byte": 2.802268974806772, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9423848390579224, + "step": 630, + "unclipped_grad_norm": 1.0907046794891357 + }, + { + "bits_per_byte": 2.773638163004386, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.922539472579956, + "step": 640, + "unclipped_grad_norm": 1.5626765489578247 + }, + { + "bits_per_byte": 2.718988439918267, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8846591711044312, + "step": 650, + "unclipped_grad_norm": 1.6085642576217651 + }, + { + "bits_per_byte": 2.628252113149558, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8217655420303345, + "step": 660, + "unclipped_grad_norm": 1.223048448562622 + }, + { + "bits_per_byte": 2.676602111734572, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8552792072296143, + "step": 670, + "unclipped_grad_norm": 1.3098845481872559 + }, + { + "bits_per_byte": 2.620622790774633, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8164772987365723, + "step": 680, + "unclipped_grad_norm": 1.1348049640655518 + }, + { + "bits_per_byte": 2.6956237368853, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8684639930725098, + "step": 690, + "unclipped_grad_norm": 0.956748902797699 + }, + { + "bits_per_byte": 2.6070459643675705, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.807066559791565, + "step": 700, + "unclipped_grad_norm": 1.1346901655197144 + }, + { + "bits_per_byte": 2.6253973731278863, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.819786787033081, + "step": 710, + "unclipped_grad_norm": 1.3684734106063843 + }, + { + "bits_per_byte": 2.6636225810739558, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8462824821472168, + "step": 720, + "unclipped_grad_norm": 1.6031205654144287 + }, + { + "bits_per_byte": 2.6068867084328713, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.806956171989441, + "step": 730, + "unclipped_grad_norm": 1.1380469799041748 + }, + { + "bits_per_byte": 2.5424343182128744, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7622811794281006, + "step": 740, + "unclipped_grad_norm": 1.2064875364303589 + }, + { + "bits_per_byte": 2.5150987077541362, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7433335781097412, + "step": 750, + "unclipped_grad_norm": 1.2007335424423218 + }, + { + "bits_per_byte": 2.45541642833847, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7019649744033813, + "step": 760, + "unclipped_grad_norm": 1.2129080295562744 + }, + { + "bits_per_byte": 2.6149697210407723, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.812558889389038, + "step": 770, + "unclipped_grad_norm": 1.221384048461914 + }, + { + "bits_per_byte": 2.484800524151635, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.72233247756958, + "step": 780, + "unclipped_grad_norm": 1.1399351358413696 + }, + { + "bits_per_byte": 2.4467844471090148, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.695981740951538, + "step": 790, + "unclipped_grad_norm": 1.4305745363235474 + }, + { + "bits_per_byte": 2.3889862376335507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.655919075012207, + "step": 800, + "unclipped_grad_norm": 0.9679463505744934 + }, + { + "bits_per_byte": 2.474951249719373, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7155054807662964, + "step": 810, + "unclipped_grad_norm": 1.4494884014129639 + }, + { + "bits_per_byte": 2.4123399337768623, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6721066236495972, + "step": 820, + "unclipped_grad_norm": 1.4326691627502441 + }, + { + "bits_per_byte": 2.4471743317785206, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6962519884109497, + "step": 830, + "unclipped_grad_norm": 1.0609163045883179 + }, + { + "bits_per_byte": 2.528181256022619, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7524017095565796, + "step": 840, + "unclipped_grad_norm": 1.2206887006759644 + }, + { + "bits_per_byte": 2.4185156587866947, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6763873100280762, + "step": 850, + "unclipped_grad_norm": 1.1635538339614868 + }, + { + "bits_per_byte": 2.438352137737211, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6901369094848633, + "step": 860, + "unclipped_grad_norm": 1.1800516843795776 + }, + { + "bits_per_byte": 2.364430554743359, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6388983726501465, + "step": 870, + "unclipped_grad_norm": 1.172108769416809 + }, + { + "bits_per_byte": 2.3856504621374155, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.65360689163208, + "step": 880, + "unclipped_grad_norm": 1.2126058340072632 + }, + { + "bits_per_byte": 2.2982358082874708, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5930156707763672, + "step": 890, + "unclipped_grad_norm": 0.9879708886146545 + }, + { + "bits_per_byte": 2.330757727565628, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.61555814743042, + "step": 900, + "unclipped_grad_norm": 1.5637768507003784 + }, + { + "bits_per_byte": 2.3422774694857202, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6235430240631104, + "step": 910, + "unclipped_grad_norm": 1.2122199535369873 + }, + { + "bits_per_byte": 2.3285455147275793, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6140247583389282, + "step": 920, + "unclipped_grad_norm": 1.228180170059204 + }, + { + "bits_per_byte": 2.3433181365060913, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6242643594741821, + "step": 930, + "unclipped_grad_norm": 1.113643765449524 + }, + { + "bits_per_byte": 2.3376019491393643, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6203022003173828, + "step": 940, + "unclipped_grad_norm": 1.0455681085586548 + }, + { + "bits_per_byte": 2.3211645353005004, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6089086532592773, + "step": 950, + "unclipped_grad_norm": 1.2410560846328735 + }, + { + "bits_per_byte": 2.362830772125033, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6377894878387451, + "step": 960, + "unclipped_grad_norm": 1.1456577777862549 + }, + { + "bits_per_byte": 2.268770708645749, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.57259202003479, + "step": 970, + "unclipped_grad_norm": 1.1831417083740234 + }, + { + "bits_per_byte": 2.35130105521119, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6297976970672607, + "step": 980, + "unclipped_grad_norm": 1.1095668077468872 + }, + { + "bits_per_byte": 2.3073481370607856, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5993318557739258, + "step": 990, + "unclipped_grad_norm": 1.1233346462249756 + }, + { + "bits_per_byte": 2.2862505093314924, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5847080945968628, + "step": 1000, + "unclipped_grad_norm": 1.2164878845214844 + }, + { + "bits_per_byte": 2.265617922690132, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5704066753387451, + "step": 1010, + "unclipped_grad_norm": 1.195701241493225 + }, + { + "bits_per_byte": 2.236535656429231, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.550248384475708, + "step": 1020, + "unclipped_grad_norm": 1.3137857913970947 + }, + { + "bits_per_byte": 2.2352426908600447, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5493521690368652, + "step": 1030, + "unclipped_grad_norm": 1.0306543111801147 + }, + { + "bits_per_byte": 2.288523947993248, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5862839221954346, + "step": 1040, + "unclipped_grad_norm": 1.111948847770691 + }, + { + "bits_per_byte": 2.267572505517188, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5717614889144897, + "step": 1050, + "unclipped_grad_norm": 1.4793521165847778 + }, + { + "bits_per_byte": 2.1784870402141108, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.510012149810791, + "step": 1060, + "unclipped_grad_norm": 1.2200031280517578 + }, + { + "bits_per_byte": 2.2691612812458586, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5728627443313599, + "step": 1070, + "unclipped_grad_norm": 1.243715524673462 + }, + { + "bits_per_byte": 2.2409759045092073, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.55332612991333, + "step": 1080, + "unclipped_grad_norm": 1.1007108688354492 + }, + { + "bits_per_byte": 2.2116298168618824, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.532984972000122, + "step": 1090, + "unclipped_grad_norm": 1.183898687362671 + }, + { + "bits_per_byte": 2.265987857372138, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5706630945205688, + "step": 1100, + "unclipped_grad_norm": 0.9869716167449951 + }, + { + "bits_per_byte": 2.1294642454989217, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4760321378707886, + "step": 1110, + "unclipped_grad_norm": 1.0796256065368652 + }, + { + "bits_per_byte": 2.1685739602183784, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.503140926361084, + "step": 1120, + "unclipped_grad_norm": 1.0218125581741333 + }, + { + "bits_per_byte": 2.111904300897876, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4638605117797852, + "step": 1130, + "unclipped_grad_norm": 1.0956017971038818 + }, + { + "bits_per_byte": 2.24492118652, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.556060791015625, + "step": 1140, + "unclipped_grad_norm": 1.0603337287902832 + }, + { + "bits_per_byte": 2.107949387858635, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4611191749572754, + "step": 1150, + "unclipped_grad_norm": 1.118997573852539 + }, + { + "bits_per_byte": 2.133223098316179, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4786375761032104, + "step": 1160, + "unclipped_grad_norm": 1.0214279890060425 + }, + { + "bits_per_byte": 2.1914696665624422, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5190110206604004, + "step": 1170, + "unclipped_grad_norm": 1.0747911930084229 + }, + { + "bits_per_byte": 2.168808888519457, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.5033037662506104, + "step": 1180, + "unclipped_grad_norm": 1.0325632095336914 + }, + { + "bits_per_byte": 2.099094345131011, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4549813270568848, + "step": 1190, + "unclipped_grad_norm": 1.1188828945159912 + }, + { + "bits_per_byte": 2.0751268429067697, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.438368320465088, + "step": 1200, + "unclipped_grad_norm": 1.219267725944519 + }, + { + "bits_per_byte": 2.1310088216863203, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4771027565002441, + "step": 1210, + "unclipped_grad_norm": 1.0049350261688232 + }, + { + "bits_per_byte": 2.151060622915717, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4910016059875488, + "step": 1220, + "unclipped_grad_norm": 1.0629826784133911 + }, + { + "bits_per_byte": 2.098358087402667, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4544709920883179, + "step": 1230, + "unclipped_grad_norm": 1.041517734527588 + }, + { + "bits_per_byte": 2.0484189691236807, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4198558330535889, + "step": 1240, + "unclipped_grad_norm": 1.308692455291748 + }, + { + "bits_per_byte": 2.1357799643865563, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.480409860610962, + "step": 1250, + "unclipped_grad_norm": 1.1052577495574951 + }, + { + "bits_per_byte": 2.115429601275262, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.466304063796997, + "step": 1260, + "unclipped_grad_norm": 1.1282013654708862 + }, + { + "bits_per_byte": 2.1198928950304534, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.469397783279419, + "step": 1270, + "unclipped_grad_norm": 1.2154173851013184 + }, + { + "bits_per_byte": 2.0054133314142284, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.3900465965270996, + "step": 1280, + "unclipped_grad_norm": 1.0942060947418213 + }, + { + "bits_per_byte": 2.1170947373009157, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4674582481384277, + "step": 1290, + "unclipped_grad_norm": 1.0796113014221191 + }, + { + "bits_per_byte": 2.0596614750105826, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4276485443115234, + "step": 1300, + "unclipped_grad_norm": 1.0578112602233887 + }, + { + "bits_per_byte": 2.0645378710931466, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4310286045074463, + "step": 1310, + "unclipped_grad_norm": 1.0866352319717407 + }, + { + "bits_per_byte": 2.0503159377626976, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.421170711517334, + "step": 1320, + "unclipped_grad_norm": 0.958570659160614 + }, + { + "bits_per_byte": 2.123195993822925, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4716873168945312, + "step": 1330, + "unclipped_grad_norm": 1.0188450813293457 + }, + { + "bits_per_byte": 2.0677759604435915, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4332730770111084, + "step": 1340, + "unclipped_grad_norm": 1.1106895208358765 + }, + { + "bits_per_byte": 2.0615133841950732, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4289321899414062, + "step": 1350, + "unclipped_grad_norm": 1.2472416162490845 + }, + { + "bits_per_byte": 2.122740239798138, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4713714122772217, + "step": 1360, + "unclipped_grad_norm": 1.2623285055160522 + }, + { + "bits_per_byte": 2.1191595610071747, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4688894748687744, + "step": 1370, + "unclipped_grad_norm": 0.9515148401260376 + }, + { + "bits_per_byte": 2.0626532852049917, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4297223091125488, + "step": 1380, + "unclipped_grad_norm": 0.9549561142921448 + }, + { + "bits_per_byte": 2.085573929033431, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.44560968875885, + "step": 1390, + "unclipped_grad_norm": 1.1485545635223389 + }, + { + "bits_per_byte": 2.0049195692236013, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3897043466567993, + "step": 1400, + "unclipped_grad_norm": 0.9498034715652466 + }, + { + "bits_per_byte": 2.044304112219137, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4170036315917969, + "step": 1410, + "unclipped_grad_norm": 1.0090101957321167 + }, + { + "bits_per_byte": 2.0692653302000634, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4343054294586182, + "step": 1420, + "unclipped_grad_norm": 1.0467784404754639 + }, + { + "bits_per_byte": 2.1090376940732947, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4618735313415527, + "step": 1430, + "unclipped_grad_norm": 1.0122960805892944 + }, + { + "bits_per_byte": 2.054903402991812, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4243505001068115, + "step": 1440, + "unclipped_grad_norm": 1.1181519031524658 + }, + { + "bits_per_byte": 1.9969074134678557, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.384150743484497, + "step": 1450, + "unclipped_grad_norm": 0.900530219078064 + }, + { + "bits_per_byte": 2.037356701054883, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4121880531311035, + "step": 1460, + "unclipped_grad_norm": 1.1993578672409058 + }, + { + "bits_per_byte": 1.9743343465768104, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.368504285812378, + "step": 1470, + "unclipped_grad_norm": 0.9825910329818726 + }, + { + "bits_per_byte": 2.0340613414817006, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.409903883934021, + "step": 1480, + "unclipped_grad_norm": 0.9332759380340576 + }, + { + "bits_per_byte": 2.041156141777744, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4148216247558594, + "step": 1490, + "unclipped_grad_norm": 0.9170629978179932 + }, + { + "bits_per_byte": 2.0456156519146167, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.4179127216339111, + "step": 1500, + "unclipped_grad_norm": 1.007337212562561 + }, + { + "bits_per_byte": 2.0521109206897528, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4224148988723755, + "step": 1510, + "unclipped_grad_norm": 0.9886259436607361 + }, + { + "bits_per_byte": 2.0030100458510716, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.388380765914917, + "step": 1520, + "unclipped_grad_norm": 0.9661282896995544 + }, + { + "bits_per_byte": 2.0761377569285413, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4390690326690674, + "step": 1530, + "unclipped_grad_norm": 0.9895579218864441 + }, + { + "bits_per_byte": 2.0208730238829373, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4007624387741089, + "step": 1540, + "unclipped_grad_norm": 0.9900161623954773 + }, + { + "bits_per_byte": 2.0817916865756563, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4429880380630493, + "step": 1550, + "unclipped_grad_norm": 1.0090205669403076 + }, + { + "bits_per_byte": 1.98651725359863, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3769488334655762, + "step": 1560, + "unclipped_grad_norm": 0.9516891837120056 + }, + { + "bits_per_byte": 2.0695575287238794, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.434507966041565, + "step": 1570, + "unclipped_grad_norm": 0.897994339466095 + }, + { + "bits_per_byte": 2.0129038637899073, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3952386379241943, + "step": 1580, + "unclipped_grad_norm": 0.9900956749916077 + }, + { + "bits_per_byte": 2.0172801343237623, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3982720375061035, + "step": 1590, + "unclipped_grad_norm": 0.9180708527565002 + }, + { + "bits_per_byte": 2.0575405849601425, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4261784553527832, + "step": 1600, + "unclipped_grad_norm": 1.0603415966033936 + }, + { + "bits_per_byte": 1.9872597027024046, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.377463459968567, + "step": 1610, + "unclipped_grad_norm": 1.0459290742874146 + }, + { + "bits_per_byte": 1.943624092542842, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3472175598144531, + "step": 1620, + "unclipped_grad_norm": 0.9182554483413696 + }, + { + "bits_per_byte": 1.9757394448343595, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3694782257080078, + "step": 1630, + "unclipped_grad_norm": 0.9694951772689819 + }, + { + "bits_per_byte": 2.015004803852847, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3966948986053467, + "step": 1640, + "unclipped_grad_norm": 0.9169139266014099 + }, + { + "bits_per_byte": 2.0755808771050477, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.438683032989502, + "step": 1650, + "unclipped_grad_norm": 0.9065545797348022 + }, + { + "bits_per_byte": 1.9985508796795006, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3852899074554443, + "step": 1660, + "unclipped_grad_norm": 0.9444847702980042 + }, + { + "bits_per_byte": 2.017943299425489, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3987317085266113, + "step": 1670, + "unclipped_grad_norm": 0.9302135109901428 + }, + { + "bits_per_byte": 2.035893988609295, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4111741781234741, + "step": 1680, + "unclipped_grad_norm": 0.9957410097122192 + }, + { + "bits_per_byte": 2.1006239588277844, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4560415744781494, + "step": 1690, + "unclipped_grad_norm": 1.046242117881775 + }, + { + "bits_per_byte": 1.9860793857695331, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3766453266143799, + "step": 1700, + "unclipped_grad_norm": 0.9597991108894348 + }, + { + "bits_per_byte": 1.902240595196328, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3185327053070068, + "step": 1710, + "unclipped_grad_norm": 0.8989817500114441 + }, + { + "bits_per_byte": 1.90692127502221, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.321777105331421, + "step": 1720, + "unclipped_grad_norm": 0.8322402238845825 + }, + { + "bits_per_byte": 1.8988062736412483, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3161522150039673, + "step": 1730, + "unclipped_grad_norm": 0.9716639518737793 + }, + { + "bits_per_byte": 1.9978667326943684, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3848156929016113, + "step": 1740, + "unclipped_grad_norm": 0.8983874917030334 + }, + { + "bits_per_byte": 2.0305991587371817, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4075040817260742, + "step": 1750, + "unclipped_grad_norm": 1.0045808553695679 + }, + { + "bits_per_byte": 2.0290275812735974, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.4064147472381592, + "step": 1760, + "unclipped_grad_norm": 1.0337824821472168 + }, + { + "bits_per_byte": 2.0294850551248924, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.4067318439483643, + "step": 1770, + "unclipped_grad_norm": 0.985150933265686 + }, + { + "bits_per_byte": 1.8961565369394051, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3143155574798584, + "step": 1780, + "unclipped_grad_norm": 0.9629419445991516 + }, + { + "bits_per_byte": 1.9830617781774944, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3745536804199219, + "step": 1790, + "unclipped_grad_norm": 0.8722974061965942 + }, + { + "bits_per_byte": 1.903928776897197, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3197028636932373, + "step": 1800, + "unclipped_grad_norm": 0.8771339058876038 + }, + { + "bits_per_byte": 1.9579086275582052, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.357118844985962, + "step": 1810, + "unclipped_grad_norm": 0.8671345710754395 + }, + { + "bits_per_byte": 1.9298429507465542, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3376652002334595, + "step": 1820, + "unclipped_grad_norm": 0.9717899560928345 + }, + { + "bits_per_byte": 1.9258892415858693, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3349246978759766, + "step": 1830, + "unclipped_grad_norm": 0.8506329655647278 + }, + { + "bits_per_byte": 1.920128166747267, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3309314250946045, + "step": 1840, + "unclipped_grad_norm": 0.9434375166893005 + }, + { + "bits_per_byte": 1.9401304369732149, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3447959423065186, + "step": 1850, + "unclipped_grad_norm": 0.9371633529663086 + }, + { + "bits_per_byte": 1.9244834553977166, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3339502811431885, + "step": 1860, + "unclipped_grad_norm": 0.9993670582771301 + }, + { + "bits_per_byte": 1.923443476307949, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.333229422569275, + "step": 1870, + "unclipped_grad_norm": 0.8980090618133545 + }, + { + "bits_per_byte": 1.9331135448180123, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3399322032928467, + "step": 1880, + "unclipped_grad_norm": 0.8608688116073608 + }, + { + "bits_per_byte": 1.937305793915444, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3428380489349365, + "step": 1890, + "unclipped_grad_norm": 0.9540092945098877 + }, + { + "bits_per_byte": 1.9529099518108777, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3536540269851685, + "step": 1900, + "unclipped_grad_norm": 0.9091746211051941 + }, + { + "bits_per_byte": 1.8773674323326432, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3012919425964355, + "step": 1910, + "unclipped_grad_norm": 0.8313417434692383 + }, + { + "bits_per_byte": 1.8889088440667445, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3092918395996094, + "step": 1920, + "unclipped_grad_norm": 0.9861888289451599 + }, + { + "bits_per_byte": 1.9783957168769355, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3713194131851196, + "step": 1930, + "unclipped_grad_norm": 0.9654703140258789 + }, + { + "bits_per_byte": 1.8474522540936138, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.280556321144104, + "step": 1940, + "unclipped_grad_norm": 0.8361129760742188 + }, + { + "bits_per_byte": 1.9440165569521108, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.347489595413208, + "step": 1950, + "unclipped_grad_norm": 0.7723413705825806 + }, + { + "bits_per_byte": 1.8547983210425678, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.285648226737976, + "step": 1960, + "unclipped_grad_norm": 0.8670309782028198 + }, + { + "bits_per_byte": 1.9366295581322517, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3423693180084229, + "step": 1970, + "unclipped_grad_norm": 0.8607841730117798 + }, + { + "bits_per_byte": 1.973360924772927, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3678295612335205, + "step": 1980, + "unclipped_grad_norm": 0.8964365124702454 + }, + { + "bits_per_byte": 1.9723059831925343, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.367098331451416, + "step": 1990, + "unclipped_grad_norm": 0.9170846343040466 + }, + { + "bits_per_byte": 1.931895563684602, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.339087963104248, + "step": 2000, + "unclipped_grad_norm": 0.8662557601928711 + }, + { + "bits_per_byte": 1.9008518352906107, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3175700902938843, + "step": 2010, + "unclipped_grad_norm": 0.8140403628349304 + }, + { + "bits_per_byte": 1.945781786880567, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3487131595611572, + "step": 2020, + "unclipped_grad_norm": 0.8827610015869141 + }, + { + "bits_per_byte": 2.023320680970017, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.4024590253829956, + "step": 2030, + "unclipped_grad_norm": 0.9354773759841919 + }, + { + "bits_per_byte": 1.9386018551723463, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3437364101409912, + "step": 2040, + "unclipped_grad_norm": 0.8753958940505981 + }, + { + "bits_per_byte": 1.9926081911615872, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3811707496643066, + "step": 2050, + "unclipped_grad_norm": 0.8750029802322388 + }, + { + "bits_per_byte": 1.948962090060322, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3509175777435303, + "step": 2060, + "unclipped_grad_norm": 0.9496069550514221 + }, + { + "bits_per_byte": 1.9059292790920332, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.321089506149292, + "step": 2070, + "unclipped_grad_norm": 0.8713699579238892 + }, + { + "bits_per_byte": 1.8954827089133246, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3138484954833984, + "step": 2080, + "unclipped_grad_norm": 0.8487783074378967 + }, + { + "bits_per_byte": 1.892261817827966, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3116159439086914, + "step": 2090, + "unclipped_grad_norm": 0.859647810459137 + }, + { + "bits_per_byte": 1.8838277886296537, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.305769920349121, + "step": 2100, + "unclipped_grad_norm": 0.8237807750701904 + }, + { + "bits_per_byte": 1.9342706440930173, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3407342433929443, + "step": 2110, + "unclipped_grad_norm": 0.885201632976532 + }, + { + "bits_per_byte": 1.89223670836094, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.311598539352417, + "step": 2120, + "unclipped_grad_norm": 0.8643916249275208 + }, + { + "bits_per_byte": 1.8591027028283622, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.288631796836853, + "step": 2130, + "unclipped_grad_norm": 0.9507161378860474 + }, + { + "bits_per_byte": 1.991344806608349, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3802950382232666, + "step": 2140, + "unclipped_grad_norm": 0.967012882232666 + }, + { + "bits_per_byte": 1.8634151677987472, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2916209697723389, + "step": 2150, + "unclipped_grad_norm": 0.8769172430038452 + }, + { + "bits_per_byte": 1.9079485273958137, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3224891424179077, + "step": 2160, + "unclipped_grad_norm": 0.8441570401191711 + }, + { + "bits_per_byte": 1.982388982047319, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3740873336791992, + "step": 2170, + "unclipped_grad_norm": 0.8798146843910217 + }, + { + "bits_per_byte": 1.855854810466818, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2863805294036865, + "step": 2180, + "unclipped_grad_norm": 0.9269494414329529 + }, + { + "bits_per_byte": 1.881138839883414, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3039060831069946, + "step": 2190, + "unclipped_grad_norm": 0.9328587055206299 + }, + { + "bits_per_byte": 1.909031158182995, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3232395648956299, + "step": 2200, + "unclipped_grad_norm": 0.8771402835845947 + }, + { + "bits_per_byte": 1.8627256893514381, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2911430597305298, + "step": 2210, + "unclipped_grad_norm": 0.9345882534980774 + }, + { + "bits_per_byte": 1.85186464098415, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2836147546768188, + "step": 2220, + "unclipped_grad_norm": 0.8119871020317078 + }, + { + "bits_per_byte": 1.929916043373171, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3377158641815186, + "step": 2230, + "unclipped_grad_norm": 0.8476319909095764 + }, + { + "bits_per_byte": 1.8588350978236197, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.288446307182312, + "step": 2240, + "unclipped_grad_norm": 0.9140865206718445 + }, + { + "bits_per_byte": 1.8803798804451561, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.303380012512207, + "step": 2250, + "unclipped_grad_norm": 0.891239583492279 + }, + { + "bits_per_byte": 1.8744335802915746, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2992583513259888, + "step": 2260, + "unclipped_grad_norm": 0.8316894769668579 + }, + { + "bits_per_byte": 1.8524942694689588, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2840511798858643, + "step": 2270, + "unclipped_grad_norm": 0.8655860424041748 + }, + { + "bits_per_byte": 1.8715530428722735, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2972617149353027, + "step": 2280, + "unclipped_grad_norm": 0.8277263045310974 + }, + { + "bits_per_byte": 1.9110787836241676, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.3246588706970215, + "step": 2290, + "unclipped_grad_norm": 0.9131273031234741 + }, + { + "bits_per_byte": 1.8344360911283637, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2715342044830322, + "step": 2300, + "unclipped_grad_norm": 0.882969856262207 + }, + { + "bits_per_byte": 1.851766782855809, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2835469245910645, + "step": 2310, + "unclipped_grad_norm": 0.854379415512085 + }, + { + "bits_per_byte": 1.8345169229742693, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.271590232849121, + "step": 2320, + "unclipped_grad_norm": 0.8358980417251587 + }, + { + "bits_per_byte": 1.902897396939973, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.3189879655838013, + "step": 2330, + "unclipped_grad_norm": 0.8988526463508606 + }, + { + "bits_per_byte": 1.820362922790911, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2617794275283813, + "step": 2340, + "unclipped_grad_norm": 0.8210547566413879 + }, + { + "bits_per_byte": 1.8788371960669168, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3023107051849365, + "step": 2350, + "unclipped_grad_norm": 0.8696999549865723 + }, + { + "bits_per_byte": 1.8997241450489033, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.3167884349822998, + "step": 2360, + "unclipped_grad_norm": 0.8768331408500671 + }, + { + "bits_per_byte": 1.9060586100454822, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3211791515350342, + "step": 2370, + "unclipped_grad_norm": 0.8498597741127014 + }, + { + "bits_per_byte": 1.8700977256806723, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.296252965927124, + "step": 2380, + "unclipped_grad_norm": 0.9666873216629028 + }, + { + "bits_per_byte": 1.9181192374025384, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3295389413833618, + "step": 2390, + "unclipped_grad_norm": 0.8190925717353821 + }, + { + "bits_per_byte": 1.8665703615114762, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2938079833984375, + "step": 2400, + "unclipped_grad_norm": 0.8776637315750122 + }, + { + "bits_per_byte": 1.8564054989148806, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2867622375488281, + "step": 2410, + "unclipped_grad_norm": 0.8437303900718689 + }, + { + "bits_per_byte": 1.8440313471853016, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2781851291656494, + "step": 2420, + "unclipped_grad_norm": 0.8381853103637695 + }, + { + "bits_per_byte": 1.8694933786455403, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2958340644836426, + "step": 2430, + "unclipped_grad_norm": 0.8620160818099976 + }, + { + "bits_per_byte": 1.9125165585853807, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.325655460357666, + "step": 2440, + "unclipped_grad_norm": 0.8893632292747498 + }, + { + "bits_per_byte": 1.8473590394968462, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2804917097091675, + "step": 2450, + "unclipped_grad_norm": 0.8294656872749329 + }, + { + "bits_per_byte": 1.8569149115267363, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2871153354644775, + "step": 2460, + "unclipped_grad_norm": 0.8894950151443481 + }, + { + "bits_per_byte": 1.8524935815383552, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.284050703048706, + "step": 2470, + "unclipped_grad_norm": 0.857410728931427 + }, + { + "bits_per_byte": 1.8769517502655078, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.3010038137435913, + "step": 2480, + "unclipped_grad_norm": 0.8339816331863403 + }, + { + "bits_per_byte": 1.8457628685141885, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2793853282928467, + "step": 2490, + "unclipped_grad_norm": 0.8252822756767273 + }, + { + "bits_per_byte": 1.8621884155501427, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2907706499099731, + "step": 2500, + "unclipped_grad_norm": 0.8138288259506226 + }, + { + "bits_per_byte": 1.815880023013521, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2586721181869507, + "step": 2510, + "unclipped_grad_norm": 0.8689138889312744 + }, + { + "bits_per_byte": 1.8820847444631592, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.304561734199524, + "step": 2520, + "unclipped_grad_norm": 0.8937636613845825 + }, + { + "bits_per_byte": 1.878350485164975, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.3019733428955078, + "step": 2530, + "unclipped_grad_norm": 0.9091646671295166 + }, + { + "bits_per_byte": 1.8572593927764145, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2873541116714478, + "step": 2540, + "unclipped_grad_norm": 0.8881837129592896 + }, + { + "bits_per_byte": 1.8423865051124497, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2770450115203857, + "step": 2550, + "unclipped_grad_norm": 1.0689516067504883 + }, + { + "bits_per_byte": 1.8515612635880279, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2834044694900513, + "step": 2560, + "unclipped_grad_norm": 0.829948365688324 + }, + { + "bits_per_byte": 1.8581186181001252, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2879496812820435, + "step": 2570, + "unclipped_grad_norm": 0.8880435824394226 + }, + { + "bits_per_byte": 1.880620656156364, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.3035469055175781, + "step": 2580, + "unclipped_grad_norm": 0.9021074175834656 + }, + { + "bits_per_byte": 1.8318979711669305, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2697749137878418, + "step": 2590, + "unclipped_grad_norm": 0.7873226404190063 + }, + { + "bits_per_byte": 1.8382764637221303, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2741961479187012, + "step": 2600, + "unclipped_grad_norm": 0.8448187708854675 + }, + { + "bits_per_byte": 1.7796977969770509, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2335925102233887, + "step": 2610, + "unclipped_grad_norm": 0.8379483222961426 + }, + { + "bits_per_byte": 1.8338702682070251, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2711420059204102, + "step": 2620, + "unclipped_grad_norm": 0.8167812824249268 + }, + { + "bits_per_byte": 1.8538771819645465, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.285009741783142, + "step": 2630, + "unclipped_grad_norm": 0.8943092226982117 + }, + { + "bits_per_byte": 1.7228497916476038, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1941884756088257, + "step": 2640, + "unclipped_grad_norm": 0.8208516240119934 + }, + { + "bits_per_byte": 1.79524330878854, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2443678379058838, + "step": 2650, + "unclipped_grad_norm": 0.8445570468902588 + }, + { + "bits_per_byte": 1.8489555544448055, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2815983295440674, + "step": 2660, + "unclipped_grad_norm": 0.8848552703857422 + }, + { + "bits_per_byte": 1.8531913151529056, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2845343351364136, + "step": 2670, + "unclipped_grad_norm": 0.8651367425918579 + }, + { + "bits_per_byte": 1.762816667898961, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2218914031982422, + "step": 2680, + "unclipped_grad_norm": 0.7578476071357727 + }, + { + "bits_per_byte": 1.9435059404616992, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3471356630325317, + "step": 2690, + "unclipped_grad_norm": 0.8484299778938293 + }, + { + "bits_per_byte": 1.8394798263302174, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.275030255317688, + "step": 2700, + "unclipped_grad_norm": 0.8363795876502991 + }, + { + "bits_per_byte": 1.7593267959476528, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2194724082946777, + "step": 2710, + "unclipped_grad_norm": 0.7895286083221436 + }, + { + "bits_per_byte": 1.7812189835239323, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2346469163894653, + "step": 2720, + "unclipped_grad_norm": 0.8936913013458252 + }, + { + "bits_per_byte": 1.806379013449256, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2520865201950073, + "step": 2730, + "unclipped_grad_norm": 0.8484287858009338 + }, + { + "bits_per_byte": 1.8120035340630734, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.255985140800476, + "step": 2740, + "unclipped_grad_norm": 0.8566655516624451 + }, + { + "bits_per_byte": 1.8444568322635362, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.278480052947998, + "step": 2750, + "unclipped_grad_norm": 0.8961714506149292 + }, + { + "bits_per_byte": 1.855966255224577, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2864577770233154, + "step": 2760, + "unclipped_grad_norm": 0.9054936766624451 + }, + { + "bits_per_byte": 1.7360374213157639, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2033294439315796, + "step": 2770, + "unclipped_grad_norm": 0.8361671566963196 + }, + { + "bits_per_byte": 1.75377691580431, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.215625524520874, + "step": 2780, + "unclipped_grad_norm": 0.7793561220169067 + }, + { + "bits_per_byte": 1.8377969760915247, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2738637924194336, + "step": 2790, + "unclipped_grad_norm": 0.7846143245697021 + }, + { + "bits_per_byte": 1.9117343814892567, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.325113296508789, + "step": 2800, + "unclipped_grad_norm": 0.872995913028717 + }, + { + "bits_per_byte": 1.8341784611173713, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2713556289672852, + "step": 2810, + "unclipped_grad_norm": 0.8531777858734131 + }, + { + "bits_per_byte": 1.8258434939259562, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.265578269958496, + "step": 2820, + "unclipped_grad_norm": 0.8563913702964783 + }, + { + "bits_per_byte": 1.8998995673527832, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.3169100284576416, + "step": 2830, + "unclipped_grad_norm": 0.864924430847168 + }, + { + "bits_per_byte": 1.8091240285396772, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2539892196655273, + "step": 2840, + "unclipped_grad_norm": 0.8685922622680664 + }, + { + "bits_per_byte": 1.8212324670736735, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.26238214969635, + "step": 2850, + "unclipped_grad_norm": 0.8504610061645508 + }, + { + "bits_per_byte": 1.7804541766755455, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.234116792678833, + "step": 2860, + "unclipped_grad_norm": 0.8544119000434875 + }, + { + "bits_per_byte": 1.8702564656674188, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2963629961013794, + "step": 2870, + "unclipped_grad_norm": 0.855370044708252 + }, + { + "bits_per_byte": 1.8717095470845586, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.297370195388794, + "step": 2880, + "unclipped_grad_norm": 0.8495773673057556 + }, + { + "bits_per_byte": 1.7666914370229, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2245771884918213, + "step": 2890, + "unclipped_grad_norm": 0.8828970789909363 + }, + { + "bits_per_byte": 1.7353923143923775, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2028822898864746, + "step": 2900, + "unclipped_grad_norm": 0.8020629286766052 + }, + { + "bits_per_byte": 1.7995504422967483, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2473533153533936, + "step": 2910, + "unclipped_grad_norm": 0.8652002811431885 + }, + { + "bits_per_byte": 1.7761088630188457, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.231104850769043, + "step": 2920, + "unclipped_grad_norm": 0.8863427042961121 + }, + { + "bits_per_byte": 1.7940502651395047, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.24354088306427, + "step": 2930, + "unclipped_grad_norm": 0.8493190407752991 + }, + { + "bits_per_byte": 1.8227453984533135, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2634308338165283, + "step": 2940, + "unclipped_grad_norm": 0.9217897653579712 + }, + { + "bits_per_byte": 1.8002454241888848, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2478350400924683, + "step": 2950, + "unclipped_grad_norm": 0.9446085095405579 + }, + { + "bits_per_byte": 1.8533994141604497, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.28467857837677, + "step": 2960, + "unclipped_grad_norm": 0.9335201382637024 + }, + { + "bits_per_byte": 1.7928699482066333, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.242722749710083, + "step": 2970, + "unclipped_grad_norm": 0.9046677947044373 + }, + { + "bits_per_byte": 1.784769565350995, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2371079921722412, + "step": 2980, + "unclipped_grad_norm": 0.8317873477935791 + }, + { + "bits_per_byte": 1.8006910312372704, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2481439113616943, + "step": 2990, + "unclipped_grad_norm": 0.8293972015380859 + }, + { + "bits_per_byte": 1.8134308180825838, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.256974458694458, + "step": 3000, + "unclipped_grad_norm": 0.8042416572570801 + }, + { + "bits_per_byte": 1.811533849443567, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.255659580230713, + "step": 3010, + "unclipped_grad_norm": 0.7810729146003723 + }, + { + "bits_per_byte": 1.7608942458276164, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2205588817596436, + "step": 3020, + "unclipped_grad_norm": 0.846473753452301 + }, + { + "bits_per_byte": 1.842709660513421, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2772690057754517, + "step": 3030, + "unclipped_grad_norm": 0.8461940884590149 + }, + { + "bits_per_byte": 1.775488005649231, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2306745052337646, + "step": 3040, + "unclipped_grad_norm": 0.8409631252288818 + }, + { + "bits_per_byte": 1.7848693152884954, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2371771335601807, + "step": 3050, + "unclipped_grad_norm": 0.8528071641921997 + }, + { + "bits_per_byte": 1.8093759831231913, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2541638612747192, + "step": 3060, + "unclipped_grad_norm": 0.9029015898704529 + }, + { + "bits_per_byte": 1.7287833650850215, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.1983013153076172, + "step": 3070, + "unclipped_grad_norm": 0.802097737789154 + }, + { + "bits_per_byte": 1.755199900257549, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2166118621826172, + "step": 3080, + "unclipped_grad_norm": 0.8510171175003052 + }, + { + "bits_per_byte": 1.8287252352238135, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.267575740814209, + "step": 3090, + "unclipped_grad_norm": 0.9035980105400085 + }, + { + "bits_per_byte": 1.7486468453117232, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2120696306228638, + "step": 3100, + "unclipped_grad_norm": 0.8420509099960327 + }, + { + "bits_per_byte": 1.8143894493784931, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.257638931274414, + "step": 3110, + "unclipped_grad_norm": 0.8073800206184387 + }, + { + "bits_per_byte": 1.7807735484581977, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2343381643295288, + "step": 3120, + "unclipped_grad_norm": 0.7948007583618164 + }, + { + "bits_per_byte": 1.8404088766101783, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2756742238998413, + "step": 3130, + "unclipped_grad_norm": 0.7875926494598389 + }, + { + "bits_per_byte": 1.8127681689288093, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2565151453018188, + "step": 3140, + "unclipped_grad_norm": 0.8293087482452393 + }, + { + "bits_per_byte": 1.7580950562021733, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.218618631362915, + "step": 3150, + "unclipped_grad_norm": 0.831396222114563 + }, + { + "bits_per_byte": 1.7956681059361712, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2446622848510742, + "step": 3160, + "unclipped_grad_norm": 0.8769922256469727 + }, + { + "bits_per_byte": 1.6987359321874784, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.177474021911621, + "step": 3170, + "unclipped_grad_norm": 0.8662227988243103 + }, + { + "bits_per_byte": 1.8090672742648926, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2539498805999756, + "step": 3180, + "unclipped_grad_norm": 0.87493896484375 + }, + { + "bits_per_byte": 1.77672765659665, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2315337657928467, + "step": 3190, + "unclipped_grad_norm": 0.8155474066734314 + }, + { + "bits_per_byte": 1.7352984118650063, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2028172016143799, + "step": 3200, + "unclipped_grad_norm": 0.8111788630485535 + }, + { + "bits_per_byte": 1.7460088034301386, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2102410793304443, + "step": 3210, + "unclipped_grad_norm": 0.8163402080535889 + }, + { + "bits_per_byte": 1.7462268774314327, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2103922367095947, + "step": 3220, + "unclipped_grad_norm": 0.8070465326309204 + }, + { + "bits_per_byte": 1.8407530138945547, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2759127616882324, + "step": 3230, + "unclipped_grad_norm": 0.9377122521400452 + }, + { + "bits_per_byte": 1.717580415207818, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1905360221862793, + "step": 3240, + "unclipped_grad_norm": 0.817809522151947 + }, + { + "bits_per_byte": 1.8646075235171788, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2924474477767944, + "step": 3250, + "unclipped_grad_norm": 0.8675336837768555 + }, + { + "bits_per_byte": 1.7240638171800442, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1950299739837646, + "step": 3260, + "unclipped_grad_norm": 0.8383887410163879 + }, + { + "bits_per_byte": 1.7558293567597067, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.217048168182373, + "step": 3270, + "unclipped_grad_norm": 0.7917745113372803 + }, + { + "bits_per_byte": 1.8139120255396979, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.257308006286621, + "step": 3280, + "unclipped_grad_norm": 0.819724440574646 + }, + { + "bits_per_byte": 1.8132829130028418, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2568719387054443, + "step": 3290, + "unclipped_grad_norm": 0.8676843047142029 + }, + { + "bits_per_byte": 1.7244280764345716, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1952824592590332, + "step": 3300, + "unclipped_grad_norm": 0.7852486968040466 + }, + { + "bits_per_byte": 1.7655542877353951, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2237889766693115, + "step": 3310, + "unclipped_grad_norm": 0.8482234477996826 + }, + { + "bits_per_byte": 1.7764401016044076, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2313344478607178, + "step": 3320, + "unclipped_grad_norm": 0.80937659740448 + }, + { + "bits_per_byte": 1.7828375122512021, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2357687950134277, + "step": 3330, + "unclipped_grad_norm": 0.8368547558784485 + }, + { + "bits_per_byte": 1.7427524839187023, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2079839706420898, + "step": 3340, + "unclipped_grad_norm": 0.8288648128509521 + }, + { + "bits_per_byte": 1.7505973005551585, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2134215831756592, + "step": 3350, + "unclipped_grad_norm": 0.7933014631271362 + }, + { + "bits_per_byte": 1.7744660847378042, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.229966163635254, + "step": 3360, + "unclipped_grad_norm": 0.8237564563751221 + }, + { + "bits_per_byte": 1.725253937124015, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.195854902267456, + "step": 3370, + "unclipped_grad_norm": 0.8081465363502502 + }, + { + "bits_per_byte": 1.7597923529835384, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2197951078414917, + "step": 3380, + "unclipped_grad_norm": 0.816970944404602 + }, + { + "bits_per_byte": 1.7815524578839552, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2348780632019043, + "step": 3390, + "unclipped_grad_norm": 0.8336570858955383 + }, + { + "bits_per_byte": 1.834212685664893, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2713793516159058, + "step": 3400, + "unclipped_grad_norm": 0.8331590890884399 + }, + { + "bits_per_byte": 1.8815787715043208, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.3042110204696655, + "step": 3410, + "unclipped_grad_norm": 0.8871732354164124 + }, + { + "bits_per_byte": 1.7373591079876445, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2042455673217773, + "step": 3420, + "unclipped_grad_norm": 0.8121070265769958 + }, + { + "bits_per_byte": 1.6796587566405214, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1642507314682007, + "step": 3430, + "unclipped_grad_norm": 0.9302552938461304 + }, + { + "bits_per_byte": 1.7533135945428855, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2153043746948242, + "step": 3440, + "unclipped_grad_norm": 0.8166707754135132 + }, + { + "bits_per_byte": 1.8162779908676177, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2589479684829712, + "step": 3450, + "unclipped_grad_norm": 0.8299344182014465 + }, + { + "bits_per_byte": 1.7445694806250678, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2092434167861938, + "step": 3460, + "unclipped_grad_norm": 0.8381974101066589 + }, + { + "bits_per_byte": 1.7494488004126965, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.212625503540039, + "step": 3470, + "unclipped_grad_norm": 0.8255388140678406 + }, + { + "bits_per_byte": 1.6905657243755898, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1718108654022217, + "step": 3480, + "unclipped_grad_norm": 0.7999947667121887 + }, + { + "bits_per_byte": 1.9018883747273607, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.3182885646820068, + "step": 3490, + "unclipped_grad_norm": 0.8489032983779907 + }, + { + "bits_per_byte": 1.7449648687394015, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.209517478942871, + "step": 3500, + "unclipped_grad_norm": 0.8267634510993958 + }, + { + "bits_per_byte": 1.784236763098622, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.236738681793213, + "step": 3510, + "unclipped_grad_norm": 0.8314446806907654 + }, + { + "bits_per_byte": 1.7790934499419189, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2331736087799072, + "step": 3520, + "unclipped_grad_norm": 0.8323742151260376 + }, + { + "bits_per_byte": 1.7501370749814495, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2131025791168213, + "step": 3530, + "unclipped_grad_norm": 0.8194237947463989 + }, + { + "bits_per_byte": 1.7620009541859187, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2213259935379028, + "step": 3540, + "unclipped_grad_norm": 0.8233645558357239 + }, + { + "bits_per_byte": 1.7622630557458336, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2215076684951782, + "step": 3550, + "unclipped_grad_norm": 0.8949882984161377 + }, + { + "bits_per_byte": 1.7110605529136091, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1860167980194092, + "step": 3560, + "unclipped_grad_norm": 0.862553060054779 + }, + { + "bits_per_byte": 1.73054515536046, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1995224952697754, + "step": 3570, + "unclipped_grad_norm": 0.8900191783905029 + }, + { + "bits_per_byte": 1.763418263211679, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2223083972930908, + "step": 3580, + "unclipped_grad_norm": 0.9058765769004822 + }, + { + "bits_per_byte": 1.7831680629061606, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2359979152679443, + "step": 3590, + "unclipped_grad_norm": 0.8453392386436462 + }, + { + "bits_per_byte": 1.7970675287662419, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.245632290840149, + "step": 3600, + "unclipped_grad_norm": 0.8932130932807922 + }, + { + "bits_per_byte": 1.7267670404863058, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1969037055969238, + "step": 3610, + "unclipped_grad_norm": 0.8229808211326599 + }, + { + "bits_per_byte": 1.6908183668897072, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1719859838485718, + "step": 3620, + "unclipped_grad_norm": 0.8186308741569519 + }, + { + "bits_per_byte": 1.7541439267812513, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2158799171447754, + "step": 3630, + "unclipped_grad_norm": 0.8367486596107483 + }, + { + "bits_per_byte": 1.7116509693540212, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.186426043510437, + "step": 3640, + "unclipped_grad_norm": 0.8415036797523499 + }, + { + "bits_per_byte": 1.7837473004742666, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2363994121551514, + "step": 3650, + "unclipped_grad_norm": 0.8471388816833496 + }, + { + "bits_per_byte": 1.6866421121788058, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1690912246704102, + "step": 3660, + "unclipped_grad_norm": 0.823422908782959 + }, + { + "bits_per_byte": 1.7654146378228945, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2236921787261963, + "step": 3670, + "unclipped_grad_norm": 0.819604218006134 + }, + { + "bits_per_byte": 1.6976106497028831, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1766940355300903, + "step": 3680, + "unclipped_grad_norm": 0.8133332133293152 + }, + { + "bits_per_byte": 1.6802502049768386, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1646606922149658, + "step": 3690, + "unclipped_grad_norm": 0.7951709032058716 + }, + { + "bits_per_byte": 1.6953377269890801, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1751185655593872, + "step": 3700, + "unclipped_grad_norm": 0.8330065011978149 + }, + { + "bits_per_byte": 1.747265652642644, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.2111122608184814, + "step": 3710, + "unclipped_grad_norm": 0.8084886074066162 + }, + { + "bits_per_byte": 1.7300491573953718, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.199178695678711, + "step": 3720, + "unclipped_grad_norm": 0.8567172288894653 + }, + { + "bits_per_byte": 1.6577672569948454, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1490767002105713, + "step": 3730, + "unclipped_grad_norm": 0.8171049952507019 + }, + { + "bits_per_byte": 1.720541440507723, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.192588448524475, + "step": 3740, + "unclipped_grad_norm": 0.8777816295623779 + }, + { + "bits_per_byte": 1.7468836791750777, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.2108474969863892, + "step": 3750, + "unclipped_grad_norm": 0.8143476843833923 + }, + { + "bits_per_byte": 1.8090201510185562, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2539172172546387, + "step": 3760, + "unclipped_grad_norm": 0.8902925252914429 + }, + { + "bits_per_byte": 1.7778630860576463, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.232320785522461, + "step": 3770, + "unclipped_grad_norm": 0.8659905791282654 + }, + { + "bits_per_byte": 1.6107073002739534, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.116457223892212, + "step": 3780, + "unclipped_grad_norm": 0.7876954674720764 + }, + { + "bits_per_byte": 1.7097620838995948, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1851167678833008, + "step": 3790, + "unclipped_grad_norm": 0.8166313767433167 + }, + { + "bits_per_byte": 1.703528744701723, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1807961463928223, + "step": 3800, + "unclipped_grad_norm": 0.7970600128173828 + }, + { + "bits_per_byte": 1.7306136044555034, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1995699405670166, + "step": 3810, + "unclipped_grad_norm": 0.9229923486709595 + }, + { + "bits_per_byte": 1.7223408949837007, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.193835735321045, + "step": 3820, + "unclipped_grad_norm": 0.88306725025177 + }, + { + "bits_per_byte": 1.7747577673136674, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.230168342590332, + "step": 3830, + "unclipped_grad_norm": 0.9060350060462952 + }, + { + "bits_per_byte": 1.6914633018304428, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1724330186843872, + "step": 3840, + "unclipped_grad_norm": 0.8194422721862793 + }, + { + "bits_per_byte": 1.648618811864849, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.142735481262207, + "step": 3850, + "unclipped_grad_norm": 0.8284996747970581 + }, + { + "bits_per_byte": 1.7050731489064712, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1818666458129883, + "step": 3860, + "unclipped_grad_norm": 0.8395226001739502 + }, + { + "bits_per_byte": 1.7881865166583373, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2394764423370361, + "step": 3870, + "unclipped_grad_norm": 0.8676077127456665 + }, + { + "bits_per_byte": 1.6805537543556115, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.164871096611023, + "step": 3880, + "unclipped_grad_norm": 0.8120661377906799 + }, + { + "bits_per_byte": 1.6503448297489083, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1439318656921387, + "step": 3890, + "unclipped_grad_norm": 0.8249826431274414 + }, + { + "bits_per_byte": 1.7267109741421247, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1968648433685303, + "step": 3900, + "unclipped_grad_norm": 0.9262217879295349 + }, + { + "bits_per_byte": 1.7407927416121205, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.2066255807876587, + "step": 3910, + "unclipped_grad_norm": 0.894594669342041 + }, + { + "bits_per_byte": 1.6851732083577864, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.168073058128357, + "step": 3920, + "unclipped_grad_norm": 0.7890235185623169 + }, + { + "bits_per_byte": 1.7178566193451037, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1907274723052979, + "step": 3930, + "unclipped_grad_norm": 0.8578466773033142 + }, + { + "bits_per_byte": 1.7552539028099199, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.2166492938995361, + "step": 3940, + "unclipped_grad_norm": 0.8101285696029663 + }, + { + "bits_per_byte": 1.7865160491705068, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2383185625076294, + "step": 3950, + "unclipped_grad_norm": 0.8577295541763306 + }, + { + "bits_per_byte": 1.7671413436375571, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2248890399932861, + "step": 3960, + "unclipped_grad_norm": 0.9096070528030396 + }, + { + "bits_per_byte": 1.6949598811051345, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1748566627502441, + "step": 3970, + "unclipped_grad_norm": 0.8878244757652283 + }, + { + "bits_per_byte": 1.7218173797944742, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1934728622436523, + "step": 3980, + "unclipped_grad_norm": 0.8564468026161194 + }, + { + "bits_per_byte": 1.7444124604648301, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.209134578704834, + "step": 3990, + "unclipped_grad_norm": 0.8601830005645752 + }, + { + "bits_per_byte": 1.6364575746570382, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1343059539794922, + "step": 4000, + "unclipped_grad_norm": 0.8214763402938843 + }, + { + "bits_per_byte": 1.7331732222482947, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2013441324234009, + "step": 4010, + "unclipped_grad_norm": 0.8387731909751892 + }, + { + "bits_per_byte": 1.7078159282224312, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1837677955627441, + "step": 4020, + "unclipped_grad_norm": 0.8044644594192505 + }, + { + "bits_per_byte": 1.7505291954254167, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.213374376296997, + "step": 4030, + "unclipped_grad_norm": 0.8809406161308289 + }, + { + "bits_per_byte": 1.6714938802908095, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1585912704467773, + "step": 4040, + "unclipped_grad_norm": 0.877934455871582 + }, + { + "bits_per_byte": 1.7160740191689108, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.189491868019104, + "step": 4050, + "unclipped_grad_norm": 0.8732415437698364 + }, + { + "bits_per_byte": 1.7110662283410876, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1860207319259644, + "step": 4060, + "unclipped_grad_norm": 0.8489102721214294 + }, + { + "bits_per_byte": 1.7342386547703899, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.202082633972168, + "step": 4070, + "unclipped_grad_norm": 0.8639312386512756 + }, + { + "bits_per_byte": 1.7989254573435127, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.246920108795166, + "step": 4080, + "unclipped_grad_norm": 0.8819956183433533 + }, + { + "bits_per_byte": 1.705004355846126, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.181818962097168, + "step": 4090, + "unclipped_grad_norm": 0.8767969012260437 + }, + { + "bits_per_byte": 1.6958074116085866, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1754441261291504, + "step": 4100, + "unclipped_grad_norm": 0.8374283313751221 + }, + { + "bits_per_byte": 1.693619276341659, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1739274263381958, + "step": 4110, + "unclipped_grad_norm": 0.8304628729820251 + }, + { + "bits_per_byte": 1.744371184628623, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.2091059684753418, + "step": 4120, + "unclipped_grad_norm": 0.9000032544136047 + }, + { + "bits_per_byte": 1.6641087732801096, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1534723043441772, + "step": 4130, + "unclipped_grad_norm": 0.8784226775169373 + }, + { + "bits_per_byte": 1.7338391390724355, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2018057107925415, + "step": 4140, + "unclipped_grad_norm": 0.864092230796814 + }, + { + "bits_per_byte": 1.6882804189109248, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.170226812362671, + "step": 4150, + "unclipped_grad_norm": 0.8413121104240417 + }, + { + "bits_per_byte": 1.698938527750195, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.177614450454712, + "step": 4160, + "unclipped_grad_norm": 0.9109639525413513 + }, + { + "bits_per_byte": 1.7387922394172843, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.205238938331604, + "step": 4170, + "unclipped_grad_norm": 0.8913268446922302 + }, + { + "bits_per_byte": 1.7814171075377263, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.2347842454910278, + "step": 4180, + "unclipped_grad_norm": 0.9075717926025391 + }, + { + "bits_per_byte": 1.694894183732505, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1748111248016357, + "step": 4190, + "unclipped_grad_norm": 0.8244585990905762 + }, + { + "bits_per_byte": 1.7220003693349923, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1935997009277344, + "step": 4200, + "unclipped_grad_norm": 0.8734019994735718 + }, + { + "bits_per_byte": 1.725182564323907, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1958054304122925, + "step": 4210, + "unclipped_grad_norm": 0.834058403968811 + }, + { + "bits_per_byte": 1.7486925926968528, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.2121013402938843, + "step": 4220, + "unclipped_grad_norm": 0.8368670344352722 + }, + { + "bits_per_byte": 1.720319754870761, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1924347877502441, + "step": 4230, + "unclipped_grad_norm": 0.8471981287002563 + }, + { + "bits_per_byte": 1.6660473617206353, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1548160314559937, + "step": 4240, + "unclipped_grad_norm": 0.8083959221839905 + }, + { + "bits_per_byte": 1.6911604403822735, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1722230911254883, + "step": 4250, + "unclipped_grad_norm": 0.8626519441604614 + }, + { + "bits_per_byte": 1.6717248529909183, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.158751368522644, + "step": 4260, + "unclipped_grad_norm": 0.8391480445861816 + }, + { + "bits_per_byte": 1.6638891514349579, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.153320074081421, + "step": 4270, + "unclipped_grad_norm": 0.8006730079650879 + }, + { + "bits_per_byte": 1.6802054894876142, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1646296977996826, + "step": 4280, + "unclipped_grad_norm": 0.8485363125801086 + }, + { + "bits_per_byte": 1.732838028061763, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.2011117935180664, + "step": 4290, + "unclipped_grad_norm": 0.8599663972854614 + }, + { + "bits_per_byte": 1.6415455094001634, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1378326416015625, + "step": 4300, + "unclipped_grad_norm": 0.8445435166358948 + }, + { + "bits_per_byte": 1.7360513519104837, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2033390998840332, + "step": 4310, + "unclipped_grad_norm": 0.9501782655715942 + }, + { + "bits_per_byte": 1.7070877536786782, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1832630634307861, + "step": 4320, + "unclipped_grad_norm": 0.8405798673629761 + }, + { + "bits_per_byte": 1.6698505860618154, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1574522256851196, + "step": 4330, + "unclipped_grad_norm": 0.8460624814033508 + }, + { + "bits_per_byte": 1.709274341101748, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1847786903381348, + "step": 4340, + "unclipped_grad_norm": 0.8663898706436157 + }, + { + "bits_per_byte": 1.6889219141986431, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1706714630126953, + "step": 4350, + "unclipped_grad_norm": 0.9172630310058594 + }, + { + "bits_per_byte": 1.675118930605696, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1611039638519287, + "step": 4360, + "unclipped_grad_norm": 0.8827696442604065 + }, + { + "bits_per_byte": 1.690404060683779, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.171698808670044, + "step": 4370, + "unclipped_grad_norm": 0.8694136738777161 + }, + { + "bits_per_byte": 1.7489961420756257, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.2123117446899414, + "step": 4380, + "unclipped_grad_norm": 0.8883715867996216 + }, + { + "bits_per_byte": 1.7051840777162777, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1819435358047485, + "step": 4390, + "unclipped_grad_norm": 0.8627059459686279 + }, + { + "bits_per_byte": 1.6575398959304046, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1489191055297852, + "step": 4400, + "unclipped_grad_norm": 0.8316671848297119 + }, + { + "bits_per_byte": 1.7285451691135765, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1981362104415894, + "step": 4410, + "unclipped_grad_norm": 0.8469527959823608 + }, + { + "bits_per_byte": 1.6753765606166884, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1612825393676758, + "step": 4420, + "unclipped_grad_norm": 0.829068124294281 + }, + { + "bits_per_byte": 1.7212594680750752, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1930861473083496, + "step": 4430, + "unclipped_grad_norm": 0.8506612777709961 + }, + { + "bits_per_byte": 1.6578714784812683, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.149148941040039, + "step": 4440, + "unclipped_grad_norm": 0.8394047021865845 + }, + { + "bits_per_byte": 1.6499258800114065, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.143641471862793, + "step": 4450, + "unclipped_grad_norm": 0.8416959047317505 + }, + { + "bits_per_byte": 1.64743299148715, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1419135332107544, + "step": 4460, + "unclipped_grad_norm": 0.8612979054450989 + }, + { + "bits_per_byte": 1.6939761403421993, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1741747856140137, + "step": 4470, + "unclipped_grad_norm": 0.9045028686523438 + }, + { + "bits_per_byte": 1.7161144350918636, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1895198822021484, + "step": 4480, + "unclipped_grad_norm": 0.9105556607246399 + }, + { + "bits_per_byte": 1.6800321309755446, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1645095348358154, + "step": 4490, + "unclipped_grad_norm": 0.8570636510848999 + }, + { + "bits_per_byte": 1.7013357939205715, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1792761087417603, + "step": 4500, + "unclipped_grad_norm": 0.8411341905593872 + }, + { + "bits_per_byte": 1.6660826181640622, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1548404693603516, + "step": 4510, + "unclipped_grad_norm": 0.8592174053192139 + }, + { + "bits_per_byte": 1.7057920363870778, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1823649406433105, + "step": 4520, + "unclipped_grad_norm": 0.8456743955612183 + }, + { + "bits_per_byte": 1.7124625554834427, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1869885921478271, + "step": 4530, + "unclipped_grad_norm": 0.8667335510253906 + }, + { + "bits_per_byte": 1.761984099886134, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.2213143110275269, + "step": 4540, + "unclipped_grad_norm": 0.8827010989189148 + }, + { + "bits_per_byte": 1.608659158884828, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1150375604629517, + "step": 4550, + "unclipped_grad_norm": 0.8445418477058411 + }, + { + "bits_per_byte": 1.6846809940110172, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1677318811416626, + "step": 4560, + "unclipped_grad_norm": 0.9384724497795105 + }, + { + "bits_per_byte": 1.6632275341770886, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.152861475944519, + "step": 4570, + "unclipped_grad_norm": 0.836455762386322 + }, + { + "bits_per_byte": 1.6185414539860559, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.121887445449829, + "step": 4580, + "unclipped_grad_norm": 0.8520491123199463 + }, + { + "bits_per_byte": 1.6763542819868433, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.161960244178772, + "step": 4590, + "unclipped_grad_norm": 0.9324198961257935 + }, + { + "bits_per_byte": 1.667847676109867, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1560639142990112, + "step": 4600, + "unclipped_grad_norm": 0.8337410092353821 + }, + { + "bits_per_byte": 1.7332526782329933, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.2013992071151733, + "step": 4610, + "unclipped_grad_norm": 0.934492290019989 + }, + { + "bits_per_byte": 1.7177494741536163, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1906532049179077, + "step": 4620, + "unclipped_grad_norm": 0.8968923091888428 + }, + { + "bits_per_byte": 1.707787379102388, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1837480068206787, + "step": 4630, + "unclipped_grad_norm": 0.9312987327575684 + }, + { + "bits_per_byte": 1.7495194852822011, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.2126744985580444, + "step": 4640, + "unclipped_grad_norm": 0.8537502884864807 + }, + { + "bits_per_byte": 1.6591016703828898, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1500016450881958, + "step": 4650, + "unclipped_grad_norm": 0.8534259796142578 + }, + { + "bits_per_byte": 1.63647786860984, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1343200206756592, + "step": 4660, + "unclipped_grad_norm": 0.862847626209259 + }, + { + "bits_per_byte": 1.758119477738596, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.2186355590820312, + "step": 4670, + "unclipped_grad_norm": 0.8828908205032349 + }, + { + "bits_per_byte": 1.7750497938548324, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.2303707599639893, + "step": 4680, + "unclipped_grad_norm": 0.9205049872398376 + }, + { + "bits_per_byte": 1.7426339878722579, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.2079018354415894, + "step": 4690, + "unclipped_grad_norm": 0.92873615026474 + }, + { + "bits_per_byte": 1.659708425175134, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.150422215461731, + "step": 4700, + "unclipped_grad_norm": 0.8619311451911926 + }, + { + "bits_per_byte": 1.7262008735996655, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1965112686157227, + "step": 4710, + "unclipped_grad_norm": 0.9467828869819641 + }, + { + "bits_per_byte": 1.6608278602495998, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.151198148727417, + "step": 4720, + "unclipped_grad_norm": 0.8912813067436218 + }, + { + "bits_per_byte": 1.5957210760430698, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.106069564819336, + "step": 4730, + "unclipped_grad_norm": 0.8848060369491577 + }, + { + "bits_per_byte": 1.6368768683598418, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.134596586227417, + "step": 4740, + "unclipped_grad_norm": 0.9091616868972778 + }, + { + "bits_per_byte": 1.6949358035340139, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.174839973449707, + "step": 4750, + "unclipped_grad_norm": 0.9046142101287842 + }, + { + "bits_per_byte": 1.665315747523865, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1543089151382446, + "step": 4760, + "unclipped_grad_norm": 0.8868388533592224 + }, + { + "bits_per_byte": 1.6325195159175816, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1315762996673584, + "step": 4770, + "unclipped_grad_norm": 0.839006781578064 + }, + { + "bits_per_byte": 1.6782909786182094, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1633026599884033, + "step": 4780, + "unclipped_grad_norm": 0.930033266544342 + }, + { + "bits_per_byte": 1.6275290953374955, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1281172037124634, + "step": 4790, + "unclipped_grad_norm": 0.8176500201225281 + }, + { + "bits_per_byte": 1.6333247386889214, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1321344375610352, + "step": 4800, + "unclipped_grad_norm": 0.855272650718689 + }, + { + "bits_per_byte": 1.6820415762682257, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1659023761749268, + "step": 4810, + "unclipped_grad_norm": 0.9120509624481201 + }, + { + "bits_per_byte": 1.6350118884938853, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1333038806915283, + "step": 4820, + "unclipped_grad_norm": 0.89643794298172 + }, + { + "bits_per_byte": 1.6668641073295827, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1553821563720703, + "step": 4830, + "unclipped_grad_norm": 0.9077607989311218 + }, + { + "bits_per_byte": 1.667841656717087, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.156059741973877, + "step": 4840, + "unclipped_grad_norm": 0.9171648621559143 + }, + { + "bits_per_byte": 1.6750915853642085, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1610850095748901, + "step": 4850, + "unclipped_grad_norm": 0.8878066539764404 + }, + { + "bits_per_byte": 1.6131449823672828, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1181468963623047, + "step": 4860, + "unclipped_grad_norm": 0.8896912932395935 + }, + { + "bits_per_byte": 1.6943018754829335, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1744005680084229, + "step": 4870, + "unclipped_grad_norm": 0.8964843153953552 + }, + { + "bits_per_byte": 1.5440334101527613, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0702424049377441, + "step": 4880, + "unclipped_grad_norm": 0.8637375235557556 + }, + { + "bits_per_byte": 1.609086879737524, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1153340339660645, + "step": 4890, + "unclipped_grad_norm": 0.8875832557678223 + }, + { + "bits_per_byte": 1.651299333461197, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1445934772491455, + "step": 4900, + "unclipped_grad_norm": 0.9340603351593018 + }, + { + "bits_per_byte": 1.7515653908968651, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.2140926122665405, + "step": 4910, + "unclipped_grad_norm": 0.9391474723815918 + }, + { + "bits_per_byte": 1.6798804422774836, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1644043922424316, + "step": 4920, + "unclipped_grad_norm": 0.8976443409919739 + }, + { + "bits_per_byte": 1.6454846000355252, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1405630111694336, + "step": 4930, + "unclipped_grad_norm": 0.8620614409446716 + }, + { + "bits_per_byte": 1.6197019929140781, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1226918697357178, + "step": 4940, + "unclipped_grad_norm": 0.8644847869873047 + }, + { + "bits_per_byte": 1.5870352642438945, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1000490188598633, + "step": 4950, + "unclipped_grad_norm": 0.8775791525840759 + }, + { + "bits_per_byte": 1.5784220291233835, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.094078779220581, + "step": 4960, + "unclipped_grad_norm": 0.8890455961227417 + }, + { + "bits_per_byte": 1.6649912162616867, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1540839672088623, + "step": 4970, + "unclipped_grad_norm": 0.9286753535270691 + }, + { + "bits_per_byte": 1.7156910138054393, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1892263889312744, + "step": 4980, + "unclipped_grad_norm": 0.9152454137802124 + }, + { + "bits_per_byte": 1.6570270436655319, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1485636234283447, + "step": 4990, + "unclipped_grad_norm": 0.9534220099449158 + }, + { + "bits_per_byte": 1.6136648859208411, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1185072660446167, + "step": 5000, + "unclipped_grad_norm": 0.8542637825012207 + }, + { + "bits_per_byte": 1.5871047452348432, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1000971794128418, + "step": 5010, + "unclipped_grad_norm": 0.8614861965179443 + }, + { + "bits_per_byte": 1.6463324745042789, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.141150712966919, + "step": 5020, + "unclipped_grad_norm": 0.8538332581520081 + }, + { + "bits_per_byte": 1.630251408718003, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1300041675567627, + "step": 5030, + "unclipped_grad_norm": 0.8974499702453613 + }, + { + "bits_per_byte": 1.6081984173631667, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1147181987762451, + "step": 5040, + "unclipped_grad_norm": 0.8664147853851318 + }, + { + "bits_per_byte": 1.566623675308893, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0859007835388184, + "step": 5050, + "unclipped_grad_norm": 0.8855953216552734 + }, + { + "bits_per_byte": 1.5903796388725728, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1023671627044678, + "step": 5060, + "unclipped_grad_norm": 0.8359810709953308 + }, + { + "bits_per_byte": 1.6634062241513352, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1529853343963623, + "step": 5070, + "unclipped_grad_norm": 0.8875454068183899 + }, + { + "bits_per_byte": 1.6210933325595582, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1236562728881836, + "step": 5080, + "unclipped_grad_norm": 0.9011101722717285 + }, + { + "bits_per_byte": 1.6245405528134524, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1260457038879395, + "step": 5090, + "unclipped_grad_norm": 0.8935633301734924 + }, + { + "bits_per_byte": 1.6269426344980533, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1277107000350952, + "step": 5100, + "unclipped_grad_norm": 0.8533574342727661 + }, + { + "bits_per_byte": 1.565103176692615, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0848468542099, + "step": 5110, + "unclipped_grad_norm": 0.9117326736450195 + }, + { + "bits_per_byte": 1.6439374441083634, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1394906044006348, + "step": 5120, + "unclipped_grad_norm": 0.8894507884979248 + }, + { + "bits_per_byte": 1.6476570848812242, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.142068862915039, + "step": 5130, + "unclipped_grad_norm": 0.8907402753829956 + }, + { + "bits_per_byte": 1.615579052824944, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.119834065437317, + "step": 5140, + "unclipped_grad_norm": 0.8878148794174194 + }, + { + "bits_per_byte": 1.6886897376199783, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1705105304718018, + "step": 5150, + "unclipped_grad_norm": 0.906775712966919 + }, + { + "bits_per_byte": 1.5749478075933039, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0916706323623657, + "step": 5160, + "unclipped_grad_norm": 0.8668916821479797 + }, + { + "bits_per_byte": 1.7387628303839868, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.2052185535430908, + "step": 5170, + "unclipped_grad_norm": 0.951082706451416 + }, + { + "bits_per_byte": 1.633963310271575, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1325770616531372, + "step": 5180, + "unclipped_grad_norm": 0.9125628471374512 + }, + { + "bits_per_byte": 1.571220427601154, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0890870094299316, + "step": 5190, + "unclipped_grad_norm": 0.837740421295166 + }, + { + "bits_per_byte": 1.600543469573263, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1094121932983398, + "step": 5200, + "unclipped_grad_norm": 0.8841894865036011 + }, + { + "bits_per_byte": 1.630884304873178, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1304428577423096, + "step": 5210, + "unclipped_grad_norm": 0.8948630690574646 + }, + { + "bits_per_byte": 1.7074470254363305, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1835120916366577, + "step": 5220, + "unclipped_grad_norm": 0.9275010228157043 + }, + { + "bits_per_byte": 1.6155556631844268, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.119817852973938, + "step": 5230, + "unclipped_grad_norm": 0.8827324509620667 + }, + { + "bits_per_byte": 1.615808649663846, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1199932098388672, + "step": 5240, + "unclipped_grad_norm": 0.8995877504348755 + }, + { + "bits_per_byte": 1.592627108154048, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1039249897003174, + "step": 5250, + "unclipped_grad_norm": 0.8981198668479919 + }, + { + "bits_per_byte": 1.60489222288298, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.112426519393921, + "step": 5260, + "unclipped_grad_norm": 0.8831214308738708 + }, + { + "bits_per_byte": 1.5994845723919007, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1086782217025757, + "step": 5270, + "unclipped_grad_norm": 0.8702658414840698 + }, + { + "bits_per_byte": 1.6097115207254575, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.115767002105713, + "step": 5280, + "unclipped_grad_norm": 0.9307021498680115 + }, + { + "bits_per_byte": 1.6134614304448702, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1183662414550781, + "step": 5290, + "unclipped_grad_norm": 0.9075614213943481 + }, + { + "bits_per_byte": 1.6247061721062332, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.1261605024337769, + "step": 5300, + "unclipped_grad_norm": 0.8755994439125061 + }, + { + "bits_per_byte": 1.6182275856482313, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.121669888496399, + "step": 5310, + "unclipped_grad_norm": 0.914459764957428 + }, + { + "bits_per_byte": 1.6139610400456268, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1187125444412231, + "step": 5320, + "unclipped_grad_norm": 0.8940036296844482 + }, + { + "bits_per_byte": 1.6078936641058377, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1145069599151611, + "step": 5330, + "unclipped_grad_norm": 0.9279354214668274 + }, + { + "bits_per_byte": 1.569059465593063, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.087589144706726, + "step": 5340, + "unclipped_grad_norm": 0.8639222979545593 + }, + { + "bits_per_byte": 1.6467650108711989, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1414505243301392, + "step": 5350, + "unclipped_grad_norm": 0.9308397769927979 + }, + { + "bits_per_byte": 1.565089418080546, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0848373174667358, + "step": 5360, + "unclipped_grad_norm": 0.9050061702728271 + }, + { + "bits_per_byte": 1.5389489150626532, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0667181015014648, + "step": 5370, + "unclipped_grad_norm": 0.8974517583847046 + }, + { + "bits_per_byte": 1.6578491207366561, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1491334438323975, + "step": 5380, + "unclipped_grad_norm": 0.9271295070648193 + }, + { + "bits_per_byte": 1.5921135679585716, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.1035690307617188, + "step": 5390, + "unclipped_grad_norm": 0.884811282157898 + }, + { + "bits_per_byte": 1.6126496723327979, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1178035736083984, + "step": 5400, + "unclipped_grad_norm": 0.9159505367279053 + }, + { + "bits_per_byte": 1.6339560870002385, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.132572054862976, + "step": 5410, + "unclipped_grad_norm": 0.9119471907615662 + }, + { + "bits_per_byte": 1.595336178870439, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1058027744293213, + "step": 5420, + "unclipped_grad_norm": 0.9820973873138428 + }, + { + "bits_per_byte": 1.649176723584248, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1431221961975098, + "step": 5430, + "unclipped_grad_norm": 0.9529035091400146 + }, + { + "bits_per_byte": 1.5866154545931386, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0997580289840698, + "step": 5440, + "unclipped_grad_norm": 0.9037122130393982 + }, + { + "bits_per_byte": 1.5834752233710345, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.097581386566162, + "step": 5450, + "unclipped_grad_norm": 0.9287867546081543 + }, + { + "bits_per_byte": 1.5614380544200777, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0823063850402832, + "step": 5460, + "unclipped_grad_norm": 0.8947009444236755 + }, + { + "bits_per_byte": 1.607795462012195, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1144388914108276, + "step": 5470, + "unclipped_grad_norm": 0.9344596862792969 + }, + { + "bits_per_byte": 1.6195384374131077, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.122578501701355, + "step": 5480, + "unclipped_grad_norm": 0.8986119031906128 + }, + { + "bits_per_byte": 1.630767700635893, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1303620338439941, + "step": 5490, + "unclipped_grad_norm": 0.9683650135993958 + }, + { + "bits_per_byte": 1.6068012303075572, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1137497425079346, + "step": 5500, + "unclipped_grad_norm": 0.913102924823761 + }, + { + "bits_per_byte": 1.7072599083121918, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1833823919296265, + "step": 5510, + "unclipped_grad_norm": 1.0086424350738525 + }, + { + "bits_per_byte": 1.6241427569420068, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1257699728012085, + "step": 5520, + "unclipped_grad_norm": 0.9219017624855042 + }, + { + "bits_per_byte": 1.5865065895751422, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0996825695037842, + "step": 5530, + "unclipped_grad_norm": 0.8849281668663025 + }, + { + "bits_per_byte": 1.663044200671269, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.152734398841858, + "step": 5540, + "unclipped_grad_norm": 1.0463494062423706 + }, + { + "bits_per_byte": 1.588076447212218, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.1007707118988037, + "step": 5550, + "unclipped_grad_norm": 0.8995247483253479 + }, + { + "bits_per_byte": 1.5728786843207734, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0902364253997803, + "step": 5560, + "unclipped_grad_norm": 0.9297628998756409 + }, + { + "bits_per_byte": 1.553519629209052, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0768177509307861, + "step": 5570, + "unclipped_grad_norm": 0.9231326580047607 + }, + { + "bits_per_byte": 1.6281497807244594, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1285474300384521, + "step": 5580, + "unclipped_grad_norm": 0.932182788848877 + }, + { + "bits_per_byte": 1.5666377778862637, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0859105587005615, + "step": 5590, + "unclipped_grad_norm": 0.9885890483856201 + }, + { + "bits_per_byte": 1.6130727496539203, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1180968284606934, + "step": 5600, + "unclipped_grad_norm": 0.9734658598899841 + }, + { + "bits_per_byte": 1.6005983320388883, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1094502210617065, + "step": 5610, + "unclipped_grad_norm": 0.8992385864257812 + }, + { + "bits_per_byte": 1.607953514068338, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1145484447479248, + "step": 5620, + "unclipped_grad_norm": 0.9051816463470459 + }, + { + "bits_per_byte": 1.594605252604272, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.1052961349487305, + "step": 5630, + "unclipped_grad_norm": 0.9290134906768799 + }, + { + "bits_per_byte": 1.5677630603708592, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0866905450820923, + "step": 5640, + "unclipped_grad_norm": 0.9029556512832642 + }, + { + "bits_per_byte": 1.5944298303003919, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1051745414733887, + "step": 5650, + "unclipped_grad_norm": 0.9501181244850159 + }, + { + "bits_per_byte": 1.676524028863245, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1620779037475586, + "step": 5660, + "unclipped_grad_norm": 0.9740756750106812 + }, + { + "bits_per_byte": 1.5742370032972879, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0911779403686523, + "step": 5670, + "unclipped_grad_norm": 0.8980613350868225 + }, + { + "bits_per_byte": 1.5945870224432805, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.105283498764038, + "step": 5680, + "unclipped_grad_norm": 0.9082752466201782 + }, + { + "bits_per_byte": 1.6598031876157593, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1504878997802734, + "step": 5690, + "unclipped_grad_norm": 0.9417368769645691 + }, + { + "bits_per_byte": 1.5763857545371678, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0926673412322998, + "step": 5700, + "unclipped_grad_norm": 0.9450368881225586 + }, + { + "bits_per_byte": 1.6005926566114097, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1094462871551514, + "step": 5710, + "unclipped_grad_norm": 0.9505442976951599 + }, + { + "bits_per_byte": 1.6062563892696238, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1133720874786377, + "step": 5720, + "unclipped_grad_norm": 0.8568877577781677 + }, + { + "bits_per_byte": 1.6007586198694923, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.1095613241195679, + "step": 5730, + "unclipped_grad_norm": 0.885662853717804 + }, + { + "bits_per_byte": 1.596347780822814, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.106503963470459, + "step": 5740, + "unclipped_grad_norm": 0.8908458948135376 + }, + { + "bits_per_byte": 1.6113288455741714, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.1168880462646484, + "step": 5750, + "unclipped_grad_norm": 0.9814403057098389 + }, + { + "bits_per_byte": 1.6378140018243932, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.1352461576461792, + "step": 5760, + "unclipped_grad_norm": 1.000325322151184 + }, + { + "bits_per_byte": 1.57939407506606, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.094752550125122, + "step": 5770, + "unclipped_grad_norm": 0.9007365703582764 + }, + { + "bits_per_byte": 1.5515046804715436, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0754210948944092, + "step": 5780, + "unclipped_grad_norm": 0.9067476987838745 + }, + { + "bits_per_byte": 1.586475632697987, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.099661111831665, + "step": 5790, + "unclipped_grad_norm": 0.9168080687522888 + }, + { + "bits_per_byte": 1.544505330546729, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0705695152282715, + "step": 5800, + "unclipped_grad_norm": 0.9089300036430359 + }, + { + "bits_per_byte": 1.6342659997370934, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.1327868700027466, + "step": 5810, + "unclipped_grad_norm": 0.9730345010757446 + }, + { + "bits_per_byte": 1.5628627586998252, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0832939147949219, + "step": 5820, + "unclipped_grad_norm": 0.9135822057723999 + }, + { + "bits_per_byte": 1.5753394120893185, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0919420719146729, + "step": 5830, + "unclipped_grad_norm": 0.9162984490394592 + }, + { + "bits_per_byte": 1.5812992988723182, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0960731506347656, + "step": 5840, + "unclipped_grad_norm": 0.946280300617218 + }, + { + "bits_per_byte": 1.5446322537430657, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.07065749168396, + "step": 5850, + "unclipped_grad_norm": 0.9933390021324158 + }, + { + "bits_per_byte": 1.596009318965916, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.106269359588623, + "step": 5860, + "unclipped_grad_norm": 1.0133196115493774 + }, + { + "bits_per_byte": 1.5986716104012721, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1081147193908691, + "step": 5870, + "unclipped_grad_norm": 0.9077220559120178 + }, + { + "bits_per_byte": 1.616603209510832, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1205439567565918, + "step": 5880, + "unclipped_grad_norm": 0.9455923438072205 + }, + { + "bits_per_byte": 1.6185034458202152, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1218611001968384, + "step": 5890, + "unclipped_grad_norm": 0.9350056648254395 + }, + { + "bits_per_byte": 1.6333419369540076, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1321463584899902, + "step": 5900, + "unclipped_grad_norm": 1.0195506811141968 + }, + { + "bits_per_byte": 1.6111236702716922, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.1167458295822144, + "step": 5910, + "unclipped_grad_norm": 0.9529072642326355 + }, + { + "bits_per_byte": 1.5460519705259381, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0716415643692017, + "step": 5920, + "unclipped_grad_norm": 0.8979566097259521 + }, + { + "bits_per_byte": 1.6630168554297817, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1527154445648193, + "step": 5930, + "unclipped_grad_norm": 0.9750632047653198 + }, + { + "bits_per_byte": 1.5225558727477122, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0553553104400635, + "step": 5940, + "unclipped_grad_norm": 0.8854946494102478 + }, + { + "bits_per_byte": 1.5768313615855534, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0929762125015259, + "step": 5950, + "unclipped_grad_norm": 0.8891376852989197 + }, + { + "bits_per_byte": 1.6173250206965033, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1210442781448364, + "step": 5960, + "unclipped_grad_norm": 1.0107448101043701 + }, + { + "bits_per_byte": 1.598686744874548, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.1081252098083496, + "step": 5970, + "unclipped_grad_norm": 1.0125014781951904 + }, + { + "bits_per_byte": 1.623099338199222, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.125046730041504, + "step": 5980, + "unclipped_grad_norm": 0.9798391461372375 + }, + { + "bits_per_byte": 1.6025701131310304, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.1108169555664062, + "step": 5990, + "unclipped_grad_norm": 0.9963014125823975 + }, + { + "bits_per_byte": 1.561719762002191, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0825016498565674, + "step": 6000, + "unclipped_grad_norm": 0.9239053130149841 + }, + { + "bits_per_byte": 1.5905753551292547, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1025028228759766, + "step": 6010, + "unclipped_grad_norm": 0.9512903094291687 + }, + { + "bits_per_byte": 1.5463665267943663, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.07185959815979, + "step": 6020, + "unclipped_grad_norm": 0.931431770324707 + }, + { + "bits_per_byte": 1.5669568057036143, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0861316919326782, + "step": 6030, + "unclipped_grad_norm": 0.9379295110702515 + }, + { + "bits_per_byte": 1.6392313108501535, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.1362285614013672, + "step": 6040, + "unclipped_grad_norm": 1.0010037422180176 + }, + { + "bits_per_byte": 1.547184132316568, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0724263191223145, + "step": 6050, + "unclipped_grad_norm": 0.9731131792068481 + }, + { + "bits_per_byte": 1.550029757257744, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0743987560272217, + "step": 6060, + "unclipped_grad_norm": 0.9350199699401855 + }, + { + "bits_per_byte": 1.6229744787946958, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.12496018409729, + "step": 6070, + "unclipped_grad_norm": 1.0221160650253296 + }, + { + "bits_per_byte": 1.5381166910151283, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0661412477493286, + "step": 6080, + "unclipped_grad_norm": 0.9525545239448547 + }, + { + "bits_per_byte": 1.54977178328145, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0742199420928955, + "step": 6090, + "unclipped_grad_norm": 0.9721336364746094 + }, + { + "bits_per_byte": 1.5613978104797757, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0822784900665283, + "step": 6100, + "unclipped_grad_norm": 0.9061561822891235 + }, + { + "bits_per_byte": 1.6307178256671429, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1303274631500244, + "step": 6110, + "unclipped_grad_norm": 0.9887846112251282 + }, + { + "bits_per_byte": 1.6160071176429416, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.1201307773590088, + "step": 6120, + "unclipped_grad_norm": 1.008078694343567 + }, + { + "bits_per_byte": 1.5519858879286577, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0757546424865723, + "step": 6130, + "unclipped_grad_norm": 0.958514928817749 + }, + { + "bits_per_byte": 1.5641512527200894, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0841870307922363, + "step": 6140, + "unclipped_grad_norm": 0.9408689141273499 + }, + { + "bits_per_byte": 1.6176376851557719, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.1212610006332397, + "step": 6150, + "unclipped_grad_norm": 0.9648155570030212 + }, + { + "bits_per_byte": 1.4733855169180825, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0212730169296265, + "step": 6160, + "unclipped_grad_norm": 0.9190199375152588 + }, + { + "bits_per_byte": 1.587719411229027, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.1005232334136963, + "step": 6170, + "unclipped_grad_norm": 0.9420804977416992 + }, + { + "bits_per_byte": 1.561453876823957, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0823173522949219, + "step": 6180, + "unclipped_grad_norm": 0.9441719055175781 + }, + { + "bits_per_byte": 1.573854341899118, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0909126996994019, + "step": 6190, + "unclipped_grad_norm": 0.9590060114860535 + }, + { + "bits_per_byte": 1.5911162405662183, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.1028777360916138, + "step": 6200, + "unclipped_grad_norm": 0.9321414232254028 + }, + { + "bits_per_byte": 1.586496958546694, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0996758937835693, + "step": 6210, + "unclipped_grad_norm": 1.0076017379760742 + }, + { + "bits_per_byte": 1.574202778749766, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0911542177200317, + "step": 6220, + "unclipped_grad_norm": 0.9875207543373108 + }, + { + "bits_per_byte": 1.5328827430014202, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0625133514404297, + "step": 6230, + "unclipped_grad_norm": 0.9433984756469727 + }, + { + "bits_per_byte": 1.5747416003949193, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0915277004241943, + "step": 6240, + "unclipped_grad_norm": 0.9963345527648926 + }, + { + "bits_per_byte": 1.6134170589209478, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.118335485458374, + "step": 6250, + "unclipped_grad_norm": 1.0228519439697266 + }, + { + "bits_per_byte": 1.4956729206260448, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0367214679718018, + "step": 6260, + "unclipped_grad_norm": 0.9954123497009277 + }, + { + "bits_per_byte": 1.574694133183281, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0914947986602783, + "step": 6270, + "unclipped_grad_norm": 0.9892840385437012 + }, + { + "bits_per_byte": 1.5724050441002972, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0899081230163574, + "step": 6280, + "unclipped_grad_norm": 0.976605236530304 + }, + { + "bits_per_byte": 1.528724890434161, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.05963134765625, + "step": 6290, + "unclipped_grad_norm": 0.9648171067237854 + }, + { + "bits_per_byte": 1.589885188751342, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.1020244359970093, + "step": 6300, + "unclipped_grad_norm": 1.012677550315857 + }, + { + "bits_per_byte": 1.549742718213454, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0741997957229614, + "step": 6310, + "unclipped_grad_norm": 0.9430392384529114 + }, + { + "bits_per_byte": 1.5003391539092545, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0399558544158936, + "step": 6320, + "unclipped_grad_norm": 0.9601491689682007 + }, + { + "bits_per_byte": 1.5786977173127166, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.094269871711731, + "step": 6330, + "unclipped_grad_norm": 0.9872254133224487 + }, + { + "bits_per_byte": 1.5486566477732553, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0734469890594482, + "step": 6340, + "unclipped_grad_norm": 0.9915545582771301 + }, + { + "bits_per_byte": 1.643350295338318, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1390836238861084, + "step": 6350, + "unclipped_grad_norm": 1.0489985942840576 + }, + { + "bits_per_byte": 1.5515856843001, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0754772424697876, + "step": 6360, + "unclipped_grad_norm": 1.0075738430023193 + }, + { + "bits_per_byte": 1.515523502153932, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.050480842590332, + "step": 6370, + "unclipped_grad_norm": 0.9415894746780396 + }, + { + "bits_per_byte": 1.5423135836441333, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0690503120422363, + "step": 6380, + "unclipped_grad_norm": 0.9571269154548645 + }, + { + "bits_per_byte": 1.6015189551689568, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1100883483886719, + "step": 6390, + "unclipped_grad_norm": 0.962506115436554 + }, + { + "bits_per_byte": 1.5875768376114616, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1004244089126587, + "step": 6400, + "unclipped_grad_norm": 0.9905776977539062 + }, + { + "bits_per_byte": 1.5643576319011245, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0843300819396973, + "step": 6410, + "unclipped_grad_norm": 1.0040770769119263 + }, + { + "bits_per_byte": 1.5604245606585432, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0816038846969604, + "step": 6420, + "unclipped_grad_norm": 0.9999781250953674 + }, + { + "bits_per_byte": 1.592441538873767, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.103796362876892, + "step": 6430, + "unclipped_grad_norm": 0.9464544653892517 + }, + { + "bits_per_byte": 1.5425199628251687, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0691933631896973, + "step": 6440, + "unclipped_grad_norm": 0.9930383563041687 + }, + { + "bits_per_byte": 1.6124689185667411, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.1176782846450806, + "step": 6450, + "unclipped_grad_norm": 1.0360738039016724 + }, + { + "bits_per_byte": 1.6512981295826408, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.1445926427841187, + "step": 6460, + "unclipped_grad_norm": 1.0094163417816162 + }, + { + "bits_per_byte": 1.6379674103489628, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.1353524923324585, + "step": 6470, + "unclipped_grad_norm": 1.026789665222168 + }, + { + "bits_per_byte": 1.5269778906666966, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0584204196929932, + "step": 6480, + "unclipped_grad_norm": 0.9595211744308472 + }, + { + "bits_per_byte": 1.6320837118802953, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1312742233276367, + "step": 6490, + "unclipped_grad_norm": 0.9658059477806091 + }, + { + "bits_per_byte": 1.5804063649490385, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.095454216003418, + "step": 6500, + "unclipped_grad_norm": 1.007287621498108 + }, + { + "bits_per_byte": 1.5282990613906247, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0593361854553223, + "step": 6510, + "unclipped_grad_norm": 0.9717181921005249 + }, + { + "bits_per_byte": 1.518525631307393, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0525617599487305, + "step": 6520, + "unclipped_grad_norm": 0.9680256843566895 + }, + { + "bits_per_byte": 1.5436008737858413, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.069942593574524, + "step": 6530, + "unclipped_grad_norm": 0.9684496521949768 + }, + { + "bits_per_byte": 1.5757463230412598, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.09222412109375, + "step": 6540, + "unclipped_grad_norm": 0.9695699214935303 + }, + { + "bits_per_byte": 1.5133627121284916, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.048983097076416, + "step": 6550, + "unclipped_grad_norm": 0.9651139974594116 + }, + { + "bits_per_byte": 1.5985231893735776, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.1080118417739868, + "step": 6560, + "unclipped_grad_norm": 0.9865891337394714 + }, + { + "bits_per_byte": 1.5411050615565203, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.068212628364563, + "step": 6570, + "unclipped_grad_norm": 1.0009613037109375 + }, + { + "bits_per_byte": 1.585936467087532, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0992873907089233, + "step": 6580, + "unclipped_grad_norm": 1.0042976140975952 + }, + { + "bits_per_byte": 1.5806860087393415, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0956480503082275, + "step": 6590, + "unclipped_grad_norm": 1.0104939937591553 + }, + { + "bits_per_byte": 1.4917766536707477, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0340207815170288, + "step": 6600, + "unclipped_grad_norm": 0.9221757054328918 + }, + { + "bits_per_byte": 1.5136690132296782, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.049195408821106, + "step": 6610, + "unclipped_grad_norm": 0.9053012132644653 + }, + { + "bits_per_byte": 1.5109461839012182, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.047308087348938, + "step": 6620, + "unclipped_grad_norm": 0.9886469841003418 + }, + { + "bits_per_byte": 1.5485409034492248, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0733667612075806, + "step": 6630, + "unclipped_grad_norm": 1.005003809928894 + }, + { + "bits_per_byte": 1.5964243131024478, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.106557011604309, + "step": 6640, + "unclipped_grad_norm": 1.033184289932251 + }, + { + "bits_per_byte": 1.579020356765735, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0944935083389282, + "step": 6650, + "unclipped_grad_norm": 1.0069372653961182 + }, + { + "bits_per_byte": 1.596521827265487, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1066246032714844, + "step": 6660, + "unclipped_grad_norm": 0.9888476729393005 + }, + { + "bits_per_byte": 1.5643971879108232, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.084357500076294, + "step": 6670, + "unclipped_grad_norm": 0.9964956045150757 + }, + { + "bits_per_byte": 1.541306109275379, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0683519840240479, + "step": 6680, + "unclipped_grad_norm": 0.9876991510391235 + }, + { + "bits_per_byte": 1.57202289865008, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0896432399749756, + "step": 6690, + "unclipped_grad_norm": 1.0232517719268799 + }, + { + "bits_per_byte": 1.634657604233108, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.1330583095550537, + "step": 6700, + "unclipped_grad_norm": 1.028996467590332 + }, + { + "bits_per_byte": 1.582099534146783, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0966278314590454, + "step": 6710, + "unclipped_grad_norm": 0.982172429561615 + }, + { + "bits_per_byte": 1.5951110535804596, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1056467294692993, + "step": 6720, + "unclipped_grad_norm": 0.9665309190750122 + }, + { + "bits_per_byte": 1.5713714283686118, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0891916751861572, + "step": 6730, + "unclipped_grad_norm": 1.033214807510376 + }, + { + "bits_per_byte": 1.5070635035753395, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0446168184280396, + "step": 6740, + "unclipped_grad_norm": 0.9829891920089722 + }, + { + "bits_per_byte": 1.5742997769648528, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0912214517593384, + "step": 6750, + "unclipped_grad_norm": 0.9734475612640381 + }, + { + "bits_per_byte": 1.5504435475157199, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0746855735778809, + "step": 6760, + "unclipped_grad_norm": 0.947333574295044 + }, + { + "bits_per_byte": 1.4574869247243718, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0102529525756836, + "step": 6770, + "unclipped_grad_norm": 0.9365482330322266 + }, + { + "bits_per_byte": 1.5535751796052808, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.076856255531311, + "step": 6780, + "unclipped_grad_norm": 1.000013828277588 + }, + { + "bits_per_byte": 1.5272709491037668, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0586235523223877, + "step": 6790, + "unclipped_grad_norm": 1.0473871231079102 + }, + { + "bits_per_byte": 1.588691973119656, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.101197361946106, + "step": 6800, + "unclipped_grad_norm": 1.0225850343704224 + }, + { + "bits_per_byte": 1.5961993597951194, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.1064010858535767, + "step": 6810, + "unclipped_grad_norm": 1.0318177938461304 + }, + { + "bits_per_byte": 1.5861701915100546, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0994493961334229, + "step": 6820, + "unclipped_grad_norm": 1.0743839740753174 + }, + { + "bits_per_byte": 1.5382866098741805, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0662590265274048, + "step": 6830, + "unclipped_grad_norm": 0.9585179090499878 + }, + { + "bits_per_byte": 1.521101759434667, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0543473958969116, + "step": 6840, + "unclipped_grad_norm": 1.0282230377197266 + }, + { + "bits_per_byte": 1.5590901472704985, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.080678939819336, + "step": 6850, + "unclipped_grad_norm": 1.064353346824646 + }, + { + "bits_per_byte": 1.5486480486407121, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0734410285949707, + "step": 6860, + "unclipped_grad_norm": 1.054447889328003 + }, + { + "bits_per_byte": 1.486493862584195, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0303590297698975, + "step": 6870, + "unclipped_grad_norm": 0.992324709892273 + }, + { + "bits_per_byte": 1.501989499426934, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0410997867584229, + "step": 6880, + "unclipped_grad_norm": 0.9187514781951904 + }, + { + "bits_per_byte": 1.4984072727921125, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0386167764663696, + "step": 6890, + "unclipped_grad_norm": 0.9562593102455139 + }, + { + "bits_per_byte": 1.6084883801125214, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.1149191856384277, + "step": 6900, + "unclipped_grad_norm": 1.046886920928955 + }, + { + "bits_per_byte": 1.5851809473022918, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0987637042999268, + "step": 6910, + "unclipped_grad_norm": 1.0160750150680542 + }, + { + "bits_per_byte": 1.5136987662282775, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0492160320281982, + "step": 6920, + "unclipped_grad_norm": 0.9592021107673645 + }, + { + "bits_per_byte": 1.5198106856746398, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.053452491760254, + "step": 6930, + "unclipped_grad_norm": 1.0287657976150513 + }, + { + "bits_per_byte": 1.494010192357503, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0355689525604248, + "step": 6940, + "unclipped_grad_norm": 0.972965657711029 + }, + { + "bits_per_byte": 1.513553956836251, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0491156578063965, + "step": 6950, + "unclipped_grad_norm": 1.0125614404678345 + }, + { + "bits_per_byte": 1.583024456843123, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0972689390182495, + "step": 6960, + "unclipped_grad_norm": 0.9876657724380493 + }, + { + "bits_per_byte": 1.5602042508827878, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.081451177597046, + "step": 6970, + "unclipped_grad_norm": 1.0233051776885986 + }, + { + "bits_per_byte": 1.5255848311947078, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0574548244476318, + "step": 6980, + "unclipped_grad_norm": 0.9819255471229553 + }, + { + "bits_per_byte": 1.5587054220805185, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0804122686386108, + "step": 6990, + "unclipped_grad_norm": 1.0355700254440308 + }, + { + "bits_per_byte": 1.6012561656784385, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.1099061965942383, + "step": 7000, + "unclipped_grad_norm": 1.0185670852661133 + }, + { + "bits_per_byte": 1.585546754400677, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0990172624588013, + "step": 7010, + "unclipped_grad_norm": 1.014304757118225 + }, + { + "bits_per_byte": 1.542434143482388, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0691338777542114, + "step": 7020, + "unclipped_grad_norm": 1.027226448059082 + }, + { + "bits_per_byte": 1.5806457647990395, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0956201553344727, + "step": 7030, + "unclipped_grad_norm": 1.0283117294311523 + }, + { + "bits_per_byte": 1.5888914729946568, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.1013356447219849, + "step": 7040, + "unclipped_grad_norm": 1.0418747663497925 + }, + { + "bits_per_byte": 1.5596535624347252, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0810694694519043, + "step": 7050, + "unclipped_grad_norm": 1.007202386856079 + }, + { + "bits_per_byte": 1.5766827685752078, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.092873215675354, + "step": 7060, + "unclipped_grad_norm": 1.057618498802185 + }, + { + "bits_per_byte": 1.5601829250340808, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0814363956451416, + "step": 7070, + "unclipped_grad_norm": 1.0584709644317627 + }, + { + "bits_per_byte": 1.5180239579148262, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0522140264511108, + "step": 7080, + "unclipped_grad_norm": 1.0032520294189453 + }, + { + "bits_per_byte": 1.5539740073726318, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0771327018737793, + "step": 7090, + "unclipped_grad_norm": 1.0915608406066895 + }, + { + "bits_per_byte": 1.5426179929361605, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0692613124847412, + "step": 7100, + "unclipped_grad_norm": 1.0186309814453125 + }, + { + "bits_per_byte": 1.5505398578002032, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0747523307800293, + "step": 7110, + "unclipped_grad_norm": 1.0057092905044556 + }, + { + "bits_per_byte": 1.58839977459584, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.1009948253631592, + "step": 7120, + "unclipped_grad_norm": 1.0634574890136719 + }, + { + "bits_per_byte": 1.5622621952830122, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0828776359558105, + "step": 7130, + "unclipped_grad_norm": 1.0760761499404907 + }, + { + "bits_per_byte": 1.5468659644124718, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0722057819366455, + "step": 7140, + "unclipped_grad_norm": 1.0254970788955688 + }, + { + "bits_per_byte": 1.5017244741619544, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.040916085243225, + "step": 7150, + "unclipped_grad_norm": 0.9808151125907898 + }, + { + "bits_per_byte": 1.5967245948108544, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.1067651510238647, + "step": 7160, + "unclipped_grad_norm": 1.0412861108779907 + }, + { + "bits_per_byte": 1.5266782968888937, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0582127571105957, + "step": 7170, + "unclipped_grad_norm": 0.993831217288971 + }, + { + "bits_per_byte": 1.5582240426407534, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0800786018371582, + "step": 7180, + "unclipped_grad_norm": 1.0170890092849731 + }, + { + "bits_per_byte": 1.5191605912543786, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.053001880645752, + "step": 7190, + "unclipped_grad_norm": 1.0318375825881958 + }, + { + "bits_per_byte": 1.5658617921655709, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0853726863861084, + "step": 7200, + "unclipped_grad_norm": 1.0288457870483398 + }, + { + "bits_per_byte": 1.6065680218329872, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.1135880947113037, + "step": 7210, + "unclipped_grad_norm": 1.079895257949829 + }, + { + "bits_per_byte": 1.5943379915648312, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.1051108837127686, + "step": 7220, + "unclipped_grad_norm": 1.0698199272155762 + }, + { + "bits_per_byte": 1.5270222621906193, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0584511756896973, + "step": 7230, + "unclipped_grad_norm": 1.0323591232299805 + }, + { + "bits_per_byte": 1.5416120664112638, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0685640573501587, + "step": 7240, + "unclipped_grad_norm": 1.0111463069915771 + }, + { + "bits_per_byte": 1.5732295289285334, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0904796123504639, + "step": 7250, + "unclipped_grad_norm": 1.0152740478515625 + }, + { + "bits_per_byte": 1.467654711026032, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0173007249832153, + "step": 7260, + "unclipped_grad_norm": 0.9989895820617676 + }, + { + "bits_per_byte": 1.5079978853174771, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.045264482498169, + "step": 7270, + "unclipped_grad_norm": 1.015647053718567 + }, + { + "bits_per_byte": 1.5841817281007788, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0980710983276367, + "step": 7280, + "unclipped_grad_norm": 1.0212509632110596 + }, + { + "bits_per_byte": 1.50208838945118, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0411683320999146, + "step": 7290, + "unclipped_grad_norm": 1.0158213376998901 + }, + { + "bits_per_byte": 1.529626423489984, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0602562427520752, + "step": 7300, + "unclipped_grad_norm": 1.0379971265792847 + }, + { + "bits_per_byte": 1.530171608493219, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0606341361999512, + "step": 7310, + "unclipped_grad_norm": 0.9976150393486023 + }, + { + "bits_per_byte": 1.4838797262910803, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0285470485687256, + "step": 7320, + "unclipped_grad_norm": 1.0280481576919556 + }, + { + "bits_per_byte": 1.510586568178264, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0470588207244873, + "step": 7330, + "unclipped_grad_norm": 1.0713517665863037 + }, + { + "bits_per_byte": 1.4810440763436543, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0265815258026123, + "step": 7340, + "unclipped_grad_norm": 1.0121647119522095 + }, + { + "bits_per_byte": 1.5558935057389116, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0784631967544556, + "step": 7350, + "unclipped_grad_norm": 1.046762228012085 + }, + { + "bits_per_byte": 1.5163350882833535, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0510433912277222, + "step": 7360, + "unclipped_grad_norm": 1.0365986824035645 + }, + { + "bits_per_byte": 1.536221614185271, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0648276805877686, + "step": 7370, + "unclipped_grad_norm": 1.0269999504089355 + }, + { + "bits_per_byte": 1.4635044256954106, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0144239664077759, + "step": 7380, + "unclipped_grad_norm": 1.0192774534225464 + }, + { + "bits_per_byte": 1.5685326827334702, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.087224006652832, + "step": 7390, + "unclipped_grad_norm": 1.030569076538086 + }, + { + "bits_per_byte": 1.49844063742638, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0386399030685425, + "step": 7400, + "unclipped_grad_norm": 1.0422602891921997 + }, + { + "bits_per_byte": 1.48524905215725, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.029496192932129, + "step": 7410, + "unclipped_grad_norm": 0.9939574003219604 + }, + { + "bits_per_byte": 1.556964785671136, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0792057514190674, + "step": 7420, + "unclipped_grad_norm": 1.0415644645690918 + }, + { + "bits_per_byte": 1.5795878995135824, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0948868989944458, + "step": 7430, + "unclipped_grad_norm": 1.0722742080688477 + }, + { + "bits_per_byte": 1.591624105334216, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.1032297611236572, + "step": 7440, + "unclipped_grad_norm": 1.0902073383331299 + }, + { + "bits_per_byte": 1.5369473809719119, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0653307437896729, + "step": 7450, + "unclipped_grad_norm": 1.0743108987808228 + }, + { + "bits_per_byte": 1.5231069051610766, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0557372570037842, + "step": 7460, + "unclipped_grad_norm": 0.9955658912658691 + }, + { + "bits_per_byte": 1.5091140527215767, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0460381507873535, + "step": 7470, + "unclipped_grad_norm": 1.048521876335144 + }, + { + "bits_per_byte": 1.5543542610136893, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.077396273612976, + "step": 7480, + "unclipped_grad_norm": 1.0729283094406128 + }, + { + "bits_per_byte": 1.6078190236353633, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.114455223083496, + "step": 7490, + "unclipped_grad_norm": 1.0465319156646729 + }, + { + "bits_per_byte": 1.5529930183321103, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0764527320861816, + "step": 7500, + "unclipped_grad_norm": 1.0308523178100586 + }, + { + "bits_per_byte": 1.5173444544612673, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0517430305480957, + "step": 7510, + "unclipped_grad_norm": 1.008348822593689 + }, + { + "bits_per_byte": 1.5327847128904286, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0624454021453857, + "step": 7520, + "unclipped_grad_norm": 1.0445557832717896 + }, + { + "bits_per_byte": 1.5854733178087586, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.098966360092163, + "step": 7530, + "unclipped_grad_norm": 1.1247494220733643 + }, + { + "bits_per_byte": 1.5176457680655788, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0519518852233887, + "step": 7540, + "unclipped_grad_norm": 0.9896391034126282 + }, + { + "bits_per_byte": 1.5701451920679599, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0883417129516602, + "step": 7550, + "unclipped_grad_norm": 1.0925025939941406 + }, + { + "bits_per_byte": 1.5354839805957203, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0643163919448853, + "step": 7560, + "unclipped_grad_norm": 1.032513976097107 + }, + { + "bits_per_byte": 1.4933951823980176, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0351426601409912, + "step": 7570, + "unclipped_grad_norm": 1.0785413980484009 + }, + { + "bits_per_byte": 1.6648412473901344, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.153980016708374, + "step": 7580, + "unclipped_grad_norm": 1.1147191524505615 + }, + { + "bits_per_byte": 1.5229436936254077, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0556241273880005, + "step": 7590, + "unclipped_grad_norm": 1.0130250453948975 + }, + { + "bits_per_byte": 1.506402230282772, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0441584587097168, + "step": 7600, + "unclipped_grad_norm": 1.0460206270217896 + }, + { + "bits_per_byte": 1.5204523529450091, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0538972616195679, + "step": 7610, + "unclipped_grad_norm": 1.0537638664245605 + }, + { + "bits_per_byte": 1.4833304137042245, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0281662940979004, + "step": 7620, + "unclipped_grad_norm": 1.0038108825683594 + }, + { + "bits_per_byte": 1.4865265392878588, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.030381679534912, + "step": 7630, + "unclipped_grad_norm": 1.040962815284729 + }, + { + "bits_per_byte": 1.4985199214284277, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0386948585510254, + "step": 7640, + "unclipped_grad_norm": 1.037416696548462 + }, + { + "bits_per_byte": 1.6218830768923203, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1242036819458008, + "step": 7650, + "unclipped_grad_norm": 1.0509169101715088 + }, + { + "bits_per_byte": 1.5436070651612726, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0699468851089478, + "step": 7660, + "unclipped_grad_norm": 1.0892055034637451 + }, + { + "bits_per_byte": 1.5646802713541432, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0845537185668945, + "step": 7670, + "unclipped_grad_norm": 1.0592975616455078 + }, + { + "bits_per_byte": 1.5230315767599987, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.055685043334961, + "step": 7680, + "unclipped_grad_norm": 1.0794378519058228 + }, + { + "bits_per_byte": 1.5199706295399424, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0535633563995361, + "step": 7690, + "unclipped_grad_norm": 1.030026912689209 + }, + { + "bits_per_byte": 1.486342001903483, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0302537679672241, + "step": 7700, + "unclipped_grad_norm": 1.021731972694397 + }, + { + "bits_per_byte": 1.565274987360827, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0849659442901611, + "step": 7710, + "unclipped_grad_norm": 1.054081678390503 + }, + { + "bits_per_byte": 1.5620626954080112, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0827393531799316, + "step": 7720, + "unclipped_grad_norm": 1.0361971855163574 + }, + { + "bits_per_byte": 1.5781238112067872, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0938720703125, + "step": 7730, + "unclipped_grad_norm": 1.0852437019348145 + }, + { + "bits_per_byte": 1.5098900384422698, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0465760231018066, + "step": 7740, + "unclipped_grad_norm": 1.0850290060043335 + }, + { + "bits_per_byte": 1.5288445903591616, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0597143173217773, + "step": 7750, + "unclipped_grad_norm": 1.0632920265197754 + }, + { + "bits_per_byte": 1.5461653070928567, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0717201232910156, + "step": 7760, + "unclipped_grad_norm": 1.0411477088928223 + }, + { + "bits_per_byte": 1.4760610510175551, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.023127555847168, + "step": 7770, + "unclipped_grad_norm": 1.0089881420135498 + }, + { + "bits_per_byte": 1.5534400012417027, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0767625570297241, + "step": 7780, + "unclipped_grad_norm": 1.0804110765457153 + }, + { + "bits_per_byte": 1.4972104455247583, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0377871990203857, + "step": 7790, + "unclipped_grad_norm": 1.0377298593521118 + }, + { + "bits_per_byte": 1.5201942069860641, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0537183284759521, + "step": 7800, + "unclipped_grad_norm": 1.0282031297683716 + }, + { + "bits_per_byte": 1.4946203868027643, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.035991907119751, + "step": 7810, + "unclipped_grad_norm": 1.041825771331787 + }, + { + "bits_per_byte": 1.4985438270168976, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.038711428642273, + "step": 7820, + "unclipped_grad_norm": 1.0459965467453003 + }, + { + "bits_per_byte": 1.5248091894393165, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0569171905517578, + "step": 7830, + "unclipped_grad_norm": 1.093446969985962 + }, + { + "bits_per_byte": 1.5216629388244325, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0547363758087158, + "step": 7840, + "unclipped_grad_norm": 1.0666427612304688 + }, + { + "bits_per_byte": 1.4598855667559556, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0119155645370483, + "step": 7850, + "unclipped_grad_norm": 1.0127599239349365 + }, + { + "bits_per_byte": 1.522777730367325, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.055509090423584, + "step": 7860, + "unclipped_grad_norm": 1.0349164009094238 + }, + { + "bits_per_byte": 1.5484055531029957, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.073272943496704, + "step": 7870, + "unclipped_grad_norm": 1.0840545892715454 + }, + { + "bits_per_byte": 1.4683410937856254, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0177764892578125, + "step": 7880, + "unclipped_grad_norm": 1.0721038579940796 + }, + { + "bits_per_byte": 1.5719193650942607, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.089571475982666, + "step": 7890, + "unclipped_grad_norm": 1.1230442523956299 + }, + { + "bits_per_byte": 1.5165904825198848, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.051220417022705, + "step": 7900, + "unclipped_grad_norm": 1.0280076265335083 + }, + { + "bits_per_byte": 1.5320310849143477, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0619230270385742, + "step": 7910, + "unclipped_grad_norm": 1.0358502864837646 + }, + { + "bits_per_byte": 1.5396064047369018, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0671738386154175, + "step": 7920, + "unclipped_grad_norm": 1.0826281309127808 + }, + { + "bits_per_byte": 1.4911463372553357, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0335838794708252, + "step": 7930, + "unclipped_grad_norm": 1.0188233852386475 + }, + { + "bits_per_byte": 1.500506321045893, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.040071725845337, + "step": 7940, + "unclipped_grad_norm": 1.0400398969650269 + }, + { + "bits_per_byte": 1.561352235077297, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0822468996047974, + "step": 7950, + "unclipped_grad_norm": 1.0635569095611572 + }, + { + "bits_per_byte": 1.4866159702663075, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0304436683654785, + "step": 7960, + "unclipped_grad_norm": 1.0361822843551636 + }, + { + "bits_per_byte": 1.5208446453716271, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0541691780090332, + "step": 7970, + "unclipped_grad_norm": 1.0433354377746582 + }, + { + "bits_per_byte": 1.555866504462726, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.078444480895996, + "step": 7980, + "unclipped_grad_norm": 1.1136473417282104 + }, + { + "bits_per_byte": 1.613144638401981, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.1181466579437256, + "step": 7990, + "unclipped_grad_norm": 1.1455442905426025 + }, + { + "bits_per_byte": 1.4895616891102856, + "learning_rate": 3e-05, + "loss_nats": 1.0324854850769043, + "step": 8000, + "unclipped_grad_norm": 1.039122223854065 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073002.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073002.json new file mode 100644 index 0000000..d489189 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073002.json @@ -0,0 +1,7366 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "68d80bba3d8494c2df7d715607540a0248ed186d1f5b4bbcc53645c92dfa1a84", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027575084823183715, + 0.00024360731185879558, + 0.00021477397240232676, + 0.00019141066877637058, + 0.00017414781905245036, + 0.00016276225505862385, + 0.00015418927068822086, + 0.00014831383305136114, + 0.0001412670244462788, + 0.00013702295836992562, + 0.0001326367782894522, + 0.00012906579650007188, + 0.00012653179874178022, + 0.00012431222421582788, + 0.000122369994642213, + 0.000120684111607261 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023138570031733252, + "first_to_last_ratio": 1.8739548634355478, + "imbalance_abs_log_ratio": 0.6280510978460953, + "last_quartile_mean": 0.00012347453230177052, + "mean": 0.0001624279166207998, + "normalized": [ + 1.6976813713346965, + 1.4997872097782008, + 1.3222725309204868, + 1.1784345496669344, + 1.072154483511671, + 1.0020583803866954, + 0.9492781407040226, + 0.9131055556023103, + 0.8697213347633911, + 0.8435924145343563, + 0.8165885584748509, + 0.79460353358706, + 0.77900278089005, + 0.7653377990807093, + 0.7533803128676146, + 0.74300104389695 + ], + "population_cv": 0.2778958958835334 + }, + "activation_output_rms_by_block": [ + 0.028587408363819122, + 0.028957132250070572, + 0.029405318200588226, + 0.029973436146974564, + 0.030426617711782455, + 0.030960483476519585, + 0.03149084374308586, + 0.03222988545894623, + 0.03307616710662842, + 0.03391994163393974, + 0.034928515553474426, + 0.035913772881031036, + 0.03729887306690216, + 0.038532089442014694, + 0.03956005349755287, + 0.0403735376894474 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02923082374036312, + "first_to_last_ratio": 0.7506412221981493, + "imbalance_abs_log_ratio": 0.2868274747935402, + "last_quartile_mean": 0.03894113842397928, + "mean": 0.033477129763923585, + "normalized": [ + 0.8539384518748725, + 0.8649825255113729, + 0.8783703503840009, + 0.8953406805883115, + 0.9088777301503161, + 0.9248249086719481, + 0.9406673739700877, + 0.962743391868635, + 0.9880227886881968, + 1.0132272949664087, + 1.0433545468140735, + 1.0727853054993168, + 1.114159825825243, + 1.1509974037122668, + 1.1817038610097486, + 1.206003560465201 + ], + "population_cv": 0.11262664842931464 + }, + "bits_per_byte": 8.031398550585404, + "branch_output_rms_by_sublayer": [ + 0.0029695050325244665, + 0.003912033978849649, + 0.0030677192844450474, + 0.0038352999836206436, + 0.00318979867734015, + 0.0038548391312360764, + 0.003840027377009392, + 0.003801720915362239, + 0.00428530341014266, + 0.00395593885332346, + 0.004158117808401585, + 0.003868808038532734, + 0.004507514648139477, + 0.0038615495432168245, + 0.0055657969787716866, + 0.003801694605499506, + 0.005897593684494495, + 0.0038540444802492857, + 0.00619681179523468, + 0.003802293911576271, + 0.006646857596933842, + 0.0038319535087794065, + 0.00662674754858017, + 0.003906778059899807, + 0.007378070615231991, + 0.003958481829613447, + 0.007767242379486561, + 0.0038770362734794617, + 0.00770326005294919, + 0.0037339557893574238, + 0.007914260029792786, + 0.003793976968154311 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007112265943118693, + 0.0061890396223896905, + 0.005352992918648162, + 0.005122911821075447, + 0.005146431820875539, + 0.004279647792521989, + 0.004250462252416264, + 0.004421502222176574, + 0.004575182382709001, + 0.004269268868925463, + 0.0040238830908244155, + 0.003973260546139901, + 0.0038964174441780434, + 0.004243476176619415, + 0.00401474940360252, + 0.003997243047861786 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005944302576307998, + "first_to_last_ratio": 1.472101165080992, + "imbalance_abs_log_ratio": 0.38669074422298993, + "last_quartile_mean": 0.004037971518065441, + "mean": 0.004679295959630182, + "normalized": [ + 1.5199435993103534, + 1.3226433369003716, + 1.1439740005398642, + 1.0948039759127195, + 1.0998303730466061, + 0.9145922440990935, + 0.9083550792867974, + 0.944907580183498, + 0.9777501620287747, + 0.9123741917070097, + 0.8599334441633468, + 0.8491150336329483, + 0.8326930969517022, + 0.9068621034508766, + 0.857981507953136, + 0.854240270832901 + ], + "population_cv": 0.1873316910212935 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02822216972708702, + 0.02833663299679756, + 0.028587408363819122, + 0.02871040627360344, + 0.028957132250070572, + 0.029123734682798386, + 0.029405318200588226, + 0.0297223012894392, + 0.029973436146974564, + 0.03016654960811138, + 0.030426617711782455, + 0.030734822154045105, + 0.030960483476519585, + 0.03127134218811989, + 0.03149084374308586, + 0.0320059098303318, + 0.03222988545894623, + 0.03279566392302513, + 0.03307616710662842, + 0.033656880259513855, + 0.03391994163393974, + 0.03468930721282959, + 0.034928515553474426, + 0.03560047596693039, + 0.035913772881031036, + 0.03701025992631912, + 0.03729887306690216, + 0.038355886936187744, + 0.038532089442014694, + 0.0393846333026886, + 0.03956005349755287, + 0.04016231372952461 + ], + "loss_nats": 5.566941261291504, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.02833663299679756, + 0.028587408363819122, + 0.02871040627360344, + 0.028957132250070572, + 0.029123734682798386, + 0.029405318200588226, + 0.0297223012894392, + 0.029973436146974564, + 0.03016654960811138, + 0.030426617711782455, + 0.030734822154045105, + 0.030960483476519585, + 0.03127134218811989, + 0.03149084374308586, + 0.0320059098303318, + 0.03222988545894623, + 0.03279566392302513, + 0.03307616710662842, + 0.033656880259513855, + 0.03391994163393974, + 0.03468930721282959, + 0.034928515553474426, + 0.03560047596693039, + 0.035913772881031036, + 0.03701025992631912, + 0.03729887306690216, + 0.038355886936187744, + 0.038532089442014694, + 0.0393846333026886, + 0.03956005349755287, + 0.04016231372952461, + 0.0403735376894474 + ] + }, + { + "activation_grad_rms_by_block": [ + 9.082402539206669e-05, + 8.28480115160346e-05, + 7.297925185412169e-05, + 6.382335413945839e-05, + 5.4197575082071126e-05, + 4.6132205170579255e-05, + 3.93990776501596e-05, + 3.41209233738482e-05, + 2.9922650355729274e-05, + 2.6855812393478118e-05, + 2.4530303562642075e-05, + 2.2887908926350065e-05, + 2.182140451623127e-05, + 2.1062218365841545e-05, + 2.0474079065024853e-05, + 2.0058148948010057e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.761866072542034e-05, + "first_to_last_ratio": 3.722010140399952, + "imbalance_abs_log_ratio": 1.3142638826655557, + "last_quartile_mean": 2.085396272377693e-05, + "mean": 4.1996059394477925e-05, + "normalized": [ + 2.1626797061823653, + 1.972756794579834, + 1.7377642785150873, + 1.5197462585704062, + 1.2905395378404856, + 1.098488901952672, + 0.938161299375155, + 0.8124791674700501, + 0.7125109066700628, + 0.6394841035254226, + 0.584109650198932, + 0.5450013467063437, + 0.5196060018693224, + 0.5015284450381319, + 0.4875238143823798, + 0.47761978712334874 + ], + "population_cv": 0.55073751796069 + }, + "activation_output_rms_by_block": [ + 0.03037329390645027, + 0.03339420258998871, + 0.03790459781885147, + 0.04353494197130203, + 0.04983566701412201, + 0.057033151388168335, + 0.06467092037200928, + 0.07435981929302216, + 0.08538646996021271, + 0.09887228161096573, + 0.11526436358690262, + 0.13244163990020752, + 0.15128590166568756, + 0.17302390933036804, + 0.19715538620948792, + 0.22326189279556274 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03630175907164812, + "first_to_last_ratio": 0.194980199104046, + "imbalance_abs_log_ratio": 1.63485726863041, + "last_quartile_mean": 0.18618177250027657, + "mean": 0.09798740246333182, + "normalized": [ + 0.30997141615031953, + 0.34080097798781084, + 0.38683133613053855, + 0.4442912137363105, + 0.5085925921219432, + 0.5820457523558795, + 0.65999219028395, + 0.7588712163366979, + 0.8714025253620276, + 1.0090305398999126, + 1.1763181867182986, + 1.351619050722045, + 1.543932157220932, + 1.7657770793048206, + 2.012048296547774, + 2.2784754691207403 + ], + "population_cv": 0.6071319453340859 + }, + "bits_per_byte": 5.218263195949167, + "branch_output_rms_by_sublayer": [ + 0.004278926644474268, + 0.007860537618398666, + 0.004119343124330044, + 0.008471532724797726, + 0.004720318131148815, + 0.009602016769349575, + 0.0057782139629125595, + 0.009444202296435833, + 0.005580307450145483, + 0.010230599902570248, + 0.00646675331518054, + 0.010376903228461742, + 0.00653326790779829, + 0.010733475908637047, + 0.008962317369878292, + 0.01155572384595871, + 0.010035770013928413, + 0.012197046540677547, + 0.011996372602880001, + 0.012902887538075447, + 0.015837425366044044, + 0.013319648802280426, + 0.01564962975680828, + 0.013529929332435131, + 0.01653609797358513, + 0.013358287513256073, + 0.01997774839401245, + 0.013715346343815327, + 0.020573746412992477, + 0.01530109066516161, + 0.024493996053934097, + 0.014329549856483936 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003576967965622096, + 0.00031529422707157425, + 0.00030800051934586363, + 0.00027582768007438345, + 0.00025927860434436177, + 0.0002499710324331713, + 0.00022865411268768897, + 0.00021433881461782036, + 0.00020517591210118428, + 0.00017936399546700514, + 0.000174300235585937, + 0.00015047843844280973, + 0.00014083116496318749, + 0.00013997136590409212, + 0.0001362051425885804, + 0.00013592122810918616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031420480576350773, + "first_to_last_ratio": 2.2730213947880955, + "imbalance_abs_log_ratio": 0.8211099574033609, + "last_quartile_mean": 0.00013823222539126155, + "mean": 0.000216956829393691, + "normalized": [ + 1.6487003315905355, + 1.4532579036700417, + 1.4196396592197809, + 1.271348225567332, + 1.19507002876537, + 1.1521694575448123, + 1.0539152573656578, + 0.9879330151386018, + 0.9456992558131047, + 0.8267266624805357, + 0.8033867201739516, + 0.6935870092835418, + 0.6491206815510496, + 0.6451576854955755, + 0.6277983641686699, + 0.6264897421714382 + ], + "population_cv": 0.3194559569879116 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028607700020074844, + 0.028983348980545998, + 0.03037329390645027, + 0.030827771872282028, + 0.03339420258998871, + 0.03403528779745102, + 0.03790459781885147, + 0.03889948129653931, + 0.04353494197130203, + 0.04432978481054306, + 0.04983566701412201, + 0.050834495574235916, + 0.057033151388168335, + 0.0581708587706089, + 0.06467092037200928, + 0.06689786911010742, + 0.07435981929302216, + 0.07719690352678299, + 0.08538646996021271, + 0.0897849053144455, + 0.09887228161096573, + 0.10603746026754379, + 0.11526436358690262, + 0.12274596095085144, + 0.13244163990020752, + 0.14162033796310425, + 0.15128590166568756, + 0.16265419125556946, + 0.17302390933036804, + 0.18535073101520538, + 0.19715538620948792, + 0.21195416152477264 + ], + "loss_nats": 3.6170244216918945, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028983348980545998, + 0.03037329390645027, + 0.030827771872282028, + 0.03339420258998871, + 0.03403528779745102, + 0.03790459781885147, + 0.03889948129653931, + 0.04353494197130203, + 0.04432978481054306, + 0.04983566701412201, + 0.050834495574235916, + 0.057033151388168335, + 0.0581708587706089, + 0.06467092037200928, + 0.06689786911010742, + 0.07435981929302216, + 0.07719690352678299, + 0.08538646996021271, + 0.0897849053144455, + 0.09887228161096573, + 0.10603746026754379, + 0.11526436358690262, + 0.12274596095085144, + 0.13244163990020752, + 0.14162033796310425, + 0.15128590166568756, + 0.16265419125556946, + 0.17302390933036804, + 0.18535073101520538, + 0.19715538620948792, + 0.21195416152477264, + 0.22326189279556274 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015646869724150747, + 0.0001555694470880553, + 0.00014404315152205527, + 0.00012938148574903607, + 0.00011473113409010693, + 0.00010209716856479645, + 9.211692668031901e-05, + 8.257795707322657e-05, + 7.45681973057799e-05, + 6.577855674549937e-05, + 5.7623117754701525e-05, + 5.006180435884744e-05, + 4.3452899262774736e-05, + 3.91966023016721e-05, + 3.658098648884334e-05, + 3.536837175488472e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014636569540016353, + "first_to_last_ratio": 3.786979944917394, + "imbalance_abs_log_ratio": 1.3315688531352086, + "last_quartile_mean": 3.864971495204372e-05, + "mean": 8.622603149888164e-05, + "normalized": [ + 1.8146341020407148, + 1.8042051151347847, + 1.670529757871595, + 1.5004921773619393, + 1.330585811450629, + 1.1840643340534658, + 1.068319219602653, + 0.9576917276344513, + 0.8647991332727294, + 0.7628619293044065, + 0.6682798309632156, + 0.5805880601091649, + 0.5039417738173225, + 0.4545796857435157, + 0.42424527550381114, + 0.41018206613560143 + ], + "population_cv": 0.48370287188714023 + }, + "activation_output_rms_by_block": [ + 0.05259466543793678, + 0.067360520362854, + 0.0797014907002449, + 0.08837522566318512, + 0.09281065315008163, + 0.09652531147003174, + 0.09793278574943542, + 0.09766053408384323, + 0.09857666492462158, + 0.10091027617454529, + 0.10140916705131531, + 0.11094733327627182, + 0.12530675530433655, + 0.14505283534526825, + 0.17291401326656342, + 0.21681295335292816 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0720079755410552, + "first_to_last_ratio": 0.43635474619550435, + "imbalance_abs_log_ratio": 0.829299728394033, + "last_quartile_mean": 0.1650216393172741, + "mean": 0.10905569908209145, + "normalized": [ + 0.4822734243200464, + 0.6176707951057974, + 0.7308328805471209, + 0.8103677882680927, + 0.8510390005406192, + 0.8851010289464333, + 0.8980070420319504, + 0.8955105960150644, + 0.9039111734125715, + 0.9253095163654427, + 0.9298841592403141, + 1.0173455785447438, + 1.1490161115744364, + 1.33008028526854, + 1.5855568734321968, + 1.98809374638663 + ], + "population_cv": 0.3579222353559594 + }, + "bits_per_byte": 3.109639980064452, + "branch_output_rms_by_sublayer": [ + 0.015709737315773964, + 0.018998395651578903, + 0.02321256324648857, + 0.011548326350748539, + 0.022954270243644714, + 0.009733532555401325, + 0.020237641409039497, + 0.00895718764513731, + 0.01714940369129181, + 0.010070553049445152, + 0.016518093645572662, + 0.010933448560535908, + 0.01985226757824421, + 0.011251374147832394, + 0.022037845104932785, + 0.012155328877270222, + 0.027303192764520645, + 0.013005394488573074, + 0.03429069370031357, + 0.01562904566526413, + 0.035662416368722916, + 0.018316252157092094, + 0.019603708758950233, + 0.021821532398462296, + 0.018347281962633133, + 0.02647775411605835, + 0.01771368645131588, + 0.02924429066479206, + 0.014302654191851616, + 0.03788068890571594, + 0.024004697799682617, + 0.04663809388875961 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0010353898057018447, + 0.0006549276400495494, + 0.000583207798204784, + 0.0004740072073606198, + 0.00043399488807089015, + 0.0004525702167816539, + 0.0003859790612774441, + 0.0004183888521327983, + 0.00036657723102634496, + 0.00038828410964056754, + 0.0003961803157723344, + 0.00030130391836939404, + 0.00026972366751228246, + 0.0002437005745447571, + 0.00023517814034756533, + 0.0002654411672498285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006868831128291994, + "first_to_last_ratio": 2.7094817103793067, + "imbalance_abs_log_ratio": 0.9967573658183029, + "last_quartile_mean": 0.00025351088741360834, + "mean": 0.0004315534121276662, + "normalized": [ + 2.3992158944987, + 1.5176050557000407, + 1.3514151013878533, + 1.0983743704484823, + 1.0056574131373146, + 1.0487003556532426, + 0.8943946460172122, + 0.9694949463382451, + 0.8494365256412354, + 0.8997359277643753, + 0.9180330977319039, + 0.6981845349892853, + 0.6250064532741785, + 0.5647054749103996, + 0.5449572028363264, + 0.6150829996712046 + ], + "population_cv": 0.44646086778633726 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04346943646669388, + 0.04716380685567856, + 0.05259466543793678, + 0.06381504237651825, + 0.067360520362854, + 0.076678566634655, + 0.0797014907002449, + 0.08585279434919357, + 0.08837522566318512, + 0.09026700258255005, + 0.09281065315008163, + 0.09405135363340378, + 0.09652531147003174, + 0.09571083635091782, + 0.09793278574943542, + 0.09508897364139557, + 0.09766053408384323, + 0.09508920460939407, + 0.09857666492462158, + 0.09573151916265488, + 0.10091027617454529, + 0.09348288178443909, + 0.10140916705131531, + 0.09920287877321243, + 0.11094733327627182, + 0.10943257063627243, + 0.12530675530433655, + 0.12546418607234955, + 0.14505283534526825, + 0.1458611935377121, + 0.17291401326656342, + 0.1839449107646942 + ], + "loss_nats": 2.155438184738159, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04716380685567856, + 0.05259466543793678, + 0.06381504237651825, + 0.067360520362854, + 0.076678566634655, + 0.0797014907002449, + 0.08585279434919357, + 0.08837522566318512, + 0.09026700258255005, + 0.09281065315008163, + 0.09405135363340378, + 0.09652531147003174, + 0.09571083635091782, + 0.09793278574943542, + 0.09508897364139557, + 0.09766053408384323, + 0.09508920460939407, + 0.09857666492462158, + 0.09573151916265488, + 0.10091027617454529, + 0.09348288178443909, + 0.10140916705131531, + 0.09920287877321243, + 0.11094733327627182, + 0.10943257063627243, + 0.12530675530433655, + 0.12546418607234955, + 0.14505283534526825, + 0.1458611935377121, + 0.17291401326656342, + 0.1839449107646942, + 0.21681295335292816 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001656424137763679, + 0.00018305814592167735, + 0.00020359577320050448, + 0.00018629668920766562, + 0.0001775996934156865, + 0.00016671977937221527, + 0.00015271134907379746, + 0.0001390485413139686, + 0.00012298233923502266, + 0.00011299804464215413, + 9.434259118279442e-05, + 8.304810035042465e-05, + 7.153192564146593e-05, + 5.851028618053533e-05, + 4.2740048229461536e-05, + 2.919154394476209e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018464825552655384, + "first_to_last_ratio": 3.6568753347836163, + "imbalance_abs_log_ratio": 1.2966090491094597, + "last_quartile_mean": 5.049345099905622e-05, + "mean": 0.0001243760790430315, + "normalized": [ + 1.3317867475068026, + 1.4718115197886492, + 1.6369367386960691, + 1.4978498328702814, + 1.4279248452127258, + 1.3404489083028075, + 1.2278192900819984, + 1.1179685224347742, + 0.9887941490137615, + 0.9085191100377042, + 0.7585268156761388, + 0.6677176269698272, + 0.5751260707994935, + 0.4704303804294394, + 0.34363559744212857, + 0.23470384473739866 + ], + "population_cv": 0.4334270573824917 + }, + "activation_output_rms_by_block": [ + 0.08618219196796417, + 0.145728200674057, + 0.16391748189926147, + 0.16589416563510895, + 0.16637063026428223, + 0.1656494140625, + 0.16352342069149017, + 0.1510169357061386, + 0.14240477979183197, + 0.14064612984657288, + 0.14404520392417908, + 0.15505897998809814, + 0.1711372584104538, + 0.19663043320178986, + 0.24908031523227692, + 0.4027986526489258 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1404305100440979, + "first_to_last_ratio": 0.5508987205974385, + "imbalance_abs_log_ratio": 0.5962042968923701, + "last_quartile_mean": 0.2549116648733616, + "mean": 0.1756302621215582, + "normalized": [ + 0.49070240473885574, + 0.8297442531469593, + 0.9333100111517798, + 0.9445648126419658, + 0.9472776972178796, + 0.9431712511358084, + 0.9310663134939208, + 0.859857144673712, + 0.8108214271938511, + 0.8008080620481458, + 0.820161640620235, + 0.8828716538655387, + 0.9744178272193509, + 1.1195703452614385, + 1.4182084125108352, + 2.293446743079724 + ], + "population_cv": 0.38002251655845937 + }, + "bits_per_byte": 2.081392170877702, + "branch_output_rms_by_sublayer": [ + 0.03403226286172867, + 0.04189683124423027, + 0.08070491254329681, + 0.007666230201721191, + 0.03132990375161171, + 0.004042334854602814, + 0.023764250800013542, + 0.004978273529559374, + 0.022670205682516098, + 0.009758620522916317, + 0.027493907138705254, + 0.012959085404872894, + 0.036309871822595596, + 0.017362121492624283, + 0.039587654173374176, + 0.02089655213057995, + 0.04225468263030052, + 0.02445071190595627, + 0.032471224665641785, + 0.02915186993777752, + 0.04851555824279785, + 0.047949694097042084, + 0.01816859096288681, + 0.05110293999314308, + 0.02429918199777603, + 0.05613848567008972, + 0.01771482080221176, + 0.06749960780143738, + 0.015686625614762306, + 0.09889734536409378, + 0.016557788476347923, + 0.2156895101070404 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00041094985323999886, + 0.0003483840396349268, + 0.0002459421498427246, + 0.00023334166050981232, + 0.000254581953755271, + 0.0003388594313501643, + 0.0004326353023247584, + 0.0004560987158898983, + 0.0004819816052452772, + 0.0004606041111206281, + 0.00053396017856909, + 0.0004147477851776299, + 0.00037952749197896316, + 0.00033346538787636917, + 0.0002981284512971557, + 0.0002588526279031963 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030965442580686565, + "first_to_last_ratio": 0.9753095285106969, + "imbalance_abs_log_ratio": 0.0250003932265083, + "last_quartile_mean": 0.0003174934897639211, + "mean": 0.0003676287966072415, + "normalized": [ + 1.117839127490983, + 0.9476516607242959, + 0.6689958787572303, + 0.6347208452201429, + 0.6924973127914547, + 0.9217434392447414, + 1.176826479093712, + 1.2406501343178897, + 1.3110550906059877, + 1.2529054181049841, + 1.4524438332820528, + 1.1281700155298995, + 1.0323660591377277, + 0.9070709121642312, + 0.8109496700163646, + 0.7041141235183029 + ], + "population_cv": 0.2454972418327454 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05505011975765228, + 0.06996718049049377, + 0.08618219196796417, + 0.1434549242258072, + 0.145728200674057, + 0.16383236646652222, + 0.16391748189926147, + 0.16620540618896484, + 0.16589416563510895, + 0.16680952906608582, + 0.16637063026428223, + 0.16661377251148224, + 0.1656494140625, + 0.1662609726190567, + 0.16352342069149017, + 0.15300041437149048, + 0.1510169357061386, + 0.14022879302501678, + 0.14240477979183197, + 0.13479726016521454, + 0.14064612984657288, + 0.13225789368152618, + 0.14404520392417908, + 0.1422654092311859, + 0.15505897998809814, + 0.1554868519306183, + 0.1711372584104538, + 0.17103315889835358, + 0.19663043320178986, + 0.19691547751426697, + 0.24908031523227692, + 0.25020092725753784 + ], + "loss_nats": 1.4427111148834229, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06996718049049377, + 0.08618219196796417, + 0.1434549242258072, + 0.145728200674057, + 0.16383236646652222, + 0.16391748189926147, + 0.16620540618896484, + 0.16589416563510895, + 0.16680952906608582, + 0.16637063026428223, + 0.16661377251148224, + 0.1656494140625, + 0.1662609726190567, + 0.16352342069149017, + 0.15300041437149048, + 0.1510169357061386, + 0.14022879302501678, + 0.14240477979183197, + 0.13479726016521454, + 0.14064612984657288, + 0.13225789368152618, + 0.14404520392417908, + 0.1422654092311859, + 0.15505897998809814, + 0.1554868519306183, + 0.1711372584104538, + 0.17103315889835358, + 0.19663043320178986, + 0.19691547751426697, + 0.24908031523227692, + 0.25020092725753784, + 0.4027986526489258 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001670715573709458, + 0.0001843688660301268, + 0.00019890567637048662, + 0.00018651749996934086, + 0.00017948838649317622, + 0.0001703005691524595, + 0.00015747325960546732, + 0.0001445972011424601, + 0.00012834250810556114, + 0.00011769140110118315, + 9.774974023457617e-05, + 8.55058096931316e-05, + 7.324934267671779e-05, + 5.983562004985288e-05, + 4.2463150748517364e-05, + 2.6351406631874852e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018421589993522502, + "first_to_last_ratio": 3.6496550331101454, + "imbalance_abs_log_ratio": 1.2946326516512365, + "last_quartile_mean": 5.047488002674072e-05, + "mean": 0.00012624449971099239, + "normalized": [ + 1.323396724240799, + 1.4604110789158873, + 1.5755591477318631, + 1.477430703090666, + 1.42175213101619, + 1.348974169506975, + 1.2473672909787434, + 1.145374265599547, + 1.016618612291004, + 0.9322497326268504, + 0.7742891013735379, + 0.6773032479741881, + 0.5802180914368962, + 0.47396615446084944, + 0.336356442028975, + 0.2087331067270282 + ], + "population_cv": 0.42845315545725604 + }, + "activation_output_rms_by_block": [ + 0.10210617631673813, + 0.1711120903491974, + 0.18304166197776794, + 0.1841333508491516, + 0.18528582155704498, + 0.18522554636001587, + 0.18429037928581238, + 0.1677425056695938, + 0.15884697437286377, + 0.1616142988204956, + 0.1705332249403, + 0.18593955039978027, + 0.20567089319229126, + 0.23390550911426544, + 0.2939614951610565, + 0.5084910988807678 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16009831987321377, + "first_to_last_ratio": 0.5156025192452343, + "imbalance_abs_log_ratio": 0.6624191218832031, + "last_quartile_mean": 0.31050724908709526, + "mean": 0.20511878607794642, + "normalized": [ + 0.4977904670220559, + 0.8342097455869972, + 0.8923690778289365, + 0.8976913054622885, + 0.9033098581552409, + 0.9030160030765246, + 0.8984568542433792, + 0.8177822659590556, + 0.7744145595347903, + 0.7879058857099571, + 0.8313876471338725, + 0.9064969326072458, + 1.002691645777072, + 1.1403417189948652, + 1.4331280950997307, + 2.479007937807988 + ], + "population_cv": 0.4251348773789681 + }, + "bits_per_byte": 1.9581638498120857, + "branch_output_rms_by_sublayer": [ + 0.052206385880708694, + 0.0456010177731514, + 0.08834584802389145, + 0.009068594314157963, + 0.02721724845468998, + 0.003163012443110347, + 0.024817200377583504, + 0.004634208977222443, + 0.025762483477592468, + 0.010066481307148933, + 0.02899768576025963, + 0.014108272269368172, + 0.04359622299671173, + 0.02107245847582817, + 0.046488646417856216, + 0.02715279720723629, + 0.04875380918383598, + 0.032857682555913925, + 0.03842322900891304, + 0.03946002945303917, + 0.05467399209737778, + 0.0617416612803936, + 0.025746947154402733, + 0.06822259724140167, + 0.032913338392972946, + 0.0761992558836937, + 0.024072155356407166, + 0.08795499801635742, + 0.022052984684705734, + 0.12569737434387207, + 0.025944042950868607, + 0.30153149366378784 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00042380566817482924, + 0.00033628063183600165, + 0.000202638705536708, + 0.00018804477448005897, + 0.00023208100123381636, + 0.0003223172253228011, + 0.0004741333361060697, + 0.0004905991387745841, + 0.0005898184863404294, + 0.0005622263144383919, + 0.000639932357888532, + 0.0005200270037733745, + 0.0004834555123403282, + 0.0004217229809514017, + 0.0003667204712485774, + 0.0003029597160326848 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002876924450068995, + "first_to_last_ratio": 0.7307130437944471, + "imbalance_abs_log_ratio": 0.3137344492502099, + "last_quartile_mean": 0.000393714670143248, + "mean": 0.0004097977077799118, + "normalized": [ + 1.0341826226183788, + 0.8206015442541381, + 0.4944847218265511, + 0.4588721969646822, + 0.5663306476044494, + 0.7865276432827353, + 1.156993626622993, + 1.1971739457314583, + 1.4392918143339164, + 1.371960613162688, + 1.561581106335074, + 1.2689846573096573, + 1.1797418657109118, + 1.029100390131744, + 0.894881704525127, + 0.739290899585495 + ], + "population_cv": 0.3269281757976474 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05859006941318512, + 0.08463093638420105, + 0.10210617631673813, + 0.16779249906539917, + 0.1711120903491974, + 0.1829725056886673, + 0.18304166197776794, + 0.18442119657993317, + 0.1841333508491516, + 0.1860278993844986, + 0.18528582155704498, + 0.1865086406469345, + 0.18522554636001587, + 0.18776042759418488, + 0.18429037928581238, + 0.17156346142292023, + 0.1677425056695938, + 0.15597018599510193, + 0.15884697437286377, + 0.1531117558479309, + 0.1616142988204956, + 0.15448127686977386, + 0.1705332249403, + 0.1696566343307495, + 0.18593955039978027, + 0.1874322146177292, + 0.20567089319229126, + 0.20625530183315277, + 0.23390550911426544, + 0.234551340341568, + 0.2939614951610565, + 0.2957085371017456 + ], + "loss_nats": 1.3572957515716553, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08463093638420105, + 0.10210617631673813, + 0.16779249906539917, + 0.1711120903491974, + 0.1829725056886673, + 0.18304166197776794, + 0.18442119657993317, + 0.1841333508491516, + 0.1860278993844986, + 0.18528582155704498, + 0.1865086406469345, + 0.18522554636001587, + 0.18776042759418488, + 0.18429037928581238, + 0.17156346142292023, + 0.1677425056695938, + 0.15597018599510193, + 0.15884697437286377, + 0.1531117558479309, + 0.1616142988204956, + 0.15448127686977386, + 0.1705332249403, + 0.1696566343307495, + 0.18593955039978027, + 0.1874322146177292, + 0.20567089319229126, + 0.20625530183315277, + 0.23390550911426544, + 0.234551340341568, + 0.2939614951610565, + 0.2957085371017456, + 0.5084910988807678 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.000266833056230098, + 0.0002936421660706401, + 0.00031041845795698464, + 0.0002957055112347007, + 0.0002881943073589355, + 0.0002774772292468697, + 0.00026020428049378097, + 0.00024136141291819513, + 0.00021430903871078044, + 0.00019666840671561658, + 0.00016092424630187452, + 0.0001381110487272963, + 0.00011530883784871548, + 9.161087655229494e-05, + 6.125834624981508e-05, + 3.126815863652155e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029164979787310585, + "first_to_last_ratio": 3.8958554703706607, + "imbalance_abs_log_ratio": 1.3599132881647322, + "last_quartile_mean": 7.486155482183676e-05, + "mean": 0.00020270596132831997, + "normalized": [ + 1.3163552491577308, + 1.4486113982362465, + 1.531373107741041, + 1.4587904040757373, + 1.4217357273087359, + 1.3688656585557626, + 1.2836538145631138, + 1.1906971622174716, + 1.05724092822148, + 0.9702152093942396, + 0.7938801860949112, + 0.6813368872936094, + 0.5688477886545793, + 0.4519397256596421, + 0.3022029833182646, + 0.15425376950743425 + ], + "population_cv": 0.4374351028998013 + }, + "activation_output_rms_by_block": [ + 0.10593125969171524, + 0.16277404129505157, + 0.16965323686599731, + 0.16991853713989258, + 0.16967523097991943, + 0.1679684966802597, + 0.16474221646785736, + 0.14505790174007416, + 0.1350426822900772, + 0.13761113584041595, + 0.1448632776737213, + 0.15946830809116364, + 0.17768383026123047, + 0.2030501663684845, + 0.25671765208244324, + 0.47381165623664856 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15206926874816418, + "first_to_last_ratio": 0.5473743911850654, + "imbalance_abs_log_ratio": 0.6026222661058785, + "last_quartile_mean": 0.2778158262372017, + "mean": 0.18399810185655951, + "normalized": [ + 0.5757193070083771, + 0.8846506548309193, + 0.9220379729691717, + 0.9234798371580865, + 0.9221575074301264, + 0.9128816818512828, + 0.8953473693782256, + 0.7883662944151337, + 0.7339351924149371, + 0.7478943231039105, + 0.7873085440123357, + 0.8666845281669334, + 0.9656829525325674, + 1.1035448970380004, + 1.3952190239580522, + 2.57508991373194 + ], + "population_cv": 0.4416592993774559 + }, + "bits_per_byte": 1.8703918160136477, + "branch_output_rms_by_sublayer": [ + 0.06665868312120438, + 0.03488925099372864, + 0.07055649906396866, + 0.008275732398033142, + 0.021501976996660233, + 0.0027294722385704517, + 0.020079251378774643, + 0.004083180334419012, + 0.019392874091863632, + 0.008691227994859219, + 0.02471928857266903, + 0.012035587802529335, + 0.03809819743037224, + 0.018656741827726364, + 0.04367014393210411, + 0.024902483448386192, + 0.04372077062726021, + 0.03141119331121445, + 0.03309999778866768, + 0.037526171654462814, + 0.0474277026951313, + 0.05794820562005043, + 0.02467101812362671, + 0.0651254877448082, + 0.02942940965294838, + 0.07373781502246857, + 0.023407138884067535, + 0.08589385449886322, + 0.019243014976382256, + 0.1193988099694252, + 0.02799036167562008, + 0.300537645816803 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.000552485054575078, + 0.0004878954173704819, + 0.00026717869748665266, + 0.00024508760509843496, + 0.0003479630309338483, + 0.0004644527429795662, + 0.0007202858637767138, + 0.0008028534351930377, + 0.0009144494502519988, + 0.0008971273720051169, + 0.001000538678877296, + 0.0008212401598758333, + 0.0007501935670996717, + 0.0006492653356624181, + 0.0005392275192896546, + 0.00040293680800761964 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00038816169363266187, + "first_to_last_ratio": 0.6630643028303428, + "imbalance_abs_log_ratio": 0.4108833058215335, + "last_quartile_mean": 0.000585405807514841, + "mean": 0.0006164487961552139, + "normalized": [ + 0.8962383542979121, + 0.7914613840005554, + 0.4334158800423393, + 0.3975798259758869, + 0.5644638015421408, + 0.7534328007068132, + 1.1684439458218279, + 1.3023846265909318, + 1.4834150962016837, + 1.4553153118320499, + 1.6230685908021032, + 1.3322114748181866, + 1.2169600650996852, + 1.0532348180609332, + 0.8747320501764496, + 0.6536419740305005 + ], + "population_cv": 0.37363268494405877 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060181986540555954, + 0.0962739959359169, + 0.10593125969171524, + 0.15956835448741913, + 0.16277404129505157, + 0.16968216001987457, + 0.16965323686599731, + 0.1704123467206955, + 0.16991853713989258, + 0.17062334716320038, + 0.16967523097991943, + 0.16979995369911194, + 0.1679684966802597, + 0.16850441694259644, + 0.16474221646785736, + 0.14905613660812378, + 0.14505790174007416, + 0.13280805945396423, + 0.1350426822900772, + 0.13030488789081573, + 0.13761113584041595, + 0.13074533641338348, + 0.1448632776737213, + 0.1443873643875122, + 0.15946830809116364, + 0.16084782779216766, + 0.17768383026123047, + 0.17834986746311188, + 0.2030501663684845, + 0.20354533195495605, + 0.25671765208244324, + 0.2584022283554077 + ], + "loss_nats": 1.2964568138122559, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0962739959359169, + 0.10593125969171524, + 0.15956835448741913, + 0.16277404129505157, + 0.16968216001987457, + 0.16965323686599731, + 0.1704123467206955, + 0.16991853713989258, + 0.17062334716320038, + 0.16967523097991943, + 0.16979995369911194, + 0.1679684966802597, + 0.16850441694259644, + 0.16474221646785736, + 0.14905613660812378, + 0.14505790174007416, + 0.13280805945396423, + 0.1350426822900772, + 0.13030488789081573, + 0.13761113584041595, + 0.13074533641338348, + 0.1448632776737213, + 0.1443873643875122, + 0.15946830809116364, + 0.16084782779216766, + 0.17768383026123047, + 0.17834986746311188, + 0.2030501663684845, + 0.20354533195495605, + 0.25671765208244324, + 0.2584022283554077, + 0.47381165623664856 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.032388224749795, + "cross_entropy_nats": 5.567627251148224, + "step": 0 + }, + { + "bits_per_byte": 5.232053968773903, + "cross_entropy_nats": 3.626583456993103, + "step": 100 + }, + { + "bits_per_byte": 3.1492899782815944, + "cross_entropy_nats": 2.1829214692115784, + "step": 500 + }, + { + "bits_per_byte": 1.9917407321684666, + "cross_entropy_nats": 1.3805694729089737, + "step": 2000 + }, + { + "bits_per_byte": 1.8310010171491868, + "cross_entropy_nats": 1.269153192639351, + "step": 4000 + }, + { + "bits_per_byte": 1.7368251448523782, + "cross_entropy_nats": 1.2038754522800446, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "final_optimizer_state": "34afebd4020a97001cb564ade8591d46cc164cbd65eb2e1cad4d8d953e6b7cce", + "final_public_parameters": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.682085667085165, + "measured_steps": 7980, + "median_ms": 21.630175004247576, + "p95_ms": 22.00388514320366, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.03605033732594, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.570165634155273, + "step": 1, + "unclipped_grad_norm": 14.13492488861084 + }, + { + "bits_per_byte": 7.579231395965356, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.253522872924805, + "step": 10, + "unclipped_grad_norm": 10.806709289550781 + }, + { + "bits_per_byte": 6.980917512225713, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.838803291320801, + "step": 20, + "unclipped_grad_norm": 4.377180576324463 + }, + { + "bits_per_byte": 6.73711765808501, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.669814109802246, + "step": 30, + "unclipped_grad_norm": 3.1878201961517334 + }, + { + "bits_per_byte": 6.622164454248309, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.590134620666504, + "step": 40, + "unclipped_grad_norm": 3.092231035232544 + }, + { + "bits_per_byte": 6.373620630666205, + "learning_rate": 3.75e-05, + "loss_nats": 4.4178571701049805, + "step": 50, + "unclipped_grad_norm": 3.1614222526550293 + }, + { + "bits_per_byte": 6.096217430338719, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.225575923919678, + "step": 60, + "unclipped_grad_norm": 2.885824203491211 + }, + { + "bits_per_byte": 5.853263042909657, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 4.057172775268555, + "step": 70, + "unclipped_grad_norm": 2.6939339637756348 + }, + { + "bits_per_byte": 5.640556620149043, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.909735918045044, + "step": 80, + "unclipped_grad_norm": 2.5527353286743164 + }, + { + "bits_per_byte": 5.452656287018279, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7794933319091797, + "step": 90, + "unclipped_grad_norm": 2.4638988971710205 + }, + { + "bits_per_byte": 5.271040199950041, + "learning_rate": 7.5e-05, + "loss_nats": 3.653606653213501, + "step": 100, + "unclipped_grad_norm": 2.4236807823181152 + }, + { + "bits_per_byte": 5.106934010531447, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.5398569107055664, + "step": 110, + "unclipped_grad_norm": 2.3251681327819824 + }, + { + "bits_per_byte": 4.82528283872765, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3446311950683594, + "step": 120, + "unclipped_grad_norm": 2.082082509994507 + }, + { + "bits_per_byte": 4.666372933122225, + "learning_rate": 9.75e-05, + "loss_nats": 3.234483242034912, + "step": 130, + "unclipped_grad_norm": 1.8279871940612793 + }, + { + "bits_per_byte": 4.4669253408472285, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0962367057800293, + "step": 140, + "unclipped_grad_norm": 1.706978440284729 + }, + { + "bits_per_byte": 4.329626087218621, + "learning_rate": 0.0001125, + "loss_nats": 3.001068115234375, + "step": 150, + "unclipped_grad_norm": 1.465320348739624 + }, + { + "bits_per_byte": 4.211352242359061, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.9190869331359863, + "step": 160, + "unclipped_grad_norm": 1.149477243423462 + }, + { + "bits_per_byte": 4.052300967014626, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.8088409900665283, + "step": 170, + "unclipped_grad_norm": 1.175323247909546 + }, + { + "bits_per_byte": 3.9086503940896553, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7092700004577637, + "step": 180, + "unclipped_grad_norm": 1.1839734315872192 + }, + { + "bits_per_byte": 3.9167359864373195, + "learning_rate": 0.0001425, + "loss_nats": 2.714874505996704, + "step": 190, + "unclipped_grad_norm": 0.9742223024368286 + }, + { + "bits_per_byte": 3.7936297730538158, + "learning_rate": 0.00015, + "loss_nats": 2.6295437812805176, + "step": 200, + "unclipped_grad_norm": 0.6956769227981567 + }, + { + "bits_per_byte": 3.735681250741498, + "learning_rate": 0.0001575, + "loss_nats": 2.589376926422119, + "step": 210, + "unclipped_grad_norm": 0.6548250317573547 + }, + { + "bits_per_byte": 3.7032831589719617, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.566920280456543, + "step": 220, + "unclipped_grad_norm": 0.6789838075637817 + }, + { + "bits_per_byte": 3.722889181170322, + "learning_rate": 0.0001725, + "loss_nats": 2.580510139465332, + "step": 230, + "unclipped_grad_norm": 0.730959415435791 + }, + { + "bits_per_byte": 3.721254658056522, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5793771743774414, + "step": 240, + "unclipped_grad_norm": 0.7681548595428467 + }, + { + "bits_per_byte": 3.588607879099055, + "learning_rate": 0.0001875, + "loss_nats": 2.487433433532715, + "step": 250, + "unclipped_grad_norm": 1.0810788869857788 + }, + { + "bits_per_byte": 3.588186865569743, + "learning_rate": 0.000195, + "loss_nats": 2.4871416091918945, + "step": 260, + "unclipped_grad_norm": 0.9089803695678711 + }, + { + "bits_per_byte": 3.5492888614934976, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.460179567337036, + "step": 270, + "unclipped_grad_norm": 0.6836683750152588 + }, + { + "bits_per_byte": 3.551394961035964, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.461639404296875, + "step": 280, + "unclipped_grad_norm": 0.6695814728736877 + }, + { + "bits_per_byte": 3.464650695559082, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.401512861251831, + "step": 290, + "unclipped_grad_norm": 0.8494355082511902 + }, + { + "bits_per_byte": 3.543094390374721, + "learning_rate": 0.000225, + "loss_nats": 2.455885887145996, + "step": 300, + "unclipped_grad_norm": 0.7996388077735901 + }, + { + "bits_per_byte": 3.442018122670838, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.3858251571655273, + "step": 310, + "unclipped_grad_norm": 0.7709511518478394 + }, + { + "bits_per_byte": 3.43619410218202, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3817882537841797, + "step": 320, + "unclipped_grad_norm": 0.8889051079750061 + }, + { + "bits_per_byte": 3.4344873463548575, + "learning_rate": 0.0002475, + "loss_nats": 2.3806052207946777, + "step": 330, + "unclipped_grad_norm": 0.977308988571167 + }, + { + "bits_per_byte": 3.4251934039022314, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3741631507873535, + "step": 340, + "unclipped_grad_norm": 0.7606520652770996 + }, + { + "bits_per_byte": 3.3985436602551338, + "learning_rate": 0.0002625, + "loss_nats": 2.3556909561157227, + "step": 350, + "unclipped_grad_norm": 0.6722884178161621 + }, + { + "bits_per_byte": 3.3928290207322647, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3517298698425293, + "step": 360, + "unclipped_grad_norm": 0.8897987008094788 + }, + { + "bits_per_byte": 3.4104885432881593, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3639705181121826, + "step": 370, + "unclipped_grad_norm": 1.0889073610305786 + }, + { + "bits_per_byte": 3.4536713231280003, + "learning_rate": 0.000285, + "loss_nats": 2.393902540206909, + "step": 380, + "unclipped_grad_norm": 1.0755773782730103 + }, + { + "bits_per_byte": 3.3990293392611703, + "learning_rate": 0.0002925, + "loss_nats": 2.356027603149414, + "step": 390, + "unclipped_grad_norm": 1.091948390007019 + }, + { + "bits_per_byte": 3.337470901107242, + "learning_rate": 0.0003, + "loss_nats": 2.3133585453033447, + "step": 400, + "unclipped_grad_norm": 0.9980842471122742 + }, + { + "bits_per_byte": 3.3053706832890004, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2911083698272705, + "step": 410, + "unclipped_grad_norm": 0.7333840727806091 + }, + { + "bits_per_byte": 3.321116382906094, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3020224571228027, + "step": 420, + "unclipped_grad_norm": 0.8894042372703552 + }, + { + "bits_per_byte": 3.3009490093353175, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.28804349899292, + "step": 430, + "unclipped_grad_norm": 0.9840139746665955 + }, + { + "bits_per_byte": 3.3110151538903176, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.295020818710327, + "step": 440, + "unclipped_grad_norm": 1.0280733108520508 + }, + { + "bits_per_byte": 3.2949894665176194, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.2839126586914062, + "step": 450, + "unclipped_grad_norm": 0.928855836391449 + }, + { + "bits_per_byte": 3.2291744577548367, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.238293170928955, + "step": 460, + "unclipped_grad_norm": 0.8088337779045105 + }, + { + "bits_per_byte": 3.242055958304461, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2472219467163086, + "step": 470, + "unclipped_grad_norm": 1.018490195274353 + }, + { + "bits_per_byte": 3.1972775234299147, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.21618390083313, + "step": 480, + "unclipped_grad_norm": 1.066487193107605 + }, + { + "bits_per_byte": 3.156052938052798, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1876091957092285, + "step": 490, + "unclipped_grad_norm": 1.2544180154800415 + }, + { + "bits_per_byte": 3.153048401142225, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1855266094207764, + "step": 500, + "unclipped_grad_norm": 0.7976559400558472 + }, + { + "bits_per_byte": 3.126950721804397, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1674370765686035, + "step": 510, + "unclipped_grad_norm": 0.9979055523872375 + }, + { + "bits_per_byte": 3.0098652770622984, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0862796306610107, + "step": 520, + "unclipped_grad_norm": 1.0761884450912476 + }, + { + "bits_per_byte": 3.0772147150360794, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.132962703704834, + "step": 530, + "unclipped_grad_norm": 1.010690450668335 + }, + { + "bits_per_byte": 3.0006050432092413, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0798609256744385, + "step": 540, + "unclipped_grad_norm": 1.0196499824523926 + }, + { + "bits_per_byte": 2.9739387892276614, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0613772869110107, + "step": 550, + "unclipped_grad_norm": 0.949600875377655 + }, + { + "bits_per_byte": 2.9792871057041928, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.065084457397461, + "step": 560, + "unclipped_grad_norm": 1.027385950088501 + }, + { + "bits_per_byte": 2.952135172716576, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.046264171600342, + "step": 570, + "unclipped_grad_norm": 1.2502241134643555 + }, + { + "bits_per_byte": 2.8483484584350953, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.9743247032165527, + "step": 580, + "unclipped_grad_norm": 0.9710299968719482 + }, + { + "bits_per_byte": 2.9027066709280014, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.012002944946289, + "step": 590, + "unclipped_grad_norm": 1.0665416717529297 + }, + { + "bits_per_byte": 2.824687945242644, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.957924485206604, + "step": 600, + "unclipped_grad_norm": 1.3016425371170044 + }, + { + "bits_per_byte": 2.889905658258981, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.0031299591064453, + "step": 610, + "unclipped_grad_norm": 1.458404779434204 + }, + { + "bits_per_byte": 2.7619725797963617, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9144535064697266, + "step": 620, + "unclipped_grad_norm": 1.0435335636138916 + }, + { + "bits_per_byte": 2.780679820661313, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9274203777313232, + "step": 630, + "unclipped_grad_norm": 1.2443503141403198 + }, + { + "bits_per_byte": 2.796278303129268, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.938232421875, + "step": 640, + "unclipped_grad_norm": 1.0311143398284912 + }, + { + "bits_per_byte": 2.704606046774563, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.874690055847168, + "step": 650, + "unclipped_grad_norm": 0.9809540510177612 + }, + { + "bits_per_byte": 2.691190368111359, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8653910160064697, + "step": 660, + "unclipped_grad_norm": 1.5349526405334473 + }, + { + "bits_per_byte": 2.7107573502479734, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8789538145065308, + "step": 670, + "unclipped_grad_norm": 1.2438547611236572 + }, + { + "bits_per_byte": 2.7307020062858824, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8927783966064453, + "step": 680, + "unclipped_grad_norm": 1.507369041442871 + }, + { + "bits_per_byte": 2.6187595307351854, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.815185785293579, + "step": 690, + "unclipped_grad_norm": 1.4796253442764282 + }, + { + "bits_per_byte": 2.562429709132788, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7761409282684326, + "step": 700, + "unclipped_grad_norm": 1.6683489084243774 + }, + { + "bits_per_byte": 2.6317965036011897, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8242223262786865, + "step": 710, + "unclipped_grad_norm": 1.1483558416366577 + }, + { + "bits_per_byte": 2.6874191325432393, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8627769947052002, + "step": 720, + "unclipped_grad_norm": 1.464730143547058 + }, + { + "bits_per_byte": 2.5059506065894417, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.736992597579956, + "step": 730, + "unclipped_grad_norm": 1.4666827917099 + }, + { + "bits_per_byte": 2.538973683312213, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7598824501037598, + "step": 740, + "unclipped_grad_norm": 1.090103030204773 + }, + { + "bits_per_byte": 2.545309180204697, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7642738819122314, + "step": 750, + "unclipped_grad_norm": 1.1345367431640625 + }, + { + "bits_per_byte": 2.5190732268155758, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7460885047912598, + "step": 760, + "unclipped_grad_norm": 1.0966333150863647 + }, + { + "bits_per_byte": 2.5090782830780327, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7391605377197266, + "step": 770, + "unclipped_grad_norm": 1.297254204750061 + }, + { + "bits_per_byte": 2.4228073138563255, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6793620586395264, + "step": 780, + "unclipped_grad_norm": 1.1724724769592285 + }, + { + "bits_per_byte": 2.510659835535367, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7402567863464355, + "step": 790, + "unclipped_grad_norm": 1.0964044332504272 + }, + { + "bits_per_byte": 2.4699436308742007, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.7120344638824463, + "step": 800, + "unclipped_grad_norm": 1.4117348194122314 + }, + { + "bits_per_byte": 2.5086447148152073, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.738860011100769, + "step": 810, + "unclipped_grad_norm": 1.889366626739502 + }, + { + "bits_per_byte": 2.4598160885128424, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7050145864486694, + "step": 820, + "unclipped_grad_norm": 1.144262671470642 + }, + { + "bits_per_byte": 2.470564488243815, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7124648094177246, + "step": 830, + "unclipped_grad_norm": 1.5465298891067505 + }, + { + "bits_per_byte": 2.4487553682879026, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.69734787940979, + "step": 840, + "unclipped_grad_norm": 1.3992143869400024 + }, + { + "bits_per_byte": 2.383892111514994, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.652388095855713, + "step": 850, + "unclipped_grad_norm": 1.1111818552017212 + }, + { + "bits_per_byte": 2.4110481720862316, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6712112426757812, + "step": 860, + "unclipped_grad_norm": 1.363444209098816 + }, + { + "bits_per_byte": 2.3893498089574745, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6561710834503174, + "step": 870, + "unclipped_grad_norm": 1.2841624021530151 + }, + { + "bits_per_byte": 2.4391853936806416, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6907144784927368, + "step": 880, + "unclipped_grad_norm": 1.406774640083313 + }, + { + "bits_per_byte": 2.4004903291324147, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6638931035995483, + "step": 890, + "unclipped_grad_norm": 1.1680277585983276 + }, + { + "bits_per_byte": 2.3361910034716855, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6193242073059082, + "step": 900, + "unclipped_grad_norm": 1.2712005376815796 + }, + { + "bits_per_byte": 2.3780436694897533, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.648334264755249, + "step": 910, + "unclipped_grad_norm": 1.335773229598999 + }, + { + "bits_per_byte": 2.3308695162886885, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.615635633468628, + "step": 920, + "unclipped_grad_norm": 1.1817855834960938 + }, + { + "bits_per_byte": 2.330381085560238, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6152970790863037, + "step": 930, + "unclipped_grad_norm": 1.0815848112106323 + }, + { + "bits_per_byte": 2.2884202424547775, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5862120389938354, + "step": 940, + "unclipped_grad_norm": 1.0937949419021606 + }, + { + "bits_per_byte": 2.296288964679704, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5916662216186523, + "step": 950, + "unclipped_grad_norm": 1.6217018365859985 + }, + { + "bits_per_byte": 2.2717374093731326, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.574648380279541, + "step": 960, + "unclipped_grad_norm": 1.6217092275619507 + }, + { + "bits_per_byte": 2.290087270289591, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5873675346374512, + "step": 970, + "unclipped_grad_norm": 1.0035263299942017 + }, + { + "bits_per_byte": 2.280217357939225, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5805262327194214, + "step": 980, + "unclipped_grad_norm": 1.3000710010528564 + }, + { + "bits_per_byte": 2.201290563857312, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5258183479309082, + "step": 990, + "unclipped_grad_norm": 1.0893296003341675 + }, + { + "bits_per_byte": 2.251006276672828, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5602786540985107, + "step": 1000, + "unclipped_grad_norm": 1.206483006477356 + }, + { + "bits_per_byte": 2.2491729416146304, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5590078830718994, + "step": 1010, + "unclipped_grad_norm": 1.1218043565750122 + }, + { + "bits_per_byte": 2.2577407733153136, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5649466514587402, + "step": 1020, + "unclipped_grad_norm": 1.0566829442977905 + }, + { + "bits_per_byte": 2.1604180269665116, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4974876642227173, + "step": 1030, + "unclipped_grad_norm": 1.102095127105713 + }, + { + "bits_per_byte": 2.2614810520062782, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5675392150878906, + "step": 1040, + "unclipped_grad_norm": 1.2485690116882324 + }, + { + "bits_per_byte": 2.2470793968056775, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5575567483901978, + "step": 1050, + "unclipped_grad_norm": 1.242668628692627 + }, + { + "bits_per_byte": 2.1863758844091876, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5154802799224854, + "step": 1060, + "unclipped_grad_norm": 1.1564915180206299 + }, + { + "bits_per_byte": 2.2939011575551245, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5900111198425293, + "step": 1070, + "unclipped_grad_norm": 1.2426202297210693 + }, + { + "bits_per_byte": 2.2229925386217375, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5408610105514526, + "step": 1080, + "unclipped_grad_norm": 1.0772974491119385 + }, + { + "bits_per_byte": 2.1433408376664382, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.485650658607483, + "step": 1090, + "unclipped_grad_norm": 1.0020748376846313 + }, + { + "bits_per_byte": 2.2307484402276976, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5462369918823242, + "step": 1100, + "unclipped_grad_norm": 1.1032538414001465 + }, + { + "bits_per_byte": 2.263051941539259, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5686280727386475, + "step": 1110, + "unclipped_grad_norm": 1.062880039215088 + }, + { + "bits_per_byte": 2.1486932817265902, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4893606901168823, + "step": 1120, + "unclipped_grad_norm": 1.1697474718093872 + }, + { + "bits_per_byte": 2.172017052888652, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5055274963378906, + "step": 1130, + "unclipped_grad_norm": 1.4915344715118408 + }, + { + "bits_per_byte": 2.1695033544636413, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5037851333618164, + "step": 1140, + "unclipped_grad_norm": 1.5511094331741333 + }, + { + "bits_per_byte": 2.1545989939745684, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4934542179107666, + "step": 1150, + "unclipped_grad_norm": 1.240431547164917 + }, + { + "bits_per_byte": 2.097475816403741, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4538594484329224, + "step": 1160, + "unclipped_grad_norm": 1.178969144821167 + }, + { + "bits_per_byte": 2.2105972330261023, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5322692394256592, + "step": 1170, + "unclipped_grad_norm": 1.1780297756195068 + }, + { + "bits_per_byte": 2.102323491383611, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4572196006774902, + "step": 1180, + "unclipped_grad_norm": 0.919143557548523 + }, + { + "bits_per_byte": 2.136596881978155, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4809761047363281, + "step": 1190, + "unclipped_grad_norm": 1.1034574508666992 + }, + { + "bits_per_byte": 2.1069721824364325, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4604418277740479, + "step": 1200, + "unclipped_grad_norm": 1.1095129251480103 + }, + { + "bits_per_byte": 2.254480670185558, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5626869201660156, + "step": 1210, + "unclipped_grad_norm": 1.2801767587661743 + }, + { + "bits_per_byte": 2.1061446019204806, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4598681926727295, + "step": 1220, + "unclipped_grad_norm": 1.0897294282913208 + }, + { + "bits_per_byte": 2.047843343191243, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4194568395614624, + "step": 1230, + "unclipped_grad_norm": 1.1728222370147705 + }, + { + "bits_per_byte": 2.0460125878728084, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4181878566741943, + "step": 1240, + "unclipped_grad_norm": 1.1321473121643066 + }, + { + "bits_per_byte": 2.217450741662985, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5370197296142578, + "step": 1250, + "unclipped_grad_norm": 1.0654293298721313 + }, + { + "bits_per_byte": 2.1496304151911416, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4900102615356445, + "step": 1260, + "unclipped_grad_norm": 1.0989246368408203 + }, + { + "bits_per_byte": 2.0288079594284456, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4062625169754028, + "step": 1270, + "unclipped_grad_norm": 1.0421466827392578 + }, + { + "bits_per_byte": 2.1597347398946334, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.497014045715332, + "step": 1280, + "unclipped_grad_norm": 1.076604962348938 + }, + { + "bits_per_byte": 2.060473061140004, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4282110929489136, + "step": 1290, + "unclipped_grad_norm": 1.1023766994476318 + }, + { + "bits_per_byte": 2.1029859685547345, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4576787948608398, + "step": 1300, + "unclipped_grad_norm": 1.0737801790237427 + }, + { + "bits_per_byte": 2.17370695441603, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.5066988468170166, + "step": 1310, + "unclipped_grad_norm": 1.1388311386108398 + }, + { + "bits_per_byte": 2.1091708086450627, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.461965799331665, + "step": 1320, + "unclipped_grad_norm": 1.289936900138855 + }, + { + "bits_per_byte": 1.9980870424701236, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3849684000015259, + "step": 1330, + "unclipped_grad_norm": 1.0864324569702148 + }, + { + "bits_per_byte": 2.0165677821838885, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3977782726287842, + "step": 1340, + "unclipped_grad_norm": 0.974994957447052 + }, + { + "bits_per_byte": 2.204823603453987, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.52826726436615, + "step": 1350, + "unclipped_grad_norm": 1.1115007400512695 + }, + { + "bits_per_byte": 2.0280969831497786, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4057697057724, + "step": 1360, + "unclipped_grad_norm": 0.916259229183197 + }, + { + "bits_per_byte": 2.124154625118834, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4723517894744873, + "step": 1370, + "unclipped_grad_norm": 1.1937594413757324 + }, + { + "bits_per_byte": 2.0873195529396886, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4468196630477905, + "step": 1380, + "unclipped_grad_norm": 0.9523230791091919 + }, + { + "bits_per_byte": 2.1050263707245707, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4590930938720703, + "step": 1390, + "unclipped_grad_norm": 0.8934341669082642 + }, + { + "bits_per_byte": 2.0970224701360665, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4535452127456665, + "step": 1400, + "unclipped_grad_norm": 0.9132420420646667 + }, + { + "bits_per_byte": 2.040656188211686, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4144750833511353, + "step": 1410, + "unclipped_grad_norm": 1.0989789962768555 + }, + { + "bits_per_byte": 2.069019738974631, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4341351985931396, + "step": 1420, + "unclipped_grad_norm": 1.0159250497817993 + }, + { + "bits_per_byte": 2.0709008852097686, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.435439109802246, + "step": 1430, + "unclipped_grad_norm": 0.9772002696990967 + }, + { + "bits_per_byte": 2.030792811202053, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4076383113861084, + "step": 1440, + "unclipped_grad_norm": 1.0684847831726074 + }, + { + "bits_per_byte": 2.0937577234747375, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.451282262802124, + "step": 1450, + "unclipped_grad_norm": 1.1867483854293823 + }, + { + "bits_per_byte": 2.0324108239813703, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4087598323822021, + "step": 1460, + "unclipped_grad_norm": 1.0155152082443237 + }, + { + "bits_per_byte": 2.0236820165194795, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4027094841003418, + "step": 1470, + "unclipped_grad_norm": 0.9900296330451965 + }, + { + "bits_per_byte": 1.9307554906920323, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.338297724723816, + "step": 1480, + "unclipped_grad_norm": 0.8600749969482422 + }, + { + "bits_per_byte": 2.042708801149734, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4158978462219238, + "step": 1490, + "unclipped_grad_norm": 1.3268300294876099 + }, + { + "bits_per_byte": 1.9310841495378313, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3385255336761475, + "step": 1500, + "unclipped_grad_norm": 0.9175688028335571 + }, + { + "bits_per_byte": 2.024790788669592, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4034780263900757, + "step": 1510, + "unclipped_grad_norm": 1.1049447059631348 + }, + { + "bits_per_byte": 2.021844897842963, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.4014360904693604, + "step": 1520, + "unclipped_grad_norm": 0.9541988968849182 + }, + { + "bits_per_byte": 1.9848854822072437, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3758177757263184, + "step": 1530, + "unclipped_grad_norm": 1.0897772312164307 + }, + { + "bits_per_byte": 2.0563004180647706, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4253188371658325, + "step": 1540, + "unclipped_grad_norm": 0.960680365562439 + }, + { + "bits_per_byte": 2.0173688773716076, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3983335494995117, + "step": 1550, + "unclipped_grad_norm": 1.0041202306747437 + }, + { + "bits_per_byte": 2.0953523466135375, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4523875713348389, + "step": 1560, + "unclipped_grad_norm": 0.9772875905036926 + }, + { + "bits_per_byte": 2.057308064416176, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4260172843933105, + "step": 1570, + "unclipped_grad_norm": 1.0609372854232788 + }, + { + "bits_per_byte": 2.019397928686487, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3997399806976318, + "step": 1580, + "unclipped_grad_norm": 1.008042573928833 + }, + { + "bits_per_byte": 2.0214770269527675, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.4011811017990112, + "step": 1590, + "unclipped_grad_norm": 0.9353208541870117 + }, + { + "bits_per_byte": 2.0397778728137297, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4138662815093994, + "step": 1600, + "unclipped_grad_norm": 0.9491218328475952 + }, + { + "bits_per_byte": 2.06155259623947, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4289593696594238, + "step": 1610, + "unclipped_grad_norm": 0.9174792170524597 + }, + { + "bits_per_byte": 2.0684931280976895, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4337701797485352, + "step": 1620, + "unclipped_grad_norm": 0.9907702803611755 + }, + { + "bits_per_byte": 1.9372454280049913, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3427962064743042, + "step": 1630, + "unclipped_grad_norm": 0.9023784399032593 + }, + { + "bits_per_byte": 2.056802951370592, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4256671667099, + "step": 1640, + "unclipped_grad_norm": 0.9690282344818115 + }, + { + "bits_per_byte": 2.0155706267741857, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3970870971679688, + "step": 1650, + "unclipped_grad_norm": 1.0204124450683594 + }, + { + "bits_per_byte": 1.9586099728084236, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.35760498046875, + "step": 1660, + "unclipped_grad_norm": 0.942615270614624 + }, + { + "bits_per_byte": 2.120022913914506, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4694879055023193, + "step": 1670, + "unclipped_grad_norm": 0.9669644236564636 + }, + { + "bits_per_byte": 2.02882687752004, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4062756299972534, + "step": 1680, + "unclipped_grad_norm": 0.8547722101211548 + }, + { + "bits_per_byte": 1.9310628236891243, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3385107517242432, + "step": 1690, + "unclipped_grad_norm": 0.9439292550086975 + }, + { + "bits_per_byte": 1.9561478691786718, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.355898380279541, + "step": 1700, + "unclipped_grad_norm": 0.8740168213844299 + }, + { + "bits_per_byte": 2.0381777462301023, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.412757158279419, + "step": 1710, + "unclipped_grad_norm": 1.1610065698623657 + }, + { + "bits_per_byte": 2.022566537045983, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.4019362926483154, + "step": 1720, + "unclipped_grad_norm": 0.8489123582839966 + }, + { + "bits_per_byte": 1.9268317065125973, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3355779647827148, + "step": 1730, + "unclipped_grad_norm": 0.9296807646751404 + }, + { + "bits_per_byte": 1.9993203300594609, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3858232498168945, + "step": 1740, + "unclipped_grad_norm": 0.9287921786308289 + }, + { + "bits_per_byte": 2.0917408829280695, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.449884295463562, + "step": 1750, + "unclipped_grad_norm": 0.9826183915138245 + }, + { + "bits_per_byte": 1.9484261921202333, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.35054612159729, + "step": 1760, + "unclipped_grad_norm": 0.8883485198020935 + }, + { + "bits_per_byte": 2.0047687404387946, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3895998001098633, + "step": 1770, + "unclipped_grad_norm": 1.015872836112976 + }, + { + "bits_per_byte": 1.9445588182502813, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3478654623031616, + "step": 1780, + "unclipped_grad_norm": 0.9133850932121277 + }, + { + "bits_per_byte": 1.9244168981118326, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3339041471481323, + "step": 1790, + "unclipped_grad_norm": 1.018885850906372 + }, + { + "bits_per_byte": 2.0032738672374952, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.388563632965088, + "step": 1800, + "unclipped_grad_norm": 1.0286787748336792 + }, + { + "bits_per_byte": 1.9831790703453829, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3746349811553955, + "step": 1810, + "unclipped_grad_norm": 0.8631220459938049 + }, + { + "bits_per_byte": 2.0513977086366246, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4219205379486084, + "step": 1820, + "unclipped_grad_norm": 0.8914260268211365 + }, + { + "bits_per_byte": 1.9040037613329734, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3197548389434814, + "step": 1830, + "unclipped_grad_norm": 1.014155626296997 + }, + { + "bits_per_byte": 1.9870995868544514, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3773524761199951, + "step": 1840, + "unclipped_grad_norm": 0.853826105594635 + }, + { + "bits_per_byte": 1.8864264464841909, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3075711727142334, + "step": 1850, + "unclipped_grad_norm": 0.9869223833084106 + }, + { + "bits_per_byte": 2.0198366564288377, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.4000440835952759, + "step": 1860, + "unclipped_grad_norm": 0.9129742980003357 + }, + { + "bits_per_byte": 2.033820393787842, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4097368717193604, + "step": 1870, + "unclipped_grad_norm": 0.9571877717971802 + }, + { + "bits_per_byte": 1.9046001971661655, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3201682567596436, + "step": 1880, + "unclipped_grad_norm": 0.9010704159736633 + }, + { + "bits_per_byte": 1.9853064957365558, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3761096000671387, + "step": 1890, + "unclipped_grad_norm": 0.9147026538848877 + }, + { + "bits_per_byte": 1.8987844318445888, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3161370754241943, + "step": 1900, + "unclipped_grad_norm": 1.0985209941864014 + }, + { + "bits_per_byte": 1.886329620251755, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3075040578842163, + "step": 1910, + "unclipped_grad_norm": 0.9578983783721924 + }, + { + "bits_per_byte": 1.954425118964979, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3547042608261108, + "step": 1920, + "unclipped_grad_norm": 0.9071367979049683 + }, + { + "bits_per_byte": 1.9264953084475098, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3353447914123535, + "step": 1930, + "unclipped_grad_norm": 0.8819456100463867 + }, + { + "bits_per_byte": 1.874642539212373, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.299403190612793, + "step": 1940, + "unclipped_grad_norm": 0.8353087902069092 + }, + { + "bits_per_byte": 1.9478835868567612, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3501700162887573, + "step": 1950, + "unclipped_grad_norm": 1.0307022333145142 + }, + { + "bits_per_byte": 1.9152383560179354, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3275420665740967, + "step": 1960, + "unclipped_grad_norm": 0.9104531407356262 + }, + { + "bits_per_byte": 1.8790318804276935, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.302445650100708, + "step": 1970, + "unclipped_grad_norm": 0.900970458984375 + }, + { + "bits_per_byte": 1.862672890677623, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2911064624786377, + "step": 1980, + "unclipped_grad_norm": 1.0306406021118164 + }, + { + "bits_per_byte": 1.9492345105792885, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3511064052581787, + "step": 1990, + "unclipped_grad_norm": 0.9717852473258972 + }, + { + "bits_per_byte": 1.8977989712551449, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3154540061950684, + "step": 2000, + "unclipped_grad_norm": 0.805345892906189 + }, + { + "bits_per_byte": 1.9588875528069163, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.357797384262085, + "step": 2010, + "unclipped_grad_norm": 0.8472830653190613 + }, + { + "bits_per_byte": 1.946676612613006, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.34933340549469, + "step": 2020, + "unclipped_grad_norm": 1.1016662120819092 + }, + { + "bits_per_byte": 1.9391907237689006, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.344144582748413, + "step": 2030, + "unclipped_grad_norm": 0.868341863155365 + }, + { + "bits_per_byte": 1.9511490214486933, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.352433443069458, + "step": 2040, + "unclipped_grad_norm": 0.8853706121444702 + }, + { + "bits_per_byte": 1.8935334575484455, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3124973773956299, + "step": 2050, + "unclipped_grad_norm": 0.8925303220748901 + }, + { + "bits_per_byte": 1.8037261810596972, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2502477169036865, + "step": 2060, + "unclipped_grad_norm": 0.8529354333877563 + }, + { + "bits_per_byte": 1.9080168045082062, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3225364685058594, + "step": 2070, + "unclipped_grad_norm": 0.9431800842285156 + }, + { + "bits_per_byte": 2.015688606872678, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3971688747406006, + "step": 2080, + "unclipped_grad_norm": 0.9268007874488831 + }, + { + "bits_per_byte": 1.8786344285215495, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3021701574325562, + "step": 2090, + "unclipped_grad_norm": 0.8720243573188782 + }, + { + "bits_per_byte": 1.9356492570223336, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3416898250579834, + "step": 2100, + "unclipped_grad_norm": 0.8494227528572083 + }, + { + "bits_per_byte": 1.8547725236449384, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2856303453445435, + "step": 2110, + "unclipped_grad_norm": 0.9150698781013489 + }, + { + "bits_per_byte": 1.8966766124756143, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.31467604637146, + "step": 2120, + "unclipped_grad_norm": 0.8588051199913025 + }, + { + "bits_per_byte": 1.924086863404827, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3336753845214844, + "step": 2130, + "unclipped_grad_norm": 0.870884358882904 + }, + { + "bits_per_byte": 1.8065243387892351, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2521872520446777, + "step": 2140, + "unclipped_grad_norm": 0.7779732346534729 + }, + { + "bits_per_byte": 1.8523434406841521, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2839466333389282, + "step": 2150, + "unclipped_grad_norm": 0.9460588693618774 + }, + { + "bits_per_byte": 1.9103203401338626, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3241331577301025, + "step": 2160, + "unclipped_grad_norm": 0.8497743606567383 + }, + { + "bits_per_byte": 1.8798684040414901, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.303025484085083, + "step": 2170, + "unclipped_grad_norm": 0.9395503401756287 + }, + { + "bits_per_byte": 1.8289068489031246, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2677016258239746, + "step": 2180, + "unclipped_grad_norm": 1.0030585527420044 + }, + { + "bits_per_byte": 1.947407538879173, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3498400449752808, + "step": 2190, + "unclipped_grad_norm": 0.8407843112945557 + }, + { + "bits_per_byte": 1.9944045499498493, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3824158906936646, + "step": 2200, + "unclipped_grad_norm": 0.9960005879402161 + }, + { + "bits_per_byte": 2.0001205653339253, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3863779306411743, + "step": 2210, + "unclipped_grad_norm": 0.9367930889129639 + }, + { + "bits_per_byte": 1.8803774726880442, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3033783435821533, + "step": 2220, + "unclipped_grad_norm": 0.9032418131828308 + }, + { + "bits_per_byte": 1.8995089947526738, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3166393041610718, + "step": 2230, + "unclipped_grad_norm": 0.9414434432983398 + }, + { + "bits_per_byte": 1.859138991167694, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2886569499969482, + "step": 2240, + "unclipped_grad_norm": 0.9088358283042908 + }, + { + "bits_per_byte": 1.8658401231759127, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2933018207550049, + "step": 2250, + "unclipped_grad_norm": 0.9021689295768738 + }, + { + "bits_per_byte": 1.8814774737229627, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3041408061981201, + "step": 2260, + "unclipped_grad_norm": 0.8373098969459534 + }, + { + "bits_per_byte": 1.9445300971475874, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.3478455543518066, + "step": 2270, + "unclipped_grad_norm": 0.9310195446014404 + }, + { + "bits_per_byte": 1.8355594818037997, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.272312879562378, + "step": 2280, + "unclipped_grad_norm": 0.8906505107879639 + }, + { + "bits_per_byte": 1.8521159076370606, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2837889194488525, + "step": 2290, + "unclipped_grad_norm": 0.8472009301185608 + }, + { + "bits_per_byte": 1.9052471958987114, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3206167221069336, + "step": 2300, + "unclipped_grad_norm": 0.8389962911605835 + }, + { + "bits_per_byte": 1.8722884406873628, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2977714538574219, + "step": 2310, + "unclipped_grad_norm": 0.9144077897071838 + }, + { + "bits_per_byte": 1.8873241959216946, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.3081934452056885, + "step": 2320, + "unclipped_grad_norm": 0.8859429955482483 + }, + { + "bits_per_byte": 1.8308435455344907, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.269044041633606, + "step": 2330, + "unclipped_grad_norm": 0.8459444046020508 + }, + { + "bits_per_byte": 1.8670001461559822, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.294105887413025, + "step": 2340, + "unclipped_grad_norm": 0.9549022912979126 + }, + { + "bits_per_byte": 1.7832834632648895, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.236077904701233, + "step": 2350, + "unclipped_grad_norm": 0.8211755752563477 + }, + { + "bits_per_byte": 1.8681859665336813, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2949278354644775, + "step": 2360, + "unclipped_grad_norm": 0.8336057662963867 + }, + { + "bits_per_byte": 1.9109924483334344, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.324599027633667, + "step": 2370, + "unclipped_grad_norm": 1.0507880449295044 + }, + { + "bits_per_byte": 1.8140403245972416, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.257396936416626, + "step": 2380, + "unclipped_grad_norm": 0.8251904249191284 + }, + { + "bits_per_byte": 1.8016899064734815, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2488362789154053, + "step": 2390, + "unclipped_grad_norm": 0.7763776183128357 + }, + { + "bits_per_byte": 1.9132727663012243, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3261796236038208, + "step": 2400, + "unclipped_grad_norm": 0.8198488354682922 + }, + { + "bits_per_byte": 1.8847781647583217, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.3064286708831787, + "step": 2410, + "unclipped_grad_norm": 0.8494725823402405 + }, + { + "bits_per_byte": 1.9646795845226739, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.3618121147155762, + "step": 2420, + "unclipped_grad_norm": 0.9295504689216614 + }, + { + "bits_per_byte": 1.8742646933284273, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.29914128780365, + "step": 2430, + "unclipped_grad_norm": 0.8960878849029541 + }, + { + "bits_per_byte": 1.802365110360769, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2493042945861816, + "step": 2440, + "unclipped_grad_norm": 0.8283300399780273 + }, + { + "bits_per_byte": 1.9114973893963676, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.324949026107788, + "step": 2450, + "unclipped_grad_norm": 0.8929033875465393 + }, + { + "bits_per_byte": 1.9097386948086446, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3237299919128418, + "step": 2460, + "unclipped_grad_norm": 0.8874521255493164 + }, + { + "bits_per_byte": 1.848397298760105, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2812113761901855, + "step": 2470, + "unclipped_grad_norm": 0.8395382761955261 + }, + { + "bits_per_byte": 1.8498309461376972, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2822051048278809, + "step": 2480, + "unclipped_grad_norm": 0.8023180961608887 + }, + { + "bits_per_byte": 1.7784197938984894, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2327066659927368, + "step": 2490, + "unclipped_grad_norm": 0.8236642479896545 + }, + { + "bits_per_byte": 1.8541779796209057, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2852182388305664, + "step": 2500, + "unclipped_grad_norm": 0.8692671656608582 + }, + { + "bits_per_byte": 1.8219728523856378, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2628953456878662, + "step": 2510, + "unclipped_grad_norm": 0.809110701084137 + }, + { + "bits_per_byte": 1.8139226884640514, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2573153972625732, + "step": 2520, + "unclipped_grad_norm": 0.8132631778717041 + }, + { + "bits_per_byte": 1.8159424527157844, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2587153911590576, + "step": 2530, + "unclipped_grad_norm": 0.8300133347511292 + }, + { + "bits_per_byte": 1.8460886036549224, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2796111106872559, + "step": 2540, + "unclipped_grad_norm": 0.8400688171386719 + }, + { + "bits_per_byte": 1.8887621428655585, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.3091901540756226, + "step": 2550, + "unclipped_grad_norm": 0.872266411781311 + }, + { + "bits_per_byte": 1.8326880594649941, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.270322561264038, + "step": 2560, + "unclipped_grad_norm": 0.8239099383354187 + }, + { + "bits_per_byte": 1.8127545822993913, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2565057277679443, + "step": 2570, + "unclipped_grad_norm": 0.8143022060394287 + }, + { + "bits_per_byte": 1.852552915552903, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.284091830253601, + "step": 2580, + "unclipped_grad_norm": 0.9730299115180969 + }, + { + "bits_per_byte": 1.8163588227135232, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.25900399684906, + "step": 2590, + "unclipped_grad_norm": 0.8007379770278931 + }, + { + "bits_per_byte": 1.8542342179477378, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2852572202682495, + "step": 2600, + "unclipped_grad_norm": 0.8254386782646179 + }, + { + "bits_per_byte": 1.7603803616668383, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2202026844024658, + "step": 2610, + "unclipped_grad_norm": 0.7949259877204895 + }, + { + "bits_per_byte": 1.8049837182028061, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2511193752288818, + "step": 2620, + "unclipped_grad_norm": 0.865028977394104 + }, + { + "bits_per_byte": 1.8277927452908351, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2669293880462646, + "step": 2630, + "unclipped_grad_norm": 0.826142430305481 + }, + { + "bits_per_byte": 1.8414027643495146, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2763631343841553, + "step": 2640, + "unclipped_grad_norm": 0.841355562210083 + }, + { + "bits_per_byte": 1.7851867952619882, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2373971939086914, + "step": 2650, + "unclipped_grad_norm": 0.788905143737793 + }, + { + "bits_per_byte": 1.7802099613113203, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.233947515487671, + "step": 2660, + "unclipped_grad_norm": 0.7999458909034729 + }, + { + "bits_per_byte": 1.815510432296817, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.258415937423706, + "step": 2670, + "unclipped_grad_norm": 0.8260414004325867 + }, + { + "bits_per_byte": 1.8079201499836377, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2531547546386719, + "step": 2680, + "unclipped_grad_norm": 0.8360624313354492 + }, + { + "bits_per_byte": 1.8778747811526886, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3016436100006104, + "step": 2690, + "unclipped_grad_norm": 0.8545008301734924 + }, + { + "bits_per_byte": 1.8673174541468243, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.294325828552246, + "step": 2700, + "unclipped_grad_norm": 0.8579524755477905 + }, + { + "bits_per_byte": 1.7489412796100003, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2122737169265747, + "step": 2710, + "unclipped_grad_norm": 0.8124077916145325 + }, + { + "bits_per_byte": 1.8645750187961658, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2924249172210693, + "step": 2720, + "unclipped_grad_norm": 0.9173712730407715 + }, + { + "bits_per_byte": 1.7690415799469403, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2262061834335327, + "step": 2730, + "unclipped_grad_norm": 0.7701802849769592 + }, + { + "bits_per_byte": 1.7960051919318623, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2448959350585938, + "step": 2740, + "unclipped_grad_norm": 0.841517448425293 + }, + { + "bits_per_byte": 1.8115809726899035, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2556922435760498, + "step": 2750, + "unclipped_grad_norm": 0.8873427510261536 + }, + { + "bits_per_byte": 1.9016966140716487, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3181556463241577, + "step": 2760, + "unclipped_grad_norm": 0.8982214331626892 + }, + { + "bits_per_byte": 1.8168608400713917, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2593519687652588, + "step": 2770, + "unclipped_grad_norm": 0.8194716572761536 + }, + { + "bits_per_byte": 1.8166995203448824, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2592401504516602, + "step": 2780, + "unclipped_grad_norm": 0.8212807774543762 + }, + { + "bits_per_byte": 1.8228974311166763, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2635362148284912, + "step": 2790, + "unclipped_grad_norm": 0.9329227209091187 + }, + { + "bits_per_byte": 1.757803201643659, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2184163331985474, + "step": 2800, + "unclipped_grad_norm": 0.8613669276237488 + }, + { + "bits_per_byte": 1.8264712306016053, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2660133838653564, + "step": 2810, + "unclipped_grad_norm": 0.8505002856254578 + }, + { + "bits_per_byte": 1.8298420905585164, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2683498859405518, + "step": 2820, + "unclipped_grad_norm": 0.8405188322067261 + }, + { + "bits_per_byte": 1.7699738978972677, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2268524169921875, + "step": 2830, + "unclipped_grad_norm": 0.8156670928001404 + }, + { + "bits_per_byte": 1.8402193517289274, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2755428552627563, + "step": 2840, + "unclipped_grad_norm": 0.9026433825492859 + }, + { + "bits_per_byte": 1.7471456087523418, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.211029052734375, + "step": 2850, + "unclipped_grad_norm": 0.7569094300270081 + }, + { + "bits_per_byte": 1.7464477031551406, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.210545301437378, + "step": 2860, + "unclipped_grad_norm": 0.8251471519470215 + }, + { + "bits_per_byte": 1.8558431156465593, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.286372423171997, + "step": 2870, + "unclipped_grad_norm": 0.8680534958839417 + }, + { + "bits_per_byte": 1.7726970711910293, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2287399768829346, + "step": 2880, + "unclipped_grad_norm": 0.8200762271881104 + }, + { + "bits_per_byte": 1.8185927053655802, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2605524063110352, + "step": 2890, + "unclipped_grad_norm": 0.856894314289093 + }, + { + "bits_per_byte": 1.732919547838272, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2011682987213135, + "step": 2900, + "unclipped_grad_norm": 0.8436375856399536 + }, + { + "bits_per_byte": 1.7766230911449254, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2314612865447998, + "step": 2910, + "unclipped_grad_norm": 0.8930034041404724 + }, + { + "bits_per_byte": 1.8530752268635733, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2844538688659668, + "step": 2920, + "unclipped_grad_norm": 0.8661487698554993 + }, + { + "bits_per_byte": 1.7993667648256269, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2472259998321533, + "step": 2930, + "unclipped_grad_norm": 0.9391220808029175 + }, + { + "bits_per_byte": 1.8410732455904613, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.276134729385376, + "step": 2940, + "unclipped_grad_norm": 0.8410946726799011 + }, + { + "bits_per_byte": 1.7968802396594523, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2455024719238281, + "step": 2950, + "unclipped_grad_norm": 0.8708962798118591 + }, + { + "bits_per_byte": 1.7886298879322615, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.239783763885498, + "step": 2960, + "unclipped_grad_norm": 0.8532378673553467 + }, + { + "bits_per_byte": 1.841481188438308, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2764174938201904, + "step": 2970, + "unclipped_grad_norm": 0.9022923111915588 + }, + { + "bits_per_byte": 1.775313615241256, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2305536270141602, + "step": 2980, + "unclipped_grad_norm": 0.7898789644241333 + }, + { + "bits_per_byte": 1.9084053133165053, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.3228057622909546, + "step": 2990, + "unclipped_grad_norm": 0.835300624370575 + }, + { + "bits_per_byte": 1.8358795415170555, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.272534728050232, + "step": 3000, + "unclipped_grad_norm": 0.8411970138549805 + }, + { + "bits_per_byte": 1.7909437425169699, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2413876056671143, + "step": 3010, + "unclipped_grad_norm": 0.8151383399963379 + }, + { + "bits_per_byte": 1.7661882157864754, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2242283821105957, + "step": 3020, + "unclipped_grad_norm": 0.8463819622993469 + }, + { + "bits_per_byte": 1.805475244618972, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.251460075378418, + "step": 3030, + "unclipped_grad_norm": 0.9359927177429199 + }, + { + "bits_per_byte": 1.7607282825695338, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.220443844795227, + "step": 3040, + "unclipped_grad_norm": 0.7992194890975952 + }, + { + "bits_per_byte": 1.7895066554863601, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.240391492843628, + "step": 3050, + "unclipped_grad_norm": 0.8191037178039551 + }, + { + "bits_per_byte": 1.7907009030139516, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2412192821502686, + "step": 3060, + "unclipped_grad_norm": 0.8547786474227905 + }, + { + "bits_per_byte": 1.763532287709201, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.222387433052063, + "step": 3070, + "unclipped_grad_norm": 0.8228699564933777 + }, + { + "bits_per_byte": 1.772629653991891, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2286932468414307, + "step": 3080, + "unclipped_grad_norm": 0.857837438583374 + }, + { + "bits_per_byte": 1.7447516102523315, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2093696594238281, + "step": 3090, + "unclipped_grad_norm": 0.8343800902366638 + }, + { + "bits_per_byte": 1.8342075261853672, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2713757753372192, + "step": 3100, + "unclipped_grad_norm": 0.847975492477417 + }, + { + "bits_per_byte": 1.8219030274293875, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2628469467163086, + "step": 3110, + "unclipped_grad_norm": 0.872373104095459 + }, + { + "bits_per_byte": 1.7073493392906405, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1834443807601929, + "step": 3120, + "unclipped_grad_norm": 0.8234026432037354 + }, + { + "bits_per_byte": 1.8002469720327425, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2478361129760742, + "step": 3130, + "unclipped_grad_norm": 0.8703857660293579 + }, + { + "bits_per_byte": 1.7476684360109647, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2113914489746094, + "step": 3140, + "unclipped_grad_norm": 0.7984506487846375 + }, + { + "bits_per_byte": 1.731277285505183, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.200029969215393, + "step": 3150, + "unclipped_grad_norm": 0.7827578783035278 + }, + { + "bits_per_byte": 1.7956350852672056, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2446393966674805, + "step": 3160, + "unclipped_grad_norm": 0.8662806153297424 + }, + { + "bits_per_byte": 1.8529331691939606, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2843554019927979, + "step": 3170, + "unclipped_grad_norm": 0.8483284711837769 + }, + { + "bits_per_byte": 1.7207951149177458, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1927642822265625, + "step": 3180, + "unclipped_grad_norm": 0.8697596788406372 + }, + { + "bits_per_byte": 1.8014959100433083, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.248701810836792, + "step": 3190, + "unclipped_grad_norm": 0.9436899423599243 + }, + { + "bits_per_byte": 1.7115792525886113, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1863763332366943, + "step": 3200, + "unclipped_grad_norm": 0.7923902273178101 + }, + { + "bits_per_byte": 1.7455042063325072, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2098913192749023, + "step": 3210, + "unclipped_grad_norm": 0.8492945432662964 + }, + { + "bits_per_byte": 1.7423042971305538, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2076733112335205, + "step": 3220, + "unclipped_grad_norm": 0.8532925844192505 + }, + { + "bits_per_byte": 1.7197866086530862, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1920652389526367, + "step": 3230, + "unclipped_grad_norm": 0.8590078949928284 + }, + { + "bits_per_byte": 1.772480717016244, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2285900115966797, + "step": 3240, + "unclipped_grad_norm": 0.8679581880569458 + }, + { + "bits_per_byte": 1.7710855937524448, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2276229858398438, + "step": 3250, + "unclipped_grad_norm": 0.8697354793548584 + }, + { + "bits_per_byte": 1.8824907955018464, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.3048431873321533, + "step": 3260, + "unclipped_grad_norm": 0.882398247718811 + }, + { + "bits_per_byte": 1.726842196904733, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1969558000564575, + "step": 3270, + "unclipped_grad_norm": 0.8395750522613525 + }, + { + "bits_per_byte": 1.6700450984399413, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1575870513916016, + "step": 3280, + "unclipped_grad_norm": 0.7638692259788513 + }, + { + "bits_per_byte": 1.7148034113443364, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1886111497879028, + "step": 3290, + "unclipped_grad_norm": 0.8540530800819397 + }, + { + "bits_per_byte": 1.79605059535169, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2449274063110352, + "step": 3300, + "unclipped_grad_norm": 0.8206289410591125 + }, + { + "bits_per_byte": 1.7794766272880411, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2334392070770264, + "step": 3310, + "unclipped_grad_norm": 0.8657307028770447 + }, + { + "bits_per_byte": 1.796555880379925, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2452776432037354, + "step": 3320, + "unclipped_grad_norm": 0.8429055213928223 + }, + { + "bits_per_byte": 1.7430792509553417, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2082104682922363, + "step": 3330, + "unclipped_grad_norm": 0.8332963585853577 + }, + { + "bits_per_byte": 1.747674627386396, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2113957405090332, + "step": 3340, + "unclipped_grad_norm": 0.8822024464607239 + }, + { + "bits_per_byte": 1.7362042444871009, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2034450769424438, + "step": 3350, + "unclipped_grad_norm": 0.7802554368972778 + }, + { + "bits_per_byte": 1.7416999500954218, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.207254409790039, + "step": 3360, + "unclipped_grad_norm": 0.8200514316558838 + }, + { + "bits_per_byte": 1.676567712456564, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1621081829071045, + "step": 3370, + "unclipped_grad_norm": 0.8079729676246643 + }, + { + "bits_per_byte": 1.7870502272840867, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2386888265609741, + "step": 3380, + "unclipped_grad_norm": 0.9383370876312256 + }, + { + "bits_per_byte": 1.7544094679941835, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2160639762878418, + "step": 3390, + "unclipped_grad_norm": 0.848736047744751 + }, + { + "bits_per_byte": 1.758961848762522, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.219219446182251, + "step": 3400, + "unclipped_grad_norm": 0.8597305417060852 + }, + { + "bits_per_byte": 1.7916406162182659, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.241870641708374, + "step": 3410, + "unclipped_grad_norm": 0.9396098256111145 + }, + { + "bits_per_byte": 1.7669203459311984, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2247358560562134, + "step": 3420, + "unclipped_grad_norm": 0.8953562378883362 + }, + { + "bits_per_byte": 1.7096045477914046, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1850075721740723, + "step": 3430, + "unclipped_grad_norm": 0.8465718626976013 + }, + { + "bits_per_byte": 1.713755865017931, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.187885046005249, + "step": 3440, + "unclipped_grad_norm": 0.8348708152770996 + }, + { + "bits_per_byte": 1.7182920794170884, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1910293102264404, + "step": 3450, + "unclipped_grad_norm": 0.9096519351005554 + }, + { + "bits_per_byte": 1.707162050183851, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.183314561843872, + "step": 3460, + "unclipped_grad_norm": 0.8655852675437927 + }, + { + "bits_per_byte": 1.7530255236026904, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2151046991348267, + "step": 3470, + "unclipped_grad_norm": 0.8270760774612427 + }, + { + "bits_per_byte": 1.8250635526042933, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2650376558303833, + "step": 3480, + "unclipped_grad_norm": 0.9349867701530457 + }, + { + "bits_per_byte": 1.751566594775421, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2140934467315674, + "step": 3490, + "unclipped_grad_norm": 0.7542157769203186 + }, + { + "bits_per_byte": 1.6978185767277763, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1768381595611572, + "step": 3500, + "unclipped_grad_norm": 0.8014698028564453 + }, + { + "bits_per_byte": 1.7200215369541647, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.192228078842163, + "step": 3510, + "unclipped_grad_norm": 0.8212472796440125 + }, + { + "bits_per_byte": 1.7550949908405227, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2165391445159912, + "step": 3520, + "unclipped_grad_norm": 0.8387999534606934 + }, + { + "bits_per_byte": 1.7221352037332687, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1936931610107422, + "step": 3530, + "unclipped_grad_norm": 0.8480421900749207 + }, + { + "bits_per_byte": 1.728760147427155, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1982852220535278, + "step": 3540, + "unclipped_grad_norm": 0.8651905059814453 + }, + { + "bits_per_byte": 1.782876724295599, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2357959747314453, + "step": 3550, + "unclipped_grad_norm": 0.9422439932823181 + }, + { + "bits_per_byte": 1.717021471592514, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1901485919952393, + "step": 3560, + "unclipped_grad_norm": 0.8451233506202698 + }, + { + "bits_per_byte": 1.7490260670568758, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2123324871063232, + "step": 3570, + "unclipped_grad_norm": 0.869125247001648 + }, + { + "bits_per_byte": 1.759480720420175, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2195791006088257, + "step": 3580, + "unclipped_grad_norm": 0.8907663226127625 + }, + { + "bits_per_byte": 1.7336618249593958, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2016828060150146, + "step": 3590, + "unclipped_grad_norm": 0.8389977216720581 + }, + { + "bits_per_byte": 1.7850502410372031, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.237302541732788, + "step": 3600, + "unclipped_grad_norm": 0.8462889790534973 + }, + { + "bits_per_byte": 1.760744448938715, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.2204550504684448, + "step": 3610, + "unclipped_grad_norm": 0.8433146476745605 + }, + { + "bits_per_byte": 1.7017459725428794, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1795604228973389, + "step": 3620, + "unclipped_grad_norm": 0.7866216897964478 + }, + { + "bits_per_byte": 1.7213356563894076, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1931389570236206, + "step": 3630, + "unclipped_grad_norm": 0.881050169467926 + }, + { + "bits_per_byte": 1.6603886165592963, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1508936882019043, + "step": 3640, + "unclipped_grad_norm": 0.8100939989089966 + }, + { + "bits_per_byte": 1.7286516263744605, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1982100009918213, + "step": 3650, + "unclipped_grad_norm": 0.8479801416397095 + }, + { + "bits_per_byte": 1.7281064413712255, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1978321075439453, + "step": 3660, + "unclipped_grad_norm": 0.8173186182975769 + }, + { + "bits_per_byte": 1.7549983365907376, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2164721488952637, + "step": 3670, + "unclipped_grad_norm": 0.8759186267852783 + }, + { + "bits_per_byte": 1.6966509865110686, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.176028847694397, + "step": 3680, + "unclipped_grad_norm": 0.8159555792808533 + }, + { + "bits_per_byte": 1.7680449404851903, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.225515365600586, + "step": 3690, + "unclipped_grad_norm": 0.8133695721626282 + }, + { + "bits_per_byte": 1.6566674279425777, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.148314356803894, + "step": 3700, + "unclipped_grad_norm": 0.8174366354942322 + }, + { + "bits_per_byte": 1.7250475579429796, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1957118511199951, + "step": 3710, + "unclipped_grad_norm": 0.8113536238670349 + }, + { + "bits_per_byte": 1.7713626578029849, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.22781503200531, + "step": 3720, + "unclipped_grad_norm": 0.8423733115196228 + }, + { + "bits_per_byte": 1.7265372716647531, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.196744441986084, + "step": 3730, + "unclipped_grad_norm": 0.8644699454307556 + }, + { + "bits_per_byte": 1.7046167069510814, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1815502643585205, + "step": 3740, + "unclipped_grad_norm": 0.8081701993942261 + }, + { + "bits_per_byte": 1.72418454900095, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1951136589050293, + "step": 3750, + "unclipped_grad_norm": 0.7984616756439209 + }, + { + "bits_per_byte": 1.7538620472164872, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2156845331192017, + "step": 3760, + "unclipped_grad_norm": 0.8375996351242065 + }, + { + "bits_per_byte": 1.7548705534811466, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2163835763931274, + "step": 3770, + "unclipped_grad_norm": 0.817222535610199 + }, + { + "bits_per_byte": 1.8002908276087126, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2478665113449097, + "step": 3780, + "unclipped_grad_norm": 0.8592872023582458 + }, + { + "bits_per_byte": 1.6997967211780003, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1782093048095703, + "step": 3790, + "unclipped_grad_norm": 0.8284241557121277 + }, + { + "bits_per_byte": 1.7288383995332977, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1983394622802734, + "step": 3800, + "unclipped_grad_norm": 0.8618725538253784 + }, + { + "bits_per_byte": 1.6712219757197955, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1584028005599976, + "step": 3810, + "unclipped_grad_norm": 0.9065259099006653 + }, + { + "bits_per_byte": 1.6792523616365327, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1639690399169922, + "step": 3820, + "unclipped_grad_norm": 0.8587555885314941 + }, + { + "bits_per_byte": 1.7510613097471863, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.2137432098388672, + "step": 3830, + "unclipped_grad_norm": 0.8374478220939636 + }, + { + "bits_per_byte": 1.6819891215597125, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1658660173416138, + "step": 3840, + "unclipped_grad_norm": 0.8430510759353638 + }, + { + "bits_per_byte": 1.6493425148596799, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1432371139526367, + "step": 3850, + "unclipped_grad_norm": 0.7972609400749207 + }, + { + "bits_per_byte": 1.629205754200757, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.129279375076294, + "step": 3860, + "unclipped_grad_norm": 0.801466703414917 + }, + { + "bits_per_byte": 1.6121130864621058, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.117431640625, + "step": 3870, + "unclipped_grad_norm": 0.7957305312156677 + }, + { + "bits_per_byte": 1.7045224604584086, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1814849376678467, + "step": 3880, + "unclipped_grad_norm": 0.8071211576461792 + }, + { + "bits_per_byte": 1.6850354502544453, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1679775714874268, + "step": 3890, + "unclipped_grad_norm": 0.8125332593917847 + }, + { + "bits_per_byte": 1.6717119542921035, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1587424278259277, + "step": 3900, + "unclipped_grad_norm": 0.8193724751472473 + }, + { + "bits_per_byte": 1.7034513525088348, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1807425022125244, + "step": 3910, + "unclipped_grad_norm": 0.8426622748374939 + }, + { + "bits_per_byte": 1.7239871129177595, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.194976806640625, + "step": 3920, + "unclipped_grad_norm": 0.8514105081558228 + }, + { + "bits_per_byte": 1.7258452134776812, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1962647438049316, + "step": 3930, + "unclipped_grad_norm": 0.8463394045829773 + }, + { + "bits_per_byte": 1.6514083704618439, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1446690559387207, + "step": 3940, + "unclipped_grad_norm": 0.7947182059288025 + }, + { + "bits_per_byte": 1.7619474675815003, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2212889194488525, + "step": 3950, + "unclipped_grad_norm": 0.8610087633132935 + }, + { + "bits_per_byte": 1.7314143556779207, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.200124979019165, + "step": 3960, + "unclipped_grad_norm": 0.8606525659561157 + }, + { + "bits_per_byte": 1.6963023776777697, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1757872104644775, + "step": 3970, + "unclipped_grad_norm": 0.8503133654594421 + }, + { + "bits_per_byte": 1.6634555831721327, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1530195474624634, + "step": 3980, + "unclipped_grad_norm": 0.8032745718955994 + }, + { + "bits_per_byte": 1.7136887917840944, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1878385543823242, + "step": 3990, + "unclipped_grad_norm": 0.8245929479598999 + }, + { + "bits_per_byte": 1.6397087346889487, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1365594863891602, + "step": 4000, + "unclipped_grad_norm": 0.8357985615730286 + }, + { + "bits_per_byte": 1.7015423450842577, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1794192790985107, + "step": 4010, + "unclipped_grad_norm": 0.8669785261154175 + }, + { + "bits_per_byte": 1.7224059044257267, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1938807964324951, + "step": 4020, + "unclipped_grad_norm": 0.8723159432411194 + }, + { + "bits_per_byte": 1.6827126525718923, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.166367530822754, + "step": 4030, + "unclipped_grad_norm": 0.8873636722564697 + }, + { + "bits_per_byte": 1.663583022316422, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1531078815460205, + "step": 4040, + "unclipped_grad_norm": 0.867052435874939 + }, + { + "bits_per_byte": 1.6966081628310037, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1759991645812988, + "step": 4050, + "unclipped_grad_norm": 0.8396835923194885 + }, + { + "bits_per_byte": 1.6707194424139744, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1580544710159302, + "step": 4060, + "unclipped_grad_norm": 0.7937865853309631 + }, + { + "bits_per_byte": 1.8029612022286594, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2497174739837646, + "step": 4070, + "unclipped_grad_norm": 0.8503637909889221 + }, + { + "bits_per_byte": 1.6233989319770252, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1252543926239014, + "step": 4080, + "unclipped_grad_norm": 0.8034883141517639 + }, + { + "bits_per_byte": 1.6627509702515477, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1525311470031738, + "step": 4090, + "unclipped_grad_norm": 0.8565743565559387 + }, + { + "bits_per_byte": 1.7695604516045933, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.2265658378601074, + "step": 4100, + "unclipped_grad_norm": 0.9050058722496033 + }, + { + "bits_per_byte": 1.6734001359929729, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1599125862121582, + "step": 4110, + "unclipped_grad_norm": 0.8419733643531799 + }, + { + "bits_per_byte": 1.686295739119968, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1688511371612549, + "step": 4120, + "unclipped_grad_norm": 0.8891176581382751 + }, + { + "bits_per_byte": 1.7589159293947416, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.219187617301941, + "step": 4130, + "unclipped_grad_norm": 0.8800218105316162 + }, + { + "bits_per_byte": 1.728608458729094, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.198180079460144, + "step": 4140, + "unclipped_grad_norm": 0.9665619730949402 + }, + { + "bits_per_byte": 1.6689839654841176, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1568515300750732, + "step": 4150, + "unclipped_grad_norm": 0.8343537449836731 + }, + { + "bits_per_byte": 1.6883550593813994, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.170278549194336, + "step": 4160, + "unclipped_grad_norm": 0.8785508275032043 + }, + { + "bits_per_byte": 1.7045477419080854, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1815024614334106, + "step": 4170, + "unclipped_grad_norm": 0.8620172739028931 + }, + { + "bits_per_byte": 1.6884666761218092, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1703559160232544, + "step": 4180, + "unclipped_grad_norm": 0.917255699634552 + }, + { + "bits_per_byte": 1.707792538581914, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1837515830993652, + "step": 4190, + "unclipped_grad_norm": 0.8766590356826782 + }, + { + "bits_per_byte": 1.696188009214946, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1757079362869263, + "step": 4200, + "unclipped_grad_norm": 0.8971944451332092 + }, + { + "bits_per_byte": 1.720210545887463, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1923590898513794, + "step": 4210, + "unclipped_grad_norm": 0.9026775360107422 + }, + { + "bits_per_byte": 1.6943541582087958, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1744368076324463, + "step": 4220, + "unclipped_grad_norm": 0.8253030180931091 + }, + { + "bits_per_byte": 1.693875014543492, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1741046905517578, + "step": 4230, + "unclipped_grad_norm": 0.9072017669677734 + }, + { + "bits_per_byte": 1.7273019065304893, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1972744464874268, + "step": 4240, + "unclipped_grad_norm": 0.8421373963356018 + }, + { + "bits_per_byte": 1.655899869371777, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.147782325744629, + "step": 4250, + "unclipped_grad_norm": 0.8799253106117249 + }, + { + "bits_per_byte": 1.6871874691646918, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1694692373275757, + "step": 4260, + "unclipped_grad_norm": 0.8348552584648132 + }, + { + "bits_per_byte": 1.6070542167869764, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1139250993728638, + "step": 4270, + "unclipped_grad_norm": 0.9256986975669861 + }, + { + "bits_per_byte": 1.6846880452997026, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1677367687225342, + "step": 4280, + "unclipped_grad_norm": 0.8782404661178589 + }, + { + "bits_per_byte": 1.655903652990096, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.147784948348999, + "step": 4290, + "unclipped_grad_norm": 0.8491708636283875 + }, + { + "bits_per_byte": 1.6260650070307003, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1271023750305176, + "step": 4300, + "unclipped_grad_norm": 0.8707545399665833 + }, + { + "bits_per_byte": 1.6222229146104252, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1244392395019531, + "step": 4310, + "unclipped_grad_norm": 0.8290966153144836 + }, + { + "bits_per_byte": 1.7449810351085826, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.2095286846160889, + "step": 4320, + "unclipped_grad_norm": 0.8566304445266724 + }, + { + "bits_per_byte": 1.683775849319526, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1671044826507568, + "step": 4330, + "unclipped_grad_norm": 0.8478822708129883 + }, + { + "bits_per_byte": 1.6416012317790432, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.137871265411377, + "step": 4340, + "unclipped_grad_norm": 0.8954984545707703 + }, + { + "bits_per_byte": 1.6099557360896828, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.115936279296875, + "step": 4350, + "unclipped_grad_norm": 0.8437728881835938 + }, + { + "bits_per_byte": 1.6930343633460747, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1735219955444336, + "step": 4360, + "unclipped_grad_norm": 0.9155061841011047 + }, + { + "bits_per_byte": 1.666669594951457, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1552473306655884, + "step": 4370, + "unclipped_grad_norm": 0.8761768341064453 + }, + { + "bits_per_byte": 1.5925817047342201, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.103893518447876, + "step": 4380, + "unclipped_grad_norm": 0.8261715769767761 + }, + { + "bits_per_byte": 1.7381402531878634, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.204787015914917, + "step": 4390, + "unclipped_grad_norm": 0.8470990061759949 + }, + { + "bits_per_byte": 1.684359214471253, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.167508840560913, + "step": 4400, + "unclipped_grad_norm": 0.8836275339126587 + }, + { + "bits_per_byte": 1.6966413554826203, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1760221719741821, + "step": 4410, + "unclipped_grad_norm": 0.868717610836029 + }, + { + "bits_per_byte": 1.7161705014360449, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.189558744430542, + "step": 4420, + "unclipped_grad_norm": 0.9417465329170227 + }, + { + "bits_per_byte": 1.6679764911153634, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1561532020568848, + "step": 4430, + "unclipped_grad_norm": 0.9034039974212646 + }, + { + "bits_per_byte": 1.6769285320580742, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.162358283996582, + "step": 4440, + "unclipped_grad_norm": 0.8574883341789246 + }, + { + "bits_per_byte": 1.6926945256279697, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1732864379882812, + "step": 4450, + "unclipped_grad_norm": 0.8669698238372803 + }, + { + "bits_per_byte": 1.689570116809745, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1711207628250122, + "step": 4460, + "unclipped_grad_norm": 0.9505860805511475 + }, + { + "bits_per_byte": 1.6610163532349456, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1513288021087646, + "step": 4470, + "unclipped_grad_norm": 0.8719004988670349 + }, + { + "bits_per_byte": 1.7079321884944145, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1838483810424805, + "step": 4480, + "unclipped_grad_norm": 0.862112283706665 + }, + { + "bits_per_byte": 1.747874987174651, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.2115346193313599, + "step": 4490, + "unclipped_grad_norm": 0.8834897875785828 + }, + { + "bits_per_byte": 1.6443725602150463, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1397922039031982, + "step": 4500, + "unclipped_grad_norm": 0.8092275261878967 + }, + { + "bits_per_byte": 1.6392098130187958, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1362136602401733, + "step": 4510, + "unclipped_grad_norm": 0.8292563557624817 + }, + { + "bits_per_byte": 1.7170035853968242, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.190136194229126, + "step": 4520, + "unclipped_grad_norm": 0.929835319519043 + }, + { + "bits_per_byte": 1.6814485800880505, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1654913425445557, + "step": 4530, + "unclipped_grad_norm": 0.9049495458602905 + }, + { + "bits_per_byte": 1.6774269377802746, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1627037525177002, + "step": 4540, + "unclipped_grad_norm": 0.9091315865516663 + }, + { + "bits_per_byte": 1.6939209339112724, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1741365194320679, + "step": 4550, + "unclipped_grad_norm": 0.8590887188911438 + }, + { + "bits_per_byte": 1.692378077550382, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1730670928955078, + "step": 4560, + "unclipped_grad_norm": 0.8829112648963928 + }, + { + "bits_per_byte": 1.680521765582551, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1648489236831665, + "step": 4570, + "unclipped_grad_norm": 0.8655811548233032 + }, + { + "bits_per_byte": 1.6629215770412038, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1526494026184082, + "step": 4580, + "unclipped_grad_norm": 0.8624271154403687 + }, + { + "bits_per_byte": 1.6390448816566183, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1360993385314941, + "step": 4590, + "unclipped_grad_norm": 0.8392674326896667 + }, + { + "bits_per_byte": 1.6424092062727966, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1384313106536865, + "step": 4600, + "unclipped_grad_norm": 0.863978385925293 + }, + { + "bits_per_byte": 1.6535189415532323, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.146131992340088, + "step": 4610, + "unclipped_grad_norm": 0.847902238368988 + }, + { + "bits_per_byte": 1.7059622992114318, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1824829578399658, + "step": 4620, + "unclipped_grad_norm": 0.884921133518219 + }, + { + "bits_per_byte": 1.687283435483873, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.169535756111145, + "step": 4630, + "unclipped_grad_norm": 0.8606892824172974 + }, + { + "bits_per_byte": 1.71471243252203, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1885480880737305, + "step": 4640, + "unclipped_grad_norm": 0.916991114616394 + }, + { + "bits_per_byte": 1.7302961244820108, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1993498802185059, + "step": 4650, + "unclipped_grad_norm": 0.8595409989356995 + }, + { + "bits_per_byte": 1.6534050890383611, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1460530757904053, + "step": 4660, + "unclipped_grad_norm": 0.8465806245803833 + }, + { + "bits_per_byte": 1.6640274254862515, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1534159183502197, + "step": 4670, + "unclipped_grad_norm": 0.8708443641662598 + }, + { + "bits_per_byte": 1.6364145789943225, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1342761516571045, + "step": 4680, + "unclipped_grad_norm": 0.8731908202171326 + }, + { + "bits_per_byte": 1.653688516446983, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.146249532699585, + "step": 4690, + "unclipped_grad_norm": 0.8683258891105652 + }, + { + "bits_per_byte": 1.6763891944649685, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1619844436645508, + "step": 4700, + "unclipped_grad_norm": 0.907905101776123 + }, + { + "bits_per_byte": 1.661433239180637, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1516177654266357, + "step": 4710, + "unclipped_grad_norm": 0.862278401851654 + }, + { + "bits_per_byte": 1.68449817645315, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1676051616668701, + "step": 4720, + "unclipped_grad_norm": 0.8695031404495239 + }, + { + "bits_per_byte": 1.7284700126951493, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1980841159820557, + "step": 4730, + "unclipped_grad_norm": 0.8831793069839478 + }, + { + "bits_per_byte": 1.66034751270574, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1508651971817017, + "step": 4740, + "unclipped_grad_norm": 0.9113041758537292 + }, + { + "bits_per_byte": 1.6300096011108898, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1298365592956543, + "step": 4750, + "unclipped_grad_norm": 0.9407509565353394 + }, + { + "bits_per_byte": 1.6017535395047338, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1102509498596191, + "step": 4760, + "unclipped_grad_norm": 0.8554308414459229 + }, + { + "bits_per_byte": 1.6115298932930302, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1170274019241333, + "step": 4770, + "unclipped_grad_norm": 0.846508264541626 + }, + { + "bits_per_byte": 1.6498288817963198, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1435742378234863, + "step": 4780, + "unclipped_grad_norm": 0.8984644412994385 + }, + { + "bits_per_byte": 1.6369418778018678, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1346416473388672, + "step": 4790, + "unclipped_grad_norm": 0.9358733296394348 + }, + { + "bits_per_byte": 1.6726428963812239, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1593877077102661, + "step": 4800, + "unclipped_grad_norm": 0.9628081321716309 + }, + { + "bits_per_byte": 1.6377882044267638, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1352282762527466, + "step": 4810, + "unclipped_grad_norm": 0.874860942363739 + }, + { + "bits_per_byte": 1.6475561310651679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1419988870620728, + "step": 4820, + "unclipped_grad_norm": 0.8858349919319153 + }, + { + "bits_per_byte": 1.624869727607204, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1262738704681396, + "step": 4830, + "unclipped_grad_norm": 0.8662204742431641 + }, + { + "bits_per_byte": 1.5894763859902412, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.101741075515747, + "step": 4840, + "unclipped_grad_norm": 0.836323082447052 + }, + { + "bits_per_byte": 1.6522122173719767, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.145226240158081, + "step": 4850, + "unclipped_grad_norm": 0.9080801010131836 + }, + { + "bits_per_byte": 1.6432429781641795, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1390092372894287, + "step": 4860, + "unclipped_grad_norm": 0.8762933611869812 + }, + { + "bits_per_byte": 1.6506518187806984, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1441446542739868, + "step": 4870, + "unclipped_grad_norm": 0.8573883771896362 + }, + { + "bits_per_byte": 1.6124140561011158, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1176402568817139, + "step": 4880, + "unclipped_grad_norm": 0.8266409635543823 + }, + { + "bits_per_byte": 1.606751183356156, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1137150526046753, + "step": 4890, + "unclipped_grad_norm": 0.9300702810287476 + }, + { + "bits_per_byte": 1.5799279092143381, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0951225757598877, + "step": 4900, + "unclipped_grad_norm": 0.8544939160346985 + }, + { + "bits_per_byte": 1.6316014725272758, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1309399604797363, + "step": 4910, + "unclipped_grad_norm": 0.8843865990638733 + }, + { + "bits_per_byte": 1.6408737451658935, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1373670101165771, + "step": 4920, + "unclipped_grad_norm": 0.8914017677307129 + }, + { + "bits_per_byte": 1.6335975031731897, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1323235034942627, + "step": 4930, + "unclipped_grad_norm": 0.9028998017311096 + }, + { + "bits_per_byte": 1.697724846183056, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.176773190498352, + "step": 4940, + "unclipped_grad_norm": 0.9006686806678772 + }, + { + "bits_per_byte": 1.5654046622795774, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0850558280944824, + "step": 4950, + "unclipped_grad_norm": 0.8810124397277832 + }, + { + "bits_per_byte": 1.6525025240866331, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1454274654388428, + "step": 4960, + "unclipped_grad_norm": 0.9115336537361145 + }, + { + "bits_per_byte": 1.721798977650832, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1934601068496704, + "step": 4970, + "unclipped_grad_norm": 0.9325219988822937 + }, + { + "bits_per_byte": 1.6130810048211617, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1181025505065918, + "step": 4980, + "unclipped_grad_norm": 0.9085396528244019 + }, + { + "bits_per_byte": 1.6527175024002116, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1455764770507812, + "step": 4990, + "unclipped_grad_norm": 0.903756856918335 + }, + { + "bits_per_byte": 1.663184366531722, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1528315544128418, + "step": 5000, + "unclipped_grad_norm": 0.9185003638267517 + }, + { + "bits_per_byte": 1.6328528182949538, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1318073272705078, + "step": 5010, + "unclipped_grad_norm": 0.9199827313423157 + }, + { + "bits_per_byte": 1.6437107709745262, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1393334865570068, + "step": 5020, + "unclipped_grad_norm": 0.8566684722900391 + }, + { + "bits_per_byte": 1.6317774107791085, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1310619115829468, + "step": 5030, + "unclipped_grad_norm": 0.9012371897697449 + }, + { + "bits_per_byte": 1.6211884389654854, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1237221956253052, + "step": 5040, + "unclipped_grad_norm": 0.9131078124046326 + }, + { + "bits_per_byte": 1.6615727171104868, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1517144441604614, + "step": 5050, + "unclipped_grad_norm": 0.9046566486358643 + }, + { + "bits_per_byte": 1.602131729353981, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1105130910873413, + "step": 5060, + "unclipped_grad_norm": 0.895842969417572 + }, + { + "bits_per_byte": 1.6828842912574533, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1664865016937256, + "step": 5070, + "unclipped_grad_norm": 0.9286125898361206 + }, + { + "bits_per_byte": 1.6860735375350533, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1686971187591553, + "step": 5080, + "unclipped_grad_norm": 0.9972010254859924 + }, + { + "bits_per_byte": 1.6658614484750525, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1546871662139893, + "step": 5090, + "unclipped_grad_norm": 0.9360038042068481 + }, + { + "bits_per_byte": 1.5959157604038465, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1062045097351074, + "step": 5100, + "unclipped_grad_norm": 0.8976820707321167 + }, + { + "bits_per_byte": 1.6220873922815453, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.124345302581787, + "step": 5110, + "unclipped_grad_norm": 0.8928531408309937 + }, + { + "bits_per_byte": 1.5977783325126909, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1074955463409424, + "step": 5120, + "unclipped_grad_norm": 0.8989635705947876 + }, + { + "bits_per_byte": 1.6638141669991817, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1532680988311768, + "step": 5130, + "unclipped_grad_norm": 0.9084221720695496 + }, + { + "bits_per_byte": 1.5980395741593514, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.10767662525177, + "step": 5140, + "unclipped_grad_norm": 0.896472156047821 + }, + { + "bits_per_byte": 1.590172399778283, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.102223515510559, + "step": 5150, + "unclipped_grad_norm": 0.8930872678756714 + }, + { + "bits_per_byte": 1.6326397317905348, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1316596269607544, + "step": 5160, + "unclipped_grad_norm": 0.9028102159500122 + }, + { + "bits_per_byte": 1.6614396025387188, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1516221761703491, + "step": 5170, + "unclipped_grad_norm": 0.9357115030288696 + }, + { + "bits_per_byte": 1.6782567540706879, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1632789373397827, + "step": 5180, + "unclipped_grad_norm": 0.8988713622093201 + }, + { + "bits_per_byte": 1.6215858908716294, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.123997688293457, + "step": 5190, + "unclipped_grad_norm": 0.8884453177452087 + }, + { + "bits_per_byte": 1.5925564232845433, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.103875994682312, + "step": 5200, + "unclipped_grad_norm": 0.9432880282402039 + }, + { + "bits_per_byte": 1.6471872282790672, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1417431831359863, + "step": 5210, + "unclipped_grad_norm": 0.932002067565918 + }, + { + "bits_per_byte": 1.634469455213064, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1329278945922852, + "step": 5220, + "unclipped_grad_norm": 0.8859370946884155 + }, + { + "bits_per_byte": 1.684877914146255, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1678683757781982, + "step": 5230, + "unclipped_grad_norm": 0.9864789843559265 + }, + { + "bits_per_byte": 1.612974375577627, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1180286407470703, + "step": 5240, + "unclipped_grad_norm": 0.9477531313896179 + }, + { + "bits_per_byte": 1.5751990742462143, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0918447971343994, + "step": 5250, + "unclipped_grad_norm": 0.8995106220245361 + }, + { + "bits_per_byte": 1.5678148271487689, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.086726427078247, + "step": 5260, + "unclipped_grad_norm": 0.9177948236465454 + }, + { + "bits_per_byte": 1.6615842399480945, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1517224311828613, + "step": 5270, + "unclipped_grad_norm": 0.9501162171363831 + }, + { + "bits_per_byte": 1.6530886409607735, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1458337306976318, + "step": 5280, + "unclipped_grad_norm": 0.9527469277381897 + }, + { + "bits_per_byte": 1.6249691335794025, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1263427734375, + "step": 5290, + "unclipped_grad_norm": 0.9307261109352112 + }, + { + "bits_per_byte": 1.5681051338634253, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0869276523590088, + "step": 5300, + "unclipped_grad_norm": 0.9039522409439087 + }, + { + "bits_per_byte": 1.6362377808292354, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1341536045074463, + "step": 5310, + "unclipped_grad_norm": 0.9477429986000061 + }, + { + "bits_per_byte": 1.60507762018061, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1125550270080566, + "step": 5320, + "unclipped_grad_norm": 0.926771342754364 + }, + { + "bits_per_byte": 1.6965259551238914, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1759421825408936, + "step": 5330, + "unclipped_grad_norm": 0.9526844620704651 + }, + { + "bits_per_byte": 1.5996515675458884, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1087939739227295, + "step": 5340, + "unclipped_grad_norm": 0.9416574835777283 + }, + { + "bits_per_byte": 1.628412054267025, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.128729224205017, + "step": 5350, + "unclipped_grad_norm": 0.9132721424102783 + }, + { + "bits_per_byte": 1.5519294776191745, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0757155418395996, + "step": 5360, + "unclipped_grad_norm": 0.8944959044456482 + }, + { + "bits_per_byte": 1.5965125402023406, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1066181659698486, + "step": 5370, + "unclipped_grad_norm": 0.9166159629821777 + }, + { + "bits_per_byte": 1.6489008634122642, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1429309844970703, + "step": 5380, + "unclipped_grad_norm": 0.9673669934272766 + }, + { + "bits_per_byte": 1.5567627060563722, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0790656805038452, + "step": 5390, + "unclipped_grad_norm": 0.8981189131736755 + }, + { + "bits_per_byte": 1.6274061277421286, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1280319690704346, + "step": 5400, + "unclipped_grad_norm": 0.95365971326828 + }, + { + "bits_per_byte": 1.5903326876088872, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1023346185684204, + "step": 5410, + "unclipped_grad_norm": 0.9384826421737671 + }, + { + "bits_per_byte": 1.5563373929607884, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0787708759307861, + "step": 5420, + "unclipped_grad_norm": 0.8839442133903503 + }, + { + "bits_per_byte": 1.6100503265476573, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.116001844406128, + "step": 5430, + "unclipped_grad_norm": 0.9247893691062927 + }, + { + "bits_per_byte": 1.5906383007794704, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1025464534759521, + "step": 5440, + "unclipped_grad_norm": 0.9033803343772888 + }, + { + "bits_per_byte": 1.6134788006926075, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1183782815933228, + "step": 5450, + "unclipped_grad_norm": 0.9274419546127319 + }, + { + "bits_per_byte": 1.588039986890235, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.100745439529419, + "step": 5460, + "unclipped_grad_norm": 0.93131422996521 + }, + { + "bits_per_byte": 1.6180344491313123, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1215360164642334, + "step": 5470, + "unclipped_grad_norm": 0.9031676054000854 + }, + { + "bits_per_byte": 1.5764667583657241, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0927234888076782, + "step": 5480, + "unclipped_grad_norm": 0.9524510502815247 + }, + { + "bits_per_byte": 1.667538279320965, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1558494567871094, + "step": 5490, + "unclipped_grad_norm": 1.0016353130340576 + }, + { + "bits_per_byte": 1.6287254066568972, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1289464235305786, + "step": 5500, + "unclipped_grad_norm": 0.9340007305145264 + }, + { + "bits_per_byte": 1.5745544832707805, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.091398000717163, + "step": 5510, + "unclipped_grad_norm": 0.919238805770874 + }, + { + "bits_per_byte": 1.54915986900968, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0737957954406738, + "step": 5520, + "unclipped_grad_norm": 0.9040963649749756 + }, + { + "bits_per_byte": 1.5578374256416139, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.079810619354248, + "step": 5530, + "unclipped_grad_norm": 0.8987631797790527 + }, + { + "bits_per_byte": 1.6786796594091595, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.163572072982788, + "step": 5540, + "unclipped_grad_norm": 0.9474910497665405 + }, + { + "bits_per_byte": 1.6142570221877617, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.11891770362854, + "step": 5550, + "unclipped_grad_norm": 0.9275277256965637 + }, + { + "bits_per_byte": 1.6535560898058186, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1461577415466309, + "step": 5560, + "unclipped_grad_norm": 0.9428616166114807 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.109092354774475, + "step": 5570, + "unclipped_grad_norm": 0.8811440467834473 + }, + { + "bits_per_byte": 1.625784675309794, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1269080638885498, + "step": 5580, + "unclipped_grad_norm": 0.9328234791755676 + }, + { + "bits_per_byte": 1.6288614449337298, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1290407180786133, + "step": 5590, + "unclipped_grad_norm": 0.9723306894302368 + }, + { + "bits_per_byte": 1.6197528997787336, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1227271556854248, + "step": 5600, + "unclipped_grad_norm": 1.0117963552474976 + }, + { + "bits_per_byte": 1.5616874292638288, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0824792385101318, + "step": 5610, + "unclipped_grad_norm": 0.901840329170227 + }, + { + "bits_per_byte": 1.6073563903045422, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1141345500946045, + "step": 5620, + "unclipped_grad_norm": 0.9225190281867981 + }, + { + "bits_per_byte": 1.5324476268947373, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0622117519378662, + "step": 5630, + "unclipped_grad_norm": 0.8962357640266418 + }, + { + "bits_per_byte": 1.5325635432014189, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0622920989990234, + "step": 5640, + "unclipped_grad_norm": 0.8800385594367981 + }, + { + "bits_per_byte": 1.6207170345194706, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1233954429626465, + "step": 5650, + "unclipped_grad_norm": 0.9619011282920837 + }, + { + "bits_per_byte": 1.5448582389462995, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0708141326904297, + "step": 5660, + "unclipped_grad_norm": 0.9848581552505493 + }, + { + "bits_per_byte": 1.6303492668463437, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.130071997642517, + "step": 5670, + "unclipped_grad_norm": 0.942057728767395 + }, + { + "bits_per_byte": 1.6277949805157295, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1283015012741089, + "step": 5680, + "unclipped_grad_norm": 1.0017359256744385 + }, + { + "bits_per_byte": 1.6320572265520623, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.131255865097046, + "step": 5690, + "unclipped_grad_norm": 0.9255995154380798 + }, + { + "bits_per_byte": 1.6423192593463953, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1383689641952515, + "step": 5700, + "unclipped_grad_norm": 0.9918621182441711 + }, + { + "bits_per_byte": 1.6211487109731362, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.123694658279419, + "step": 5710, + "unclipped_grad_norm": 1.0109361410140991 + }, + { + "bits_per_byte": 1.6355581053930257, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1336824893951416, + "step": 5720, + "unclipped_grad_norm": 0.9666855335235596 + }, + { + "bits_per_byte": 1.5611546270114558, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0821099281311035, + "step": 5730, + "unclipped_grad_norm": 0.9373083114624023 + }, + { + "bits_per_byte": 1.6439162902423072, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.13947594165802, + "step": 5740, + "unclipped_grad_norm": 0.923280656337738 + }, + { + "bits_per_byte": 1.5432997321641806, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0697338581085205, + "step": 5750, + "unclipped_grad_norm": 0.9531146287918091 + }, + { + "bits_per_byte": 1.5628747974853856, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0833022594451904, + "step": 5760, + "unclipped_grad_norm": 0.9156620502471924 + }, + { + "bits_per_byte": 1.5744538734200257, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0913282632827759, + "step": 5770, + "unclipped_grad_norm": 0.9679534435272217 + }, + { + "bits_per_byte": 1.6685440338632107, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1565465927124023, + "step": 5780, + "unclipped_grad_norm": 0.9836651682853699 + }, + { + "bits_per_byte": 1.5013796489469744, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0406770706176758, + "step": 5790, + "unclipped_grad_norm": 0.9446811676025391 + }, + { + "bits_per_byte": 1.6245429605705646, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.1260473728179932, + "step": 5800, + "unclipped_grad_norm": 0.9269219040870667 + }, + { + "bits_per_byte": 1.5340637478648953, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.063331961631775, + "step": 5810, + "unclipped_grad_norm": 0.87967449426651 + }, + { + "bits_per_byte": 1.5414010436986552, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0684177875518799, + "step": 5820, + "unclipped_grad_norm": 0.9367535710334778 + }, + { + "bits_per_byte": 1.6061841565562613, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.1133220195770264, + "step": 5830, + "unclipped_grad_norm": 0.9435480833053589 + }, + { + "bits_per_byte": 1.5806048329281341, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0955917835235596, + "step": 5840, + "unclipped_grad_norm": 0.9508634805679321 + }, + { + "bits_per_byte": 1.5486303344276733, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.073428750038147, + "step": 5850, + "unclipped_grad_norm": 0.9543959498405457 + }, + { + "bits_per_byte": 1.563623781929893, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.083821415901184, + "step": 5860, + "unclipped_grad_norm": 0.900787353515625 + }, + { + "bits_per_byte": 1.6201161271373559, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.122978925704956, + "step": 5870, + "unclipped_grad_norm": 0.9706017374992371 + }, + { + "bits_per_byte": 1.5594879431419442, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.080954670906067, + "step": 5880, + "unclipped_grad_norm": 0.9371631741523743 + }, + { + "bits_per_byte": 1.6214689426690427, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1239166259765625, + "step": 5890, + "unclipped_grad_norm": 1.0151838064193726 + }, + { + "bits_per_byte": 1.6434954486956461, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1391842365264893, + "step": 5900, + "unclipped_grad_norm": 1.0114811658859253 + }, + { + "bits_per_byte": 1.5747577667641004, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.091538906097412, + "step": 5910, + "unclipped_grad_norm": 0.9295753836631775 + }, + { + "bits_per_byte": 1.5403519495283922, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0676906108856201, + "step": 5920, + "unclipped_grad_norm": 0.9814701080322266 + }, + { + "bits_per_byte": 1.6536778535226295, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1462421417236328, + "step": 5930, + "unclipped_grad_norm": 0.9983100891113281 + }, + { + "bits_per_byte": 1.5856325737434576, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.099076747894287, + "step": 5940, + "unclipped_grad_norm": 0.964913010597229 + }, + { + "bits_per_byte": 1.5932624120663352, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.104365348815918, + "step": 5950, + "unclipped_grad_norm": 0.9470323920249939 + }, + { + "bits_per_byte": 1.5376077943512252, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0657885074615479, + "step": 5960, + "unclipped_grad_norm": 0.97988361120224 + }, + { + "bits_per_byte": 1.5721553252912444, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0897350311279297, + "step": 5970, + "unclipped_grad_norm": 0.9889833927154541 + }, + { + "bits_per_byte": 1.552403117839651, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0760438442230225, + "step": 5980, + "unclipped_grad_norm": 0.9984597563743591 + }, + { + "bits_per_byte": 1.542794103170644, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0693833827972412, + "step": 5990, + "unclipped_grad_norm": 0.9274381399154663 + }, + { + "bits_per_byte": 1.648261775881658, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1424880027770996, + "step": 6000, + "unclipped_grad_norm": 0.9919959902763367 + }, + { + "bits_per_byte": 1.595717292424751, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1060669422149658, + "step": 6010, + "unclipped_grad_norm": 1.009833574295044 + }, + { + "bits_per_byte": 1.5605256864572503, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0816739797592163, + "step": 6020, + "unclipped_grad_norm": 0.9331597685813904 + }, + { + "bits_per_byte": 1.5469388850564376, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.072256326675415, + "step": 6030, + "unclipped_grad_norm": 0.9432128667831421 + }, + { + "bits_per_byte": 1.5694490062972672, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0878591537475586, + "step": 6040, + "unclipped_grad_norm": 0.9656267166137695 + }, + { + "bits_per_byte": 1.5872409554943265, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.100191593170166, + "step": 6050, + "unclipped_grad_norm": 0.9372153878211975 + }, + { + "bits_per_byte": 1.5733865490887713, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0905884504318237, + "step": 6060, + "unclipped_grad_norm": 0.9818388819694519 + }, + { + "bits_per_byte": 1.4709371719003996, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0195759534835815, + "step": 6070, + "unclipped_grad_norm": 0.9397172331809998 + }, + { + "bits_per_byte": 1.5520641400348003, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0758088827133179, + "step": 6080, + "unclipped_grad_norm": 1.003225564956665 + }, + { + "bits_per_byte": 1.548305115234892, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0732033252716064, + "step": 6090, + "unclipped_grad_norm": 0.9615646600723267 + }, + { + "bits_per_byte": 1.6580690865471095, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.149285912513733, + "step": 6100, + "unclipped_grad_norm": 1.030853271484375 + }, + { + "bits_per_byte": 1.5849845431150065, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0986275672912598, + "step": 6110, + "unclipped_grad_norm": 1.0033804178237915 + }, + { + "bits_per_byte": 1.5391577020008007, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0668628215789795, + "step": 6120, + "unclipped_grad_norm": 0.9861558079719543 + }, + { + "bits_per_byte": 1.6089128332948508, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.115213394165039, + "step": 6130, + "unclipped_grad_norm": 0.9338588714599609 + }, + { + "bits_per_byte": 1.5704974125369269, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0885858535766602, + "step": 6140, + "unclipped_grad_norm": 0.9351722598075867 + }, + { + "bits_per_byte": 1.5301674809095982, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.060631275177002, + "step": 6150, + "unclipped_grad_norm": 0.9382454752922058 + }, + { + "bits_per_byte": 1.594941134721407, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1055289506912231, + "step": 6160, + "unclipped_grad_norm": 0.9520907998085022 + }, + { + "bits_per_byte": 1.53183020917814, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.061783790588379, + "step": 6170, + "unclipped_grad_norm": 0.9805156588554382 + }, + { + "bits_per_byte": 1.5374516341042417, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0656802654266357, + "step": 6180, + "unclipped_grad_norm": 0.9368215799331665 + }, + { + "bits_per_byte": 1.5636041759076946, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0838078260421753, + "step": 6190, + "unclipped_grad_norm": 1.028733491897583 + }, + { + "bits_per_byte": 1.544540243024854, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0705937147140503, + "step": 6200, + "unclipped_grad_norm": 0.9240782856941223 + }, + { + "bits_per_byte": 1.5709191139968426, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0888781547546387, + "step": 6210, + "unclipped_grad_norm": 0.9899703860282898 + }, + { + "bits_per_byte": 1.604423398176728, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.1121015548706055, + "step": 6220, + "unclipped_grad_norm": 0.9920126795768738 + }, + { + "bits_per_byte": 1.531685743751415, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0616836547851562, + "step": 6230, + "unclipped_grad_norm": 0.9534332156181335 + }, + { + "bits_per_byte": 1.5783843649228444, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0940526723861694, + "step": 6240, + "unclipped_grad_norm": 0.9837333559989929 + }, + { + "bits_per_byte": 1.5384501653751512, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0663723945617676, + "step": 6250, + "unclipped_grad_norm": 0.9426394104957581 + }, + { + "bits_per_byte": 1.5118525324712653, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0479363203048706, + "step": 6260, + "unclipped_grad_norm": 0.9544605612754822 + }, + { + "bits_per_byte": 1.5631276119821538, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.08347749710083, + "step": 6270, + "unclipped_grad_norm": 0.9525503516197205 + }, + { + "bits_per_byte": 1.563399516553168, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0836659669876099, + "step": 6280, + "unclipped_grad_norm": 0.9565526247024536 + }, + { + "bits_per_byte": 1.5618869291388295, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0826175212860107, + "step": 6290, + "unclipped_grad_norm": 0.9895926713943481 + }, + { + "bits_per_byte": 1.5244160370994442, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0566446781158447, + "step": 6300, + "unclipped_grad_norm": 0.9698922038078308 + }, + { + "bits_per_byte": 1.5852545558768611, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0988147258758545, + "step": 6310, + "unclipped_grad_norm": 1.004137396812439 + }, + { + "bits_per_byte": 1.55847651317222, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0802536010742188, + "step": 6320, + "unclipped_grad_norm": 0.9893865585327148 + }, + { + "bits_per_byte": 1.5695085122944659, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0879004001617432, + "step": 6330, + "unclipped_grad_norm": 0.9685529470443726 + }, + { + "bits_per_byte": 1.4838095573695282, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0284984111785889, + "step": 6340, + "unclipped_grad_norm": 0.916477382183075 + }, + { + "bits_per_byte": 1.5924496220583575, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.103801965713501, + "step": 6350, + "unclipped_grad_norm": 0.9532552361488342 + }, + { + "bits_per_byte": 1.6071991981616538, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.114025592803955, + "step": 6360, + "unclipped_grad_norm": 0.9765400290489197 + }, + { + "bits_per_byte": 1.53600216432277, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0646755695343018, + "step": 6370, + "unclipped_grad_norm": 1.0022367238998413 + }, + { + "bits_per_byte": 1.5289498437414897, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0597872734069824, + "step": 6380, + "unclipped_grad_norm": 0.9823477864265442 + }, + { + "bits_per_byte": 1.6382666601614642, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1355599164962769, + "step": 6390, + "unclipped_grad_norm": 0.9935681223869324 + }, + { + "bits_per_byte": 1.5996092598137763, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1087646484375, + "step": 6400, + "unclipped_grad_norm": 1.055086374282837 + }, + { + "bits_per_byte": 1.61815156931655, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.1216171979904175, + "step": 6410, + "unclipped_grad_norm": 1.0302053689956665 + }, + { + "bits_per_byte": 1.538316190890129, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0662795305252075, + "step": 6420, + "unclipped_grad_norm": 1.0392645597457886 + }, + { + "bits_per_byte": 1.5869190039719114, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.099968433380127, + "step": 6430, + "unclipped_grad_norm": 0.9931192994117737 + }, + { + "bits_per_byte": 1.5112646957706162, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.047528862953186, + "step": 6440, + "unclipped_grad_norm": 0.9700445532798767 + }, + { + "bits_per_byte": 1.5261810950452492, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0578681230545044, + "step": 6450, + "unclipped_grad_norm": 0.9057909846305847 + }, + { + "bits_per_byte": 1.548995453595455, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0736818313598633, + "step": 6460, + "unclipped_grad_norm": 0.9796255826950073 + }, + { + "bits_per_byte": 1.5656127612871213, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0852000713348389, + "step": 6470, + "unclipped_grad_norm": 1.0005474090576172 + }, + { + "bits_per_byte": 1.5309916217725328, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0612025260925293, + "step": 6480, + "unclipped_grad_norm": 0.9810837507247925 + }, + { + "bits_per_byte": 1.6052196778502228, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1126534938812256, + "step": 6490, + "unclipped_grad_norm": 1.013913869857788 + }, + { + "bits_per_byte": 1.648418280093943, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.1425964832305908, + "step": 6500, + "unclipped_grad_norm": 1.0267438888549805 + }, + { + "bits_per_byte": 1.6007087449007422, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.1095267534255981, + "step": 6510, + "unclipped_grad_norm": 1.015332818031311 + }, + { + "bits_per_byte": 1.5803760960024866, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.095433235168457, + "step": 6520, + "unclipped_grad_norm": 1.0154883861541748 + }, + { + "bits_per_byte": 1.5618320666732044, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.082579493522644, + "step": 6530, + "unclipped_grad_norm": 0.9579733610153198 + }, + { + "bits_per_byte": 1.5301038473287791, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0605871677398682, + "step": 6540, + "unclipped_grad_norm": 0.9741249084472656 + }, + { + "bits_per_byte": 1.5804765338705906, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0955028533935547, + "step": 6550, + "unclipped_grad_norm": 0.9633719325065613 + }, + { + "bits_per_byte": 1.594095152061813, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.1049425601959229, + "step": 6560, + "unclipped_grad_norm": 1.0173745155334473 + }, + { + "bits_per_byte": 1.5579932419232956, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.079918622970581, + "step": 6570, + "unclipped_grad_norm": 0.988563060760498 + }, + { + "bits_per_byte": 1.5634815522776295, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0837228298187256, + "step": 6580, + "unclipped_grad_norm": 0.980972409248352 + }, + { + "bits_per_byte": 1.5783241709950424, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0940109491348267, + "step": 6590, + "unclipped_grad_norm": 1.0017447471618652 + }, + { + "bits_per_byte": 1.4742578129232586, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.021877646446228, + "step": 6600, + "unclipped_grad_norm": 0.9747391939163208 + }, + { + "bits_per_byte": 1.5148066784651357, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0499839782714844, + "step": 6610, + "unclipped_grad_norm": 0.9631054401397705 + }, + { + "bits_per_byte": 1.6442318784066405, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1396946907043457, + "step": 6620, + "unclipped_grad_norm": 1.0293588638305664 + }, + { + "bits_per_byte": 1.593316414618706, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.104402780532837, + "step": 6630, + "unclipped_grad_norm": 0.9871839880943298 + }, + { + "bits_per_byte": 1.541715771949734, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0686359405517578, + "step": 6640, + "unclipped_grad_norm": 0.9633845090866089 + }, + { + "bits_per_byte": 1.535648567992596, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0644304752349854, + "step": 6650, + "unclipped_grad_norm": 0.9820731282234192 + }, + { + "bits_per_byte": 1.568001256342304, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0868556499481201, + "step": 6660, + "unclipped_grad_norm": 1.0003972053527832 + }, + { + "bits_per_byte": 1.5410072034281794, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0681447982788086, + "step": 6670, + "unclipped_grad_norm": 1.0044373273849487 + }, + { + "bits_per_byte": 1.5651386051186926, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0848714113235474, + "step": 6680, + "unclipped_grad_norm": 1.0404566526412964 + }, + { + "bits_per_byte": 1.5524736307265046, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0760927200317383, + "step": 6690, + "unclipped_grad_norm": 1.033932089805603 + }, + { + "bits_per_byte": 1.565877786552101, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0853837728500366, + "step": 6700, + "unclipped_grad_norm": 0.9689918756484985 + }, + { + "bits_per_byte": 1.5892436934636238, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.1015797853469849, + "step": 6710, + "unclipped_grad_norm": 1.0171167850494385 + }, + { + "bits_per_byte": 1.6393943504031716, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1363415718078613, + "step": 6720, + "unclipped_grad_norm": 1.0878738164901733 + }, + { + "bits_per_byte": 1.4992198908174392, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.039180040359497, + "step": 6730, + "unclipped_grad_norm": 0.971998393535614 + }, + { + "bits_per_byte": 1.5310298019210244, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0612289905548096, + "step": 6740, + "unclipped_grad_norm": 0.973441481590271 + }, + { + "bits_per_byte": 1.5191564636707577, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0529990196228027, + "step": 6750, + "unclipped_grad_norm": 0.9789940714836121 + }, + { + "bits_per_byte": 1.5175835103459665, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0519087314605713, + "step": 6760, + "unclipped_grad_norm": 0.9706876277923584 + }, + { + "bits_per_byte": 1.5403151452411075, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0676651000976562, + "step": 6770, + "unclipped_grad_norm": 1.0128401517868042 + }, + { + "bits_per_byte": 1.5160401380371238, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0508389472961426, + "step": 6780, + "unclipped_grad_norm": 0.994110107421875 + }, + { + "bits_per_byte": 1.5638086632795705, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0839495658874512, + "step": 6790, + "unclipped_grad_norm": 0.9864916801452637 + }, + { + "bits_per_byte": 1.5414822195098625, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0684740543365479, + "step": 6800, + "unclipped_grad_norm": 1.023742914199829 + }, + { + "bits_per_byte": 1.5324796156677978, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0622339248657227, + "step": 6810, + "unclipped_grad_norm": 1.0296359062194824 + }, + { + "bits_per_byte": 1.5483856031154957, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0732591152191162, + "step": 6820, + "unclipped_grad_norm": 0.9808952212333679 + }, + { + "bits_per_byte": 1.549455679169164, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0740008354187012, + "step": 6830, + "unclipped_grad_norm": 0.996810793876648 + }, + { + "bits_per_byte": 1.5259360197677698, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0576982498168945, + "step": 6840, + "unclipped_grad_norm": 0.9733325242996216 + }, + { + "bits_per_byte": 1.6203460679415593, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.1231383085250854, + "step": 6850, + "unclipped_grad_norm": 1.0648897886276245 + }, + { + "bits_per_byte": 1.568673364541876, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0873215198516846, + "step": 6860, + "unclipped_grad_norm": 1.0356100797653198 + }, + { + "bits_per_byte": 1.4980275350990075, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0383535623550415, + "step": 6870, + "unclipped_grad_norm": 0.9937211275100708 + }, + { + "bits_per_byte": 1.521422507078526, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0545697212219238, + "step": 6880, + "unclipped_grad_norm": 1.069592833518982 + }, + { + "bits_per_byte": 1.5148947335823775, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0500450134277344, + "step": 6890, + "unclipped_grad_norm": 1.0049850940704346 + }, + { + "bits_per_byte": 1.5822056474423651, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0967013835906982, + "step": 6900, + "unclipped_grad_norm": 1.0225300788879395 + }, + { + "bits_per_byte": 1.5606532975841907, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.081762433052063, + "step": 6910, + "unclipped_grad_norm": 1.021737813949585 + }, + { + "bits_per_byte": 1.5130916674707318, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.048795223236084, + "step": 6920, + "unclipped_grad_norm": 1.0000710487365723 + }, + { + "bits_per_byte": 1.5005744261756349, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.040118932723999, + "step": 6930, + "unclipped_grad_norm": 1.0489689111709595 + }, + { + "bits_per_byte": 1.568557792200496, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0872414112091064, + "step": 6940, + "unclipped_grad_norm": 1.0565794706344604 + }, + { + "bits_per_byte": 1.484857619643886, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0292248725891113, + "step": 6950, + "unclipped_grad_norm": 1.0331164598464966 + }, + { + "bits_per_byte": 1.498764136792653, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0388641357421875, + "step": 6960, + "unclipped_grad_norm": 1.0262477397918701 + }, + { + "bits_per_byte": 1.4731473209466375, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0211079120635986, + "step": 6970, + "unclipped_grad_norm": 0.9956484436988831 + }, + { + "bits_per_byte": 1.5915678670073838, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1031907796859741, + "step": 6980, + "unclipped_grad_norm": 1.0641905069351196 + }, + { + "bits_per_byte": 1.5488172795691613, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0735583305358887, + "step": 6990, + "unclipped_grad_norm": 1.0450271368026733 + }, + { + "bits_per_byte": 1.5813185609292149, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0960865020751953, + "step": 7000, + "unclipped_grad_norm": 1.0753706693649292 + }, + { + "bits_per_byte": 1.4760245906955722, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0231022834777832, + "step": 7010, + "unclipped_grad_norm": 0.977759838104248 + }, + { + "bits_per_byte": 1.5697365612895098, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0880584716796875, + "step": 7020, + "unclipped_grad_norm": 1.0638940334320068 + }, + { + "bits_per_byte": 1.5177874817698898, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0520501136779785, + "step": 7030, + "unclipped_grad_norm": 1.0119246244430542 + }, + { + "bits_per_byte": 1.5230066392756236, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.055667757987976, + "step": 7040, + "unclipped_grad_norm": 1.0224649906158447 + }, + { + "bits_per_byte": 1.5100723400521843, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0467023849487305, + "step": 7050, + "unclipped_grad_norm": 1.0288535356521606 + }, + { + "bits_per_byte": 1.4969820525644124, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0376288890838623, + "step": 7060, + "unclipped_grad_norm": 0.9655771851539612 + }, + { + "bits_per_byte": 1.5399431467672913, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.067407250404358, + "step": 7070, + "unclipped_grad_norm": 0.986509382724762 + }, + { + "bits_per_byte": 1.5402030125527448, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0675873756408691, + "step": 7080, + "unclipped_grad_norm": 1.0742125511169434 + }, + { + "bits_per_byte": 1.5012656244494524, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0405980348587036, + "step": 7090, + "unclipped_grad_norm": 1.0513901710510254 + }, + { + "bits_per_byte": 1.5043721470719873, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0427513122558594, + "step": 7100, + "unclipped_grad_norm": 1.0241730213165283 + }, + { + "bits_per_byte": 1.4832691878805173, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0281238555908203, + "step": 7110, + "unclipped_grad_norm": 0.991193950176239 + }, + { + "bits_per_byte": 1.5470104298391967, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0723059177398682, + "step": 7120, + "unclipped_grad_norm": 1.0775213241577148 + }, + { + "bits_per_byte": 1.564362103450047, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0843331813812256, + "step": 7130, + "unclipped_grad_norm": 1.0102075338363647 + }, + { + "bits_per_byte": 1.530404644985138, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0607956647872925, + "step": 7140, + "unclipped_grad_norm": 1.0219485759735107 + }, + { + "bits_per_byte": 1.5796152447550695, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0949058532714844, + "step": 7150, + "unclipped_grad_norm": 1.0659542083740234 + }, + { + "bits_per_byte": 1.568367407405991, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0871094465255737, + "step": 7160, + "unclipped_grad_norm": 1.0356900691986084 + }, + { + "bits_per_byte": 1.5848232233884971, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0985157489776611, + "step": 7170, + "unclipped_grad_norm": 1.0675727128982544 + }, + { + "bits_per_byte": 1.5187308066098724, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0527039766311646, + "step": 7180, + "unclipped_grad_norm": 0.9808175563812256 + }, + { + "bits_per_byte": 1.5693248348233442, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.087773084640503, + "step": 7190, + "unclipped_grad_norm": 1.0627371072769165 + }, + { + "bits_per_byte": 1.5123168856285947, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0482581853866577, + "step": 7200, + "unclipped_grad_norm": 1.0047924518585205 + }, + { + "bits_per_byte": 1.5017243021793036, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0409159660339355, + "step": 7210, + "unclipped_grad_norm": 1.0128343105316162 + }, + { + "bits_per_byte": 1.5058675362212395, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0437878370285034, + "step": 7220, + "unclipped_grad_norm": 1.020466685295105 + }, + { + "bits_per_byte": 1.5492668422185167, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0738699436187744, + "step": 7230, + "unclipped_grad_norm": 1.0896223783493042 + }, + { + "bits_per_byte": 1.5598331123222258, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0811939239501953, + "step": 7240, + "unclipped_grad_norm": 1.0040799379348755 + }, + { + "bits_per_byte": 1.5635771746315092, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0837891101837158, + "step": 7250, + "unclipped_grad_norm": 1.0716665983200073 + }, + { + "bits_per_byte": 1.530704410745592, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0610034465789795, + "step": 7260, + "unclipped_grad_norm": 1.0284863710403442 + }, + { + "bits_per_byte": 1.5407076096503765, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0679371356964111, + "step": 7270, + "unclipped_grad_norm": 1.0729715824127197 + }, + { + "bits_per_byte": 1.5375348737072594, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0657379627227783, + "step": 7280, + "unclipped_grad_norm": 1.0802364349365234 + }, + { + "bits_per_byte": 1.5291964668628268, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0599582195281982, + "step": 7290, + "unclipped_grad_norm": 1.0581231117248535 + }, + { + "bits_per_byte": 1.5223175047936162, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.055190086364746, + "step": 7300, + "unclipped_grad_norm": 1.0030993223190308 + }, + { + "bits_per_byte": 1.518025505758684, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0522150993347168, + "step": 7310, + "unclipped_grad_norm": 1.0415269136428833 + }, + { + "bits_per_byte": 1.5002450793992326, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0398906469345093, + "step": 7320, + "unclipped_grad_norm": 1.0742172002792358 + }, + { + "bits_per_byte": 1.5556656287265183, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0783052444458008, + "step": 7330, + "unclipped_grad_norm": 1.098557472229004 + }, + { + "bits_per_byte": 1.5089695872948519, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0459380149841309, + "step": 7340, + "unclipped_grad_norm": 1.0178015232086182 + }, + { + "bits_per_byte": 1.5128952632834465, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.048659086227417, + "step": 7350, + "unclipped_grad_norm": 1.0542628765106201 + }, + { + "bits_per_byte": 1.5165268489390655, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0511763095855713, + "step": 7360, + "unclipped_grad_norm": 1.0132694244384766 + }, + { + "bits_per_byte": 1.4757258568310236, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0228952169418335, + "step": 7370, + "unclipped_grad_norm": 1.0333343744277954 + }, + { + "bits_per_byte": 1.4706363742440405, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0193674564361572, + "step": 7380, + "unclipped_grad_norm": 1.0362695455551147 + }, + { + "bits_per_byte": 1.4849742238811712, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0293056964874268, + "step": 7390, + "unclipped_grad_norm": 1.0089579820632935 + }, + { + "bits_per_byte": 1.4991997688472884, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0391660928726196, + "step": 7400, + "unclipped_grad_norm": 1.0394701957702637 + }, + { + "bits_per_byte": 1.5501245196983695, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0744644403457642, + "step": 7410, + "unclipped_grad_norm": 1.044879674911499 + }, + { + "bits_per_byte": 1.5165729402894967, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.051208257675171, + "step": 7420, + "unclipped_grad_norm": 1.0228228569030762 + }, + { + "bits_per_byte": 1.4806290822071222, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0262938737869263, + "step": 7430, + "unclipped_grad_norm": 1.033545970916748 + }, + { + "bits_per_byte": 1.4643294264715996, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.014995813369751, + "step": 7440, + "unclipped_grad_norm": 1.0252758264541626 + }, + { + "bits_per_byte": 1.5693753977226979, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0878081321716309, + "step": 7450, + "unclipped_grad_norm": 1.039492130279541 + }, + { + "bits_per_byte": 1.4916665847741957, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0339444875717163, + "step": 7460, + "unclipped_grad_norm": 1.036054253578186 + }, + { + "bits_per_byte": 1.466042717639495, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0161833763122559, + "step": 7470, + "unclipped_grad_norm": 1.0448886156082153 + }, + { + "bits_per_byte": 1.5608368030726611, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0818896293640137, + "step": 7480, + "unclipped_grad_norm": 1.0651273727416992 + }, + { + "bits_per_byte": 1.5231578120257319, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0557725429534912, + "step": 7490, + "unclipped_grad_norm": 1.077128291130066 + }, + { + "bits_per_byte": 1.5418131141301226, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0687034130096436, + "step": 7500, + "unclipped_grad_norm": 1.0931757688522339 + }, + { + "bits_per_byte": 1.528281175194935, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.059323787689209, + "step": 7510, + "unclipped_grad_norm": 1.0170495510101318 + }, + { + "bits_per_byte": 1.541195696413525, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0682754516601562, + "step": 7520, + "unclipped_grad_norm": 1.072600245475769 + }, + { + "bits_per_byte": 1.5347975978361268, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.063840627670288, + "step": 7530, + "unclipped_grad_norm": 1.0730668306350708 + }, + { + "bits_per_byte": 1.5420986053305548, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0689013004302979, + "step": 7540, + "unclipped_grad_norm": 1.0506013631820679 + }, + { + "bits_per_byte": 1.5157248938380923, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.050620436668396, + "step": 7550, + "unclipped_grad_norm": 1.036077857017517 + }, + { + "bits_per_byte": 1.443441617576359, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.00051748752594, + "step": 7560, + "unclipped_grad_norm": 1.0049386024475098 + }, + { + "bits_per_byte": 1.4652058500603964, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0156033039093018, + "step": 7570, + "unclipped_grad_norm": 1.0102496147155762 + }, + { + "bits_per_byte": 1.4617899306489595, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0132355690002441, + "step": 7580, + "unclipped_grad_norm": 1.0245665311813354 + }, + { + "bits_per_byte": 1.5365071053857031, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0650255680084229, + "step": 7590, + "unclipped_grad_norm": 1.048311471939087 + }, + { + "bits_per_byte": 1.567407228266224, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0864439010620117, + "step": 7600, + "unclipped_grad_norm": 1.0463460683822632 + }, + { + "bits_per_byte": 1.5308137917115408, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0610792636871338, + "step": 7610, + "unclipped_grad_norm": 1.0807271003723145 + }, + { + "bits_per_byte": 1.5336785067269625, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0630649328231812, + "step": 7620, + "unclipped_grad_norm": 1.048875331878662 + }, + { + "bits_per_byte": 1.510936380890119, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0473012924194336, + "step": 7630, + "unclipped_grad_norm": 1.0328357219696045 + }, + { + "bits_per_byte": 1.4914947741059836, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.033825397491455, + "step": 7640, + "unclipped_grad_norm": 1.0975395441055298 + }, + { + "bits_per_byte": 1.462546310347454, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0137598514556885, + "step": 7650, + "unclipped_grad_norm": 1.0314407348632812 + }, + { + "bits_per_byte": 1.4939462148113822, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.035524606704712, + "step": 7660, + "unclipped_grad_norm": 1.1010568141937256 + }, + { + "bits_per_byte": 1.5304318182439745, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0608144998550415, + "step": 7670, + "unclipped_grad_norm": 1.0511856079101562 + }, + { + "bits_per_byte": 1.5306942637691912, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.060996413230896, + "step": 7680, + "unclipped_grad_norm": 1.062692403793335 + }, + { + "bits_per_byte": 1.5040376408160592, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.042519450187683, + "step": 7690, + "unclipped_grad_norm": 1.0179529190063477 + }, + { + "bits_per_byte": 1.5428006385113768, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0693879127502441, + "step": 7700, + "unclipped_grad_norm": 1.0779086351394653 + }, + { + "bits_per_byte": 1.4839846357081066, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0286197662353516, + "step": 7710, + "unclipped_grad_norm": 1.0490714311599731 + }, + { + "bits_per_byte": 1.496933553456869, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.037595272064209, + "step": 7720, + "unclipped_grad_norm": 0.9670684337615967 + }, + { + "bits_per_byte": 1.4854798528747077, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.029656171798706, + "step": 7730, + "unclipped_grad_norm": 0.9940106868743896 + }, + { + "bits_per_byte": 1.5407578285844283, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.06797194480896, + "step": 7740, + "unclipped_grad_norm": 1.0928970575332642 + }, + { + "bits_per_byte": 1.4753865350608713, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0226600170135498, + "step": 7750, + "unclipped_grad_norm": 1.013372778892517 + }, + { + "bits_per_byte": 1.4801974057534566, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0259946584701538, + "step": 7760, + "unclipped_grad_norm": 1.0438604354858398 + }, + { + "bits_per_byte": 1.538979355991856, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0667392015457153, + "step": 7770, + "unclipped_grad_norm": 1.1242064237594604 + }, + { + "bits_per_byte": 1.51728718423853, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0517033338546753, + "step": 7780, + "unclipped_grad_norm": 1.1380839347839355 + }, + { + "bits_per_byte": 1.5213089985289567, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0544910430908203, + "step": 7790, + "unclipped_grad_norm": 1.054468035697937 + }, + { + "bits_per_byte": 1.5100510142034773, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0466876029968262, + "step": 7800, + "unclipped_grad_norm": 1.0544580221176147 + }, + { + "bits_per_byte": 1.498489308516574, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0386736392974854, + "step": 7810, + "unclipped_grad_norm": 1.0938669443130493 + }, + { + "bits_per_byte": 1.5401904578192318, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.067578673362732, + "step": 7820, + "unclipped_grad_norm": 1.1261787414550781 + }, + { + "bits_per_byte": 1.5279041892242438, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0590624809265137, + "step": 7830, + "unclipped_grad_norm": 1.0831990242004395 + }, + { + "bits_per_byte": 1.5007298984920148, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.040226697921753, + "step": 7840, + "unclipped_grad_norm": 1.0495885610580444 + }, + { + "bits_per_byte": 1.485421722738716, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.029615879058838, + "step": 7850, + "unclipped_grad_norm": 1.0680148601531982 + }, + { + "bits_per_byte": 1.5337288976436654, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0630998611450195, + "step": 7860, + "unclipped_grad_norm": 1.059203028678894 + }, + { + "bits_per_byte": 1.528788867980282, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.059675693511963, + "step": 7870, + "unclipped_grad_norm": 1.0786525011062622 + }, + { + "bits_per_byte": 1.4810511276323397, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0265864133834839, + "step": 7880, + "unclipped_grad_norm": 1.0475893020629883 + }, + { + "bits_per_byte": 1.50947349646188, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0462872982025146, + "step": 7890, + "unclipped_grad_norm": 1.0943796634674072 + }, + { + "bits_per_byte": 1.4635482812713807, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0144543647766113, + "step": 7900, + "unclipped_grad_norm": 1.0396785736083984 + }, + { + "bits_per_byte": 1.530891011921778, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.061132788658142, + "step": 7910, + "unclipped_grad_norm": 1.1283469200134277 + }, + { + "bits_per_byte": 1.5486817572402813, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0734643936157227, + "step": 7920, + "unclipped_grad_norm": 1.0839927196502686 + }, + { + "bits_per_byte": 1.5089941808139253, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0459550619125366, + "step": 7930, + "unclipped_grad_norm": 1.0844900608062744 + }, + { + "bits_per_byte": 1.5398654106091012, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.067353367805481, + "step": 7940, + "unclipped_grad_norm": 1.0818886756896973 + }, + { + "bits_per_byte": 1.473885126518839, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0216193199157715, + "step": 7950, + "unclipped_grad_norm": 1.0441930294036865 + }, + { + "bits_per_byte": 1.5289575829607784, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0597926378250122, + "step": 7960, + "unclipped_grad_norm": 1.1338133811950684 + }, + { + "bits_per_byte": 1.498011712695128, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0383425951004028, + "step": 7970, + "unclipped_grad_norm": 1.0597225427627563 + }, + { + "bits_per_byte": 1.5385745088317249, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0664585828781128, + "step": 7980, + "unclipped_grad_norm": 1.1261745691299438 + }, + { + "bits_per_byte": 1.500032680825417, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.039743423461914, + "step": 7990, + "unclipped_grad_norm": 1.0900131464004517 + }, + { + "bits_per_byte": 1.5133369147308622, + "learning_rate": 3e-05, + "loss_nats": 1.0489652156829834, + "step": 8000, + "unclipped_grad_norm": 1.0884610414505005 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073003.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073003.json new file mode 100644 index 0000000..5c94459 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073003.json @@ -0,0 +1,7366 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "14cb0082a87883b57c59c7a7d75d194347e04cee762e0db4354d50b7946c7443", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00031658337684348226, + 0.0002727926184888929, + 0.00024126221251208335, + 0.00021358189405873418, + 0.0001949657453224063, + 0.00017900438979268074, + 0.00016492439317516983, + 0.00015548785449936986, + 0.00014800082135479897, + 0.0001436655584257096, + 0.00013764512550551444, + 0.0001334012922598049, + 0.00013005948858335614, + 0.00012698007049039006, + 0.00012402374704834074, + 0.000122041383292526 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002610550254757982, + "first_to_last_ratio": 2.075552313213765, + "imbalance_abs_log_ratio": 0.7302272933071482, + "last_quartile_mean": 0.00012577617235365324, + "mean": 0.00017527624822832877, + "normalized": [ + 1.8061966754963605, + 1.5563581560322512, + 1.3764683746413606, + 1.2185444189820027, + 1.1123340857252286, + 1.021270089941082, + 0.9409397727427746, + 0.887101681323182, + 0.8443860640033859, + 0.8196521769370567, + 0.7853039239304516, + 0.7610916687697798, + 0.7420257444918054, + 0.7244568033255466, + 0.7075901515576575, + 0.6962802121000742 + ], + "population_cv": 0.32194911387722014 + }, + "activation_output_rms_by_block": [ + 0.0284949392080307, + 0.028932223096489906, + 0.029410898685455322, + 0.029886392876505852, + 0.030463071539998055, + 0.031026815995573997, + 0.031805604696273804, + 0.03248070180416107, + 0.03315235301852226, + 0.034095361828804016, + 0.034913208335638046, + 0.03547592833638191, + 0.03669917955994606, + 0.03757266327738762, + 0.038674499839544296, + 0.039735738188028336 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029181113466620445, + "first_to_last_ratio": 0.7644934703880544, + "imbalance_abs_log_ratio": 0.2685417946723145, + "last_quartile_mean": 0.03817052021622658, + "mean": 0.03330122376792133, + "normalized": [ + 0.8556724343409726, + 0.8688036000755015, + 0.8831777141411389, + 0.8974562942427076, + 0.9147733354274736, + 0.9317019762337311, + 0.9550881648653333, + 0.9753606062804618, + 0.9955295712122606, + 1.0238471134399472, + 1.0484061660601651, + 1.065304043587595, + 1.102036964638457, + 1.1282667429651916, + 1.1613537120758601, + 1.193221560413203 + ], + "population_cv": 0.1043395636179365 + }, + "bits_per_byte": 8.022455452740537, + "branch_output_rms_by_sublayer": [ + 0.003436586121097207, + 0.0038391812704503536, + 0.003560001263394952, + 0.0038144157733768225, + 0.004080869723111391, + 0.0038664413150399923, + 0.004083882085978985, + 0.003824100596830249, + 0.004302335437387228, + 0.0039005596190690994, + 0.005134448409080505, + 0.003882436081767082, + 0.005223615560680628, + 0.0038928077556192875, + 0.00572318397462368, + 0.0038727927021682262, + 0.005704221315681934, + 0.003765099449083209, + 0.0069826096296310425, + 0.003909832797944546, + 0.006672552786767483, + 0.0038559692911803722, + 0.0066586704924702644, + 0.00382705288939178, + 0.007329727057367563, + 0.0038791403640061617, + 0.007807865273207426, + 0.00388751900754869, + 0.008622666820883751, + 0.0039123171009123325, + 0.008360983803868294, + 0.0037942787166684866 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007247792155837073, + 0.0070599825250150985, + 0.007115739546591776, + 0.006247289691276946, + 0.006127370040903895, + 0.0065579111453454445, + 0.0059278656112939756, + 0.005972691938247438, + 0.0055663256066952145, + 0.006567633669083912, + 0.005568350992310226, + 0.005243662280059585, + 0.00519845242839581, + 0.005229357864659918, + 0.005505343111599652, + 0.004934663781812996 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006917700979680223, + "first_to_last_ratio": 1.3260037536012093, + "imbalance_abs_log_ratio": 0.2821697225298069, + "last_quartile_mean": 0.005216954296617094, + "mean": 0.00600440202432056, + "normalized": [ + 1.2070797602292813, + 1.1758011033270186, + 1.1850871273725165, + 1.0404515996718042, + 1.0204796441153106, + 1.092183887551653, + 0.9872532830552356, + 0.9947188602720667, + 0.927040791763953, + 1.0938031201911544, + 0.9273781085536696, + 0.8733029965049588, + 0.8657735453655023, + 0.8709206751111334, + 0.9168844939596827, + 0.821841002955059 + ], + "population_cv": 0.119259291012628 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028142500668764114, + 0.02826075814664364, + 0.0284949392080307, + 0.028706196695566177, + 0.028932223096489906, + 0.029190780594944954, + 0.029410898685455322, + 0.02965042181313038, + 0.029886392876505852, + 0.03018481843173504, + 0.030463071539998055, + 0.030850933864712715, + 0.031026815995573997, + 0.031584836542606354, + 0.031805604696273804, + 0.03225463256239891, + 0.03248070180416107, + 0.03296073526144028, + 0.03315235301852226, + 0.03385457023978233, + 0.034095361828804016, + 0.03471403941512108, + 0.034913208335638046, + 0.03529877960681915, + 0.03547592833638191, + 0.03643115609884262, + 0.03669917955994606, + 0.03748895600438118, + 0.03757266327738762, + 0.03844280168414116, + 0.038674499839544296, + 0.03956866264343262 + ], + "loss_nats": 5.560742378234863, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.02826075814664364, + 0.0284949392080307, + 0.028706196695566177, + 0.028932223096489906, + 0.029190780594944954, + 0.029410898685455322, + 0.02965042181313038, + 0.029886392876505852, + 0.03018481843173504, + 0.030463071539998055, + 0.030850933864712715, + 0.031026815995573997, + 0.031584836542606354, + 0.031805604696273804, + 0.03225463256239891, + 0.03248070180416107, + 0.03296073526144028, + 0.03315235301852226, + 0.03385457023978233, + 0.034095361828804016, + 0.03471403941512108, + 0.034913208335638046, + 0.03529877960681915, + 0.03547592833638191, + 0.03643115609884262, + 0.03669917955994606, + 0.03748895600438118, + 0.03757266327738762, + 0.03844280168414116, + 0.038674499839544296, + 0.03956866264343262, + 0.039735738188028336 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.706054748268798e-05, + 6.966097134863958e-05, + 6.121626211097464e-05, + 5.34596256329678e-05, + 4.684355008066632e-05, + 4.049177005072124e-05, + 3.4754324587993324e-05, + 3.020181247848086e-05, + 2.6929572413791902e-05, + 2.4378721718676388e-05, + 2.2718366381013766e-05, + 2.1439738702611066e-05, + 2.0652638340834528e-05, + 2.0034611225128174e-05, + 1.9586463167797774e-05, + 1.9270755728939548e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.53493516438175e-05, + "first_to_last_ratio": 3.2861795625404726, + "imbalance_abs_log_ratio": 1.1897256628392332, + "last_quartile_mean": 1.9886117115675006e-05, + "mean": 3.6793733215745306e-05, + "normalized": [ + 2.0943932769972666, + 1.893283590990143, + 1.663768711699472, + 1.45295464636599, + 1.2731393633256098, + 1.1005072470709065, + 0.9445718482603106, + 0.8208412095991516, + 0.7319064976605258, + 0.6625780965396546, + 0.6174520603223757, + 0.5827008250806217, + 0.5613085853434561, + 0.5445115098175108, + 0.5323314993058669, + 0.5237510316211383 + ], + "population_cv": 0.5070097402008534 + }, + "activation_output_rms_by_block": [ + 0.03020417131483555, + 0.03345569595694542, + 0.03829808160662651, + 0.04440607130527496, + 0.05133094638586044, + 0.059150952845811844, + 0.06815594434738159, + 0.07766561955213547, + 0.08808834105730057, + 0.10039259493350983, + 0.11500746011734009, + 0.132473424077034, + 0.1519528478384018, + 0.17349907755851746, + 0.19798777997493744, + 0.22741958498954773 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03659100504592061, + "first_to_last_ratio": 0.19492869311536967, + "imbalance_abs_log_ratio": 1.6351214636257212, + "last_quartile_mean": 0.1877148225903511, + "mean": 0.09934303711634129, + "normalized": [ + 0.30403913743308697, + 0.33676940959400337, + 0.3855134966507554, + 0.44699731953303096, + 0.5167040174717673, + 0.5954212249071846, + 0.6860666466997954, + 0.7817922806324186, + 0.8867087579992116, + 1.010564986209747, + 1.1576801299385893, + 1.3334948067059142, + 1.5295772330822623, + 1.746464398459366, + 1.9929708787045903, + 2.2892352759782764 + ], + "population_cv": 0.6013462998472742 + }, + "bits_per_byte": 5.091903414776641, + "branch_output_rms_by_sublayer": [ + 0.0044672503136098385, + 0.008439179509878159, + 0.004589807707816362, + 0.008912671357393265, + 0.005376614164561033, + 0.009684479795396328, + 0.004654362332075834, + 0.010457071475684643, + 0.00613226555287838, + 0.009992391802370548, + 0.005980229936540127, + 0.011163929477334023, + 0.008545844815671444, + 0.010891783982515335, + 0.00866044219583273, + 0.011625969782471657, + 0.010422462597489357, + 0.011488622985780239, + 0.010880537331104279, + 0.012680526822805405, + 0.014226034283638, + 0.012272600084543228, + 0.01669861003756523, + 0.013891858980059624, + 0.01948261633515358, + 0.012507432140409946, + 0.01911107264459133, + 0.013616994023323059, + 0.02317427285015583, + 0.013388233259320259, + 0.027934540063142776, + 0.013881584629416466 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00027208547857035954, + 0.0002379277587721014, + 0.0002494129668009364, + 0.00019949675850635897, + 0.0002037090250478814, + 0.0001757519905869641, + 0.00017904243730119992, + 0.00016456038574532694, + 0.0001493779474782176, + 0.00013508785060359837, + 0.0001355896704444294, + 0.00013648423482290936, + 0.00013660889397982336, + 0.00013630230473358484, + 0.0001407085634636149, + 0.00014945333748469326 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023973074066243908, + "first_to_last_ratio": 1.703016825392403, + "imbalance_abs_log_ratio": 0.532401281486357, + "last_quartile_mean": 0.0001407682749154291, + "mean": 0.00017509997527137498, + "normalized": [ + 1.5538864477203589, + 1.358810921608379, + 1.4244032097342618, + 1.1393305921212904, + 1.163386943557058, + 1.0037236745155367, + 1.0225154916425019, + 0.9398081609679643, + 0.8531009056209594, + 0.7714898325612858, + 0.774355737254038, + 0.7794646150656627, + 0.7801765463878734, + 0.7784256081266694, + 0.8035898534282527, + 0.8535314596879079 + ], + "population_cv": 0.2495872477003493 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028559360653162003, + 0.028688903898000717, + 0.03020417131483555, + 0.03060002811253071, + 0.03345569595694542, + 0.034075960516929626, + 0.03829808160662651, + 0.03887331485748291, + 0.04440607130527496, + 0.04547364264726639, + 0.05133094638586044, + 0.05236440151929855, + 0.059150952845811844, + 0.061456043273210526, + 0.06815594434738159, + 0.07020121067762375, + 0.07766561955213547, + 0.08064334839582443, + 0.08808834105730057, + 0.09190437197685242, + 0.10039259493350983, + 0.10653280466794968, + 0.11500746011734009, + 0.1223740428686142, + 0.132473424077034, + 0.14304450154304504, + 0.1519528478384018, + 0.1632441133260727, + 0.17349907755851746, + 0.1877153217792511, + 0.19798777997493744, + 0.21634350717067719 + ], + "loss_nats": 3.5294384956359863, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028688903898000717, + 0.03020417131483555, + 0.03060002811253071, + 0.03345569595694542, + 0.034075960516929626, + 0.03829808160662651, + 0.03887331485748291, + 0.04440607130527496, + 0.04547364264726639, + 0.05133094638586044, + 0.05236440151929855, + 0.059150952845811844, + 0.061456043273210526, + 0.06815594434738159, + 0.07020121067762375, + 0.07766561955213547, + 0.08064334839582443, + 0.08808834105730057, + 0.09190437197685242, + 0.10039259493350983, + 0.10653280466794968, + 0.11500746011734009, + 0.1223740428686142, + 0.132473424077034, + 0.14304450154304504, + 0.1519528478384018, + 0.1632441133260727, + 0.17349907755851746, + 0.1877153217792511, + 0.19798777997493744, + 0.21634350717067719, + 0.22741958498954773 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015226140385493636, + 0.00015351864567492157, + 0.00014040769019629806, + 0.00012548675294965506, + 0.00010927267430815846, + 9.748537559062243e-05, + 8.759690535953268e-05, + 7.801534957252443e-05, + 7.03658297425136e-05, + 6.262787064770237e-05, + 5.5923766922205687e-05, + 4.834980791201815e-05, + 4.202092532068491e-05, + 3.7704347050748765e-05, + 3.55200027115643e-05, + 3.515387652441859e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014291862316895276, + "first_to_last_ratio": 3.801048653306501, + "imbalance_abs_log_ratio": 1.3352769900584454, + "last_quartile_mean": 3.759978790185414e-05, + "mean": 8.323195152115659e-05, + "normalized": [ + 1.8293624151805847, + 1.8444676938266782, + 1.686944588348479, + 1.5076752455787101, + 1.3128693045288338, + 1.1712494277614383, + 1.0524432475581997, + 0.9373245267797649, + 0.845418477597842, + 0.752449864542502, + 0.671902627538302, + 0.5809044126488876, + 0.504865313773205, + 0.4530032801305249, + 0.4267592200158317, + 0.4223603541902161 + ], + "population_cv": 0.48987170747402153 + }, + "activation_output_rms_by_block": [ + 0.05162646621465683, + 0.06355457007884979, + 0.07474122941493988, + 0.08428152650594711, + 0.08889172971248627, + 0.09293187409639359, + 0.0973934531211853, + 0.09854210913181305, + 0.10090499371290207, + 0.10472240298986435, + 0.11040307581424713, + 0.1186341941356659, + 0.130356103181839, + 0.149788036942482, + 0.17963695526123047, + 0.2188950628042221 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0685509480535984, + "first_to_last_ratio": 0.40402744210990815, + "imbalance_abs_log_ratio": 0.9062724773132301, + "last_quartile_mean": 0.1696690395474434, + "mean": 0.1103314864449203, + "normalized": [ + 0.4679214236856113, + 0.576032936079199, + 0.6774242949427878, + 0.7638936918340364, + 0.8056787103730612, + 0.8422969461468014, + 0.8827348951725225, + 0.8931458489957648, + 0.9145620798218456, + 0.9491615346100125, + 1.0006488571090046, + 1.0752523867689434, + 1.1814950326706182, + 1.3576182263914227, + 1.6281567578708265, + 1.9839763775275423 + ], + "population_cv": 0.3749688391491054 + }, + "bits_per_byte": 3.129900568231996, + "branch_output_rms_by_sublayer": [ + 0.014640412293374538, + 0.020567819476127625, + 0.01939546875655651, + 0.012996131554245949, + 0.0218212753534317, + 0.010227516293525696, + 0.02053862437605858, + 0.010292341001331806, + 0.02414301224052906, + 0.009784159250557423, + 0.019980086013674736, + 0.010961603373289108, + 0.01658599078655243, + 0.011390469036996365, + 0.021502232179045677, + 0.012452049180865288, + 0.020629581063985825, + 0.014398678205907345, + 0.02130005694925785, + 0.015666954219341278, + 0.016051532700657845, + 0.0179822389036417, + 0.018369123339653015, + 0.02199208177626133, + 0.01977314054965973, + 0.026316894218325615, + 0.016924018040299416, + 0.03431078419089317, + 0.013718036003410816, + 0.04166935011744499, + 0.022049644961953163, + 0.04074697569012642 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0009002454290164998, + 0.0005806619682142476, + 0.0004904299887450306, + 0.00047545530221616405, + 0.00040490888378542537, + 0.0003357588676961225, + 0.0003168596431849841, + 0.00029935138955839515, + 0.00029222591787988014, + 0.00027100955094896284, + 0.0002328601817845373, + 0.0002299739939536432, + 0.0002128774552133389, + 0.00018778710115789697, + 0.00017922956471394958, + 0.0001849955243427181 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006116981720479855, + "first_to_last_ratio": 3.1988832674328185, + "imbalance_abs_log_ratio": 1.1628017699711202, + "last_quartile_mean": 0.0001912224113569759, + "mean": 0.0003496644226507373, + "normalized": [ + 2.5745983025436674, + 1.6606263909046375, + 1.4025733159444054, + 1.359747436161308, + 1.157992799827591, + 0.9602317134548619, + 0.9061821067837941, + 0.8561105167322174, + 0.8357324879224854, + 0.7750561206498867, + 0.6659533160945287, + 0.6576991511182508, + 0.6088050182502319, + 0.537049493724068, + 0.512575924525708, + 0.5290659053623568 + ], + "population_cv": 0.5241885524055848 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04364648833870888, + 0.045909684151411057, + 0.05162646621465683, + 0.0588536262512207, + 0.06355457007884979, + 0.07136592268943787, + 0.07474122941493988, + 0.08130704611539841, + 0.08428152650594711, + 0.08667806535959244, + 0.08889172971248627, + 0.09049459546804428, + 0.09293187409639359, + 0.09443560987710953, + 0.0973934531211853, + 0.09589134901762009, + 0.09854210913181305, + 0.09718576818704605, + 0.10090499371290207, + 0.09916814416646957, + 0.10472240298986435, + 0.10267213732004166, + 0.11040307581424713, + 0.10699481517076492, + 0.1186341941356659, + 0.11436477303504944, + 0.130356103181839, + 0.12692348659038544, + 0.149788036942482, + 0.15056778490543365, + 0.17963695526123047, + 0.18913158774375916 + ], + "loss_nats": 2.1694817543029785, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.045909684151411057, + 0.05162646621465683, + 0.0588536262512207, + 0.06355457007884979, + 0.07136592268943787, + 0.07474122941493988, + 0.08130704611539841, + 0.08428152650594711, + 0.08667806535959244, + 0.08889172971248627, + 0.09049459546804428, + 0.09293187409639359, + 0.09443560987710953, + 0.0973934531211853, + 0.09589134901762009, + 0.09854210913181305, + 0.09718576818704605, + 0.10090499371290207, + 0.09916814416646957, + 0.10472240298986435, + 0.10267213732004166, + 0.11040307581424713, + 0.10699481517076492, + 0.1186341941356659, + 0.11436477303504944, + 0.130356103181839, + 0.12692348659038544, + 0.149788036942482, + 0.15056778490543365, + 0.17963695526123047, + 0.18913158774375916, + 0.2188950628042221 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001594285131432116, + 0.00017026996647473425, + 0.00018808410095516592, + 0.00017561657296027988, + 0.0001699679996818304, + 0.00016689945186953992, + 0.00015736353816464543, + 0.00014705183275509626, + 0.0001390052493661642, + 0.00012381834676489234, + 0.000104674712929409, + 9.053898247657344e-05, + 7.686029857723042e-05, + 6.236408080440015e-05, + 4.532423918135464e-05, + 3.028271021321416e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00017334978838334791, + "first_to_last_ratio": 3.227644485017083, + "imbalance_abs_log_ratio": 1.171752609589353, + "last_quartile_mean": 5.370783219404984e-05, + "mean": 0.00012547191226985888, + "normalized": [ + 1.2706310939162266, + 1.3570365143437513, + 1.499013584415959, + 1.3996484933024078, + 1.3546298658162754, + 1.3301738122120965, + 1.2541734266884814, + 1.1719900501621776, + 1.1078594950174865, + 0.9868212297473388, + 0.834248167863097, + 0.7215876512812414, + 0.6125697551490491, + 0.4970361866349061, + 0.3612301618857412, + 0.24135051156376403 + ], + "population_cv": 0.3906610370884368 + }, + "activation_output_rms_by_block": [ + 0.08273108303546906, + 0.13354359567165375, + 0.1704990118741989, + 0.17198410630226135, + 0.17284893989562988, + 0.17146942019462585, + 0.16773979365825653, + 0.15899653732776642, + 0.14656992256641388, + 0.13979844748973846, + 0.1417049616575241, + 0.149912029504776, + 0.16512760519981384, + 0.19090111553668976, + 0.2512649893760681, + 0.390801340341568 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.13968944922089577, + "first_to_last_ratio": 0.5598242338035286, + "imbalance_abs_log_ratio": 0.5801324127278619, + "last_quartile_mean": 0.24952376261353493, + "mean": 0.17536830622702837, + "normalized": [ + 0.4717561845432151, + 0.7615035951751215, + 0.9722338975748231, + 0.9807023287298795, + 0.9856338560507224, + 0.9777674420407808, + 0.9565000499069877, + 0.906643513577263, + 0.8357834190213785, + 0.7971705406606262, + 0.8080420271270434, + 0.8548410641014165, + 0.9416046077678534, + 1.0885725000363118, + 1.4327844910059484, + 2.228460482680629 + ], + "population_cv": 0.3692959780115816 + }, + "bits_per_byte": 2.102960171157105, + "branch_output_rms_by_sublayer": [ + 0.027171069756150246, + 0.04243958741426468, + 0.07314105331897736, + 0.007419656030833721, + 0.04879630357027054, + 0.0038894445169717073, + 0.024768881499767303, + 0.005487008485943079, + 0.02289147861301899, + 0.00865041371434927, + 0.025303618982434273, + 0.011297631077468395, + 0.03125236928462982, + 0.014728116802871227, + 0.035565298050642014, + 0.019090818241238594, + 0.043166082352399826, + 0.02257433719933033, + 0.04396531730890274, + 0.02886364422738552, + 0.04178466275334358, + 0.03710934892296791, + 0.035476140677928925, + 0.04854284226894379, + 0.026270829141139984, + 0.058104008436203, + 0.018585238605737686, + 0.06995242089033127, + 0.01513067539781332, + 0.10811682045459747, + 0.016200197860598564, + 0.20113423466682434 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004582000765955337, + 0.0003576182203712263, + 0.0003389913908322493, + 0.00022415790052185902, + 0.00024709072048410314, + 0.00027475536769928975, + 0.00033936291802973487, + 0.00039410580789665003, + 0.0004814106753566502, + 0.0005191302840338224, + 0.0005153457080868455, + 0.0004830357094583275, + 0.00041821493179685683, + 0.0003600919520715316, + 0.0003136989944772832, + 0.0002645229655850915 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034474189708021705, + "first_to_last_ratio": 1.0165412954471984, + "imbalance_abs_log_ratio": 0.016405978393597878, + "last_quartile_mean": 0.00033913221098269076, + "mean": 0.00037435835145606595, + "normalized": [ + 1.2239611452859689, + 0.9552831370804766, + 0.9055264548359695, + 0.5987789497683099, + 0.6600379543371862, + 0.7339367924627015, + 0.9065188921518208, + 1.052750142647483, + 1.2859621629494962, + 1.3867201893978494, + 1.3766106888824827, + 1.2903030146904997, + 1.1171513341967951, + 0.9618910615215468, + 0.8379644617440793, + 0.7066036180473336 + ], + "population_cv": 0.2506116126131003 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.055472806096076965, + 0.06610024720430374, + 0.08273108303546906, + 0.13110069930553436, + 0.13354359567165375, + 0.1698971837759018, + 0.1704990118741989, + 0.17238332331180573, + 0.17198410630226135, + 0.17322638630867004, + 0.17284893989562988, + 0.1726299524307251, + 0.17146942019462585, + 0.170275017619133, + 0.16773979365825653, + 0.16162297129631042, + 0.15899653732776642, + 0.14726711809635162, + 0.14656992256641388, + 0.13564367592334747, + 0.13979844748973846, + 0.13359025120735168, + 0.1417049616575241, + 0.13732580840587616, + 0.149912029504776, + 0.14843478798866272, + 0.16512760519981384, + 0.1652906984090805, + 0.19090111553668976, + 0.191492959856987, + 0.2512649893760681, + 0.25225311517715454 + ], + "loss_nats": 1.4576609134674072, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06610024720430374, + 0.08273108303546906, + 0.13110069930553436, + 0.13354359567165375, + 0.1698971837759018, + 0.1704990118741989, + 0.17238332331180573, + 0.17198410630226135, + 0.17322638630867004, + 0.17284893989562988, + 0.1726299524307251, + 0.17146942019462585, + 0.170275017619133, + 0.16773979365825653, + 0.16162297129631042, + 0.15899653732776642, + 0.14726711809635162, + 0.14656992256641388, + 0.13564367592334747, + 0.13979844748973846, + 0.13359025120735168, + 0.1417049616575241, + 0.13732580840587616, + 0.149912029504776, + 0.14843478798866272, + 0.16512760519981384, + 0.1652906984090805, + 0.19090111553668976, + 0.191492959856987, + 0.2512649893760681, + 0.25225311517715454, + 0.390801340341568 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00016188873269129544, + 0.00017867473070509732, + 0.00019946519751101732, + 0.00018539158918429166, + 0.00018016427929978818, + 0.0001743910543154925, + 0.00016342199523933232, + 0.00015197545872069895, + 0.0001410306867910549, + 0.00012255505134817213, + 0.00010388438386144117, + 8.946077286964282e-05, + 7.552531315013766e-05, + 6.115865835454315e-05, + 4.3820087739732116e-05, + 2.6790259653353132e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018135506252292544, + "first_to_last_ratio": 3.499469999703496, + "imbalance_abs_log_ratio": 1.2526115283727475, + "last_quartile_mean": 5.1823579724441515e-05, + "mean": 0.00012872489071469317, + "normalized": [ + 1.257633483256217, + 1.3880355983452597, + 1.5495464506014887, + 1.4402155492615254, + 1.3996071645467982, + 1.3547578354679994, + 1.269544641537448, + 1.1806221615486823, + 1.0955976424453635, + 0.9520695699777604, + 0.8070263900374272, + 0.6949764911273015, + 0.58671879797927, + 0.475111363583098, + 0.3404165852962757, + 0.208120274988085 + ], + "population_cv": 0.4148571258695616 + }, + "activation_output_rms_by_block": [ + 0.09483804553747177, + 0.15792693197727203, + 0.18785393238067627, + 0.19029200077056885, + 0.19116424024105072, + 0.18977893888950348, + 0.18693965673446655, + 0.177143394947052, + 0.16181440651416779, + 0.1581212431192398, + 0.165044903755188, + 0.18009524047374725, + 0.1996685266494751, + 0.2291477918624878, + 0.29672855138778687, + 0.4953230619430542 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15772772766649723, + "first_to_last_ratio": 0.516772448690399, + "imbalance_abs_log_ratio": 0.6601526392924331, + "last_quartile_mean": 0.305216982960701, + "mean": 0.20386755419895053, + "normalized": [ + 0.46519440481893015, + 0.774654567264559, + 0.9214508562620668, + 0.9334099362611985, + 0.9376883977059789, + 0.9308932931245242, + 0.9169662012623924, + 0.8689141126114707, + 0.7937231951890498, + 0.7756076916728606, + 0.8095692539388771, + 0.8833933441806815, + 0.9794031592424085, + 1.1240032420208796, + 1.4554966951642292, + 2.429631649279893 + ], + "population_cv": 0.4175842970930077 + }, + "bits_per_byte": 1.9507894057257391, + "branch_output_rms_by_sublayer": [ + 0.03931046277284622, + 0.047202516347169876, + 0.08608254790306091, + 0.0076385182328522205, + 0.041328247636556625, + 0.004047342110425234, + 0.034119248390197754, + 0.0045016431249678135, + 0.029622135683894157, + 0.008596057072281837, + 0.02270515263080597, + 0.012007404118776321, + 0.02946399338543415, + 0.01667790301144123, + 0.038835883140563965, + 0.02453153021633625, + 0.05270649492740631, + 0.02996903844177723, + 0.05063421651721001, + 0.03910558298230171, + 0.04952584207057953, + 0.05079483985900879, + 0.03944792598485947, + 0.06468769162893295, + 0.0361105240881443, + 0.07707343250513077, + 0.028930658474564552, + 0.0911106988787651, + 0.02081119269132614, + 0.1352817416191101, + 0.02383771352469921, + 0.28301650285720825 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00041699002778082224, + 0.00029759414151732746, + 0.000265991514011251, + 0.0001953867836934275, + 0.00023560860135608575, + 0.00025046058190266666, + 0.00033611798182792255, + 0.0004636386291792892, + 0.0005491632897177901, + 0.0005951115480573408, + 0.0005882338016339408, + 0.0005513321353847482, + 0.0004945066783331371, + 0.00043544338635105514, + 0.0003756714670850321, + 0.0003149065782712711 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002939906167507071, + "first_to_last_ratio": 0.7256661946909654, + "imbalance_abs_log_ratio": 0.3206651568188065, + "last_quartile_mean": 0.00040513202751012384, + "mean": 0.00039788482163144424, + "normalized": [ + 1.0480169262829406, + 0.7479404222988562, + 0.6685138501152368, + 0.4910636774036378, + 0.5921527752428994, + 0.6294801115451073, + 0.8447620103972311, + 1.165258396332474, + 1.380206682592364, + 1.4956879873356546, + 1.4784022150480887, + 1.3856576210274245, + 1.2428387599846482, + 1.094395570470896, + 0.9441713949898092, + 0.7914515989327312 + ], + "population_cv": 0.32327688011764555 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.059103816747665405, + 0.07607264071702957, + 0.09483804553747177, + 0.15534481406211853, + 0.15792693197727203, + 0.18721190094947815, + 0.18785393238067627, + 0.19058401882648468, + 0.19029200077056885, + 0.1914086490869522, + 0.19116424024105072, + 0.19123691320419312, + 0.18977893888950348, + 0.19008098542690277, + 0.18693965673446655, + 0.18065746128559113, + 0.177143394947052, + 0.16187894344329834, + 0.16181440651416779, + 0.15150673687458038, + 0.1581212431192398, + 0.15270332992076874, + 0.165044903755188, + 0.16355812549591064, + 0.18009524047374725, + 0.17952413856983185, + 0.1996685266494751, + 0.2009342908859253, + 0.2291477918624878, + 0.23023094236850739, + 0.29672855138778687, + 0.29810962080955505 + ], + "loss_nats": 1.3521841764450073, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.07607264071702957, + 0.09483804553747177, + 0.15534481406211853, + 0.15792693197727203, + 0.18721190094947815, + 0.18785393238067627, + 0.19058401882648468, + 0.19029200077056885, + 0.1914086490869522, + 0.19116424024105072, + 0.19123691320419312, + 0.18977893888950348, + 0.19008098542690277, + 0.18693965673446655, + 0.18065746128559113, + 0.177143394947052, + 0.16187894344329834, + 0.16181440651416779, + 0.15150673687458038, + 0.1581212431192398, + 0.15270332992076874, + 0.165044903755188, + 0.16355812549591064, + 0.18009524047374725, + 0.17952413856983185, + 0.1996685266494751, + 0.2009342908859253, + 0.2291477918624878, + 0.23023094236850739, + 0.29672855138778687, + 0.29810962080955505, + 0.4953230619430542 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00025817437563091516, + 0.00027957011479884386, + 0.0003037556598428637, + 0.0002888414019253105, + 0.00028409145306795835, + 0.00028240776737220585, + 0.00026678573340177536, + 0.0002519928675610572, + 0.00023668054200243205, + 0.00020713888807222247, + 0.00017311860574409366, + 0.0001455697201890871, + 0.00011939310206798837, + 9.376799425808713e-05, + 6.36211916571483e-05, + 3.1806663173483685e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002825853880494833, + "first_to_last_ratio": 3.662935915109688, + "imbalance_abs_log_ratio": 1.2982649884510844, + "last_quartile_mean": 7.714723778917687e-05, + "mean": 0.00020541975504784205, + "normalized": [ + 1.2568137644346165, + 1.3609699550743417, + 1.478707146603886, + 1.4061033314835745, + 1.3829801958521164, + 1.3747838775605266, + 1.2987345513075956, + 1.226721682646191, + 1.1521800420183999, + 1.0083688787574498, + 0.8427553898298366, + 0.7086451843698482, + 0.5812152878878755, + 0.4564701882554999, + 0.30971311226776116, + 0.15483741165048096 + ], + "population_cv": 0.41926708997335715 + }, + "activation_output_rms_by_block": [ + 0.0951872169971466, + 0.14984266459941864, + 0.17071209847927094, + 0.17147330939769745, + 0.17152385413646698, + 0.17004960775375366, + 0.1669105589389801, + 0.15593038499355316, + 0.1393769234418869, + 0.1344291716814041, + 0.13867342472076416, + 0.1521374136209488, + 0.16997921466827393, + 0.1961352825164795, + 0.25573307275772095, + 0.4589640200138092 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1468038223683834, + "first_to_last_ratio": 0.5433095785892573, + "imbalance_abs_log_ratio": 0.6100759951930727, + "last_quartile_mean": 0.2702028974890709, + "mean": 0.18106613866984844, + "normalized": [ + 0.5257041305260761, + 0.8275576300472066, + 0.9428162534052995, + 0.9470203024009789, + 0.94729945309774, + 0.9391574206142317, + 0.9218209443529402, + 0.861179158836942, + 0.7697569764605372, + 0.7424313163629063, + 0.7658716629154935, + 0.8402311704363035, + 0.9387686505990491, + 1.0832245275529278, + 1.4123738134385158, + 2.534786588952852 + ], + "population_cv": 0.4350736575110673 + }, + "bits_per_byte": 1.8670605120664352, + "branch_output_rms_by_sublayer": [ + 0.051385972648859024, + 0.036380402743816376, + 0.07075809687376022, + 0.0071685765869915485, + 0.030611267313361168, + 0.003892687615007162, + 0.023587750270962715, + 0.004152141511440277, + 0.018766717985272408, + 0.007657408714294434, + 0.01761440560221672, + 0.010538195259869099, + 0.02535838447511196, + 0.014725946821272373, + 0.03455089405179024, + 0.021837059408426285, + 0.046537015587091446, + 0.02783774584531784, + 0.04248657822608948, + 0.03665589541196823, + 0.044122546911239624, + 0.04793383553624153, + 0.034650206565856934, + 0.06113889440894127, + 0.031748466193675995, + 0.07430189102888107, + 0.0256944727152586, + 0.0880647599697113, + 0.019152166321873665, + 0.12767614424228668, + 0.024809958413243294, + 0.2864501476287842 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005002075550754018, + 0.0003412904728571966, + 0.0002592425753327288, + 0.0002585649119364213, + 0.00030071075691208866, + 0.0003425364164715924, + 0.0005240790570344413, + 0.0006975962934822548, + 0.0008810870404936847, + 0.000990036099264455, + 0.0009754970927650687, + 0.0008971247584696028, + 0.0007874986100445877, + 0.000666994667011454, + 0.0005527789362036235, + 0.0004234912235697502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00033982637880043714, + "first_to_last_ratio": 0.5592092980363276, + "imbalance_abs_log_ratio": 0.5812314608110569, + "last_quartile_mean": 0.0006076908592073538, + "mean": 0.0005874210291827721, + "normalized": [ + 0.8515315765445054, + 0.580998050635001, + 0.44132327998776366, + 0.4401696553086297, + 0.5119169079296352, + 0.5831190908301898, + 0.8921693827739654, + 1.1875575759566523, + 1.4999242395517653, + 1.6853943765714456, + 1.6606438045335101, + 1.5272261527948612, + 1.3406033678095697, + 1.135462698602033, + 0.9410268082718403, + 0.7209330318986312 + ], + "population_cv": 0.42806100897327076 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060872070491313934, + 0.0852167010307312, + 0.0951872169971466, + 0.14736860990524292, + 0.14984266459941864, + 0.1702466458082199, + 0.17071209847927094, + 0.17185615003108978, + 0.17147330939769745, + 0.1719079613685608, + 0.17152385413646698, + 0.17163211107254028, + 0.17004960775375366, + 0.16988061368465424, + 0.1669105589389801, + 0.1597038209438324, + 0.15593038499355316, + 0.14019693434238434, + 0.1393769234418869, + 0.12975449860095978, + 0.1344291716814041, + 0.1281583607196808, + 0.13867342472076416, + 0.13766533136367798, + 0.1521374136209488, + 0.15170666575431824, + 0.16997921466827393, + 0.1708848923444748, + 0.1961352825164795, + 0.19691267609596252, + 0.25573307275772095, + 0.25681471824645996 + ], + "loss_nats": 1.2941477298736572, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0852167010307312, + 0.0951872169971466, + 0.14736860990524292, + 0.14984266459941864, + 0.1702466458082199, + 0.17071209847927094, + 0.17185615003108978, + 0.17147330939769745, + 0.1719079613685608, + 0.17152385413646698, + 0.17163211107254028, + 0.17004960775375366, + 0.16988061368465424, + 0.1669105589389801, + 0.1597038209438324, + 0.15593038499355316, + 0.14019693434238434, + 0.1393769234418869, + 0.12975449860095978, + 0.1344291716814041, + 0.1281583607196808, + 0.13867342472076416, + 0.13766533136367798, + 0.1521374136209488, + 0.15170666575431824, + 0.16997921466827393, + 0.1708848923444748, + 0.1961352825164795, + 0.19691267609596252, + 0.25573307275772095, + 0.25681471824645996, + 0.4589640200138092 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.018429854831792, + "cross_entropy_nats": 5.557952046394348, + "step": 0 + }, + { + "bits_per_byte": 5.108882101013433, + "cross_entropy_nats": 3.5412072241306305, + "step": 100 + }, + { + "bits_per_byte": 3.154257052216826, + "cross_entropy_nats": 2.186364382505417, + "step": 500 + }, + { + "bits_per_byte": 1.977161870343983, + "cross_entropy_nats": 1.37046417593956, + "step": 2000 + }, + { + "bits_per_byte": 1.8329381437371801, + "cross_entropy_nats": 1.2704959064722061, + "step": 4000 + }, + { + "bits_per_byte": 1.7363699927668699, + "cross_entropy_nats": 1.2035599648952484, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "final_optimizer_state": "de12f865acf312961efff64f13ff04f15f5b52fc60a03097833a902e65a3e5f3", + "final_public_parameters": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.482708441361407, + "measured_steps": 7980, + "median_ms": 21.420384509838186, + "p95_ms": 21.788083588762674, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.034161279888863, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.568856239318848, + "step": 1, + "unclipped_grad_norm": 18.562814712524414 + }, + { + "bits_per_byte": 7.448981367230313, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.163240432739258, + "step": 10, + "unclipped_grad_norm": 12.974101066589355 + }, + { + "bits_per_byte": 6.770661154309292, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6930646896362305, + "step": 20, + "unclipped_grad_norm": 4.495212554931641 + }, + { + "bits_per_byte": 6.551721736316109, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.54130744934082, + "step": 30, + "unclipped_grad_norm": 3.012619972229004 + }, + { + "bits_per_byte": 6.419676896636662, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.4497809410095215, + "step": 40, + "unclipped_grad_norm": 2.8561618328094482 + }, + { + "bits_per_byte": 6.250720452498435, + "learning_rate": 3.75e-05, + "loss_nats": 4.332669258117676, + "step": 50, + "unclipped_grad_norm": 2.8167924880981445 + }, + { + "bits_per_byte": 5.9407609363626195, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.11782169342041, + "step": 60, + "unclipped_grad_norm": 2.718289375305176 + }, + { + "bits_per_byte": 5.7053297580129, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9546332359313965, + "step": 70, + "unclipped_grad_norm": 2.7082417011260986 + }, + { + "bits_per_byte": 5.509554339299127, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.818932056427002, + "step": 80, + "unclipped_grad_norm": 2.5339207649230957 + }, + { + "bits_per_byte": 5.312337705971124, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.682231903076172, + "step": 90, + "unclipped_grad_norm": 2.2857937812805176 + }, + { + "bits_per_byte": 5.088000784463262, + "learning_rate": 7.5e-05, + "loss_nats": 3.5267333984375, + "step": 100, + "unclipped_grad_norm": 2.166426658630371 + }, + { + "bits_per_byte": 4.935865619440625, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.421281337738037, + "step": 110, + "unclipped_grad_norm": 2.0095791816711426 + }, + { + "bits_per_byte": 4.7340020148863085, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.281360149383545, + "step": 120, + "unclipped_grad_norm": 1.7170522212982178 + }, + { + "bits_per_byte": 4.584287677657216, + "learning_rate": 9.75e-05, + "loss_nats": 3.177586078643799, + "step": 130, + "unclipped_grad_norm": 1.5826245546340942 + }, + { + "bits_per_byte": 4.439357553809826, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0771281719207764, + "step": 140, + "unclipped_grad_norm": 1.4182634353637695 + }, + { + "bits_per_byte": 4.232185532753978, + "learning_rate": 0.0001125, + "loss_nats": 2.9335274696350098, + "step": 150, + "unclipped_grad_norm": 1.1410893201828003 + }, + { + "bits_per_byte": 4.110705931282335, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8493242263793945, + "step": 160, + "unclipped_grad_norm": 0.887942910194397 + }, + { + "bits_per_byte": 4.028603133586938, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.792414903640747, + "step": 170, + "unclipped_grad_norm": 0.9387072920799255 + }, + { + "bits_per_byte": 3.9322216482863084, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7256083488464355, + "step": 180, + "unclipped_grad_norm": 0.7101461887359619 + }, + { + "bits_per_byte": 3.8536510302743325, + "learning_rate": 0.0001425, + "loss_nats": 2.671147346496582, + "step": 190, + "unclipped_grad_norm": 0.90241938829422 + }, + { + "bits_per_byte": 3.794223801129896, + "learning_rate": 0.00015, + "loss_nats": 2.629955530166626, + "step": 200, + "unclipped_grad_norm": 0.8553160429000854 + }, + { + "bits_per_byte": 3.7222494057091122, + "learning_rate": 0.0001575, + "loss_nats": 2.580066680908203, + "step": 210, + "unclipped_grad_norm": 0.7590162754058838 + }, + { + "bits_per_byte": 3.70871849866983, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.570687770843506, + "step": 220, + "unclipped_grad_norm": 0.8053720593452454 + }, + { + "bits_per_byte": 3.6584135732924583, + "learning_rate": 0.0001725, + "loss_nats": 2.5358190536499023, + "step": 230, + "unclipped_grad_norm": 0.757310152053833 + }, + { + "bits_per_byte": 3.6516113154855327, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.53110408782959, + "step": 240, + "unclipped_grad_norm": 0.6297867894172668 + }, + { + "bits_per_byte": 3.5768769424837026, + "learning_rate": 0.0001875, + "loss_nats": 2.479302167892456, + "step": 250, + "unclipped_grad_norm": 0.5848793387413025 + }, + { + "bits_per_byte": 3.595435590338308, + "learning_rate": 0.000195, + "loss_nats": 2.492166042327881, + "step": 260, + "unclipped_grad_norm": 0.7728506326675415 + }, + { + "bits_per_byte": 3.581017252820574, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4821720123291016, + "step": 270, + "unclipped_grad_norm": 0.6333379149436951 + }, + { + "bits_per_byte": 3.5262535052369333, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4442126750946045, + "step": 280, + "unclipped_grad_norm": 0.6880936026573181 + }, + { + "bits_per_byte": 3.5151664716664115, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.436527729034424, + "step": 290, + "unclipped_grad_norm": 0.6548541188240051 + }, + { + "bits_per_byte": 3.500701698832944, + "learning_rate": 0.000225, + "loss_nats": 2.426501512527466, + "step": 300, + "unclipped_grad_norm": 0.6513921618461609 + }, + { + "bits_per_byte": 3.4243479371905896, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.373577117919922, + "step": 310, + "unclipped_grad_norm": 0.6903060674667358 + }, + { + "bits_per_byte": 3.495896847533139, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.423171043395996, + "step": 320, + "unclipped_grad_norm": 1.2031209468841553 + }, + { + "bits_per_byte": 3.486328076804434, + "learning_rate": 0.0002475, + "loss_nats": 2.4165384769439697, + "step": 330, + "unclipped_grad_norm": 0.8043278455734253 + }, + { + "bits_per_byte": 3.4281229563770284, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3761937618255615, + "step": 340, + "unclipped_grad_norm": 0.7758461236953735 + }, + { + "bits_per_byte": 3.3969462853939203, + "learning_rate": 0.0002625, + "loss_nats": 2.354583740234375, + "step": 350, + "unclipped_grad_norm": 0.7247553467750549 + }, + { + "bits_per_byte": 3.3922525348865724, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.351330280303955, + "step": 360, + "unclipped_grad_norm": 0.7819244265556335 + }, + { + "bits_per_byte": 3.4131862631495933, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.365840435028076, + "step": 370, + "unclipped_grad_norm": 0.8280189037322998 + }, + { + "bits_per_byte": 3.3082022056528055, + "learning_rate": 0.000285, + "loss_nats": 2.2930710315704346, + "step": 380, + "unclipped_grad_norm": 0.7057737112045288 + }, + { + "bits_per_byte": 3.3912378372464818, + "learning_rate": 0.0002925, + "loss_nats": 2.3506269454956055, + "step": 390, + "unclipped_grad_norm": 0.7360323071479797 + }, + { + "bits_per_byte": 3.361113700086654, + "learning_rate": 0.0003, + "loss_nats": 2.3297464847564697, + "step": 400, + "unclipped_grad_norm": 0.6539704203605652 + }, + { + "bits_per_byte": 3.4030207126223946, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.3587942123413086, + "step": 410, + "unclipped_grad_norm": 0.791742205619812 + }, + { + "bits_per_byte": 3.334176745412616, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.311075210571289, + "step": 420, + "unclipped_grad_norm": 0.8703852891921997 + }, + { + "bits_per_byte": 3.3591761435420335, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.3284034729003906, + "step": 430, + "unclipped_grad_norm": 0.8373144865036011 + }, + { + "bits_per_byte": 3.313656119476967, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.296851396560669, + "step": 440, + "unclipped_grad_norm": 1.3769525289535522 + }, + { + "bits_per_byte": 3.3562916505217624, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.326404094696045, + "step": 450, + "unclipped_grad_norm": 0.7585985660552979 + }, + { + "bits_per_byte": 3.2828990861619642, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2755322456359863, + "step": 460, + "unclipped_grad_norm": 0.9068207144737244 + }, + { + "bits_per_byte": 3.2566393991670237, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2573304176330566, + "step": 470, + "unclipped_grad_norm": 0.9806933999061584 + }, + { + "bits_per_byte": 3.2517141600116144, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2539165019989014, + "step": 480, + "unclipped_grad_norm": 1.2366756200790405 + }, + { + "bits_per_byte": 3.1895775161854854, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.2108466625213623, + "step": 490, + "unclipped_grad_norm": 0.7806023955345154 + }, + { + "bits_per_byte": 3.123298498230674, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.164905548095703, + "step": 500, + "unclipped_grad_norm": 1.1635692119598389 + }, + { + "bits_per_byte": 3.1673697404448724, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.195453405380249, + "step": 510, + "unclipped_grad_norm": 1.5141944885253906 + }, + { + "bits_per_byte": 3.1538006532570986, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1860480308532715, + "step": 520, + "unclipped_grad_norm": 1.1977208852767944 + }, + { + "bits_per_byte": 2.9931695452818388, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.07470703125, + "step": 530, + "unclipped_grad_norm": 1.6449142694473267 + }, + { + "bits_per_byte": 2.988713818763285, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0716185569763184, + "step": 540, + "unclipped_grad_norm": 1.1201231479644775 + }, + { + "bits_per_byte": 2.9807878263156216, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.066124677658081, + "step": 550, + "unclipped_grad_norm": 1.0647740364074707 + }, + { + "bits_per_byte": 2.990274733302516, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0727005004882812, + "step": 560, + "unclipped_grad_norm": 1.5665196180343628 + }, + { + "bits_per_byte": 2.920768976852217, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0245227813720703, + "step": 570, + "unclipped_grad_norm": 1.1736958026885986 + }, + { + "bits_per_byte": 2.9347649249794325, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.034224033355713, + "step": 580, + "unclipped_grad_norm": 1.3881648778915405 + }, + { + "bits_per_byte": 2.871172963961702, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9901454448699951, + "step": 590, + "unclipped_grad_norm": 0.9850356578826904 + }, + { + "bits_per_byte": 2.835541942321247, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9654479026794434, + "step": 600, + "unclipped_grad_norm": 1.141566514968872 + }, + { + "bits_per_byte": 2.8507572474430796, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.975994348526001, + "step": 610, + "unclipped_grad_norm": 1.0074148178100586 + }, + { + "bits_per_byte": 2.8309881856917016, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9622914791107178, + "step": 620, + "unclipped_grad_norm": 1.614481806755066 + }, + { + "bits_per_byte": 2.744016559150379, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9020073413848877, + "step": 630, + "unclipped_grad_norm": 1.23131263256073 + }, + { + "bits_per_byte": 2.7596193411846057, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9128223657608032, + "step": 640, + "unclipped_grad_norm": 1.2868338823318481 + }, + { + "bits_per_byte": 2.7369749014934524, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8971264362335205, + "step": 650, + "unclipped_grad_norm": 1.4851713180541992 + }, + { + "bits_per_byte": 2.7511890956046128, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.906978964805603, + "step": 660, + "unclipped_grad_norm": 0.9212717413902283 + }, + { + "bits_per_byte": 2.7285928110557007, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8913164138793945, + "step": 670, + "unclipped_grad_norm": 1.196967601776123 + }, + { + "bits_per_byte": 2.7059170705220903, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8755987882614136, + "step": 680, + "unclipped_grad_norm": 1.0040963888168335 + }, + { + "bits_per_byte": 2.678678114313137, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8567181825637817, + "step": 690, + "unclipped_grad_norm": 1.19021475315094 + }, + { + "bits_per_byte": 2.6497494285594567, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8366663455963135, + "step": 700, + "unclipped_grad_norm": 1.2448054552078247 + }, + { + "bits_per_byte": 2.6450725323518935, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8334245681762695, + "step": 710, + "unclipped_grad_norm": 1.2336615324020386 + }, + { + "bits_per_byte": 2.640725326886034, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8304113149642944, + "step": 720, + "unclipped_grad_norm": 1.0797665119171143 + }, + { + "bits_per_byte": 2.6141132474394757, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8119652271270752, + "step": 730, + "unclipped_grad_norm": 1.052964448928833 + }, + { + "bits_per_byte": 2.5497685183589187, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7673648595809937, + "step": 740, + "unclipped_grad_norm": 1.2775253057479858 + }, + { + "bits_per_byte": 2.5706501358787284, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7818388938903809, + "step": 750, + "unclipped_grad_norm": 1.240071415901184 + }, + { + "bits_per_byte": 2.470865801848127, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7126736640930176, + "step": 760, + "unclipped_grad_norm": 1.5425997972488403 + }, + { + "bits_per_byte": 2.521409439144896, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7477078437805176, + "step": 770, + "unclipped_grad_norm": 1.3344701528549194 + }, + { + "bits_per_byte": 2.472837410957618, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7140402793884277, + "step": 780, + "unclipped_grad_norm": 1.1049633026123047 + }, + { + "bits_per_byte": 2.473506767434776, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7145042419433594, + "step": 790, + "unclipped_grad_norm": 1.1233389377593994 + }, + { + "bits_per_byte": 2.401472866016794, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.664574146270752, + "step": 800, + "unclipped_grad_norm": 1.229269027709961 + }, + { + "bits_per_byte": 2.4415747486490784, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6923706531524658, + "step": 810, + "unclipped_grad_norm": 1.1753153800964355 + }, + { + "bits_per_byte": 2.4350285730092875, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6878331899642944, + "step": 820, + "unclipped_grad_norm": 1.128970742225647 + }, + { + "bits_per_byte": 2.4442745323023227, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.694242000579834, + "step": 830, + "unclipped_grad_norm": 1.1320509910583496 + }, + { + "bits_per_byte": 2.411402456347009, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6714568138122559, + "step": 840, + "unclipped_grad_norm": 1.2860697507858276 + }, + { + "bits_per_byte": 2.3597125546822393, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6356281042099, + "step": 850, + "unclipped_grad_norm": 1.1755540370941162 + }, + { + "bits_per_byte": 2.4397532803937905, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6911081075668335, + "step": 860, + "unclipped_grad_norm": 1.4042041301727295 + }, + { + "bits_per_byte": 2.4029735866282227, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.665614366531372, + "step": 870, + "unclipped_grad_norm": 1.2927662134170532 + }, + { + "bits_per_byte": 2.3685887512759196, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6417806148529053, + "step": 880, + "unclipped_grad_norm": 1.1862192153930664 + }, + { + "bits_per_byte": 2.3211230874816424, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6088799238204956, + "step": 890, + "unclipped_grad_norm": 1.135436773300171 + }, + { + "bits_per_byte": 2.364436574136139, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6389025449752808, + "step": 900, + "unclipped_grad_norm": 1.3485081195831299 + }, + { + "bits_per_byte": 2.347153177637681, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.626922607421875, + "step": 910, + "unclipped_grad_norm": 1.1310322284698486 + }, + { + "bits_per_byte": 2.338854326802947, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6211702823638916, + "step": 920, + "unclipped_grad_norm": 1.1499736309051514 + }, + { + "bits_per_byte": 2.28766523861749, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5856887102127075, + "step": 930, + "unclipped_grad_norm": 1.0895946025848389 + }, + { + "bits_per_byte": 2.3220005429663444, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6094881296157837, + "step": 940, + "unclipped_grad_norm": 1.0576175451278687 + }, + { + "bits_per_byte": 2.3596927766773903, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6356143951416016, + "step": 950, + "unclipped_grad_norm": 1.078567385673523 + }, + { + "bits_per_byte": 2.2833796029406397, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5827181339263916, + "step": 960, + "unclipped_grad_norm": 1.1438634395599365 + }, + { + "bits_per_byte": 2.2449070839426293, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5560510158538818, + "step": 970, + "unclipped_grad_norm": 1.2200326919555664 + }, + { + "bits_per_byte": 2.3829726922634817, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6517508029937744, + "step": 980, + "unclipped_grad_norm": 1.0979547500610352 + }, + { + "bits_per_byte": 2.2178129371257023, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5372707843780518, + "step": 990, + "unclipped_grad_norm": 0.9582167863845825 + }, + { + "bits_per_byte": 2.2343387500471095, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5487256050109863, + "step": 1000, + "unclipped_grad_norm": 1.1553164720535278 + }, + { + "bits_per_byte": 2.2444941535979077, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5557647943496704, + "step": 1010, + "unclipped_grad_norm": 1.5419076681137085 + }, + { + "bits_per_byte": 2.204213924956678, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278446674346924, + "step": 1020, + "unclipped_grad_norm": 1.2040239572525024 + }, + { + "bits_per_byte": 2.2117641353122064, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5330780744552612, + "step": 1030, + "unclipped_grad_norm": 1.308585286140442 + }, + { + "bits_per_byte": 2.2977618241016926, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5926871299743652, + "step": 1040, + "unclipped_grad_norm": 1.4551756381988525 + }, + { + "bits_per_byte": 2.2190181915429488, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5381062030792236, + "step": 1050, + "unclipped_grad_norm": 1.5392372608184814 + }, + { + "bits_per_byte": 2.226578548874878, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.543346643447876, + "step": 1060, + "unclipped_grad_norm": 1.1143345832824707 + }, + { + "bits_per_byte": 2.276697561006667, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.578086495399475, + "step": 1070, + "unclipped_grad_norm": 1.1605373620986938 + }, + { + "bits_per_byte": 2.2501855754629103, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5597097873687744, + "step": 1080, + "unclipped_grad_norm": 1.0904817581176758 + }, + { + "bits_per_byte": 2.178866605924565, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5102752447128296, + "step": 1090, + "unclipped_grad_norm": 1.1060513257980347 + }, + { + "bits_per_byte": 2.2050501046051734, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5284242630004883, + "step": 1100, + "unclipped_grad_norm": 1.2513083219528198 + }, + { + "bits_per_byte": 2.140159674573429, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.483445644378662, + "step": 1110, + "unclipped_grad_norm": 1.0150246620178223 + }, + { + "bits_per_byte": 2.197250347423243, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5230178833007812, + "step": 1120, + "unclipped_grad_norm": 1.0046241283416748 + }, + { + "bits_per_byte": 2.212073188135807, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.533292293548584, + "step": 1130, + "unclipped_grad_norm": 0.9860795736312866 + }, + { + "bits_per_byte": 2.1513696757393173, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4912158250808716, + "step": 1140, + "unclipped_grad_norm": 1.0693174600601196 + }, + { + "bits_per_byte": 2.217324850362554, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5369324684143066, + "step": 1150, + "unclipped_grad_norm": 1.2721859216690063 + }, + { + "bits_per_byte": 2.1164305403032837, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4669978618621826, + "step": 1160, + "unclipped_grad_norm": 1.2611263990402222 + }, + { + "bits_per_byte": 2.1545339845325424, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4934091567993164, + "step": 1170, + "unclipped_grad_norm": 1.396773099899292 + }, + { + "bits_per_byte": 2.157210722510571, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4952645301818848, + "step": 1180, + "unclipped_grad_norm": 1.0693202018737793 + }, + { + "bits_per_byte": 2.146851003570548, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4880837202072144, + "step": 1190, + "unclipped_grad_norm": 1.1468050479888916 + }, + { + "bits_per_byte": 2.1677991783762414, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.5026038885116577, + "step": 1200, + "unclipped_grad_norm": 1.1243788003921509 + }, + { + "bits_per_byte": 2.171953075342531, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5054831504821777, + "step": 1210, + "unclipped_grad_norm": 1.0842303037643433 + }, + { + "bits_per_byte": 2.1393570315418518, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4828892946243286, + "step": 1220, + "unclipped_grad_norm": 1.0247055292129517 + }, + { + "bits_per_byte": 2.133679196306267, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4789537191390991, + "step": 1230, + "unclipped_grad_norm": 1.0191700458526611 + }, + { + "bits_per_byte": 2.0961321159525497, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.452928066253662, + "step": 1240, + "unclipped_grad_norm": 1.0066498517990112 + }, + { + "bits_per_byte": 2.0423632880041502, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4156583547592163, + "step": 1250, + "unclipped_grad_norm": 1.1115199327468872 + }, + { + "bits_per_byte": 2.102061045858394, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4570376873016357, + "step": 1260, + "unclipped_grad_norm": 1.1423614025115967 + }, + { + "bits_per_byte": 2.1196135952054522, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4692041873931885, + "step": 1270, + "unclipped_grad_norm": 0.9267144203186035 + }, + { + "bits_per_byte": 2.1628902775726644, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4992012977600098, + "step": 1280, + "unclipped_grad_norm": 1.1472139358520508 + }, + { + "bits_per_byte": 2.104273774644395, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.458571434020996, + "step": 1290, + "unclipped_grad_norm": 1.0092620849609375 + }, + { + "bits_per_byte": 2.0607607881148975, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.428410530090332, + "step": 1300, + "unclipped_grad_norm": 1.0175256729125977 + }, + { + "bits_per_byte": 2.090551794880004, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.449060082435608, + "step": 1310, + "unclipped_grad_norm": 1.0536231994628906 + }, + { + "bits_per_byte": 2.0678022737891735, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4332913160324097, + "step": 1320, + "unclipped_grad_norm": 1.0326532125473022 + }, + { + "bits_per_byte": 2.079288479092348, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4412529468536377, + "step": 1330, + "unclipped_grad_norm": 1.0093759298324585 + }, + { + "bits_per_byte": 2.077430378532426, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.439965009689331, + "step": 1340, + "unclipped_grad_norm": 0.9901224970817566 + }, + { + "bits_per_byte": 2.085392143371469, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.445483684539795, + "step": 1350, + "unclipped_grad_norm": 0.9441472291946411 + }, + { + "bits_per_byte": 2.0594795173659697, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4275224208831787, + "step": 1360, + "unclipped_grad_norm": 1.1019998788833618 + }, + { + "bits_per_byte": 2.0796826633281253, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.441526174545288, + "step": 1370, + "unclipped_grad_norm": 1.3928191661834717 + }, + { + "bits_per_byte": 2.0298482824835147, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4069836139678955, + "step": 1380, + "unclipped_grad_norm": 1.0244042873382568 + }, + { + "bits_per_byte": 2.0804338835470944, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.442046880722046, + "step": 1390, + "unclipped_grad_norm": 1.095209002494812 + }, + { + "bits_per_byte": 2.195902519388431, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.5220836400985718, + "step": 1400, + "unclipped_grad_norm": 1.1566489934921265 + }, + { + "bits_per_byte": 2.0701701309262526, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4349325895309448, + "step": 1410, + "unclipped_grad_norm": 0.889271080493927 + }, + { + "bits_per_byte": 2.060555784795069, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4282684326171875, + "step": 1420, + "unclipped_grad_norm": 0.933581531047821 + }, + { + "bits_per_byte": 2.0393702739311848, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.413583755493164, + "step": 1430, + "unclipped_grad_norm": 1.1070539951324463 + }, + { + "bits_per_byte": 2.055492615553668, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4247589111328125, + "step": 1440, + "unclipped_grad_norm": 1.024774193763733 + }, + { + "bits_per_byte": 2.0178352943207476, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3986568450927734, + "step": 1450, + "unclipped_grad_norm": 0.9543858766555786 + }, + { + "bits_per_byte": 2.144940448302113, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4867594242095947, + "step": 1460, + "unclipped_grad_norm": 0.9325331449508667 + }, + { + "bits_per_byte": 2.1008117638825268, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4561717510223389, + "step": 1470, + "unclipped_grad_norm": 0.8672807216644287 + }, + { + "bits_per_byte": 2.0709982273901573, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4355065822601318, + "step": 1480, + "unclipped_grad_norm": 0.9597178101539612 + }, + { + "bits_per_byte": 2.0747749664031043, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.438124418258667, + "step": 1490, + "unclipped_grad_norm": 1.0223972797393799 + }, + { + "bits_per_byte": 1.9616643846877473, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3597221374511719, + "step": 1500, + "unclipped_grad_norm": 0.9415771961212158 + }, + { + "bits_per_byte": 1.978057942950641, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.371085286140442, + "step": 1510, + "unclipped_grad_norm": 1.0318301916122437 + }, + { + "bits_per_byte": 2.0205063568712975, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.4005082845687866, + "step": 1520, + "unclipped_grad_norm": 0.9729053378105164 + }, + { + "bits_per_byte": 2.0668845743641695, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4326552152633667, + "step": 1530, + "unclipped_grad_norm": 0.9350422620773315 + }, + { + "bits_per_byte": 2.045209256910628, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4176310300827026, + "step": 1540, + "unclipped_grad_norm": 0.9831364154815674 + }, + { + "bits_per_byte": 2.101370363532529, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4565589427947998, + "step": 1550, + "unclipped_grad_norm": 1.1195578575134277 + }, + { + "bits_per_byte": 1.9633480948396942, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.360889196395874, + "step": 1560, + "unclipped_grad_norm": 1.038403034210205 + }, + { + "bits_per_byte": 1.9669683296403562, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.363398551940918, + "step": 1570, + "unclipped_grad_norm": 0.8551042675971985 + }, + { + "bits_per_byte": 1.997066325437253, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.384260892868042, + "step": 1580, + "unclipped_grad_norm": 0.9270023107528687 + }, + { + "bits_per_byte": 2.0065728384463455, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.390850305557251, + "step": 1590, + "unclipped_grad_norm": 0.9501463174819946 + }, + { + "bits_per_byte": 1.9863786355820345, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3768527507781982, + "step": 1600, + "unclipped_grad_norm": 0.8800983428955078 + }, + { + "bits_per_byte": 1.9619350853802053, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3599097728729248, + "step": 1610, + "unclipped_grad_norm": 0.911471962928772 + }, + { + "bits_per_byte": 1.9591475905750209, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3579776287078857, + "step": 1620, + "unclipped_grad_norm": 0.9253746271133423 + }, + { + "bits_per_byte": 1.9975408255709834, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3845897912979126, + "step": 1630, + "unclipped_grad_norm": 0.932426393032074 + }, + { + "bits_per_byte": 1.9590067367839643, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3578799962997437, + "step": 1640, + "unclipped_grad_norm": 0.9996587634086609 + }, + { + "bits_per_byte": 1.9568615971797525, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3563930988311768, + "step": 1650, + "unclipped_grad_norm": 0.9020648002624512 + }, + { + "bits_per_byte": 1.9902011219801112, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.379502296447754, + "step": 1660, + "unclipped_grad_norm": 0.9605674743652344 + }, + { + "bits_per_byte": 1.9362635790512155, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3421156406402588, + "step": 1670, + "unclipped_grad_norm": 0.9840894937515259 + }, + { + "bits_per_byte": 1.9741790462430813, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3683966398239136, + "step": 1680, + "unclipped_grad_norm": 0.9542845487594604 + }, + { + "bits_per_byte": 1.975246198591685, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3691363334655762, + "step": 1690, + "unclipped_grad_norm": 0.8774324655532837 + }, + { + "bits_per_byte": 1.936552509904665, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.342315912246704, + "step": 1700, + "unclipped_grad_norm": 0.9619165658950806 + }, + { + "bits_per_byte": 1.9815170300074447, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3734829425811768, + "step": 1710, + "unclipped_grad_norm": 1.017353892326355 + }, + { + "bits_per_byte": 1.9011681113855474, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3177893161773682, + "step": 1720, + "unclipped_grad_norm": 0.890689492225647 + }, + { + "bits_per_byte": 1.9742225578537496, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.36842679977417, + "step": 1730, + "unclipped_grad_norm": 0.9096551537513733 + }, + { + "bits_per_byte": 1.9832260216090685, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3746675252914429, + "step": 1740, + "unclipped_grad_norm": 0.9014292359352112 + }, + { + "bits_per_byte": 1.9548246346629332, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3549811840057373, + "step": 1750, + "unclipped_grad_norm": 0.8635335564613342 + }, + { + "bits_per_byte": 1.9935654465962895, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3818342685699463, + "step": 1760, + "unclipped_grad_norm": 0.9282572865486145 + }, + { + "bits_per_byte": 1.9777825987266096, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.370894432067871, + "step": 1770, + "unclipped_grad_norm": 0.9273641705513 + }, + { + "bits_per_byte": 2.0529696300655105, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.4230101108551025, + "step": 1780, + "unclipped_grad_norm": 0.9726579189300537 + }, + { + "bits_per_byte": 1.9025646105105534, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3187572956085205, + "step": 1790, + "unclipped_grad_norm": 0.9649468660354614 + }, + { + "bits_per_byte": 2.051186341958714, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.4217740297317505, + "step": 1800, + "unclipped_grad_norm": 1.1441059112548828 + }, + { + "bits_per_byte": 1.9169516471858308, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3287296295166016, + "step": 1810, + "unclipped_grad_norm": 1.0408838987350464 + }, + { + "bits_per_byte": 1.9713771048952242, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3664544820785522, + "step": 1820, + "unclipped_grad_norm": 0.9737800359725952 + }, + { + "bits_per_byte": 1.9989835880290714, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.385589838027954, + "step": 1830, + "unclipped_grad_norm": 0.8691428899765015 + }, + { + "bits_per_byte": 1.9804913254776988, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.372771978378296, + "step": 1840, + "unclipped_grad_norm": 0.915309727191925 + }, + { + "bits_per_byte": 1.918886795973339, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.330070972442627, + "step": 1850, + "unclipped_grad_norm": 0.9071885347366333 + }, + { + "bits_per_byte": 1.9458363053808903, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3487509489059448, + "step": 1860, + "unclipped_grad_norm": 0.8704239726066589 + }, + { + "bits_per_byte": 2.024965007094916, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4035987854003906, + "step": 1870, + "unclipped_grad_norm": 0.9987708330154419 + }, + { + "bits_per_byte": 1.9165194547842126, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3284300565719604, + "step": 1880, + "unclipped_grad_norm": 0.868863582611084 + }, + { + "bits_per_byte": 1.9462939512148363, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3490681648254395, + "step": 1890, + "unclipped_grad_norm": 0.8501539826393127 + }, + { + "bits_per_byte": 1.9456569274760405, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3486266136169434, + "step": 1900, + "unclipped_grad_norm": 0.8660019040107727 + }, + { + "bits_per_byte": 1.9970934986960893, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.384279727935791, + "step": 1910, + "unclipped_grad_norm": 0.8706849217414856 + }, + { + "bits_per_byte": 1.9090278905126286, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3232372999191284, + "step": 1920, + "unclipped_grad_norm": 0.9953081607818604 + }, + { + "bits_per_byte": 1.9556608143114282, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3555607795715332, + "step": 1930, + "unclipped_grad_norm": 0.8388954401016235 + }, + { + "bits_per_byte": 1.9678466450383125, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3640073537826538, + "step": 1940, + "unclipped_grad_norm": 0.9170766472816467 + }, + { + "bits_per_byte": 1.9103982482747035, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.324187159538269, + "step": 1950, + "unclipped_grad_norm": 0.942866861820221 + }, + { + "bits_per_byte": 1.9025671902503163, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3187590837478638, + "step": 1960, + "unclipped_grad_norm": 0.8135740756988525 + }, + { + "bits_per_byte": 1.9075963069268465, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3222450017929077, + "step": 1970, + "unclipped_grad_norm": 0.9107059836387634 + }, + { + "bits_per_byte": 1.9379792779762228, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3433048725128174, + "step": 1980, + "unclipped_grad_norm": 0.9617857336997986 + }, + { + "bits_per_byte": 1.9109520324104816, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3245710134506226, + "step": 1990, + "unclipped_grad_norm": 0.9190393090248108 + }, + { + "bits_per_byte": 1.9811283492164946, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.373213529586792, + "step": 2000, + "unclipped_grad_norm": 0.878118097782135 + }, + { + "bits_per_byte": 1.9843879363982975, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.375472903251648, + "step": 2010, + "unclipped_grad_norm": 1.0113674402236938 + }, + { + "bits_per_byte": 1.8836764438968945, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3056650161743164, + "step": 2020, + "unclipped_grad_norm": 0.8797336220741272 + }, + { + "bits_per_byte": 1.9153360421636256, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3276097774505615, + "step": 2030, + "unclipped_grad_norm": 0.8666315674781799 + }, + { + "bits_per_byte": 1.906092662610353, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3212027549743652, + "step": 2040, + "unclipped_grad_norm": 0.8591557145118713 + }, + { + "bits_per_byte": 1.8798058023565762, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3029820919036865, + "step": 2050, + "unclipped_grad_norm": 0.9546817541122437 + }, + { + "bits_per_byte": 1.8574308594793247, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2874729633331299, + "step": 2060, + "unclipped_grad_norm": 0.8160566687583923 + }, + { + "bits_per_byte": 1.9859665651505671, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3765671253204346, + "step": 2070, + "unclipped_grad_norm": 0.8717494606971741 + }, + { + "bits_per_byte": 1.952941940583938, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.353676199913025, + "step": 2080, + "unclipped_grad_norm": 0.9155706763267517 + }, + { + "bits_per_byte": 1.8808098370723132, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.303678035736084, + "step": 2090, + "unclipped_grad_norm": 0.8717590570449829 + }, + { + "bits_per_byte": 1.8310299747280259, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.269173264503479, + "step": 2100, + "unclipped_grad_norm": 0.9355071783065796 + }, + { + "bits_per_byte": 1.868735623085839, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2953088283538818, + "step": 2110, + "unclipped_grad_norm": 0.839803159236908 + }, + { + "bits_per_byte": 1.8925072370707472, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3117860555648804, + "step": 2120, + "unclipped_grad_norm": 1.064001202583313 + }, + { + "bits_per_byte": 1.9234560310414621, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.333238124847412, + "step": 2130, + "unclipped_grad_norm": 0.9675858020782471 + }, + { + "bits_per_byte": 1.9467920129717349, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3494133949279785, + "step": 2140, + "unclipped_grad_norm": 0.9227033257484436 + }, + { + "bits_per_byte": 1.8425244351984418, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2771406173706055, + "step": 2150, + "unclipped_grad_norm": 0.9172095060348511 + }, + { + "bits_per_byte": 1.9385437250363546, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.343696117401123, + "step": 2160, + "unclipped_grad_norm": 0.9335131049156189 + }, + { + "bits_per_byte": 1.8590810330143535, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2886167764663696, + "step": 2170, + "unclipped_grad_norm": 0.8850465416908264 + }, + { + "bits_per_byte": 1.9348510855396792, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3411365747451782, + "step": 2180, + "unclipped_grad_norm": 0.8827643990516663 + }, + { + "bits_per_byte": 1.9013160164652894, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3178918361663818, + "step": 2190, + "unclipped_grad_norm": 0.9066281318664551 + }, + { + "bits_per_byte": 1.8761559865399657, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3004522323608398, + "step": 2200, + "unclipped_grad_norm": 0.844115674495697 + }, + { + "bits_per_byte": 1.8719651133037407, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2975473403930664, + "step": 2210, + "unclipped_grad_norm": 0.9099118709564209 + }, + { + "bits_per_byte": 1.8935372411667646, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3125, + "step": 2220, + "unclipped_grad_norm": 1.0520122051239014 + }, + { + "bits_per_byte": 1.8237762624625853, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2641453742980957, + "step": 2230, + "unclipped_grad_norm": 0.8068414330482483 + }, + { + "bits_per_byte": 1.9063850331168197, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3214054107666016, + "step": 2240, + "unclipped_grad_norm": 0.8128858804702759 + }, + { + "bits_per_byte": 1.9127232817317177, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.325798749923706, + "step": 2250, + "unclipped_grad_norm": 0.8365156650543213 + }, + { + "bits_per_byte": 1.965086839439917, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3620944023132324, + "step": 2260, + "unclipped_grad_norm": 0.9489718675613403 + }, + { + "bits_per_byte": 1.8335648270190927, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.270930290222168, + "step": 2270, + "unclipped_grad_norm": 0.8451220989227295 + }, + { + "bits_per_byte": 1.8897092513238603, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.3098466396331787, + "step": 2280, + "unclipped_grad_norm": 0.8222805261611938 + }, + { + "bits_per_byte": 1.8508822760824215, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2829338312149048, + "step": 2290, + "unclipped_grad_norm": 0.8723782896995544 + }, + { + "bits_per_byte": 1.839125714052091, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.274784803390503, + "step": 2300, + "unclipped_grad_norm": 0.8368175625801086 + }, + { + "bits_per_byte": 1.846592168856649, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2799601554870605, + "step": 2310, + "unclipped_grad_norm": 0.8264262080192566 + }, + { + "bits_per_byte": 1.893433707610945, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.3124282360076904, + "step": 2320, + "unclipped_grad_norm": 0.8465461134910583 + }, + { + "bits_per_byte": 1.8498577754312318, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2822237014770508, + "step": 2330, + "unclipped_grad_norm": 0.8474167585372925 + }, + { + "bits_per_byte": 1.8556085313107824, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2862098217010498, + "step": 2340, + "unclipped_grad_norm": 0.8308321237564087 + }, + { + "bits_per_byte": 1.9187394068415498, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.329968810081482, + "step": 2350, + "unclipped_grad_norm": 0.8866831660270691 + }, + { + "bits_per_byte": 1.8564204614055058, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.286772608757019, + "step": 2360, + "unclipped_grad_norm": 0.8822014927864075 + }, + { + "bits_per_byte": 1.767254508221825, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2249674797058105, + "step": 2370, + "unclipped_grad_norm": 0.8126892447471619 + }, + { + "bits_per_byte": 1.8880877988915257, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.308722734451294, + "step": 2380, + "unclipped_grad_norm": 0.8927111625671387 + }, + { + "bits_per_byte": 1.8776909316989163, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3015161752700806, + "step": 2390, + "unclipped_grad_norm": 0.864246666431427 + }, + { + "bits_per_byte": 1.9130975159799952, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3260581493377686, + "step": 2400, + "unclipped_grad_norm": 0.9290322065353394 + }, + { + "bits_per_byte": 1.848231851449975, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2810966968536377, + "step": 2410, + "unclipped_grad_norm": 0.9126319885253906 + }, + { + "bits_per_byte": 1.8675582298580322, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2944927215576172, + "step": 2420, + "unclipped_grad_norm": 0.839264452457428 + }, + { + "bits_per_byte": 1.8763154144573155, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.3005627393722534, + "step": 2430, + "unclipped_grad_norm": 0.8645790219306946 + }, + { + "bits_per_byte": 1.8291109923096986, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2678431272506714, + "step": 2440, + "unclipped_grad_norm": 0.8184406161308289 + }, + { + "bits_per_byte": 1.9499289765234726, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3515877723693848, + "step": 2450, + "unclipped_grad_norm": 0.9486745595932007 + }, + { + "bits_per_byte": 1.82168426549749, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2626953125, + "step": 2460, + "unclipped_grad_norm": 0.8475955128669739 + }, + { + "bits_per_byte": 1.8165447359591058, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2591328620910645, + "step": 2470, + "unclipped_grad_norm": 0.8975434899330139 + }, + { + "bits_per_byte": 1.9011218480524652, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.317757248878479, + "step": 2480, + "unclipped_grad_norm": 0.9190422296524048 + }, + { + "bits_per_byte": 1.8940924011637497, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.31288480758667, + "step": 2490, + "unclipped_grad_norm": 0.8232344388961792 + }, + { + "bits_per_byte": 1.8466803959565414, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2800213098526, + "step": 2500, + "unclipped_grad_norm": 0.7958891987800598 + }, + { + "bits_per_byte": 1.815657993411257, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2585182189941406, + "step": 2510, + "unclipped_grad_norm": 0.9414010643959045 + }, + { + "bits_per_byte": 1.8382814512190053, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2741996049880981, + "step": 2520, + "unclipped_grad_norm": 0.8523510694503784 + }, + { + "bits_per_byte": 1.8459320994426374, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2795026302337646, + "step": 2530, + "unclipped_grad_norm": 0.7949919700622559 + }, + { + "bits_per_byte": 1.8449038151731287, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2787898778915405, + "step": 2540, + "unclipped_grad_norm": 0.8316088318824768 + }, + { + "bits_per_byte": 1.8031964744950397, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2498805522918701, + "step": 2550, + "unclipped_grad_norm": 1.0299166440963745 + }, + { + "bits_per_byte": 1.7815225329027051, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2348573207855225, + "step": 2560, + "unclipped_grad_norm": 0.8839435577392578 + }, + { + "bits_per_byte": 1.8260208080389957, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.265701174736023, + "step": 2570, + "unclipped_grad_norm": 0.9318905472755432 + }, + { + "bits_per_byte": 1.7994856048373729, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.247308373451233, + "step": 2580, + "unclipped_grad_norm": 0.8335656523704529 + }, + { + "bits_per_byte": 1.796869232769797, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2454948425292969, + "step": 2590, + "unclipped_grad_norm": 0.8446885347366333 + }, + { + "bits_per_byte": 1.824228920799656, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2644591331481934, + "step": 2600, + "unclipped_grad_norm": 0.8396548628807068 + }, + { + "bits_per_byte": 1.8444822856958638, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2784976959228516, + "step": 2610, + "unclipped_grad_norm": 0.8530864119529724 + }, + { + "bits_per_byte": 1.7628988756060733, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2219483852386475, + "step": 2620, + "unclipped_grad_norm": 0.8868519067764282 + }, + { + "bits_per_byte": 1.7732109553518074, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2290961742401123, + "step": 2630, + "unclipped_grad_norm": 0.7921182513237 + }, + { + "bits_per_byte": 1.7536496486426716, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2155373096466064, + "step": 2640, + "unclipped_grad_norm": 0.8794499039649963 + }, + { + "bits_per_byte": 1.832483744075769, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2701809406280518, + "step": 2650, + "unclipped_grad_norm": 0.8381353616714478 + }, + { + "bits_per_byte": 1.733659417202284, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.201681137084961, + "step": 2660, + "unclipped_grad_norm": 0.7765254974365234 + }, + { + "bits_per_byte": 1.8051608603331948, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2512421607971191, + "step": 2670, + "unclipped_grad_norm": 0.8680312633514404 + }, + { + "bits_per_byte": 1.923795868759567, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.3334736824035645, + "step": 2680, + "unclipped_grad_norm": 0.9085569977760315 + }, + { + "bits_per_byte": 1.8284822237381442, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2674072980880737, + "step": 2690, + "unclipped_grad_norm": 0.8529220819473267 + }, + { + "bits_per_byte": 1.7587946816258833, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2191035747528076, + "step": 2700, + "unclipped_grad_norm": 0.8387420773506165 + }, + { + "bits_per_byte": 1.8291632750355609, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2678793668746948, + "step": 2710, + "unclipped_grad_norm": 0.7956082820892334 + }, + { + "bits_per_byte": 1.872990129902883, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.298257827758789, + "step": 2720, + "unclipped_grad_norm": 0.7902501821517944 + }, + { + "bits_per_byte": 1.7687435340129951, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2259995937347412, + "step": 2730, + "unclipped_grad_norm": 0.8540245294570923 + }, + { + "bits_per_byte": 1.8104493268472261, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2549078464508057, + "step": 2740, + "unclipped_grad_norm": 0.8416579961776733 + }, + { + "bits_per_byte": 1.7908353934469263, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2413125038146973, + "step": 2750, + "unclipped_grad_norm": 0.7708072662353516 + }, + { + "bits_per_byte": 1.9183405790741987, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3296923637390137, + "step": 2760, + "unclipped_grad_norm": 0.9251072406768799 + }, + { + "bits_per_byte": 1.8352038216818154, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.272066354751587, + "step": 2770, + "unclipped_grad_norm": 0.8847647905349731 + }, + { + "bits_per_byte": 1.8372603902208329, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2734918594360352, + "step": 2780, + "unclipped_grad_norm": 0.8566639423370361 + }, + { + "bits_per_byte": 1.8040237110456898, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2504539489746094, + "step": 2790, + "unclipped_grad_norm": 0.8849772214889526 + }, + { + "bits_per_byte": 1.8313404034128333, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2693884372711182, + "step": 2800, + "unclipped_grad_norm": 0.8829222321510315 + }, + { + "bits_per_byte": 1.7673478948012433, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2250322103500366, + "step": 2810, + "unclipped_grad_norm": 0.7698772549629211 + }, + { + "bits_per_byte": 1.8410065163219265, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2760884761810303, + "step": 2820, + "unclipped_grad_norm": 0.875986635684967 + }, + { + "bits_per_byte": 1.7626952481474518, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2218072414398193, + "step": 2830, + "unclipped_grad_norm": 0.815294623374939 + }, + { + "bits_per_byte": 1.808952733819418, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2538704872131348, + "step": 2840, + "unclipped_grad_norm": 0.7992775440216064 + }, + { + "bits_per_byte": 1.7569166310784614, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.217801809310913, + "step": 2850, + "unclipped_grad_norm": 0.8472921848297119 + }, + { + "bits_per_byte": 1.792434144169347, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2424206733703613, + "step": 2860, + "unclipped_grad_norm": 0.8268847465515137 + }, + { + "bits_per_byte": 1.793870887234655, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2434165477752686, + "step": 2870, + "unclipped_grad_norm": 0.8292641639709473 + }, + { + "bits_per_byte": 1.79901058875569, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2469791173934937, + "step": 2880, + "unclipped_grad_norm": 0.8117106556892395 + }, + { + "bits_per_byte": 1.7566032786885895, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2175846099853516, + "step": 2890, + "unclipped_grad_norm": 0.8036413788795471 + }, + { + "bits_per_byte": 1.7856263829175936, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2377018928527832, + "step": 2900, + "unclipped_grad_norm": 0.7987298369407654 + }, + { + "bits_per_byte": 1.7581136303284666, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2186315059661865, + "step": 2910, + "unclipped_grad_norm": 0.8029706478118896 + }, + { + "bits_per_byte": 1.8051701473963413, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2512485980987549, + "step": 2920, + "unclipped_grad_norm": 0.7970700263977051 + }, + { + "bits_per_byte": 1.756548588205615, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2175467014312744, + "step": 2930, + "unclipped_grad_norm": 0.8035452961921692 + }, + { + "bits_per_byte": 1.7718779178249697, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2281721830368042, + "step": 2940, + "unclipped_grad_norm": 0.8500884771347046 + }, + { + "bits_per_byte": 1.8224836408587004, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.263249397277832, + "step": 2950, + "unclipped_grad_norm": 0.8720842599868774 + }, + { + "bits_per_byte": 1.8354507887684544, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2722375392913818, + "step": 2960, + "unclipped_grad_norm": 0.8293792009353638 + }, + { + "bits_per_byte": 1.8121727649915225, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.256102442741394, + "step": 2970, + "unclipped_grad_norm": 0.8630385994911194 + }, + { + "bits_per_byte": 1.8230131754407068, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2636164426803589, + "step": 2980, + "unclipped_grad_norm": 0.7837274670600891 + }, + { + "bits_per_byte": 1.755593396562723, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2168846130371094, + "step": 2990, + "unclipped_grad_norm": 0.8064510226249695 + }, + { + "bits_per_byte": 1.8344553531852605, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.271547555923462, + "step": 3000, + "unclipped_grad_norm": 0.793539822101593 + }, + { + "bits_per_byte": 1.7345470196633868, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2022963762283325, + "step": 3010, + "unclipped_grad_norm": 0.8690526485443115 + }, + { + "bits_per_byte": 1.8061236192127248, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2519094944000244, + "step": 3020, + "unclipped_grad_norm": 0.9625511169433594 + }, + { + "bits_per_byte": 1.7339495519342893, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.201882243156433, + "step": 3030, + "unclipped_grad_norm": 0.7808479070663452 + }, + { + "bits_per_byte": 1.725635050678327, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1961190700531006, + "step": 3040, + "unclipped_grad_norm": 0.8198943138122559 + }, + { + "bits_per_byte": 1.8739061095013785, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2988927364349365, + "step": 3050, + "unclipped_grad_norm": 0.8317142724990845 + }, + { + "bits_per_byte": 1.8166181725510242, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2591837644577026, + "step": 3060, + "unclipped_grad_norm": 0.8308979868888855 + }, + { + "bits_per_byte": 1.7248370511783235, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.195565938949585, + "step": 3070, + "unclipped_grad_norm": 0.8185245394706726 + }, + { + "bits_per_byte": 1.8189339189448919, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.260788917541504, + "step": 3080, + "unclipped_grad_norm": 0.8484242558479309 + }, + { + "bits_per_byte": 1.7918220579149262, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.24199640750885, + "step": 3090, + "unclipped_grad_norm": 0.8042747974395752 + }, + { + "bits_per_byte": 1.7567866121944091, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2177116870880127, + "step": 3100, + "unclipped_grad_norm": 0.8309313058853149 + }, + { + "bits_per_byte": 1.9063415215061514, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.3213752508163452, + "step": 3110, + "unclipped_grad_norm": 0.8743271231651306 + }, + { + "bits_per_byte": 1.781927724028138, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.235138177871704, + "step": 3120, + "unclipped_grad_norm": 0.8567306995391846 + }, + { + "bits_per_byte": 1.731061619261001, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1998804807662964, + "step": 3130, + "unclipped_grad_norm": 0.8699595332145691 + }, + { + "bits_per_byte": 1.7586823769548698, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.219025731086731, + "step": 3140, + "unclipped_grad_norm": 0.7885372042655945 + }, + { + "bits_per_byte": 1.744890228268927, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.209465742111206, + "step": 3150, + "unclipped_grad_norm": 0.7910884022712708 + }, + { + "bits_per_byte": 1.7638991267034914, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2226417064666748, + "step": 3160, + "unclipped_grad_norm": 0.8217395544052124 + }, + { + "bits_per_byte": 1.816583260072899, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2591595649719238, + "step": 3170, + "unclipped_grad_norm": 0.8992500305175781 + }, + { + "bits_per_byte": 1.8054272614593814, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2514268159866333, + "step": 3180, + "unclipped_grad_norm": 0.8302581906318665 + }, + { + "bits_per_byte": 1.7809063190646637, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.234430193901062, + "step": 3190, + "unclipped_grad_norm": 0.8399643898010254 + }, + { + "bits_per_byte": 1.7853408917171614, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.237504005432129, + "step": 3200, + "unclipped_grad_norm": 0.8274973630905151 + }, + { + "bits_per_byte": 1.7397025435883011, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2058699131011963, + "step": 3210, + "unclipped_grad_norm": 0.8199328184127808 + }, + { + "bits_per_byte": 1.7292408389363165, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1986184120178223, + "step": 3220, + "unclipped_grad_norm": 0.81409752368927 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2331104278564453, + "step": 3230, + "unclipped_grad_norm": 0.816062867641449 + }, + { + "bits_per_byte": 1.7934072220079287, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2430951595306396, + "step": 3240, + "unclipped_grad_norm": 0.9001877307891846 + }, + { + "bits_per_byte": 1.765244718963842, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2235743999481201, + "step": 3250, + "unclipped_grad_norm": 0.8105912804603577 + }, + { + "bits_per_byte": 1.7706184888727015, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2272992134094238, + "step": 3260, + "unclipped_grad_norm": 0.8049122095108032 + }, + { + "bits_per_byte": 1.8044983831620713, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2507829666137695, + "step": 3270, + "unclipped_grad_norm": 0.8440065979957581 + }, + { + "bits_per_byte": 1.793506112032175, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2431637048721313, + "step": 3280, + "unclipped_grad_norm": 0.8675442337989807 + }, + { + "bits_per_byte": 1.7443828794488816, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2091140747070312, + "step": 3290, + "unclipped_grad_norm": 0.8013374209403992 + }, + { + "bits_per_byte": 1.7796785349201543, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.233579158782959, + "step": 3300, + "unclipped_grad_norm": 0.8424150943756104 + }, + { + "bits_per_byte": 1.7358951916635004, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.203230857849121, + "step": 3310, + "unclipped_grad_norm": 0.7571165561676025 + }, + { + "bits_per_byte": 1.7447595214542713, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2093751430511475, + "step": 3320, + "unclipped_grad_norm": 0.7720819115638733 + }, + { + "bits_per_byte": 1.7607893364105902, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2204861640930176, + "step": 3330, + "unclipped_grad_norm": 0.8520592451095581 + }, + { + "bits_per_byte": 1.6976324914995427, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1767091751098633, + "step": 3340, + "unclipped_grad_norm": 0.80418860912323 + }, + { + "bits_per_byte": 1.7457931371859567, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2100915908813477, + "step": 3350, + "unclipped_grad_norm": 0.8466405868530273 + }, + { + "bits_per_byte": 1.7522414546974068, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2145612239837646, + "step": 3360, + "unclipped_grad_norm": 0.7725750803947449 + }, + { + "bits_per_byte": 1.7121540186077948, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.186774730682373, + "step": 3370, + "unclipped_grad_norm": 0.779502809047699 + }, + { + "bits_per_byte": 1.7592602386617688, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2194262742996216, + "step": 3380, + "unclipped_grad_norm": 0.8456810712814331 + }, + { + "bits_per_byte": 1.7687302913488787, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2259904146194458, + "step": 3390, + "unclipped_grad_norm": 0.8990693688392639 + }, + { + "bits_per_byte": 1.7458971866897286, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2101637125015259, + "step": 3400, + "unclipped_grad_norm": 0.8017266392707825 + }, + { + "bits_per_byte": 1.738469084016313, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.205014944076538, + "step": 3410, + "unclipped_grad_norm": 0.8378265500068665 + }, + { + "bits_per_byte": 1.7225225086630118, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1939616203308105, + "step": 3420, + "unclipped_grad_norm": 0.8268731832504272 + }, + { + "bits_per_byte": 1.7602646173428078, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.2201224565505981, + "step": 3430, + "unclipped_grad_norm": 0.8129444122314453 + }, + { + "bits_per_byte": 1.768388733804265, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.225753664970398, + "step": 3440, + "unclipped_grad_norm": 0.8220692873001099 + }, + { + "bits_per_byte": 1.8502017407329574, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2824621200561523, + "step": 3450, + "unclipped_grad_norm": 0.8645455837249756 + }, + { + "bits_per_byte": 1.8368216624784819, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2731877565383911, + "step": 3460, + "unclipped_grad_norm": 0.8615949153900146 + }, + { + "bits_per_byte": 1.785747458703801, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.237785816192627, + "step": 3470, + "unclipped_grad_norm": 0.8510363101959229 + }, + { + "bits_per_byte": 1.7659735814381987, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2240796089172363, + "step": 3480, + "unclipped_grad_norm": 0.8455166816711426 + }, + { + "bits_per_byte": 1.6893135186946577, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1709429025650024, + "step": 3490, + "unclipped_grad_norm": 0.8278471231460571 + }, + { + "bits_per_byte": 1.7031111708254283, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.180506706237793, + "step": 3500, + "unclipped_grad_norm": 0.814669132232666 + }, + { + "bits_per_byte": 1.7385170671759038, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2050482034683228, + "step": 3510, + "unclipped_grad_norm": 0.9123345017433167 + }, + { + "bits_per_byte": 1.700717172325418, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.178847312927246, + "step": 3520, + "unclipped_grad_norm": 0.7641705870628357 + }, + { + "bits_per_byte": 1.742950951897798, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2081215381622314, + "step": 3530, + "unclipped_grad_norm": 0.855086624622345 + }, + { + "bits_per_byte": 1.733061433525234, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2012666463851929, + "step": 3540, + "unclipped_grad_norm": 0.8321052193641663 + }, + { + "bits_per_byte": 1.7782098030817857, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2325611114501953, + "step": 3550, + "unclipped_grad_norm": 0.8241682052612305 + }, + { + "bits_per_byte": 1.7088610667917246, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1844922304153442, + "step": 3560, + "unclipped_grad_norm": 0.7756084203720093 + }, + { + "bits_per_byte": 1.7644671853992913, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.223035454750061, + "step": 3570, + "unclipped_grad_norm": 0.8851686716079712 + }, + { + "bits_per_byte": 1.8350844657221166, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2719836235046387, + "step": 3580, + "unclipped_grad_norm": 0.8648787140846252 + }, + { + "bits_per_byte": 1.758608252432348, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2189743518829346, + "step": 3590, + "unclipped_grad_norm": 0.8309826254844666 + }, + { + "bits_per_byte": 1.7417491371335687, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2072885036468506, + "step": 3600, + "unclipped_grad_norm": 0.9055862426757812 + }, + { + "bits_per_byte": 1.7217079988285255, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.193397045135498, + "step": 3610, + "unclipped_grad_norm": 0.8098222613334656 + }, + { + "bits_per_byte": 1.7173626851718258, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.190385103225708, + "step": 3620, + "unclipped_grad_norm": 0.8190554976463318 + }, + { + "bits_per_byte": 1.7436209963055593, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2085859775543213, + "step": 3630, + "unclipped_grad_norm": 0.8753142356872559 + }, + { + "bits_per_byte": 1.6961333187319716, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1756700277328491, + "step": 3640, + "unclipped_grad_norm": 0.841607391834259 + }, + { + "bits_per_byte": 1.7716670670950119, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.228026032447815, + "step": 3650, + "unclipped_grad_norm": 0.8459575772285461 + }, + { + "bits_per_byte": 1.6801827877777005, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.164613962173462, + "step": 3660, + "unclipped_grad_norm": 0.8714450001716614 + }, + { + "bits_per_byte": 1.6935544388822839, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1738824844360352, + "step": 3670, + "unclipped_grad_norm": 0.7989607453346252 + }, + { + "bits_per_byte": 1.7486903569223915, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.2120997905731201, + "step": 3680, + "unclipped_grad_norm": 0.8391386270523071 + }, + { + "bits_per_byte": 1.7119070515211559, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1866035461425781, + "step": 3690, + "unclipped_grad_norm": 0.8321404457092285 + }, + { + "bits_per_byte": 1.6629633688253633, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.152678370475769, + "step": 3700, + "unclipped_grad_norm": 0.8227249383926392 + }, + { + "bits_per_byte": 1.726790602109474, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1969200372695923, + "step": 3710, + "unclipped_grad_norm": 0.8174790143966675 + }, + { + "bits_per_byte": 1.7194202856067484, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1918113231658936, + "step": 3720, + "unclipped_grad_norm": 0.822218120098114 + }, + { + "bits_per_byte": 1.767769424278508, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2253243923187256, + "step": 3730, + "unclipped_grad_norm": 0.8156639933586121 + }, + { + "bits_per_byte": 1.7041993050574373, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1812609434127808, + "step": 3740, + "unclipped_grad_norm": 0.7992775440216064 + }, + { + "bits_per_byte": 1.6957810982630046, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1754258871078491, + "step": 3750, + "unclipped_grad_norm": 0.8203845620155334 + }, + { + "bits_per_byte": 1.6945959658159089, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1746044158935547, + "step": 3760, + "unclipped_grad_norm": 0.8056831359863281 + }, + { + "bits_per_byte": 1.7464552703917786, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2105505466461182, + "step": 3770, + "unclipped_grad_norm": 0.9159992337226868 + }, + { + "bits_per_byte": 1.6766162115641075, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.1621417999267578, + "step": 3780, + "unclipped_grad_norm": 0.7913095951080322 + }, + { + "bits_per_byte": 1.7293030966559289, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1986615657806396, + "step": 3790, + "unclipped_grad_norm": 0.8222495913505554 + }, + { + "bits_per_byte": 1.6807076788281337, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.164977788925171, + "step": 3800, + "unclipped_grad_norm": 0.829034686088562 + }, + { + "bits_per_byte": 1.6730726810257301, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1596856117248535, + "step": 3810, + "unclipped_grad_norm": 0.8109799027442932 + }, + { + "bits_per_byte": 1.7832320404522815, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.2360422611236572, + "step": 3820, + "unclipped_grad_norm": 0.8464360237121582 + }, + { + "bits_per_byte": 1.7013447370184165, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.179282307624817, + "step": 3830, + "unclipped_grad_norm": 0.849337637424469 + }, + { + "bits_per_byte": 1.7095980124506718, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1850030422210693, + "step": 3840, + "unclipped_grad_norm": 0.8549852967262268 + }, + { + "bits_per_byte": 1.7414235739754853, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.207062840461731, + "step": 3850, + "unclipped_grad_norm": 0.8230528235435486 + }, + { + "bits_per_byte": 1.6685270075807752, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1565347909927368, + "step": 3860, + "unclipped_grad_norm": 0.8480414748191833 + }, + { + "bits_per_byte": 1.7316664822440857, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2002997398376465, + "step": 3870, + "unclipped_grad_norm": 0.8396956324577332 + }, + { + "bits_per_byte": 1.7011863409969716, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1791725158691406, + "step": 3880, + "unclipped_grad_norm": 0.8941359519958496 + }, + { + "bits_per_byte": 1.7358762735719053, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.2032177448272705, + "step": 3890, + "unclipped_grad_norm": 0.894798755645752 + }, + { + "bits_per_byte": 1.723932594417436, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1949390172958374, + "step": 3900, + "unclipped_grad_norm": 0.856995165348053 + }, + { + "bits_per_byte": 1.7425093004503822, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.207815408706665, + "step": 3910, + "unclipped_grad_norm": 0.815839409828186 + }, + { + "bits_per_byte": 1.6972770033602091, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1764627695083618, + "step": 3920, + "unclipped_grad_norm": 0.7732813358306885 + }, + { + "bits_per_byte": 1.6785904004133616, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1635102033615112, + "step": 3930, + "unclipped_grad_norm": 0.8116664886474609 + }, + { + "bits_per_byte": 1.6934890854749558, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1738371849060059, + "step": 3940, + "unclipped_grad_norm": 0.8463517427444458 + }, + { + "bits_per_byte": 1.7793810049341614, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2333729267120361, + "step": 3950, + "unclipped_grad_norm": 0.8989379405975342 + }, + { + "bits_per_byte": 1.7841349493693113, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2366681098937988, + "step": 3960, + "unclipped_grad_norm": 0.8764044642448425 + }, + { + "bits_per_byte": 1.7434844420807742, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.208491325378418, + "step": 3970, + "unclipped_grad_norm": 0.842854380607605 + }, + { + "bits_per_byte": 1.7365705675334386, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.203698992729187, + "step": 3980, + "unclipped_grad_norm": 0.8656630516052246 + }, + { + "bits_per_byte": 1.7578855813334224, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2184734344482422, + "step": 3990, + "unclipped_grad_norm": 0.8737640976905823 + }, + { + "bits_per_byte": 1.7628734221737457, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.221930742263794, + "step": 4000, + "unclipped_grad_norm": 0.8737133741378784 + }, + { + "bits_per_byte": 1.7856131402534772, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2376927137374878, + "step": 4010, + "unclipped_grad_norm": 0.8777877688407898 + }, + { + "bits_per_byte": 1.6452555191445761, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.140404224395752, + "step": 4020, + "unclipped_grad_norm": 0.7845610976219177 + }, + { + "bits_per_byte": 1.6881839366437907, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.170159935951233, + "step": 4030, + "unclipped_grad_norm": 0.8164089322090149 + }, + { + "bits_per_byte": 1.7054205538612142, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1821074485778809, + "step": 4040, + "unclipped_grad_norm": 0.8447781205177307 + }, + { + "bits_per_byte": 1.6980753468255143, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1770161390304565, + "step": 4050, + "unclipped_grad_norm": 0.8900085091590881 + }, + { + "bits_per_byte": 1.715704084486905, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1892354488372803, + "step": 4060, + "unclipped_grad_norm": 0.864895761013031 + }, + { + "bits_per_byte": 1.6694995694714043, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1572089195251465, + "step": 4070, + "unclipped_grad_norm": 0.8258009552955627 + }, + { + "bits_per_byte": 1.7115804564671673, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1863771677017212, + "step": 4080, + "unclipped_grad_norm": 0.8420462012290955 + }, + { + "bits_per_byte": 1.656621852540099, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.148282766342163, + "step": 4090, + "unclipped_grad_norm": 0.8618301153182983 + }, + { + "bits_per_byte": 1.6356337777594054, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.133734941482544, + "step": 4100, + "unclipped_grad_norm": 0.7950634360313416 + }, + { + "bits_per_byte": 1.7423063609223641, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.2076747417449951, + "step": 4110, + "unclipped_grad_norm": 0.9093177914619446 + }, + { + "bits_per_byte": 1.6838860901987291, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1671808958053589, + "step": 4120, + "unclipped_grad_norm": 0.8371159434318542 + }, + { + "bits_per_byte": 1.6537941137946128, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1463227272033691, + "step": 4130, + "unclipped_grad_norm": 0.8010553121566772 + }, + { + "bits_per_byte": 1.6821887934173643, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1660044193267822, + "step": 4140, + "unclipped_grad_norm": 0.8702026009559631 + }, + { + "bits_per_byte": 1.7209915191050311, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1929004192352295, + "step": 4150, + "unclipped_grad_norm": 0.8466006517410278 + }, + { + "bits_per_byte": 1.6966901985554654, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1760560274124146, + "step": 4160, + "unclipped_grad_norm": 0.869064450263977 + }, + { + "bits_per_byte": 1.6446666505480219, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.13999605178833, + "step": 4170, + "unclipped_grad_norm": 0.8484132289886475 + }, + { + "bits_per_byte": 1.6167019275524273, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.120612382888794, + "step": 4180, + "unclipped_grad_norm": 0.8314529657363892 + }, + { + "bits_per_byte": 1.7785682149261839, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.2328095436096191, + "step": 4190, + "unclipped_grad_norm": 0.8785865902900696 + }, + { + "bits_per_byte": 1.6748427264684103, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1609125137329102, + "step": 4200, + "unclipped_grad_norm": 0.8246812224388123 + }, + { + "bits_per_byte": 1.679696764806362, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1642770767211914, + "step": 4210, + "unclipped_grad_norm": 0.846137285232544 + }, + { + "bits_per_byte": 1.73478487166953, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.2024612426757812, + "step": 4220, + "unclipped_grad_norm": 0.9032177925109863 + }, + { + "bits_per_byte": 1.6973831166557916, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1765363216400146, + "step": 4230, + "unclipped_grad_norm": 0.8501660227775574 + }, + { + "bits_per_byte": 1.7410484798139536, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.2068028450012207, + "step": 4240, + "unclipped_grad_norm": 0.8855612874031067 + }, + { + "bits_per_byte": 1.756008390699255, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.2171722650527954, + "step": 4250, + "unclipped_grad_norm": 0.849215567111969 + }, + { + "bits_per_byte": 1.7259122867115178, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1963112354278564, + "step": 4260, + "unclipped_grad_norm": 0.8852341175079346 + }, + { + "bits_per_byte": 1.7235939605778872, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.194704294204712, + "step": 4270, + "unclipped_grad_norm": 0.8789628148078918 + }, + { + "bits_per_byte": 1.751234840241907, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.213863492012024, + "step": 4280, + "unclipped_grad_norm": 0.923163115978241 + }, + { + "bits_per_byte": 1.688365378340451, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.170285701751709, + "step": 4290, + "unclipped_grad_norm": 0.8880194425582886 + }, + { + "bits_per_byte": 1.629311007583085, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.129352331161499, + "step": 4300, + "unclipped_grad_norm": 0.8913658261299133 + }, + { + "bits_per_byte": 1.771773524355896, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2280998229980469, + "step": 4310, + "unclipped_grad_norm": 0.9543031454086304 + }, + { + "bits_per_byte": 1.6490332900534286, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1430227756500244, + "step": 4320, + "unclipped_grad_norm": 0.8419790267944336 + }, + { + "bits_per_byte": 1.6371308867351662, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1347726583480835, + "step": 4330, + "unclipped_grad_norm": 0.8086377382278442 + }, + { + "bits_per_byte": 1.6438610838113803, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1394376754760742, + "step": 4340, + "unclipped_grad_norm": 0.8752440810203552 + }, + { + "bits_per_byte": 1.5801602577756537, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0952836275100708, + "step": 4350, + "unclipped_grad_norm": 0.8255703449249268 + }, + { + "bits_per_byte": 1.7363834504092999, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.2035692930221558, + "step": 4360, + "unclipped_grad_norm": 0.9419499039649963 + }, + { + "bits_per_byte": 1.6755530148164737, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1614048480987549, + "step": 4370, + "unclipped_grad_norm": 0.8658137917518616 + }, + { + "bits_per_byte": 1.644153798283149, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1396405696868896, + "step": 4380, + "unclipped_grad_norm": 0.8454349040985107 + }, + { + "bits_per_byte": 1.6710379542833722, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1582752466201782, + "step": 4390, + "unclipped_grad_norm": 0.8288133144378662 + }, + { + "bits_per_byte": 1.6521485837911574, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1451821327209473, + "step": 4400, + "unclipped_grad_norm": 0.8934062719345093 + }, + { + "bits_per_byte": 1.6486657631285346, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1427680253982544, + "step": 4410, + "unclipped_grad_norm": 0.945034921169281 + }, + { + "bits_per_byte": 1.6360498037918425, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1340233087539673, + "step": 4420, + "unclipped_grad_norm": 0.8596903681755066 + }, + { + "bits_per_byte": 1.6605176035474434, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1509830951690674, + "step": 4430, + "unclipped_grad_norm": 0.8684608340263367 + }, + { + "bits_per_byte": 1.6658086498012377, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1546505689620972, + "step": 4440, + "unclipped_grad_norm": 0.8835514783859253 + }, + { + "bits_per_byte": 1.6458364765391906, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1408069133758545, + "step": 4450, + "unclipped_grad_norm": 0.8609577417373657 + }, + { + "bits_per_byte": 1.6156653881156773, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1198939085006714, + "step": 4460, + "unclipped_grad_norm": 0.8464177250862122 + }, + { + "bits_per_byte": 1.7173920942051233, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1904054880142212, + "step": 4470, + "unclipped_grad_norm": 0.940384030342102 + }, + { + "bits_per_byte": 1.6389153787205186, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1360095739364624, + "step": 4480, + "unclipped_grad_norm": 0.8790894746780396 + }, + { + "bits_per_byte": 1.590423322465892, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1023974418640137, + "step": 4490, + "unclipped_grad_norm": 0.8159270286560059 + }, + { + "bits_per_byte": 1.6820450159212428, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1659047603607178, + "step": 4500, + "unclipped_grad_norm": 0.8648387789726257 + }, + { + "bits_per_byte": 1.6443777196945721, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1397957801818848, + "step": 4510, + "unclipped_grad_norm": 0.874046266078949 + }, + { + "bits_per_byte": 1.6401994011918604, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1368995904922485, + "step": 4520, + "unclipped_grad_norm": 0.8307194709777832 + }, + { + "bits_per_byte": 1.6802708428949422, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.164674997329712, + "step": 4530, + "unclipped_grad_norm": 0.8542748689651489 + }, + { + "bits_per_byte": 1.6174216749462882, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.121111273765564, + "step": 4540, + "unclipped_grad_norm": 0.7951624989509583 + }, + { + "bits_per_byte": 1.6783786897701496, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1633634567260742, + "step": 4550, + "unclipped_grad_norm": 0.9085310697555542 + }, + { + "bits_per_byte": 1.678087351159588, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1631615161895752, + "step": 4560, + "unclipped_grad_norm": 0.8672650456428528 + }, + { + "bits_per_byte": 1.6438235915934922, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1394116878509521, + "step": 4570, + "unclipped_grad_norm": 0.8864105343818665 + }, + { + "bits_per_byte": 1.5942666187647232, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.105061411857605, + "step": 4580, + "unclipped_grad_norm": 0.8459784984588623 + }, + { + "bits_per_byte": 1.7013707063986967, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.179300308227539, + "step": 4590, + "unclipped_grad_norm": 0.864471971988678 + }, + { + "bits_per_byte": 1.685760529110483, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1684801578521729, + "step": 4600, + "unclipped_grad_norm": 0.8711076378822327 + }, + { + "bits_per_byte": 1.6521852160957913, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1452075242996216, + "step": 4610, + "unclipped_grad_norm": 0.8594344258308411 + }, + { + "bits_per_byte": 1.5889760884588813, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1013942956924438, + "step": 4620, + "unclipped_grad_norm": 0.8217349052429199 + }, + { + "bits_per_byte": 1.6772566749559206, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.162585735321045, + "step": 4630, + "unclipped_grad_norm": 0.8906145095825195 + }, + { + "bits_per_byte": 1.6271617403952525, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.127862572669983, + "step": 4640, + "unclipped_grad_norm": 0.8258018493652344 + }, + { + "bits_per_byte": 1.61576359020932, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1199619770050049, + "step": 4650, + "unclipped_grad_norm": 0.9025488495826721 + }, + { + "bits_per_byte": 1.6536071686531248, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1461931467056274, + "step": 4660, + "unclipped_grad_norm": 0.905563473701477 + }, + { + "bits_per_byte": 1.6824091031931194, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1661571264266968, + "step": 4670, + "unclipped_grad_norm": 0.8757683038711548 + }, + { + "bits_per_byte": 1.6350793056930237, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1333506107330322, + "step": 4680, + "unclipped_grad_norm": 0.8878620862960815 + }, + { + "bits_per_byte": 1.6161242378281793, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1202119588851929, + "step": 4690, + "unclipped_grad_norm": 0.8490695953369141 + }, + { + "bits_per_byte": 1.7156000349831328, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.189163327217102, + "step": 4700, + "unclipped_grad_norm": 0.8805282115936279 + }, + { + "bits_per_byte": 1.6530434095235966, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.14580237865448, + "step": 4710, + "unclipped_grad_norm": 0.9434417486190796 + }, + { + "bits_per_byte": 1.6037863744379321, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1116600036621094, + "step": 4720, + "unclipped_grad_norm": 0.8885965347290039 + }, + { + "bits_per_byte": 1.7271340514632472, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1971580982208252, + "step": 4730, + "unclipped_grad_norm": 0.9324085116386414 + }, + { + "bits_per_byte": 1.6733065774309035, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1598477363586426, + "step": 4740, + "unclipped_grad_norm": 0.8685431480407715 + }, + { + "bits_per_byte": 1.7120205600707252, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1866822242736816, + "step": 4750, + "unclipped_grad_norm": 0.9381123185157776 + }, + { + "bits_per_byte": 1.5687653752600876, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0873852968215942, + "step": 4760, + "unclipped_grad_norm": 0.8193861246109009 + }, + { + "bits_per_byte": 1.681102207029213, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1652512550354004, + "step": 4770, + "unclipped_grad_norm": 0.8750576376914978 + }, + { + "bits_per_byte": 1.6366992102815003, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.134473443031311, + "step": 4780, + "unclipped_grad_norm": 0.8815762996673584 + }, + { + "bits_per_byte": 1.6734204299457747, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1599266529083252, + "step": 4790, + "unclipped_grad_norm": 0.9372621774673462 + }, + { + "bits_per_byte": 1.6264474964462192, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1273674964904785, + "step": 4800, + "unclipped_grad_norm": 0.8206531405448914 + }, + { + "bits_per_byte": 1.6050609378634764, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1125434637069702, + "step": 4810, + "unclipped_grad_norm": 0.867507815361023 + }, + { + "bits_per_byte": 1.6801829597603513, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1646140813827515, + "step": 4820, + "unclipped_grad_norm": 0.9113548398017883 + }, + { + "bits_per_byte": 1.6755392562044045, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1613953113555908, + "step": 4830, + "unclipped_grad_norm": 0.9283871650695801 + }, + { + "bits_per_byte": 1.6755334087942753, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.161391258239746, + "step": 4840, + "unclipped_grad_norm": 0.8912690281867981 + }, + { + "bits_per_byte": 1.6043515094286673, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1120517253875732, + "step": 4850, + "unclipped_grad_norm": 0.8777358531951904 + }, + { + "bits_per_byte": 1.6436579723007114, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1392968893051147, + "step": 4860, + "unclipped_grad_norm": 0.9008035063743591 + }, + { + "bits_per_byte": 1.639443709423969, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1363757848739624, + "step": 4870, + "unclipped_grad_norm": 0.8822212219238281 + }, + { + "bits_per_byte": 1.8005314313372698, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.2480332851409912, + "step": 4880, + "unclipped_grad_norm": 0.953372061252594 + }, + { + "bits_per_byte": 1.587463501044543, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1003458499908447, + "step": 4890, + "unclipped_grad_norm": 0.898897111415863 + }, + { + "bits_per_byte": 1.6710314189426394, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1582707166671753, + "step": 4900, + "unclipped_grad_norm": 0.8573687672615051 + }, + { + "bits_per_byte": 1.6542582949692914, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1466444730758667, + "step": 4910, + "unclipped_grad_norm": 0.8744381070137024 + }, + { + "bits_per_byte": 1.5936046575415521, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.104602575302124, + "step": 4920, + "unclipped_grad_norm": 0.8256805539131165 + }, + { + "bits_per_byte": 1.5994107918346805, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1086270809173584, + "step": 4930, + "unclipped_grad_norm": 0.8097874522209167 + }, + { + "bits_per_byte": 1.6321965324992613, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.131352424621582, + "step": 4940, + "unclipped_grad_norm": 0.9138422012329102 + }, + { + "bits_per_byte": 1.6452641182771193, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1404101848602295, + "step": 4950, + "unclipped_grad_norm": 0.890544593334198 + }, + { + "bits_per_byte": 1.6569370967391306, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1485012769699097, + "step": 4960, + "unclipped_grad_norm": 0.8706549406051636 + }, + { + "bits_per_byte": 1.6359316517106997, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.133941411972046, + "step": 4970, + "unclipped_grad_norm": 0.9410244226455688 + }, + { + "bits_per_byte": 1.6744512939550464, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1606411933898926, + "step": 4980, + "unclipped_grad_norm": 0.9115959405899048 + }, + { + "bits_per_byte": 1.637535905877948, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1350533962249756, + "step": 4990, + "unclipped_grad_norm": 0.8625267744064331 + }, + { + "bits_per_byte": 1.6562579372508734, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1480305194854736, + "step": 5000, + "unclipped_grad_norm": 0.9143829345703125 + }, + { + "bits_per_byte": 1.691347557506412, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1723527908325195, + "step": 5010, + "unclipped_grad_norm": 0.8908807635307312 + }, + { + "bits_per_byte": 1.640470617832271, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1370875835418701, + "step": 5020, + "unclipped_grad_norm": 0.9010583162307739 + }, + { + "bits_per_byte": 1.5978712031441566, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1075599193572998, + "step": 5030, + "unclipped_grad_norm": 0.8482128381729126 + }, + { + "bits_per_byte": 1.6379517599277342, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1353416442871094, + "step": 5040, + "unclipped_grad_norm": 0.8523857593536377 + }, + { + "bits_per_byte": 1.624761722502462, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1261990070343018, + "step": 5050, + "unclipped_grad_norm": 0.8824069499969482 + }, + { + "bits_per_byte": 1.6994286783051538, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1779541969299316, + "step": 5060, + "unclipped_grad_norm": 0.9452667832374573 + }, + { + "bits_per_byte": 1.6270504676201443, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1277854442596436, + "step": 5070, + "unclipped_grad_norm": 0.8711076378822327 + }, + { + "bits_per_byte": 1.7018825267676645, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1796550750732422, + "step": 5080, + "unclipped_grad_norm": 0.9225792288780212 + }, + { + "bits_per_byte": 1.6371355302667394, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1347758769989014, + "step": 5090, + "unclipped_grad_norm": 0.9880733489990234 + }, + { + "bits_per_byte": 1.6193408293472662, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1224415302276611, + "step": 5100, + "unclipped_grad_norm": 0.8858473896980286 + }, + { + "bits_per_byte": 1.648101144085752, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1423766613006592, + "step": 5110, + "unclipped_grad_norm": 0.8950205445289612 + }, + { + "bits_per_byte": 1.5621068949492831, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0827699899673462, + "step": 5120, + "unclipped_grad_norm": 0.8509450554847717 + }, + { + "bits_per_byte": 1.6969497203756174, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1762359142303467, + "step": 5130, + "unclipped_grad_norm": 0.9032918810844421 + }, + { + "bits_per_byte": 1.6594853636769649, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1502676010131836, + "step": 5140, + "unclipped_grad_norm": 0.9204651713371277 + }, + { + "bits_per_byte": 1.6102843949354817, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1161640882492065, + "step": 5150, + "unclipped_grad_norm": 0.8756265640258789 + }, + { + "bits_per_byte": 1.6252881613967531, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1265639066696167, + "step": 5160, + "unclipped_grad_norm": 0.9101666212081909 + }, + { + "bits_per_byte": 1.6855433150224433, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1683295965194702, + "step": 5170, + "unclipped_grad_norm": 0.9160894155502319 + }, + { + "bits_per_byte": 1.6357713638800955, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1338303089141846, + "step": 5180, + "unclipped_grad_norm": 0.9035648107528687 + }, + { + "bits_per_byte": 1.5386909410863592, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0665392875671387, + "step": 5190, + "unclipped_grad_norm": 0.8579265475273132 + }, + { + "bits_per_byte": 1.6228207263048244, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1248536109924316, + "step": 5200, + "unclipped_grad_norm": 0.8737999200820923 + }, + { + "bits_per_byte": 1.5871657990758994, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1001394987106323, + "step": 5210, + "unclipped_grad_norm": 0.8650596737861633 + }, + { + "bits_per_byte": 1.6000187505054806, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1090484857559204, + "step": 5220, + "unclipped_grad_norm": 0.857958197593689 + }, + { + "bits_per_byte": 1.6870952864638293, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1694053411483765, + "step": 5230, + "unclipped_grad_norm": 0.9485795497894287 + }, + { + "bits_per_byte": 1.65525063486477, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1473323106765747, + "step": 5240, + "unclipped_grad_norm": 0.9584298133850098 + }, + { + "bits_per_byte": 1.648363245645667, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1425583362579346, + "step": 5250, + "unclipped_grad_norm": 0.9243587851524353 + }, + { + "bits_per_byte": 1.5949392429122475, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.105527639389038, + "step": 5260, + "unclipped_grad_norm": 0.97037672996521 + }, + { + "bits_per_byte": 1.6406128474845345, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1371861696243286, + "step": 5270, + "unclipped_grad_norm": 0.9107886552810669 + }, + { + "bits_per_byte": 1.5702697075071845, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.088428020477295, + "step": 5280, + "unclipped_grad_norm": 0.9339945316314697 + }, + { + "bits_per_byte": 1.5822967982473224, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.0967645645141602, + "step": 5290, + "unclipped_grad_norm": 0.9108784198760986 + }, + { + "bits_per_byte": 1.5641275191142703, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0841705799102783, + "step": 5300, + "unclipped_grad_norm": 0.9299371838569641 + }, + { + "bits_per_byte": 1.568704665384333, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0873432159423828, + "step": 5310, + "unclipped_grad_norm": 0.841905951499939 + }, + { + "bits_per_byte": 1.5839983945949592, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0979440212249756, + "step": 5320, + "unclipped_grad_norm": 0.9190206527709961 + }, + { + "bits_per_byte": 1.6212247273048175, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1237473487854004, + "step": 5330, + "unclipped_grad_norm": 0.9011974930763245 + }, + { + "bits_per_byte": 1.6958863516453326, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1754988431930542, + "step": 5340, + "unclipped_grad_norm": 0.9635267853736877 + }, + { + "bits_per_byte": 1.5503265993131332, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0746045112609863, + "step": 5350, + "unclipped_grad_norm": 0.8525294661521912 + }, + { + "bits_per_byte": 1.6023642498979476, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.110674262046814, + "step": 5360, + "unclipped_grad_norm": 0.9040093421936035 + }, + { + "bits_per_byte": 1.6756675552619482, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1614842414855957, + "step": 5370, + "unclipped_grad_norm": 0.9546249508857727 + }, + { + "bits_per_byte": 1.5762295942901843, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0925590991973877, + "step": 5380, + "unclipped_grad_norm": 0.9253390431404114 + }, + { + "bits_per_byte": 1.5803738602280255, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0954316854476929, + "step": 5390, + "unclipped_grad_norm": 0.9185824394226074 + }, + { + "bits_per_byte": 1.6346944085203927, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1330838203430176, + "step": 5400, + "unclipped_grad_norm": 0.9316674470901489 + }, + { + "bits_per_byte": 1.5477485793766999, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0728175640106201, + "step": 5410, + "unclipped_grad_norm": 0.9008721709251404 + }, + { + "bits_per_byte": 1.6909000586488672, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1720426082611084, + "step": 5420, + "unclipped_grad_norm": 0.9377346038818359 + }, + { + "bits_per_byte": 1.6161049757712826, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1201986074447632, + "step": 5430, + "unclipped_grad_norm": 0.9362180233001709 + }, + { + "bits_per_byte": 1.6092089874196365, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1154186725616455, + "step": 5440, + "unclipped_grad_norm": 0.8880205154418945 + }, + { + "bits_per_byte": 1.6243469003485809, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1259114742279053, + "step": 5450, + "unclipped_grad_norm": 0.9006999731063843 + }, + { + "bits_per_byte": 1.6430875058477994, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.1389014720916748, + "step": 5460, + "unclipped_grad_norm": 0.9130494594573975 + }, + { + "bits_per_byte": 1.6231913489174337, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1251105070114136, + "step": 5470, + "unclipped_grad_norm": 0.9382801651954651 + }, + { + "bits_per_byte": 1.5843513029945295, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0981886386871338, + "step": 5480, + "unclipped_grad_norm": 0.9530964493751526 + }, + { + "bits_per_byte": 1.6238423032509495, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1255617141723633, + "step": 5490, + "unclipped_grad_norm": 1.0083582401275635 + }, + { + "bits_per_byte": 1.6558995254064752, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1477820873260498, + "step": 5500, + "unclipped_grad_norm": 0.9359451532363892 + }, + { + "bits_per_byte": 1.6193308543535163, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1224346160888672, + "step": 5510, + "unclipped_grad_norm": 0.9424907565116882 + }, + { + "bits_per_byte": 1.622570491547819, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1246801614761353, + "step": 5520, + "unclipped_grad_norm": 0.9104092121124268 + }, + { + "bits_per_byte": 1.6496316176957802, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.1434375047683716, + "step": 5530, + "unclipped_grad_norm": 0.9198633432388306 + }, + { + "bits_per_byte": 1.6739231352342467, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1602751016616821, + "step": 5540, + "unclipped_grad_norm": 1.0088523626327515 + }, + { + "bits_per_byte": 1.5961751102413477, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.10638427734375, + "step": 5550, + "unclipped_grad_norm": 0.8725464344024658 + }, + { + "bits_per_byte": 1.5912085952497315, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1029417514801025, + "step": 5560, + "unclipped_grad_norm": 0.9294795989990234 + }, + { + "bits_per_byte": 1.6449143055652642, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1401677131652832, + "step": 5570, + "unclipped_grad_norm": 0.9388754963874817 + }, + { + "bits_per_byte": 1.5525186901810306, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0761239528656006, + "step": 5580, + "unclipped_grad_norm": 0.9197003245353699 + }, + { + "bits_per_byte": 1.6195045568308877, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1225550174713135, + "step": 5590, + "unclipped_grad_norm": 0.9434337615966797 + }, + { + "bits_per_byte": 1.5951824263805676, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.105696201324463, + "step": 5600, + "unclipped_grad_norm": 0.9404297471046448 + }, + { + "bits_per_byte": 1.6108896018838679, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1165835857391357, + "step": 5610, + "unclipped_grad_norm": 0.9016186594963074 + }, + { + "bits_per_byte": 1.5417312503883118, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0686466693878174, + "step": 5620, + "unclipped_grad_norm": 0.9296513795852661 + }, + { + "bits_per_byte": 1.5687026015925227, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0873417854309082, + "step": 5630, + "unclipped_grad_norm": 0.9855212569236755 + }, + { + "bits_per_byte": 1.548401425519375, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0732700824737549, + "step": 5640, + "unclipped_grad_norm": 0.9301762580871582 + }, + { + "bits_per_byte": 1.6637563808284919, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1532280445098877, + "step": 5650, + "unclipped_grad_norm": 1.0132923126220703 + }, + { + "bits_per_byte": 1.614469592744228, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1190650463104248, + "step": 5660, + "unclipped_grad_norm": 0.9275335073471069 + }, + { + "bits_per_byte": 1.6100616774026144, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.1160097122192383, + "step": 5670, + "unclipped_grad_norm": 0.9310563802719116 + }, + { + "bits_per_byte": 1.6255531866617328, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1267476081848145, + "step": 5680, + "unclipped_grad_norm": 0.9579228758811951 + }, + { + "bits_per_byte": 1.6417694308115869, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1379878520965576, + "step": 5690, + "unclipped_grad_norm": 0.9637964367866516 + }, + { + "bits_per_byte": 1.6007787418396433, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1095752716064453, + "step": 5700, + "unclipped_grad_norm": 0.955059826374054 + }, + { + "bits_per_byte": 1.6229964925740061, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1249754428863525, + "step": 5710, + "unclipped_grad_norm": 0.9087291955947876 + }, + { + "bits_per_byte": 1.6524653758340466, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1454017162322998, + "step": 5720, + "unclipped_grad_norm": 0.9723272919654846 + }, + { + "bits_per_byte": 1.5500029279642094, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0743801593780518, + "step": 5730, + "unclipped_grad_norm": 0.9230121374130249 + }, + { + "bits_per_byte": 1.6318054439511993, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1310813426971436, + "step": 5740, + "unclipped_grad_norm": 0.9736878275871277 + }, + { + "bits_per_byte": 1.5815806624891298, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0962681770324707, + "step": 5750, + "unclipped_grad_norm": 0.9681398868560791 + }, + { + "bits_per_byte": 1.5890875332166405, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.1014715433120728, + "step": 5760, + "unclipped_grad_norm": 0.9480424523353577 + }, + { + "bits_per_byte": 1.528980628635994, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.059808611869812, + "step": 5770, + "unclipped_grad_norm": 0.9915046691894531 + }, + { + "bits_per_byte": 1.610684942529341, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1164417266845703, + "step": 5780, + "unclipped_grad_norm": 0.9259656667709351 + }, + { + "bits_per_byte": 1.5796823179889061, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0949523448944092, + "step": 5790, + "unclipped_grad_norm": 0.9487527012825012 + }, + { + "bits_per_byte": 1.5821124328455975, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0966367721557617, + "step": 5800, + "unclipped_grad_norm": 0.9434526562690735 + }, + { + "bits_per_byte": 1.6104013431380684, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.116245150566101, + "step": 5810, + "unclipped_grad_norm": 0.9532226920127869 + }, + { + "bits_per_byte": 1.6033849669308184, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.1113817691802979, + "step": 5820, + "unclipped_grad_norm": 0.914564311504364 + }, + { + "bits_per_byte": 1.5785037208825432, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0941354036331177, + "step": 5830, + "unclipped_grad_norm": 0.9744407534599304 + }, + { + "bits_per_byte": 1.5595204478629572, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.080977201461792, + "step": 5840, + "unclipped_grad_norm": 0.9682079553604126 + }, + { + "bits_per_byte": 1.5459948722858516, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0716019868850708, + "step": 5850, + "unclipped_grad_norm": 0.908660352230072 + }, + { + "bits_per_byte": 1.5771918372217617, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0932260751724243, + "step": 5860, + "unclipped_grad_norm": 0.9774399995803833 + }, + { + "bits_per_byte": 1.617923176356204, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.121458888053894, + "step": 5870, + "unclipped_grad_norm": 0.9498292803764343 + }, + { + "bits_per_byte": 1.6179303996275405, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1214638948440552, + "step": 5880, + "unclipped_grad_norm": 0.929435670375824 + }, + { + "bits_per_byte": 1.6226986186227117, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1247689723968506, + "step": 5890, + "unclipped_grad_norm": 0.9540143013000488 + }, + { + "bits_per_byte": 1.5934973403674138, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1045281887054443, + "step": 5900, + "unclipped_grad_norm": 1.0085638761520386 + }, + { + "bits_per_byte": 1.5806963276983932, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0956552028656006, + "step": 5910, + "unclipped_grad_norm": 1.0136828422546387 + }, + { + "bits_per_byte": 1.602695832448811, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.1109040975570679, + "step": 5920, + "unclipped_grad_norm": 1.0095367431640625 + }, + { + "bits_per_byte": 1.6274630539995643, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1280714273452759, + "step": 5930, + "unclipped_grad_norm": 0.947801411151886 + }, + { + "bits_per_byte": 1.6238797954688375, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.1255877017974854, + "step": 5940, + "unclipped_grad_norm": 0.9719098806381226 + }, + { + "bits_per_byte": 1.597647625698035, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.1074049472808838, + "step": 5950, + "unclipped_grad_norm": 0.9620362520217896 + }, + { + "bits_per_byte": 1.588192019553598, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1008508205413818, + "step": 5960, + "unclipped_grad_norm": 0.9313361644744873 + }, + { + "bits_per_byte": 1.619017845928946, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.1222176551818848, + "step": 5970, + "unclipped_grad_norm": 1.0069472789764404 + }, + { + "bits_per_byte": 1.5640157303912094, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0840930938720703, + "step": 5980, + "unclipped_grad_norm": 0.9776687026023865 + }, + { + "bits_per_byte": 1.5433833157324999, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0697917938232422, + "step": 5990, + "unclipped_grad_norm": 0.9335829615592957 + }, + { + "bits_per_byte": 1.6083124418606887, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1147972345352173, + "step": 6000, + "unclipped_grad_norm": 0.9547005295753479 + }, + { + "bits_per_byte": 1.591078920330981, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1028518676757812, + "step": 6010, + "unclipped_grad_norm": 0.9792258739471436 + }, + { + "bits_per_byte": 1.553183919074568, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.076585054397583, + "step": 6020, + "unclipped_grad_norm": 0.9365350604057312 + }, + { + "bits_per_byte": 1.621642989111716, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.124037265777588, + "step": 6030, + "unclipped_grad_norm": 0.9456943869590759 + }, + { + "bits_per_byte": 1.55026812521184, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.074563980102539, + "step": 6040, + "unclipped_grad_norm": 0.986164391040802 + }, + { + "bits_per_byte": 1.573968538379291, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0909918546676636, + "step": 6050, + "unclipped_grad_norm": 0.9421037435531616 + }, + { + "bits_per_byte": 1.6258569080231564, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.1269581317901611, + "step": 6060, + "unclipped_grad_norm": 0.9926829934120178 + }, + { + "bits_per_byte": 1.5943486544891847, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.1051182746887207, + "step": 6070, + "unclipped_grad_norm": 0.9500434994697571 + }, + { + "bits_per_byte": 1.5776276412590482, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.093528151512146, + "step": 6080, + "unclipped_grad_norm": 0.9243369102478027 + }, + { + "bits_per_byte": 1.583595095278686, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.097664475440979, + "step": 6090, + "unclipped_grad_norm": 0.9602344036102295 + }, + { + "bits_per_byte": 1.5816112754009832, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0962893962860107, + "step": 6100, + "unclipped_grad_norm": 0.9188515543937683 + }, + { + "bits_per_byte": 1.5481370881849987, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0730868577957153, + "step": 6110, + "unclipped_grad_norm": 0.9290796518325806 + }, + { + "bits_per_byte": 1.557837081676312, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.079810380935669, + "step": 6120, + "unclipped_grad_norm": 0.9658544063568115 + }, + { + "bits_per_byte": 1.5877624068917426, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.100553035736084, + "step": 6130, + "unclipped_grad_norm": 0.9524391889572144 + }, + { + "bits_per_byte": 1.6144964220377627, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.1190836429595947, + "step": 6140, + "unclipped_grad_norm": 0.9724568724632263 + }, + { + "bits_per_byte": 1.5513643426284394, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0753238201141357, + "step": 6150, + "unclipped_grad_norm": 0.9667541980743408 + }, + { + "bits_per_byte": 1.6439771721007128, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.139518141746521, + "step": 6160, + "unclipped_grad_norm": 1.0203982591629028 + }, + { + "bits_per_byte": 1.5065515112237209, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0442619323730469, + "step": 6170, + "unclipped_grad_norm": 0.892124593257904 + }, + { + "bits_per_byte": 1.5721656442502963, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0897421836853027, + "step": 6180, + "unclipped_grad_norm": 0.9671148657798767 + }, + { + "bits_per_byte": 1.5890949284706275, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.1014766693115234, + "step": 6190, + "unclipped_grad_norm": 1.0115299224853516 + }, + { + "bits_per_byte": 1.5390813417038176, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.066809892654419, + "step": 6200, + "unclipped_grad_norm": 0.9446512460708618 + }, + { + "bits_per_byte": 1.5691769297436022, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0876705646514893, + "step": 6210, + "unclipped_grad_norm": 0.9595397710800171 + }, + { + "bits_per_byte": 1.5550289489530242, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0778639316558838, + "step": 6220, + "unclipped_grad_norm": 0.927939772605896 + }, + { + "bits_per_byte": 1.5332487220824564, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0627670288085938, + "step": 6230, + "unclipped_grad_norm": 0.9231003522872925 + }, + { + "bits_per_byte": 1.5454189023881122, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0712027549743652, + "step": 6240, + "unclipped_grad_norm": 0.9556372761726379 + }, + { + "bits_per_byte": 1.547023500520662, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.072314977645874, + "step": 6250, + "unclipped_grad_norm": 0.9621478915214539 + }, + { + "bits_per_byte": 1.6040983509665974, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.1118762493133545, + "step": 6260, + "unclipped_grad_norm": 0.9583669304847717 + }, + { + "bits_per_byte": 1.562768856172454, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0832288265228271, + "step": 6270, + "unclipped_grad_norm": 0.9386000633239746 + }, + { + "bits_per_byte": 1.6650833989625493, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1541478633880615, + "step": 6280, + "unclipped_grad_norm": 0.9854296445846558 + }, + { + "bits_per_byte": 1.5598107545776136, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0811784267425537, + "step": 6290, + "unclipped_grad_norm": 0.9466595649719238 + }, + { + "bits_per_byte": 1.5877661905100615, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.100555658340454, + "step": 6300, + "unclipped_grad_norm": 1.077492117881775 + }, + { + "bits_per_byte": 1.6323781461785725, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.1314783096313477, + "step": 6310, + "unclipped_grad_norm": 1.0482228994369507 + }, + { + "bits_per_byte": 1.5889704130314029, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.1013903617858887, + "step": 6320, + "unclipped_grad_norm": 0.9478169083595276 + }, + { + "bits_per_byte": 1.5225548408518068, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0553545951843262, + "step": 6330, + "unclipped_grad_norm": 0.9578908085823059 + }, + { + "bits_per_byte": 1.580892387920377, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0957911014556885, + "step": 6340, + "unclipped_grad_norm": 1.0281808376312256 + }, + { + "bits_per_byte": 1.5773070655978398, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0933059453964233, + "step": 6350, + "unclipped_grad_norm": 0.9984999299049377 + }, + { + "bits_per_byte": 1.5368639693862434, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0652729272842407, + "step": 6360, + "unclipped_grad_norm": 0.9870114922523499 + }, + { + "bits_per_byte": 1.5938194638724799, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.104751467704773, + "step": 6370, + "unclipped_grad_norm": 0.9943422675132751 + }, + { + "bits_per_byte": 1.54489022771936, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0708363056182861, + "step": 6380, + "unclipped_grad_norm": 0.9544562697410583 + }, + { + "bits_per_byte": 1.5900604390725714, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1021459102630615, + "step": 6390, + "unclipped_grad_norm": 0.9881879091262817 + }, + { + "bits_per_byte": 1.5576724942794364, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0796962976455688, + "step": 6400, + "unclipped_grad_norm": 0.9512554407119751 + }, + { + "bits_per_byte": 1.5486053969432982, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.073411464691162, + "step": 6410, + "unclipped_grad_norm": 0.9827495217323303 + }, + { + "bits_per_byte": 1.5454973264769056, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0712571144104004, + "step": 6420, + "unclipped_grad_norm": 1.0222079753875732 + }, + { + "bits_per_byte": 1.5609757650545584, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0819859504699707, + "step": 6430, + "unclipped_grad_norm": 1.0177998542785645 + }, + { + "bits_per_byte": 1.5622200595335507, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0828484296798706, + "step": 6440, + "unclipped_grad_norm": 0.9765349626541138 + }, + { + "bits_per_byte": 1.5515923916234835, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.07548189163208, + "step": 6450, + "unclipped_grad_norm": 1.028122901916504 + }, + { + "bits_per_byte": 1.5316869476299713, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.061684489250183, + "step": 6460, + "unclipped_grad_norm": 1.1802783012390137 + }, + { + "bits_per_byte": 1.566511026672578, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0858227014541626, + "step": 6470, + "unclipped_grad_norm": 0.9929527044296265 + }, + { + "bits_per_byte": 1.5861486936786968, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.099434494972229, + "step": 6480, + "unclipped_grad_norm": 0.996606171131134 + }, + { + "bits_per_byte": 1.5680002244463989, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0868549346923828, + "step": 6490, + "unclipped_grad_norm": 0.9814039468765259 + }, + { + "bits_per_byte": 1.568809918766661, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.087416172027588, + "step": 6500, + "unclipped_grad_norm": 1.001226782798767 + }, + { + "bits_per_byte": 1.513912368680649, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0493640899658203, + "step": 6510, + "unclipped_grad_norm": 0.9533669948577881 + }, + { + "bits_per_byte": 1.4992986588715345, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0392346382141113, + "step": 6520, + "unclipped_grad_norm": 1.0103793144226074 + }, + { + "bits_per_byte": 1.5872271968822576, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.100182056427002, + "step": 6530, + "unclipped_grad_norm": 1.0064735412597656 + }, + { + "bits_per_byte": 1.551986575859261, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0757551193237305, + "step": 6540, + "unclipped_grad_norm": 0.9814162254333496 + }, + { + "bits_per_byte": 1.6102771716641453, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.1161590814590454, + "step": 6550, + "unclipped_grad_norm": 1.0333638191223145 + }, + { + "bits_per_byte": 1.5466836628025573, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0720794200897217, + "step": 6560, + "unclipped_grad_norm": 0.9711053967475891 + }, + { + "bits_per_byte": 1.4930591282982317, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.034909725189209, + "step": 6570, + "unclipped_grad_norm": 0.9939249157905579 + }, + { + "bits_per_byte": 1.5761067986774682, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0924739837646484, + "step": 6580, + "unclipped_grad_norm": 0.9629629850387573 + }, + { + "bits_per_byte": 1.5551524324963437, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0779495239257812, + "step": 6590, + "unclipped_grad_norm": 1.007703185081482 + }, + { + "bits_per_byte": 1.4715650805586997, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0200111865997314, + "step": 6600, + "unclipped_grad_norm": 0.9724071621894836 + }, + { + "bits_per_byte": 1.5956089433547074, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.1059918403625488, + "step": 6610, + "unclipped_grad_norm": 0.9970710277557373 + }, + { + "bits_per_byte": 1.6125643689379698, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1177444458007812, + "step": 6620, + "unclipped_grad_norm": 1.0349808931350708 + }, + { + "bits_per_byte": 1.59399093057539, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.104870319366455, + "step": 6630, + "unclipped_grad_norm": 1.039212703704834 + }, + { + "bits_per_byte": 1.5557672704731782, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0783756971359253, + "step": 6640, + "unclipped_grad_norm": 1.023613452911377 + }, + { + "bits_per_byte": 1.5432283593640725, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.069684386253357, + "step": 6650, + "unclipped_grad_norm": 0.9931045174598694 + }, + { + "bits_per_byte": 1.5957387902561087, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1060818433761597, + "step": 6660, + "unclipped_grad_norm": 1.01187002658844 + }, + { + "bits_per_byte": 1.5766402888604447, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.092843770980835, + "step": 6670, + "unclipped_grad_norm": 1.0118197202682495 + }, + { + "bits_per_byte": 1.5937730285567469, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.1047192811965942, + "step": 6680, + "unclipped_grad_norm": 1.0237627029418945 + }, + { + "bits_per_byte": 1.5627511419594151, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0832165479660034, + "step": 6690, + "unclipped_grad_norm": 1.0074224472045898 + }, + { + "bits_per_byte": 1.5937723406261435, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.104718804359436, + "step": 6700, + "unclipped_grad_norm": 1.01242196559906 + }, + { + "bits_per_byte": 1.5383141270983187, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.066278100013733, + "step": 6710, + "unclipped_grad_norm": 0.9823194146156311 + }, + { + "bits_per_byte": 1.5376188012408802, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.065796136856079, + "step": 6720, + "unclipped_grad_norm": 1.027299404144287 + }, + { + "bits_per_byte": 1.5285384612406259, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.059502124786377, + "step": 6730, + "unclipped_grad_norm": 0.9929317235946655 + }, + { + "bits_per_byte": 1.5454023920536293, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0711913108825684, + "step": 6740, + "unclipped_grad_norm": 0.9901378154754639 + }, + { + "bits_per_byte": 1.5351463786520767, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.064082384109497, + "step": 6750, + "unclipped_grad_norm": 1.026999831199646 + }, + { + "bits_per_byte": 1.501298817101069, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.040621042251587, + "step": 6760, + "unclipped_grad_norm": 0.9864338040351868 + }, + { + "bits_per_byte": 1.5231093129181885, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.055738925933838, + "step": 6770, + "unclipped_grad_norm": 0.9597692489624023 + }, + { + "bits_per_byte": 1.5314098835794312, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0614924430847168, + "step": 6780, + "unclipped_grad_norm": 0.9848454594612122 + }, + { + "bits_per_byte": 1.6182862317321756, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.1217105388641357, + "step": 6790, + "unclipped_grad_norm": 0.994420051574707 + }, + { + "bits_per_byte": 1.6079612532876268, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.1145538091659546, + "step": 6800, + "unclipped_grad_norm": 1.0759928226470947 + }, + { + "bits_per_byte": 1.5649406530875496, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0847342014312744, + "step": 6810, + "unclipped_grad_norm": 1.0181862115859985 + }, + { + "bits_per_byte": 1.5209997737227055, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.054276704788208, + "step": 6820, + "unclipped_grad_norm": 0.9867210984230042 + }, + { + "bits_per_byte": 1.5408068436399243, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.068005919456482, + "step": 6830, + "unclipped_grad_norm": 1.0285433530807495 + }, + { + "bits_per_byte": 1.5661266454478995, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0855562686920166, + "step": 6840, + "unclipped_grad_norm": 0.9897852540016174 + }, + { + "bits_per_byte": 1.5403562490946636, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0676935911178589, + "step": 6850, + "unclipped_grad_norm": 0.9739700555801392 + }, + { + "bits_per_byte": 1.5589986525002395, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.080615520477295, + "step": 6860, + "unclipped_grad_norm": 1.0206658840179443 + }, + { + "bits_per_byte": 1.5592929148158658, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0808194875717163, + "step": 6870, + "unclipped_grad_norm": 1.015761137008667 + }, + { + "bits_per_byte": 1.5959116328202259, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.1062016487121582, + "step": 6880, + "unclipped_grad_norm": 1.0070849657058716 + }, + { + "bits_per_byte": 1.4912815156189139, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.033677577972412, + "step": 6890, + "unclipped_grad_norm": 0.9983416199684143 + }, + { + "bits_per_byte": 1.5029766798428865, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0417840480804443, + "step": 6900, + "unclipped_grad_norm": 0.9461802840232849 + }, + { + "bits_per_byte": 1.691495806551456, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1724555492401123, + "step": 6910, + "unclipped_grad_norm": 1.0603437423706055 + }, + { + "bits_per_byte": 1.4926446501096524, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0346224308013916, + "step": 6920, + "unclipped_grad_norm": 0.9669191241264343 + }, + { + "bits_per_byte": 1.5121873826924952, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.048168420791626, + "step": 6930, + "unclipped_grad_norm": 1.012385606765747 + }, + { + "bits_per_byte": 1.5487118542041824, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.073485255241394, + "step": 6940, + "unclipped_grad_norm": 1.0546537637710571 + }, + { + "bits_per_byte": 1.5353376233598361, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0642149448394775, + "step": 6950, + "unclipped_grad_norm": 0.9847853779792786 + }, + { + "bits_per_byte": 1.5735119244412503, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0906753540039062, + "step": 6960, + "unclipped_grad_norm": 1.0091943740844727 + }, + { + "bits_per_byte": 1.5592397721767493, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0807826519012451, + "step": 6970, + "unclipped_grad_norm": 1.0198149681091309 + }, + { + "bits_per_byte": 1.6070454456717824, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1139190196990967, + "step": 6980, + "unclipped_grad_norm": 1.0826783180236816 + }, + { + "bits_per_byte": 1.5476336949659235, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0727379322052002, + "step": 6990, + "unclipped_grad_norm": 1.0041176080703735 + }, + { + "bits_per_byte": 1.5120640711318265, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.048082947731018, + "step": 7000, + "unclipped_grad_norm": 1.0215020179748535 + }, + { + "bits_per_byte": 1.5377993830242862, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0659213066101074, + "step": 7010, + "unclipped_grad_norm": 1.0378373861312866 + }, + { + "bits_per_byte": 1.5074898485668284, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.044912338256836, + "step": 7020, + "unclipped_grad_norm": 0.982413113117218 + }, + { + "bits_per_byte": 1.5236237130269192, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0560954809188843, + "step": 7030, + "unclipped_grad_norm": 0.9606820344924927 + }, + { + "bits_per_byte": 1.6050487270952651, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.112534999847412, + "step": 7040, + "unclipped_grad_norm": 1.0613288879394531 + }, + { + "bits_per_byte": 1.5624771735965908, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.083026647567749, + "step": 7050, + "unclipped_grad_norm": 1.0451524257659912 + }, + { + "bits_per_byte": 1.5626594752065053, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0831530094146729, + "step": 7060, + "unclipped_grad_norm": 1.073291540145874 + }, + { + "bits_per_byte": 1.519674131449855, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0533578395843506, + "step": 7070, + "unclipped_grad_norm": 1.009161353111267 + }, + { + "bits_per_byte": 1.4484239549718545, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.003970980644226, + "step": 7080, + "unclipped_grad_norm": 0.9980688691139221 + }, + { + "bits_per_byte": 1.539162345532374, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0668660402297974, + "step": 7090, + "unclipped_grad_norm": 1.0500024557113647 + }, + { + "bits_per_byte": 1.5110017342974469, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.047346591949463, + "step": 7100, + "unclipped_grad_norm": 1.038030743598938 + }, + { + "bits_per_byte": 1.5757193217650745, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0922054052352905, + "step": 7110, + "unclipped_grad_norm": 1.0732139348983765 + }, + { + "bits_per_byte": 1.5620382738715888, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0827224254608154, + "step": 7120, + "unclipped_grad_norm": 1.0506101846694946 + }, + { + "bits_per_byte": 1.4981061311704518, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0384080410003662, + "step": 7130, + "unclipped_grad_norm": 1.0116451978683472 + }, + { + "bits_per_byte": 1.6131864301861407, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.1181756258010864, + "step": 7140, + "unclipped_grad_norm": 1.0758471488952637 + }, + { + "bits_per_byte": 1.623974385926812, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.1256532669067383, + "step": 7150, + "unclipped_grad_norm": 1.0434077978134155 + }, + { + "bits_per_byte": 1.5482879169698056, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0731914043426514, + "step": 7160, + "unclipped_grad_norm": 1.0636146068572998 + }, + { + "bits_per_byte": 1.5506334163622726, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.074817180633545, + "step": 7170, + "unclipped_grad_norm": 1.0567522048950195 + }, + { + "bits_per_byte": 1.468200068011918, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0176787376403809, + "step": 7180, + "unclipped_grad_norm": 0.9437705874443054 + }, + { + "bits_per_byte": 1.4717380951054677, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0201311111450195, + "step": 7190, + "unclipped_grad_norm": 1.010156512260437 + }, + { + "bits_per_byte": 1.5698447383769025, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.088133454322815, + "step": 7200, + "unclipped_grad_norm": 1.056505560874939 + }, + { + "bits_per_byte": 1.56738452655631, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.086428165435791, + "step": 7210, + "unclipped_grad_norm": 1.0318564176559448 + }, + { + "bits_per_byte": 1.523764394835325, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0561929941177368, + "step": 7220, + "unclipped_grad_norm": 1.0528209209442139 + }, + { + "bits_per_byte": 1.5338092135416184, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0631555318832397, + "step": 7230, + "unclipped_grad_norm": 1.025399923324585 + }, + { + "bits_per_byte": 1.490086580160719, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0328493118286133, + "step": 7240, + "unclipped_grad_norm": 0.9961045980453491 + }, + { + "bits_per_byte": 1.5541721313864256, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0772700309753418, + "step": 7250, + "unclipped_grad_norm": 1.0621839761734009 + }, + { + "bits_per_byte": 1.5116575041451867, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.04780113697052, + "step": 7260, + "unclipped_grad_norm": 1.0816527605056763 + }, + { + "bits_per_byte": 1.5896555919124404, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.101865291595459, + "step": 7270, + "unclipped_grad_norm": 1.059293270111084 + }, + { + "bits_per_byte": 1.5429449319554507, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0694879293441772, + "step": 7280, + "unclipped_grad_norm": 1.0573333501815796 + }, + { + "bits_per_byte": 1.5487371356538593, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.073502779006958, + "step": 7290, + "unclipped_grad_norm": 1.0477967262268066 + }, + { + "bits_per_byte": 1.5296157605656304, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.060248851776123, + "step": 7300, + "unclipped_grad_norm": 1.0505754947662354 + }, + { + "bits_per_byte": 1.57645196785775, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0927132368087769, + "step": 7310, + "unclipped_grad_norm": 1.0852391719818115 + }, + { + "bits_per_byte": 1.5598148821612343, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.081181287765503, + "step": 7320, + "unclipped_grad_norm": 1.079657793045044 + }, + { + "bits_per_byte": 1.5641130725715977, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.084160566329956, + "step": 7330, + "unclipped_grad_norm": 1.0734436511993408 + }, + { + "bits_per_byte": 1.5372211773520854, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0655205249786377, + "step": 7340, + "unclipped_grad_norm": 1.0373533964157104 + }, + { + "bits_per_byte": 1.585016531888067, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0986497402191162, + "step": 7350, + "unclipped_grad_norm": 1.034577488899231 + }, + { + "bits_per_byte": 1.5482229075277794, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0731463432312012, + "step": 7360, + "unclipped_grad_norm": 0.9948747754096985 + }, + { + "bits_per_byte": 1.5566985565276004, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0790212154388428, + "step": 7370, + "unclipped_grad_norm": 1.0546287298202515 + }, + { + "bits_per_byte": 1.614560055618582, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.1191277503967285, + "step": 7380, + "unclipped_grad_norm": 1.1286121606826782 + }, + { + "bits_per_byte": 1.5888789182611438, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.1013269424438477, + "step": 7390, + "unclipped_grad_norm": 1.063069224357605 + }, + { + "bits_per_byte": 1.5174531474966126, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0518183708190918, + "step": 7400, + "unclipped_grad_norm": 1.038301944732666 + }, + { + "bits_per_byte": 1.5128250943618944, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0486104488372803, + "step": 7410, + "unclipped_grad_norm": 0.9619468450546265 + }, + { + "bits_per_byte": 1.6064108296900985, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.1134791374206543, + "step": 7420, + "unclipped_grad_norm": 1.0911381244659424 + }, + { + "bits_per_byte": 1.5182607780250643, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0523781776428223, + "step": 7430, + "unclipped_grad_norm": 1.0977153778076172 + }, + { + "bits_per_byte": 1.527175670715189, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0585575103759766, + "step": 7440, + "unclipped_grad_norm": 1.035995602607727 + }, + { + "bits_per_byte": 1.4906346888690187, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0332292318344116, + "step": 7450, + "unclipped_grad_norm": 1.1210148334503174 + }, + { + "bits_per_byte": 1.5053576076614312, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0434343814849854, + "step": 7460, + "unclipped_grad_norm": 1.0129456520080566 + }, + { + "bits_per_byte": 1.555529418467035, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0782108306884766, + "step": 7470, + "unclipped_grad_norm": 1.0514966249465942 + }, + { + "bits_per_byte": 1.4886658314819414, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0318645238876343, + "step": 7480, + "unclipped_grad_norm": 1.032109260559082 + }, + { + "bits_per_byte": 1.51988842183283, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0535063743591309, + "step": 7490, + "unclipped_grad_norm": 1.0880528688430786 + }, + { + "bits_per_byte": 1.5815053340880518, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0962159633636475, + "step": 7500, + "unclipped_grad_norm": 1.0377991199493408 + }, + { + "bits_per_byte": 1.5370581377990675, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0654075145721436, + "step": 7510, + "unclipped_grad_norm": 1.0226150751113892 + }, + { + "bits_per_byte": 1.5380518535557528, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.066096305847168, + "step": 7520, + "unclipped_grad_norm": 1.0527992248535156 + }, + { + "bits_per_byte": 1.562738587225902, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0832078456878662, + "step": 7530, + "unclipped_grad_norm": 1.072679042816162 + }, + { + "bits_per_byte": 1.6011416252329638, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.1098268032073975, + "step": 7540, + "unclipped_grad_norm": 1.109021544456482 + }, + { + "bits_per_byte": 1.5189222233002826, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0528366565704346, + "step": 7550, + "unclipped_grad_norm": 1.0227179527282715 + }, + { + "bits_per_byte": 1.5563348132210255, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0787690877914429, + "step": 7560, + "unclipped_grad_norm": 1.0419527292251587 + }, + { + "bits_per_byte": 1.5726612982500827, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.090085744857788, + "step": 7570, + "unclipped_grad_norm": 1.03652822971344 + }, + { + "bits_per_byte": 1.5146027070412125, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0498425960540771, + "step": 7580, + "unclipped_grad_norm": 1.0102300643920898 + }, + { + "bits_per_byte": 1.5228090312097822, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0555307865142822, + "step": 7590, + "unclipped_grad_norm": 1.049957513809204 + }, + { + "bits_per_byte": 1.5153031923781766, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0503281354904175, + "step": 7600, + "unclipped_grad_norm": 1.020882248878479 + }, + { + "bits_per_byte": 1.4615728885435706, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.013085126876831, + "step": 7610, + "unclipped_grad_norm": 1.0289204120635986 + }, + { + "bits_per_byte": 1.5704728190178536, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0885688066482544, + "step": 7620, + "unclipped_grad_norm": 1.062748670578003 + }, + { + "bits_per_byte": 1.5187743182205407, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.052734136581421, + "step": 7630, + "unclipped_grad_norm": 1.036866307258606 + }, + { + "bits_per_byte": 1.5251964943690597, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0571856498718262, + "step": 7640, + "unclipped_grad_norm": 1.0375516414642334 + }, + { + "bits_per_byte": 1.625032595177571, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1263867616653442, + "step": 7650, + "unclipped_grad_norm": 1.15651273727417 + }, + { + "bits_per_byte": 1.5325501285546517, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0622828006744385, + "step": 7660, + "unclipped_grad_norm": 1.0661544799804688 + }, + { + "bits_per_byte": 1.5264065643005305, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0580244064331055, + "step": 7670, + "unclipped_grad_norm": 1.0767730474472046 + }, + { + "bits_per_byte": 1.490845023651024, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0333750247955322, + "step": 7680, + "unclipped_grad_norm": 1.040376901626587 + }, + { + "bits_per_byte": 1.53468460523451, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0637623071670532, + "step": 7690, + "unclipped_grad_norm": 1.0646589994430542 + }, + { + "bits_per_byte": 1.5464913861988927, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.071946144104004, + "step": 7700, + "unclipped_grad_norm": 1.1067028045654297 + }, + { + "bits_per_byte": 1.4665762078224713, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0165531635284424, + "step": 7710, + "unclipped_grad_norm": 1.0252745151519775 + }, + { + "bits_per_byte": 1.4804371495687594, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0261608362197876, + "step": 7720, + "unclipped_grad_norm": 1.0019656419754028 + }, + { + "bits_per_byte": 1.5129640563437916, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0487067699432373, + "step": 7730, + "unclipped_grad_norm": 1.0048695802688599 + }, + { + "bits_per_byte": 1.5117082390271912, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0478363037109375, + "step": 7740, + "unclipped_grad_norm": 1.034838080406189 + }, + { + "bits_per_byte": 1.5481713127325205, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.073110580444336, + "step": 7750, + "unclipped_grad_norm": 1.047455072402954 + }, + { + "bits_per_byte": 1.5013908278192805, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0406848192214966, + "step": 7760, + "unclipped_grad_norm": 1.096320629119873 + }, + { + "bits_per_byte": 1.5005262710333933, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0400855541229248, + "step": 7770, + "unclipped_grad_norm": 1.0564236640930176 + }, + { + "bits_per_byte": 1.6389760885962732, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.1360516548156738, + "step": 7780, + "unclipped_grad_norm": 1.134232997894287 + }, + { + "bits_per_byte": 1.5295576304296388, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0602085590362549, + "step": 7790, + "unclipped_grad_norm": 1.0709397792816162 + }, + { + "bits_per_byte": 1.5624484524938966, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.083006739616394, + "step": 7800, + "unclipped_grad_norm": 1.099833607673645 + }, + { + "bits_per_byte": 1.476959144420361, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0237500667572021, + "step": 7810, + "unclipped_grad_norm": 1.057004690170288 + }, + { + "bits_per_byte": 1.5143643390871164, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0496773719787598, + "step": 7820, + "unclipped_grad_norm": 1.0959510803222656 + }, + { + "bits_per_byte": 1.5593489811600472, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0808583498001099, + "step": 7830, + "unclipped_grad_norm": 1.063362956047058 + }, + { + "bits_per_byte": 1.4978546925348903, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.038233757019043, + "step": 7840, + "unclipped_grad_norm": 1.0525480508804321 + }, + { + "bits_per_byte": 1.4918236049344333, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0340533256530762, + "step": 7850, + "unclipped_grad_norm": 1.030691146850586 + }, + { + "bits_per_byte": 1.5064691315339576, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.044204831123352, + "step": 7860, + "unclipped_grad_norm": 1.0765224695205688 + }, + { + "bits_per_byte": 1.5562261201856802, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0786937475204468, + "step": 7870, + "unclipped_grad_norm": 1.0955042839050293 + }, + { + "bits_per_byte": 1.5252102529811287, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0571951866149902, + "step": 7880, + "unclipped_grad_norm": 1.043196201324463 + }, + { + "bits_per_byte": 1.5900611270031748, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.1021463871002197, + "step": 7890, + "unclipped_grad_norm": 1.104681372642517 + }, + { + "bits_per_byte": 1.5606699799013244, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0817739963531494, + "step": 7900, + "unclipped_grad_norm": 1.0641751289367676 + }, + { + "bits_per_byte": 1.5589052659208211, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0805507898330688, + "step": 7910, + "unclipped_grad_norm": 1.1519651412963867 + }, + { + "bits_per_byte": 1.481516856650876, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0269092321395874, + "step": 7920, + "unclipped_grad_norm": 1.0578134059906006 + }, + { + "bits_per_byte": 1.4791508913229565, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0252692699432373, + "step": 7930, + "unclipped_grad_norm": 1.0473592281341553 + }, + { + "bits_per_byte": 1.5474438261193708, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0726063251495361, + "step": 7940, + "unclipped_grad_norm": 1.1337641477584839 + }, + { + "bits_per_byte": 1.4870933941051028, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0307745933532715, + "step": 7950, + "unclipped_grad_norm": 1.0619428157806396 + }, + { + "bits_per_byte": 1.5210572159280935, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.054316520690918, + "step": 7960, + "unclipped_grad_norm": 1.0414892435073853 + }, + { + "bits_per_byte": 1.5042326691421377, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0426546335220337, + "step": 7970, + "unclipped_grad_norm": 1.0393376350402832 + }, + { + "bits_per_byte": 1.5355154534208282, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.064338207244873, + "step": 7980, + "unclipped_grad_norm": 1.0560662746429443 + }, + { + "bits_per_byte": 1.5624764856659872, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0830261707305908, + "step": 7990, + "unclipped_grad_norm": 1.1011888980865479 + }, + { + "bits_per_byte": 1.4795096471326563, + "learning_rate": 3e-05, + "loss_nats": 1.0255179405212402, + "step": 8000, + "unclipped_grad_norm": 1.0237547159194946 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073001.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073001.json new file mode 100644 index 0000000..e881d24 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073001.json @@ -0,0 +1,9616 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "6b2a0f3320fece089c65a158888f4aafe0879ff220d9835ed3614be007b68068", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027713447343558073, + 0.00024001447309274226, + 0.00020889485313091427, + 0.00019106797117274255, + 0.00017709653184283525, + 0.00016345064796041697, + 0.0001520135992905125, + 0.00014396758342627436, + 0.00013824697816744447, + 0.0001333381951553747, + 0.00012879272981081158, + 0.00012578140012919903, + 0.00012338913802523166, + 0.0001212825154652819, + 0.00011928759340662509, + 0.00011723516217898577 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022927794270799495, + "first_to_last_ratio": 1.9059069547229377, + "imbalance_abs_log_ratio": 0.6449579870010378, + "last_quartile_mean": 0.00012029860226903111, + "mean": 0.00016006211535568582, + "normalized": [ + 1.7314182860806244, + 1.4995083162520277, + 1.3050861702453052, + 1.1937113960299495, + 1.1064237870985023, + 1.021170110098828, + 0.9497162957812465, + 0.8994482117542517, + 0.8637083038683806, + 0.8330403159990363, + 0.8046421823465957, + 0.7858286756343993, + 0.7708828397715447, + 0.7577215582573746, + 0.7452581339534802, + 0.7324354168284537 + ], + "population_cv": 0.28697332073733334 + }, + "activation_output_rms_by_block": [ + 0.005004490725696087, + 0.007150472607463598, + 0.0056586493737995625, + 0.007886149920523167, + 0.005775847006589174, + 0.008285986259579659, + 0.006597128231078386, + 0.009296777658164501, + 0.007501539774239063, + 0.01081777922809124, + 0.007698386441916227, + 0.011321330443024635, + 0.009009703993797302, + 0.013792803511023521, + 0.00953624676913023, + 0.012954792007803917 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0064249406568706036, + "first_to_last_ratio": 0.567404514268178, + "imbalance_abs_log_ratio": 0.5666828005795043, + "last_quartile_mean": 0.011323386570438743, + "mean": 0.008643005246995017, + "normalized": [ + 0.5790220626599802, + 0.8273132322752738, + 0.6547085431617609, + 0.9124314628022479, + 0.6682683674867963, + 0.9586927258271091, + 0.7632910130850682, + 1.0756417926966766, + 0.8679318778438997, + 1.251622429808468, + 0.8907071350665657, + 1.3098835554867694, + 1.04242722714183, + 1.5958342170183197, + 1.103348487778111, + 1.4988758698611242 + ], + "population_cv": 0.28641618387108014 + }, + "bits_per_byte": 8.092078157393424, + "branch_output_rms_by_sublayer": [ + 0.0033052791841328144, + 0.003737553022801876, + 0.0033692142460495234, + 0.0038969451561570168, + 0.004222586750984192, + 0.003809630637988448, + 0.004224879201501608, + 0.003801452461630106, + 0.004351162351667881, + 0.0037875697016716003, + 0.005037518683820963, + 0.0037579576019197702, + 0.005422801710665226, + 0.003782435320317745, + 0.005750134587287903, + 0.003670723643153906, + 0.006515308283269405, + 0.003754077944904566, + 0.006654165219515562, + 0.0036852110642939806, + 0.006751408334821463, + 0.003720880253240466, + 0.007695567328482866, + 0.003703791182488203, + 0.008140962570905685, + 0.003749529365450144, + 0.009426429867744446, + 0.0036915375385433435, + 0.008798080496490002, + 0.003752112854272127, + 0.008380400948226452, + 0.0036098985001444817 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007836087849617155, + 0.00707663314616961, + 0.006860545956961424, + 0.006477463491545069, + 0.00630808320659222, + 0.0065403273634858875, + 0.006076989561376891, + 0.0053044433268254945, + 0.006002429844215364, + 0.005811298045810947, + 0.00541486468124986, + 0.005552686577658202, + 0.005724443119797102, + 0.005942467552368739, + 0.006312250195815329, + 0.004848514440357715 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007062682611073314, + "first_to_last_ratio": 1.237564932158175, + "imbalance_abs_log_ratio": 0.21314568451503188, + "last_quartile_mean": 0.005706918827084721, + "mean": 0.006130595522490438, + "normalized": [ + 1.2781935818257821, + 1.154314147819503, + 1.119066807097797, + 1.0565798163950184, + 1.0289511326347756, + 1.0668339379905794, + 0.9912559944760846, + 0.865241118479574, + 0.9790940899942118, + 0.947917380047643, + 0.883252640201938, + 0.9057336366895933, + 0.9337499267072272, + 0.9693132633801169, + 1.0296308364592772, + 0.7908716898008789 + ], + "population_cv": 0.1166547247084935 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02823692187666893, + 0.014202049933373928, + 0.014330258592963219, + 0.014431006275117397, + 0.0145475585013628, + 0.009805543348193169, + 0.009861846454441547, + 0.009975572116672993, + 0.010043247602880001, + 0.007605585735291243, + 0.007654537446796894, + 0.0077590313740074635, + 0.007819762453436852, + 0.006375181954354048, + 0.006421996746212244, + 0.0065260399132966995, + 0.006563509814441204, + 0.005602568853646517, + 0.0056375316344201565, + 0.0057973419316112995, + 0.005820456892251968, + 0.005056646186858416, + 0.00509054446592927, + 0.005272198468446732, + 0.005295110866427422, + 0.0046798414550721645, + 0.004703808110207319, + 0.004869809839874506, + 0.004889858886599541, + 0.004459173884242773, + 0.00446688337251544, + 0.00453129643574357 + ], + "loss_nats": 5.609001159667969, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0033052791841328144, + 0.005004490725696087, + 0.005999269895255566, + 0.007150472607463598, + 0.004222586750984192, + 0.0056586493737995625, + 0.006890070624649525, + 0.007886149920523167, + 0.004351162351667881, + 0.005775847006589174, + 0.007411441300064325, + 0.008285986259579659, + 0.005422801710665226, + 0.006597128231078386, + 0.008578632958233356, + 0.009296777658164501, + 0.006515308283269405, + 0.007501539774239063, + 0.010256913490593433, + 0.01081777922809124, + 0.006751408334821463, + 0.007698386441916227, + 0.01081965770572424, + 0.011321330443024635, + 0.008140962570905685, + 0.009009703993797302, + 0.013233783654868603, + 0.013792803511023521, + 0.008798080496490002, + 0.00953624676913023, + 0.01231932733207941, + 0.012954792007803917 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.295638690469787e-05, + 7.495813042623922e-05, + 6.424964522011578e-05, + 5.6337856221944094e-05, + 4.736588743980974e-05, + 4.000150511274114e-05, + 3.3406991860829294e-05, + 2.9257240385049954e-05, + 2.6081905161845498e-05, + 2.4221646526711993e-05, + 2.3017932107904926e-05, + 2.205361488449853e-05, + 2.1097952412674204e-05, + 2.0617559130187146e-05, + 2.0049723389092833e-05, + 1.9771381630562246e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.962550469324924e-05, + "first_to_last_ratio": 3.415668082811133, + "imbalance_abs_log_ratio": 1.2283731057520397, + "last_quartile_mean": 2.0384154140629107e-05, + "mean": 3.784033492593153e-05, + "normalized": [ + 2.1922741187961536, + 1.980905575306399, + 1.6979142850050797, + 1.4888308026929338, + 1.2517301322127163, + 1.0571128715176512, + 0.8828408079954885, + 0.7731760419752606, + 0.6892620060815551, + 0.640101272203941, + 0.6082909190143296, + 0.5828070741885916, + 0.557551946989138, + 0.5448566766267753, + 0.5298505795030106, + 0.5224948898909759 + ], + "population_cv": 0.5389270020989329 + }, + "activation_output_rms_by_block": [ + 0.00972363818436861, + 0.018413539975881577, + 0.011301152408123016, + 0.020593363791704178, + 0.012286243960261345, + 0.022828614339232445, + 0.014672646299004555, + 0.02904798462986946, + 0.01802690140902996, + 0.03463379293680191, + 0.0215163491666317, + 0.04299422353506088, + 0.024622667580842972, + 0.048123423010110855, + 0.032037004828453064, + 0.06016550958156586 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.015007923590019345, + "first_to_last_ratio": 0.36394181302547757, + "imbalance_abs_log_ratio": 1.010761278449072, + "last_quartile_mean": 0.04123715125024319, + "mean": 0.0263116909773089, + "normalized": [ + 0.3695558066851058, + 0.6998235116002747, + 0.4295106847320868, + 0.7826697193070493, + 0.4669499946186262, + 0.8676224708978132, + 0.5576474089657784, + 1.1039953553316026, + 0.6851289574879962, + 1.3162891342358027, + 0.8177486268437599, + 1.6340349836176982, + 0.9358071133503911, + 1.8289749241739084, + 1.2175958153385753, + 2.2866454928135314 + ], + "population_cv": 0.5242218345998277 + }, + "bits_per_byte": 5.094476963164152, + "branch_output_rms_by_sublayer": [ + 0.004319524392485619, + 0.008673875592648983, + 0.004246409051120281, + 0.009991174563765526, + 0.005452096462249756, + 0.010073051787912846, + 0.006387368775904179, + 0.010035603307187557, + 0.0060663470067083836, + 0.01051078736782074, + 0.007258581463247538, + 0.010655198246240616, + 0.007839187048375607, + 0.011261194944381714, + 0.010483811609447002, + 0.011669226922094822, + 0.01079322025179863, + 0.012236819602549076, + 0.012645936571061611, + 0.012139025144279003, + 0.013178177177906036, + 0.012943001464009285, + 0.017010189592838287, + 0.012494352646172047, + 0.016437800601124763, + 0.012577332556247711, + 0.018056757748126984, + 0.01252446137368679, + 0.022778941318392754, + 0.013186860829591751, + 0.022304536774754524, + 0.012990611605346203 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00047768131303286524, + 0.00037801938737084786, + 0.0003757481810571057, + 0.0003849183374966076, + 0.0003205908784645981, + 0.0003135875371373823, + 0.0002702256708410831, + 0.00025052984822069896, + 0.00022846424191684566, + 0.00020301436013832287, + 0.00018877625003321395, + 0.00018901092530252878, + 0.00017529332745812984, + 0.0001779656532508539, + 0.00018745944157007413, + 0.00017444725355961893 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004040918047393566, + "first_to_last_ratio": 2.260129748343848, + "imbalance_abs_log_ratio": 0.8154222224078733, + "last_quartile_mean": 0.0001787914189596692, + "mean": 0.00026848328792817357, + "normalized": [ + 1.779184532188304, + 1.407981071328276, + 1.39952167584311, + 1.4336770845848084, + 1.1940813185749002, + 1.167996487071014, + 1.0064897257715932, + 0.9331301406280542, + 0.8509439961043904, + 0.7561526890740202, + 0.703121045223927, + 0.7039951229779867, + 0.6529021929477618, + 0.6628556087202881, + 0.6982164253747687, + 0.649750883586796 + ], + "population_cv": 0.3443291255737099 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6925109028816223, + "mean_weights": [ + 0.516845703125, + 0.4831542670726776 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6916897296905518, + "mean_weights": [ + 0.5256155133247375, + 0.47438448667526245 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6921632289886475, + "mean_weights": [ + 0.5142087936401367, + 0.4857912063598633 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927341222763062, + "mean_weights": [ + 0.5126981735229492, + 0.4873018264770508 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0980398654937744, + "mean_weights": [ + 0.3447725176811218, + 0.324658066034317, + 0.33056944608688354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098333477973938, + "mean_weights": [ + 0.3432988226413727, + 0.3276415765285492, + 0.3290596008300781 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0971107482910156, + "mean_weights": [ + 0.354239284992218, + 0.3338475227355957, + 0.3119131922721863 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098445177078247, + "mean_weights": [ + 0.33804112672805786, + 0.3361448645591736, + 0.3258139491081238 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3858705759048462, + "mean_weights": [ + 0.25839337706565857, + 0.24941343069076538, + 0.24149224162101746, + 0.2507009506225586 + ], + "sources": 4 + }, + { + "entropy_mean": 1.38621187210083, + "mean_weights": [ + 0.2517760992050171, + 0.2535240054130554, + 0.24683426320552826, + 0.24786563217639923 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385884165763855, + "mean_weights": [ + 0.2573123574256897, + 0.2553749680519104, + 0.24531587958335876, + 0.24199676513671875 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861134052276611, + "mean_weights": [ + 0.2514210641384125, + 0.2551700472831726, + 0.24895039200782776, + 0.24445852637290955 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6088017225265503, + "mean_weights": [ + 0.20787756145000458, + 0.20764237642288208, + 0.1969079077243805, + 0.1933688223361969, + 0.19420333206653595 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093868017196655, + "mean_weights": [ + 0.1996297687292099, + 0.200723797082901, + 0.20193718373775482, + 0.19867070019245148, + 0.199038565158844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.607616901397705, + "mean_weights": [ + 0.20617185533046722, + 0.2148669958114624, + 0.20431196689605713, + 0.19360044598579407, + 0.18104872107505798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094070672988892, + "mean_weights": [ + 0.1992160826921463, + 0.20051369071006775, + 0.198721244931221, + 0.19902712106704712, + 0.20252184569835663 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7898081541061401, + "mean_weights": [ + 0.17206060886383057, + 0.18212765455245972, + 0.17207278311252594, + 0.1640574187040329, + 0.15447911620140076, + 0.15520241856575012 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917404174804688, + "mean_weights": [ + 0.16550754010677338, + 0.16615179181098938, + 0.16667258739471436, + 0.16704785823822021, + 0.16769331693649292, + 0.16692689061164856 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7875205278396606, + "mean_weights": [ + 0.1741587072610855, + 0.18743160367012024, + 0.17662332952022552, + 0.16453337669372559, + 0.15226861834526062, + 0.14498436450958252 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916977405548096, + "mean_weights": [ + 0.1655852198600769, + 0.16538481414318085, + 0.16465555131435394, + 0.16688036918640137, + 0.16788282990455627, + 0.16961121559143066 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941636562347412, + "mean_weights": [ + 0.15022630989551544, + 0.1620834320783615, + 0.15417584776878357, + 0.14502081274986267, + 0.1326843798160553, + 0.12827157974243164, + 0.12753766775131226 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458396434783936, + "mean_weights": [ + 0.14176656305789948, + 0.14105038344860077, + 0.14134296774864197, + 0.14280946552753448, + 0.145027756690979, + 0.14322641491889954, + 0.14477643370628357 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395363330841064, + "mean_weights": [ + 0.1491325944662094, + 0.16522708535194397, + 0.15644297003746033, + 0.1476437896490097, + 0.13585588335990906, + 0.12666445970535278, + 0.11903323233127594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456722736358643, + "mean_weights": [ + 0.14162756502628326, + 0.1389600932598114, + 0.139722540974617, + 0.1427886188030243, + 0.14271582663059235, + 0.14634007215499878, + 0.14784526824951172 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0722482204437256, + "mean_weights": [ + 0.13311916589736938, + 0.1474711298942566, + 0.1391790360212326, + 0.13093729317188263, + 0.12214824557304382, + 0.11368396133184433, + 0.1066313236951828, + 0.10682985186576843 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079228401184082, + "mean_weights": [ + 0.12360984086990356, + 0.12237860262393951, + 0.12229418754577637, + 0.12343960255384445, + 0.1245395839214325, + 0.12723875045776367, + 0.12765389680862427, + 0.12884552776813507 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0708298683166504, + "mean_weights": [ + 0.13347145915031433, + 0.14857617020606995, + 0.14165782928466797, + 0.1326693594455719, + 0.12101326882839203, + 0.11364143341779709, + 0.10599445551633835, + 0.10297602415084839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079227924346924, + "mean_weights": [ + 0.12400860339403152, + 0.12165012210607529, + 0.12233424186706543, + 0.12391965091228485, + 0.12502582371234894, + 0.12579283118247986, + 0.12804390490055084, + 0.12922482192516327 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1904914379119873, + "mean_weights": [ + 0.1202651709318161, + 0.13243305683135986, + 0.1248447597026825, + 0.11750722676515579, + 0.10865370184183121, + 0.1034422218799591, + 0.09818274527788162, + 0.09678354859352112, + 0.09788758307695389 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969027519226074, + "mean_weights": [ + 0.10986955463886261, + 0.108255535364151, + 0.10776711255311966, + 0.10903829336166382, + 0.11005579680204391, + 0.11137478053569794, + 0.11346087604761124, + 0.11412093043327332, + 0.1160571351647377 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1876320838928223, + "mean_weights": [ + 0.12014202773571014, + 0.1339646577835083, + 0.12840712070465088, + 0.12060266733169556, + 0.11135218292474747, + 0.10444460809230804, + 0.09650979936122894, + 0.09281164407730103, + 0.09176531434059143 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028673440217971802, + 0.01493624784052372, + 0.015959296375513077, + 0.015908053144812584, + 0.017468810081481934, + 0.011937789618968964, + 0.013394257053732872, + 0.013926870189607143, + 0.015381631441414356, + 0.011804917827248573, + 0.013283023610711098, + 0.013736946508288383, + 0.015293780714273453, + 0.01275910809636116, + 0.014138885773718357, + 0.014833617955446243, + 0.01643991842865944, + 0.014291302300989628, + 0.01566910557448864, + 0.016281748190522194, + 0.01796886697411537, + 0.016028419137001038, + 0.017591506242752075, + 0.01816963590681553, + 0.020232705399394035, + 0.018053309991955757, + 0.019960105419158936, + 0.01998855359852314, + 0.02254442311823368, + 0.020374728366732597, + 0.022746482864022255, + 0.022396961227059364 + ], + "loss_nats": 3.531222343444824, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1935343742370605, + "mean_weights": [ + 0.0999753400683403, + 0.09610259532928467, + 0.10262835025787354, + 0.1083034798502922, + 0.11600279062986374, + 0.11817552894353867, + 0.12095586210489273, + 0.11944535374641418, + 0.11841070652008057 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004319524392485619, + 0.00972363818436861, + 0.010906110517680645, + 0.018413539975881577, + 0.005452096462249756, + 0.011301152408123016, + 0.013437297195196152, + 0.020593363791704178, + 0.0060663470067083836, + 0.012286243960261345, + 0.014960306696593761, + 0.022828614339232445, + 0.007839187048375607, + 0.014672646299004555, + 0.020714404061436653, + 0.02904798462986946, + 0.01079322025179863, + 0.01802690140902996, + 0.026133937761187553, + 0.03463379293680191, + 0.013178177177906036, + 0.0215163491666317, + 0.03426522761583328, + 0.04299422353506088, + 0.016437800601124763, + 0.024622667580842972, + 0.03885738551616669, + 0.048123423010110855, + 0.022778941318392754, + 0.032037004828453064, + 0.050110094249248505, + 0.06016550958156586 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015777233056724072, + 0.00015884912863839418, + 0.00014271712279878557, + 0.0001244439627043903, + 0.00010494918387848884, + 9.392325591761619e-05, + 8.538185647921637e-05, + 7.606096187373623e-05, + 7.664377335458994e-05, + 6.531321560032666e-05, + 6.149107503006235e-05, + 5.037344453739934e-05, + 4.3381573050282896e-05, + 3.9420847315341234e-05, + 3.5541354009183124e-05, + 3.5066059353994206e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001459456361772027, + "first_to_last_ratio": 3.805378902507801, + "imbalance_abs_log_ratio": 1.3364155665156117, + "last_quartile_mean": 3.8352458432200365e-05, + "mean": 8.445807156931551e-05, + "normalized": [ + 1.8680550909542795, + 1.8808045896243943, + 1.6897985017531016, + 1.4734407309105797, + 1.2426187566022755, + 1.1120696242813515, + 1.0109377930698158, + 0.9005765874172524, + 0.907477188745515, + 0.7733211804004252, + 0.7280662923921495, + 0.5964313842526867, + 0.513646265468887, + 0.46675050214695213, + 0.42081654658683526, + 0.4151889653934991 + ], + "population_cv": 0.4877143313799778 + }, + "activation_output_rms_by_block": [ + 0.025262881070375443, + 0.04275903478264809, + 0.027615655213594437, + 0.03926205262541771, + 0.023017436265945435, + 0.03517371043562889, + 0.029550904408097267, + 0.04735429957509041, + 0.028369002044200897, + 0.04305245727300644, + 0.030713967978954315, + 0.05133935809135437, + 0.032636724412441254, + 0.06983363628387451, + 0.0380333848297596, + 0.08374716341495514 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03372490592300892, + "first_to_last_ratio": 0.601556641750376, + "imbalance_abs_log_ratio": 0.508234580502337, + "last_quartile_mean": 0.056062727235257626, + "mean": 0.04048260429408401, + "normalized": [ + 0.6240428823910242, + 1.056232313317272, + 0.6821610342304507, + 0.9698499716125064, + 0.5685759764548824, + 0.868859873246076, + 0.7299654981044719, + 1.1697443976451536, + 0.7007701836106094, + 1.0634804263148154, + 0.7586954573335775, + 1.2681831860026092, + 0.8061913254234652, + 1.7250282560027828, + 0.939499458914941, + 2.0687197593953623 + ], + "population_cv": 0.3940384688106746 + }, + "bits_per_byte": 3.0824777281177833, + "branch_output_rms_by_sublayer": [ + 0.014225655235350132, + 0.01885763369500637, + 0.016379276290535927, + 0.015279849991202354, + 0.023593612015247345, + 0.011461553163826466, + 0.02286384254693985, + 0.009474686346948147, + 0.01935836300253868, + 0.010116237215697765, + 0.017970046028494835, + 0.009506942704319954, + 0.022477619349956512, + 0.013764957897365093, + 0.025794917717576027, + 0.012601078487932682, + 0.02379908226430416, + 0.014191905967891216, + 0.0178237184882164, + 0.0165114663541317, + 0.023045850917696953, + 0.019447455182671547, + 0.020932989194989204, + 0.02472713589668274, + 0.017093855887651443, + 0.028764601796865463, + 0.021727578714489937, + 0.03578802943229675, + 0.008686712011694908, + 0.036414168775081635, + 0.008637740276753902, + 0.04455644637346268 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006504433508330581, + 0.00048523602227392907, + 0.0003980638932136608, + 0.0004387300445530515, + 0.000365155189968218, + 0.0002788779263831531, + 0.00029885208568229324, + 0.00026779671325771547, + 0.0002902285411340871, + 0.00023612702925164195, + 0.00026277969610552434, + 0.0002548116213942568, + 0.0001837802586554031, + 0.00016271499692659642, + 0.00013374164438063855, + 0.00014095740133328614 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004931183277184248, + "first_to_last_ratio": 3.1752920249892207, + "imbalance_abs_log_ratio": 1.1553996044896453, + "last_quartile_mean": 0.00015529857532398106, + "mean": 0.0003030185259591571, + "normalized": [ + 2.1465464818501867, + 1.6013411085609086, + 1.3136619022010376, + 1.4478654173513685, + 1.20505896071, + 0.9203329251913204, + 0.9862502127100129, + 0.8837635006310174, + 0.9577914096684838, + 0.7792494815431475, + 0.8672067005597657, + 0.8409110320489186, + 0.6064984247206531, + 0.536980359242221, + 0.4413645797968972, + 0.46517750321406204 + ], + "population_cv": 0.440031122144889 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6679806113243103, + "mean_weights": [ + 0.4191037714481354, + 0.580896258354187 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6774038076400757, + "mean_weights": [ + 0.4775225520133972, + 0.522477388381958 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686615526676178, + "mean_weights": [ + 0.49171629548072815, + 0.5082836747169495 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6786561012268066, + "mean_weights": [ + 0.48390668630599976, + 0.516093373298645 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0939385890960693, + "mean_weights": [ + 0.32296979427337646, + 0.3461979627609253, + 0.33083218336105347 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0876491069793701, + "mean_weights": [ + 0.3022151589393616, + 0.3410899341106415, + 0.35669487714767456 + ], + "sources": 3 + }, + { + "entropy_mean": 1.094268560409546, + "mean_weights": [ + 0.3278208076953888, + 0.3347434997558594, + 0.33743569254875183 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0785255432128906, + "mean_weights": [ + 0.30184268951416016, + 0.3319981098175049, + 0.3661591410636902 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3751009702682495, + "mean_weights": [ + 0.2335343211889267, + 0.2411448359489441, + 0.23957431316375732, + 0.2857465147972107 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3803684711456299, + "mean_weights": [ + 0.23392163217067719, + 0.24959179759025574, + 0.25211477279663086, + 0.264371782541275 + ], + "sources": 4 + }, + { + "entropy_mean": 1.377173662185669, + "mean_weights": [ + 0.2529916763305664, + 0.22492024302482605, + 0.24044713377952576, + 0.2816409468650818 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786447048187256, + "mean_weights": [ + 0.23216357827186584, + 0.24167919158935547, + 0.2545858919620514, + 0.2715713381767273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6062697172164917, + "mean_weights": [ + 0.20799711346626282, + 0.1897224485874176, + 0.18646025657653809, + 0.19469907879829407, + 0.22112110257148743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6025722026824951, + "mean_weights": [ + 0.20026245713233948, + 0.18544277548789978, + 0.1946466565132141, + 0.19751906394958496, + 0.22212904691696167 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6060729026794434, + "mean_weights": [ + 0.20172038674354553, + 0.20138651132583618, + 0.18545961380004883, + 0.19016142189502716, + 0.2212720513343811 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601726770401001, + "mean_weights": [ + 0.20073023438453674, + 0.1886606216430664, + 0.19257165491580963, + 0.19004762172698975, + 0.22798986732959747 + ], + "sources": 5 + }, + { + "entropy_mean": 1.788693904876709, + "mean_weights": [ + 0.16298983991146088, + 0.17133885622024536, + 0.15878048539161682, + 0.15357322990894318, + 0.17383432388305664, + 0.17948326468467712 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7880157232284546, + "mean_weights": [ + 0.16720382869243622, + 0.15951547026634216, + 0.1605137139558792, + 0.15542668104171753, + 0.17394030094146729, + 0.1834000051021576 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7797719240188599, + "mean_weights": [ + 0.15782463550567627, + 0.17797376215457916, + 0.15211433172225952, + 0.14025422930717468, + 0.16258476674556732, + 0.20924828946590424 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7884608507156372, + "mean_weights": [ + 0.16717270016670227, + 0.17295360565185547, + 0.1665436029434204, + 0.1547294557094574, + 0.1560850888490677, + 0.18251556158065796 + ], + "sources": 6 + }, + { + "entropy_mean": 1.927689790725708, + "mean_weights": [ + 0.1226624995470047, + 0.15493766963481903, + 0.12924720346927643, + 0.12375974655151367, + 0.1207568496465683, + 0.16289259493350983, + 0.18574345111846924 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9359079599380493, + "mean_weights": [ + 0.1374971866607666, + 0.15181128680706024, + 0.14001096785068512, + 0.13405972719192505, + 0.11918757855892181, + 0.14624495804309845, + 0.17118829488754272 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9246108531951904, + "mean_weights": [ + 0.12061166763305664, + 0.16360490024089813, + 0.1358117014169693, + 0.1262153834104538, + 0.1155187338590622, + 0.15234649181365967, + 0.18589113652706146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937231183052063, + "mean_weights": [ + 0.1359395533800125, + 0.16017206013202667, + 0.1499013900756836, + 0.1417488306760788, + 0.12114370614290237, + 0.13951142132282257, + 0.15158303081989288 + ], + "sources": 7 + }, + { + "entropy_mean": 2.052294969558716, + "mean_weights": [ + 0.0988478884100914, + 0.15235479176044464, + 0.12090037763118744, + 0.11009317636489868, + 0.09369668364524841, + 0.12056069076061249, + 0.13413047790527344, + 0.1694159209728241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.061406135559082, + "mean_weights": [ + 0.1149308905005455, + 0.14702443778514862, + 0.13390076160430908, + 0.12344712018966675, + 0.0966324657201767, + 0.11953848600387573, + 0.1299745738506317, + 0.1345512568950653 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0511577129364014, + "mean_weights": [ + 0.09969542920589447, + 0.16962948441505432, + 0.1319108009338379, + 0.11863738298416138, + 0.09143088757991791, + 0.12074249982833862, + 0.14481820166110992, + 0.12313534319400787 + ], + "sources": 8 + }, + { + "entropy_mean": 2.069282054901123, + "mean_weights": [ + 0.12384641170501709, + 0.13645249605178833, + 0.13362589478492737, + 0.12747827172279358, + 0.10400287806987762, + 0.11596200615167618, + 0.12676866352558136, + 0.13186340034008026 + ], + "sources": 8 + }, + { + "entropy_mean": 2.165989875793457, + "mean_weights": [ + 0.08691617846488953, + 0.1522500216960907, + 0.1198691874742508, + 0.10617777705192566, + 0.08211278915405273, + 0.11005747318267822, + 0.12842655181884766, + 0.08939606696367264, + 0.12479399144649506 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907525062561035, + "mean_weights": [ + 0.1032973974943161, + 0.12799270451068878, + 0.11672255396842957, + 0.11367214471101761, + 0.10446079820394516, + 0.11008220911026001, + 0.12035597860813141, + 0.1041758805513382, + 0.09924033284187317 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1538312435150146, + "mean_weights": [ + 0.09009353071451187, + 0.16798722743988037, + 0.13133394718170166, + 0.11392836272716522, + 0.08597822487354279, + 0.10755141079425812, + 0.13270318508148193, + 0.09302861988544464, + 0.07739552855491638 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04356693848967552, + 0.020052054896950722, + 0.025759555399417877, + 0.02808709815144539, + 0.03107464499771595, + 0.023514779284596443, + 0.025064069777727127, + 0.026588447391986847, + 0.02779453620314598, + 0.02102271094918251, + 0.022771678864955902, + 0.023088837042450905, + 0.024504151195287704, + 0.02003801427781582, + 0.021661879494786263, + 0.021999599412083626, + 0.02357223629951477, + 0.01831224001944065, + 0.01920042186975479, + 0.01775558479130268, + 0.01933184079825878, + 0.014507194049656391, + 0.0161737073212862, + 0.015147161670029163, + 0.017492808401584625, + 0.01415062416344881, + 0.01712540164589882, + 0.017588311806321144, + 0.021003147587180138, + 0.01741618849337101, + 0.020855708047747612, + 0.02042423188686371 + ], + "loss_nats": 2.136610746383667, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1821343898773193, + "mean_weights": [ + 0.12100096046924591, + 0.09246665984392166, + 0.10230658948421478, + 0.10203823447227478, + 0.09980043023824692, + 0.10532397031784058, + 0.11232466995716095, + 0.1358221024274826, + 0.12891638278961182 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.014225655235350132, + 0.025262881070375443, + 0.03417503461241722, + 0.04275903478264809, + 0.023593612015247345, + 0.027615655213594437, + 0.035288337618112564, + 0.03926205262541771, + 0.01935836300253868, + 0.023017436265945435, + 0.030542491003870964, + 0.03517371043562889, + 0.022477619349956512, + 0.029550904408097267, + 0.04085070639848709, + 0.04735429957509041, + 0.02379908226430416, + 0.028369002044200897, + 0.03469245135784149, + 0.04305245727300644, + 0.023045850917696953, + 0.030713967978954315, + 0.037714507430791855, + 0.05133935809135437, + 0.017093855887651443, + 0.032636724412441254, + 0.041826508939266205, + 0.06983363628387451, + 0.008686712011694908, + 0.0380333848297596, + 0.04225953668355942, + 0.08374716341495514 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.445879600709304e-05, + 7.514825847465545e-05, + 8.149984205374494e-05, + 7.616886432515457e-05, + 7.93620856711641e-05, + 7.107412966433913e-05, + 6.609321280848235e-05, + 6.663892418146133e-05, + 0.00013160602247808129, + 0.00010466711682965979, + 0.00017213824321515858, + 0.0001231937640113756, + 0.00011169029312441126, + 7.821891631465405e-05, + 5.0732542149489745e-05, + 4.2012750782305375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.6818940215162e-05, + "first_to_last_ratio": 1.087107257387618, + "imbalance_abs_log_ratio": 0.08352027612092149, + "last_quartile_mean": 7.066362559271511e-05, + "mean": 8.779398513070191e-05, + "normalized": [ + 0.8481081693266763, + 0.8559613550151489, + 0.9283078098392884, + 0.8675863638238926, + 0.9039581191469443, + 0.8095557976839601, + 0.752821650709751, + 0.7590374680253287, + 1.499032334415107, + 1.1921900648869996, + 1.9607065672994624, + 1.403214170401, + 1.2721861635297007, + 0.8909370749967309, + 0.5778589737550069, + 0.47853791714500205 + ], + "population_cv": 0.365612445298668 + }, + "activation_output_rms_by_block": [ + 0.10262583196163177, + 0.17122715711593628, + 0.1563396155834198, + 0.23706908524036407, + 0.04811444878578186, + 0.07098451256752014, + 0.05486894026398659, + 0.08282698690891266, + 0.05263432487845421, + 0.09547916799783707, + 0.05760422348976135, + 0.09091533720493317, + 0.05383981764316559, + 0.1016460508108139, + 0.07459317892789841, + 0.18281812965869904 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16681542247533798, + "first_to_last_ratio": 1.6160480792916094, + "imbalance_abs_log_ratio": 0.4799837116932076, + "last_quartile_mean": 0.10322429426014423, + "mean": 0.10209917556494474, + "normalized": [ + 1.005158282559804, + 1.6770669906831872, + 1.531252477978855, + 2.3219490650006835, + 0.4712520793586283, + 0.6952505950683928, + 0.5374082597668455, + 0.8112405066016116, + 0.5155215464494499, + 0.9351610085930936, + 0.5641987133688422, + 0.8904610316574242, + 0.5273286228341615, + 0.9955619156411052, + 0.7305953110311854, + 1.7905935934067303 + ], + "population_cv": 0.5286889875537891 + }, + "bits_per_byte": 2.0839361382492645, + "branch_output_rms_by_sublayer": [ + 0.05184043571352959, + 0.0848206952214241, + 0.0633782222867012, + 0.04741951450705528, + 0.15336942672729492, + 0.017866143956780434, + 0.08626767247915268, + 0.013284381479024887, + 0.043669264763593674, + 0.012495848350226879, + 0.03683627396821976, + 0.02719804272055626, + 0.04707624390721321, + 0.01943410001695156, + 0.031199999153614044, + 0.02732040360569954, + 0.04137895256280899, + 0.029010025784373283, + 0.04094765707850456, + 0.04834925755858421, + 0.03695840761065483, + 0.041233282536268234, + 0.026400581002235413, + 0.05263383314013481, + 0.021495802327990532, + 0.04698725789785385, + 0.018388623371720314, + 0.06700151413679123, + 0.015488670207560062, + 0.0718570277094841, + 0.015049737878143787, + 0.12199355661869049 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003583741677462985, + 0.0002652231972340971, + 0.00033417461006948805, + 0.0002275253539728081, + 0.0001893834795064905, + 0.00020636019253143534, + 0.0002096112726133392, + 0.0001923796657115476, + 0.00047635112247231166, + 0.000565001794959564, + 0.0009549288128880833, + 0.0007263576654831343, + 0.0005471283815528999, + 0.0004183078461688176, + 0.0002607600761312772, + 0.0002482750049477455 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029632433225567294, + "first_to_last_ratio": 0.8038795478406102, + "imbalance_abs_log_ratio": 0.21830583714537038, + "last_quartile_mean": 0.00036861782720018504, + "mean": 0.0003862589152493336, + "normalized": [ + 0.9278081452565691, + 0.6866461504529756, + 0.8651570148323317, + 0.5890488089470723, + 0.49030189862217616, + 0.5342535392308757, + 0.5426703807678671, + 0.49805883596852335, + 1.2332430493282536, + 1.462754056032209, + 2.472250542804088, + 1.8804942405387945, + 1.4164809146210218, + 1.0829726632945058, + 0.6750914110628468, + 0.6427683482398892 + ], + "population_cv": 0.550745478202196 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4576239287853241, + "mean_weights": [ + 0.8160475492477417, + 0.18395249545574188 + ], + "sources": 2 + }, + { + "entropy_mean": 0.622429370880127, + "mean_weights": [ + 0.626032829284668, + 0.37396717071533203 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5034195184707642, + "mean_weights": [ + 0.7919335961341858, + 0.20806637406349182 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6750276684761047, + "mean_weights": [ + 0.4605996012687683, + 0.5394004583358765 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9881983399391174, + "mean_weights": [ + 0.5066143870353699, + 0.2128569334745407, + 0.2805286943912506 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0713173151016235, + "mean_weights": [ + 0.2439693659543991, + 0.33285707235336304, + 0.42317354679107666 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0050503015518188, + "mean_weights": [ + 0.46526122093200684, + 0.19465598464012146, + 0.3400827646255493 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091313123703003, + "mean_weights": [ + 0.29163461923599243, + 0.3559022545814514, + 0.35246312618255615 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3032760620117188, + "mean_weights": [ + 0.25388818979263306, + 0.13041767477989197, + 0.27188149094581604, + 0.34381261467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3785908222198486, + "mean_weights": [ + 0.2572174668312073, + 0.24623332917690277, + 0.2131483256816864, + 0.28340083360671997 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2377310991287231, + "mean_weights": [ + 0.36895960569381714, + 0.12289825826883316, + 0.12992563843727112, + 0.3782165050506592 + ], + "sources": 4 + }, + { + "entropy_mean": 1.370632529258728, + "mean_weights": [ + 0.2267947793006897, + 0.24114219844341278, + 0.21443253755569458, + 0.31763046979904175 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5590848922729492, + "mean_weights": [ + 0.22265344858169556, + 0.11684703826904297, + 0.1529940664768219, + 0.24088799953460693, + 0.26661744713783264 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5987765789031982, + "mean_weights": [ + 0.19780412316322327, + 0.19732677936553955, + 0.1696196049451828, + 0.187029629945755, + 0.24821987748146057 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5600793361663818, + "mean_weights": [ + 0.19789859652519226, + 0.13498559594154358, + 0.15099790692329407, + 0.2153477668762207, + 0.300770103931427 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6049537658691406, + "mean_weights": [ + 0.21525533497333527, + 0.2069544643163681, + 0.18156641721725464, + 0.19566501677036285, + 0.20055876672267914 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7536540031433105, + "mean_weights": [ + 0.1706119179725647, + 0.12355935573577881, + 0.11797749251127243, + 0.17735670506954193, + 0.15967294573783875, + 0.250821590423584 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7486846446990967, + "mean_weights": [ + 0.1656067669391632, + 0.16547001898288727, + 0.14594469964504242, + 0.13893218338489532, + 0.11710066348314285, + 0.26694566011428833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.502744197845459, + "mean_weights": [ + 0.14790554344654083, + 0.12157414853572845, + 0.05193060636520386, + 0.11843410134315491, + 0.09838350117206573, + 0.4617721140384674 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7703553438186646, + "mean_weights": [ + 0.17112275958061218, + 0.18443143367767334, + 0.1734921634197235, + 0.14492332935333252, + 0.11953528970479965, + 0.2064950168132782 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6120144128799438, + "mean_weights": [ + 0.08869323134422302, + 0.07363369315862656, + 0.0430244579911232, + 0.08643164485692978, + 0.0696420818567276, + 0.21115350723266602, + 0.4274213910102844 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8458226919174194, + "mean_weights": [ + 0.12519317865371704, + 0.13179585337638855, + 0.1208491250872612, + 0.10168728232383728, + 0.08638136088848114, + 0.13217970728874207, + 0.3019134998321533 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5814783573150635, + "mean_weights": [ + 0.09110043942928314, + 0.07703252136707306, + 0.04712430387735367, + 0.08346284925937653, + 0.06837025284767151, + 0.1662226915359497, + 0.4666869342327118 + ], + "sources": 7 + }, + { + "entropy_mean": 1.864192008972168, + "mean_weights": [ + 0.13281965255737305, + 0.1363598257303238, + 0.09730516374111176, + 0.10794492065906525, + 0.09621791541576385, + 0.14952389895915985, + 0.27982860803604126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7858508825302124, + "mean_weights": [ + 0.0760645866394043, + 0.06870019435882568, + 0.041292984038591385, + 0.06624549627304077, + 0.05510660633444786, + 0.12923449277877808, + 0.3349960446357727, + 0.22835959494113922 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9164016246795654, + "mean_weights": [ + 0.10010629147291183, + 0.09289439767599106, + 0.06040576845407486, + 0.08482401818037033, + 0.06471984833478928, + 0.12172460556030273, + 0.20661231875419617, + 0.26871275901794434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7764818668365479, + "mean_weights": [ + 0.07597766816616058, + 0.07146216928958893, + 0.04444725811481476, + 0.06381020694971085, + 0.052670422941446304, + 0.10254879295825958, + 0.2695360779762268, + 0.3195474147796631 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9026113748550415, + "mean_weights": [ + 0.09874680638313293, + 0.09077301621437073, + 0.052286967635154724, + 0.08586552739143372, + 0.061087287962436676, + 0.1283089518547058, + 0.24689394235610962, + 0.2360374927520752 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0007686614990234, + "mean_weights": [ + 0.0738106444478035, + 0.07652983069419861, + 0.04215084761381149, + 0.06585545837879181, + 0.053016483783721924, + 0.10676231980323792, + 0.23121783137321472, + 0.19090119004249573, + 0.1597554087638855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.05625581741333, + "mean_weights": [ + 0.08243454247713089, + 0.09636277705430984, + 0.04735179618000984, + 0.07971908152103424, + 0.057041361927986145, + 0.1233949214220047, + 0.21250130236148834, + 0.17240752279758453, + 0.12878668308258057 + ], + "sources": 9 + }, + { + "entropy_mean": 2.023190498352051, + "mean_weights": [ + 0.08014722913503647, + 0.08487560600042343, + 0.04445280879735947, + 0.06905653327703476, + 0.057849444448947906, + 0.1067991554737091, + 0.20276135206222534, + 0.2105080783367157, + 0.14354978501796722 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05538630113005638, + 0.04719197377562523, + 0.05779172107577324, + 0.05863569676876068, + 0.10090170800685883, + 0.07870100438594818, + 0.1107865646481514, + 0.10731016099452972, + 0.12961514294147491, + 0.08656422793865204, + 0.08832070231437683, + 0.06243300065398216, + 0.09176014363765717, + 0.06557498127222061, + 0.0774947926402092, + 0.06959337741136551, + 0.08217275887727737, + 0.054640792310237885, + 0.061008501797914505, + 0.04349073767662048, + 0.07030727714300156, + 0.028683170676231384, + 0.04603966698050499, + 0.03691008314490318, + 0.04661448672413826, + 0.035560719668865204, + 0.03472456708550453, + 0.035963866859674454, + 0.04182035103440285, + 0.0347055159509182, + 0.03997879475355148, + 0.04123349115252495 + ], + "loss_nats": 1.444474458694458, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.9153306484222412, + "mean_weights": [ + 0.0914863795042038, + 0.05659400671720505, + 0.05275696888566017, + 0.06721317768096924, + 0.05635443329811096, + 0.0678911805152893, + 0.08835235983133316, + 0.2647843658924103, + 0.25456711649894714 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.05184043571352959, + 0.10262583196163177, + 0.1408688724040985, + 0.17122715711593628, + 0.15336942672729492, + 0.1563396155834198, + 0.2344740480184555, + 0.23706908524036407, + 0.043669264763593674, + 0.04811444878578186, + 0.059994228184223175, + 0.07098451256752014, + 0.04707624390721321, + 0.05486894026398659, + 0.069931261241436, + 0.08282698690891266, + 0.04137895256280899, + 0.05263432487845421, + 0.07298346608877182, + 0.09547916799783707, + 0.03695840761065483, + 0.05760422348976135, + 0.0662730410695076, + 0.09091533720493317, + 0.021495802327990532, + 0.05383981764316559, + 0.058252494782209396, + 0.1016460508108139, + 0.015488670207560062, + 0.07459317892789841, + 0.07778036594390869, + 0.18281812965869904 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.599572978098877e-05, + 5.6567834690213203e-05, + 7.252454088302329e-05, + 6.816006498411298e-05, + 6.961339386180043e-05, + 6.354664219543338e-05, + 5.091843559057452e-05, + 5.004410559195094e-05, + 0.00011432960309321061, + 8.808868005871773e-05, + 0.0001649916812311858, + 0.00011419985094107687, + 0.0001274511159863323, + 8.248462108895183e-05, + 5.212613905314356e-05, + 4.046529647894204e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.331204258458456e-05, + "first_to_last_ratio": 0.8371088393670095, + "imbalance_abs_log_ratio": 0.1778011818604578, + "last_quartile_mean": 7.563179315184243e-05, + "mean": 7.946923346935364e-05, + "normalized": [ + 0.7046214910652542, + 0.711820565276094, + 0.9126115569114117, + 0.8576912348147676, + 0.8759791786420843, + 0.7996382929745935, + 0.6407314298584572, + 0.6297293104160852, + 1.438664978910366, + 1.1084626869175487, + 2.076170538310438, + 1.4370322444989567, + 1.6037793548805563, + 1.0379440883969395, + 0.6559285496725645, + 0.5091944984538828 + ], + "population_cv": 0.418216483754139 + }, + "activation_output_rms_by_block": [ + 0.16897788643836975, + 0.2546124756336212, + 0.24958282709121704, + 0.3488057255744934, + 0.05895345285534859, + 0.09218393266201019, + 0.05982525274157524, + 0.10435724258422852, + 0.07404384016990662, + 0.13799543678760529, + 0.072679303586483, + 0.12079861760139465, + 0.06610427051782608, + 0.11929895728826523, + 0.08813745528459549, + 0.2320432960987091 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.25549472868442535, + "first_to_last_ratio": 2.0213831070680737, + "imbalance_abs_log_ratio": 0.7037819835779293, + "last_quartile_mean": 0.12639599479734898, + "mean": 0.1405249983072281, + "normalized": [ + 1.2024756340429583, + 1.8118660644063134, + 1.776074222364032, + 2.4821613931772033, + 0.419522886073689, + 0.6559966822448885, + 0.425726763652258, + 0.7426240444143152, + 0.5269086714950565, + 0.9819992061904, + 0.5171983950327836, + 0.8596236901372816, + 0.47040933153618064, + 0.8489518500291555, + 0.6272012549105437, + 1.6512599102929408 + ], + "population_cv": 0.5965464726510664 + }, + "bits_per_byte": 1.9646943750306483, + "branch_output_rms_by_sublayer": [ + 0.07875782251358032, + 0.1447884738445282, + 0.07949890196323395, + 0.06182052940130234, + 0.24504326283931732, + 0.02123360149562359, + 0.10607437044382095, + 0.01836485043168068, + 0.053805794566869736, + 0.015964502468705177, + 0.03685670718550682, + 0.03905382379889488, + 0.05105716362595558, + 0.02389056608080864, + 0.045697249472141266, + 0.039935220032930374, + 0.059221696108579636, + 0.040779970586299896, + 0.054663438349962234, + 0.080495685338974, + 0.04156494140625, + 0.05664549022912979, + 0.0355471596121788, + 0.07668386399745941, + 0.026702450588345528, + 0.05748681351542473, + 0.024266516789793968, + 0.08028528094291687, + 0.019828684628009796, + 0.08536289632320404, + 0.0202633123844862, + 0.16591297090053558 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00033760869044818646, + 0.0001988249533593431, + 0.00022180290878374173, + 0.00015787488110689858, + 0.0001349156990162873, + 0.00018159413720443885, + 0.00013882426062051804, + 0.00016817749051027943, + 0.0004737077931871952, + 0.0005939057773758512, + 0.0010653831085878915, + 0.0008603315741687424, + 0.000762827758322017, + 0.0005453805470031765, + 0.00031779802289983946, + 0.00029451637801964313 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022902785842454247, + "first_to_last_ratio": 0.47701150875195986, + "imbalance_abs_log_ratio": 0.7402146610222566, + "last_quartile_mean": 0.000480130676561169, + "mean": 0.00040334212378837815, + "normalized": [ + 0.8370280973313858, + 0.49294368634717844, + 0.5499125821534951, + 0.3914167943185893, + 0.33449444295353004, + 0.45022358562210574, + 0.34418488036066147, + 0.41695989729681016, + 1.1744565351565805, + 1.4724615713271154, + 2.6413881559935133, + 2.1330070018179734, + 1.89126727245144, + 1.3521537048516206, + 0.7879118102392371, + 0.7301899817787623 + ], + "population_cv": 0.6909894412084847 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.27112114429473877, + "mean_weights": [ + 0.9155579805374146, + 0.08444204181432724 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5555346012115479, + "mean_weights": [ + 0.7120839953422546, + 0.28791600465774536 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4612463414669037, + "mean_weights": [ + 0.8216869831085205, + 0.1783129870891571 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6770275831222534, + "mean_weights": [ + 0.48782873153686523, + 0.5121712684631348 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9962381720542908, + "mean_weights": [ + 0.465767502784729, + 0.19465303421020508, + 0.3395794630050659 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0414454936981201, + "mean_weights": [ + 0.22931429743766785, + 0.28697463870048523, + 0.4837110638618469 + ], + "sources": 3 + }, + { + "entropy_mean": 0.989799976348877, + "mean_weights": [ + 0.40070968866348267, + 0.17750780284404755, + 0.4217824935913086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.085202932357788, + "mean_weights": [ + 0.2848654091358185, + 0.3230235278606415, + 0.39211106300354004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2675609588623047, + "mean_weights": [ + 0.23358023166656494, + 0.10898500680923462, + 0.35628458857536316, + 0.3011501431465149 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3760833740234375, + "mean_weights": [ + 0.2731572389602661, + 0.2245836853981018, + 0.22189702093601227, + 0.280362069606781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2111725807189941, + "mean_weights": [ + 0.41297993063926697, + 0.11292652785778046, + 0.1222275048494339, + 0.3518660068511963 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647139072418213, + "mean_weights": [ + 0.23418807983398438, + 0.22505050897598267, + 0.2112433910369873, + 0.32951802015304565 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5535553693771362, + "mean_weights": [ + 0.21889275312423706, + 0.10524161159992218, + 0.1673421710729599, + 0.2413085401058197, + 0.26721489429473877 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5950126647949219, + "mean_weights": [ + 0.2038668543100357, + 0.17965060472488403, + 0.17214898765087128, + 0.19203421473503113, + 0.25229936838150024 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5395398139953613, + "mean_weights": [ + 0.2043805718421936, + 0.12131387740373611, + 0.1313788890838623, + 0.23064203560352325, + 0.3122846484184265 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6000796556472778, + "mean_weights": [ + 0.22543779015541077, + 0.20209266245365143, + 0.18844717741012573, + 0.20001280307769775, + 0.18400955200195312 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7217694520950317, + "mean_weights": [ + 0.1735105812549591, + 0.1090376153588295, + 0.10670766234397888, + 0.18005813658237457, + 0.14383366703987122, + 0.28685232996940613 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7343032360076904, + "mean_weights": [ + 0.16843369603157043, + 0.15050029754638672, + 0.15599164366722107, + 0.14163106679916382, + 0.10262320935726166, + 0.2808200716972351 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4446122646331787, + "mean_weights": [ + 0.1531161069869995, + 0.1061544418334961, + 0.04045919328927994, + 0.12559477984905243, + 0.08803897351026535, + 0.4866364896297455 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7605862617492676, + "mean_weights": [ + 0.1779545247554779, + 0.16784369945526123, + 0.1857004165649414, + 0.14843404293060303, + 0.10648007690906525, + 0.21358723938465118 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5263525247573853, + "mean_weights": [ + 0.08794503659009933, + 0.0601879358291626, + 0.033082835376262665, + 0.08346094191074371, + 0.05888189375400543, + 0.20315150916576385, + 0.4732898473739624 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7944886684417725, + "mean_weights": [ + 0.12276870757341385, + 0.11536462604999542, + 0.1304849088191986, + 0.09876619279384613, + 0.07423829287290573, + 0.11369137465953827, + 0.3446859121322632 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4918596744537354, + "mean_weights": [ + 0.09284307807683945, + 0.06384406983852386, + 0.03825954347848892, + 0.08267752081155777, + 0.05671031028032303, + 0.1521964967250824, + 0.5134689807891846 + ], + "sources": 7 + }, + { + "entropy_mean": 1.830867052078247, + "mean_weights": [ + 0.13634362816810608, + 0.12310001999139786, + 0.09669966250658035, + 0.1085672378540039, + 0.08431269228458405, + 0.13993649184703827, + 0.3110402822494507 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6549372673034668, + "mean_weights": [ + 0.06966839730739594, + 0.049915459007024765, + 0.029968995600938797, + 0.05769501253962517, + 0.040842454880476, + 0.10442843288183212, + 0.30951738357543945, + 0.33796387910842896 + ], + "sources": 8 + }, + { + "entropy_mean": 1.859694480895996, + "mean_weights": [ + 0.09671731293201447, + 0.07844845205545425, + 0.05707312002778053, + 0.0829043835401535, + 0.06170058995485306, + 0.10576222091913223, + 0.19003936648368835, + 0.3273545503616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.677255392074585, + "mean_weights": [ + 0.07509785890579224, + 0.05865459889173508, + 0.03770594298839569, + 0.059113647788763046, + 0.042673639953136444, + 0.08797852694988251, + 0.24025270342826843, + 0.39852309226989746 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8695217370986938, + "mean_weights": [ + 0.0984041839838028, + 0.08118931949138641, + 0.05154065042734146, + 0.08640529215335846, + 0.05755901336669922, + 0.11663993448019028, + 0.23645231127738953, + 0.27180927991867065 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8854753971099854, + "mean_weights": [ + 0.06882007420063019, + 0.0593036487698555, + 0.03391250967979431, + 0.060251303017139435, + 0.038275185972452164, + 0.09127645194530487, + 0.20795494318008423, + 0.23203802108764648, + 0.20816785097122192 + ], + "sources": 9 + }, + { + "entropy_mean": 2.032858371734619, + "mean_weights": [ + 0.08241330087184906, + 0.08549442887306213, + 0.041513457894325256, + 0.08080939948558807, + 0.054553937166929245, + 0.12166906148195267, + 0.2086121290922165, + 0.17801183462142944, + 0.14692245423793793 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9540549516677856, + "mean_weights": [ + 0.07821957767009735, + 0.07357673346996307, + 0.037884071469306946, + 0.06657841801643372, + 0.044617801904678345, + 0.09479035437107086, + 0.1742672622203827, + 0.2624226212501526, + 0.16764315962791443 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.058635130524635315, + 0.05468355491757393, + 0.07153555005788803, + 0.0688173845410347, + 0.13985061645507812, + 0.1219802051782608, + 0.16997404396533966, + 0.17499428987503052, + 0.1912512183189392, + 0.14618010818958282, + 0.12251991033554077, + 0.0794699639081955, + 0.12536334991455078, + 0.09043660014867783, + 0.10440115630626678, + 0.08707059919834137, + 0.11446695029735565, + 0.06907524168491364, + 0.0854191705584526, + 0.06048329547047615, + 0.10043052583932877, + 0.03875526040792465, + 0.06678051501512527, + 0.05217510461807251, + 0.06635956466197968, + 0.04468905180692673, + 0.04571816325187683, + 0.04713807255029678, + 0.05554596707224846, + 0.0446290485560894, + 0.052154671400785446, + 0.054206155240535736 + ], + "loss_nats": 1.3618223667144775, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.836374282836914, + "mean_weights": [ + 0.08612492680549622, + 0.0513404943048954, + 0.04786331206560135, + 0.0643724650144577, + 0.0475543811917305, + 0.059286996722221375, + 0.07609974592924118, + 0.2768789529800415, + 0.2904787063598633 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.07875782251358032, + 0.16897788643836975, + 0.21303832530975342, + 0.2546124756336212, + 0.24504326283931732, + 0.24958282709121704, + 0.345289409160614, + 0.3488057255744934, + 0.053805794566869736, + 0.05895345285534859, + 0.07887569069862366, + 0.09218393266201019, + 0.05105716362595558, + 0.05982525274157524, + 0.08438026905059814, + 0.10435724258422852, + 0.059221696108579636, + 0.07404384016990662, + 0.09824100881814957, + 0.13799543678760529, + 0.04156494140625, + 0.072679303586483, + 0.08400582522153854, + 0.12079861760139465, + 0.026702450588345528, + 0.06610427051782608, + 0.07228080183267593, + 0.11929895728826523, + 0.019828684628009796, + 0.08813745528459549, + 0.09191317111253738, + 0.2320432960987091 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.052410364849493e-05, + 7.120969530660659e-05, + 9.769770258571953e-05, + 9.110000246437266e-05, + 9.818621765589342e-05, + 8.837205677991733e-05, + 7.198825915111229e-05, + 6.996823503868654e-05, + 0.00017213664250448346, + 0.00013030946138314903, + 0.0002552570658735931, + 0.00017121800919994712, + 0.00020642777963075787, + 0.00012888041965197772, + 9.31880422285758e-05, + 5.8820565755013376e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.263287600129843e-05, + "first_to_last_ratio": 0.6782682211585493, + "imbalance_abs_log_ratio": 0.3882124628447938, + "last_quartile_mean": 0.00012182920181658119, + "mean": 0.0001172052661786438, + "normalized": [ + 0.6017144617120052, + 0.6075639570500958, + 0.8335606903260565, + 0.7772688500661601, + 0.8377287203651615, + 0.7539939088165287, + 0.6142066947861206, + 0.5969717685896574, + 1.4686766910465736, + 1.1118055155007445, + 2.1778634543991506, + 1.4608388750977905, + 1.7612500390223207, + 1.0996128745233913, + 0.7950840885130448, + 0.5018594101851985 + ], + "population_cv": 0.46767509062432555 + }, + "activation_output_rms_by_block": [ + 0.17702145874500275, + 0.2651609778404236, + 0.2740711271762848, + 0.3786091208457947, + 0.05278070271015167, + 0.09138669073581696, + 0.05598878860473633, + 0.10357969254255295, + 0.06921031326055527, + 0.1325109452009201, + 0.06760682165622711, + 0.11279971897602081, + 0.060408592224121094, + 0.10416899621486664, + 0.073752261698246, + 0.19092032313346863 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.27371567115187645, + "first_to_last_ratio": 2.5506400527811586, + "imbalance_abs_log_ratio": 0.9363443287654816, + "last_quartile_mean": 0.10731254331767559, + "mean": 0.13812353322282434, + "normalized": [ + 1.2816169309789325, + 1.9197378727103604, + 1.984246426234597, + 2.7410906165788056, + 0.38212679243445447, + 0.661630126333285, + 0.40535300030599286, + 0.7499061899571856, + 0.5010754622740751, + 0.9593654470679528, + 0.4894663500039642, + 0.8166582214056853, + 0.43735191834883375, + 0.7541726871902314, + 0.5339586960845188, + 1.3822432620911254 + ], + "population_cv": 0.6672200194546598 + }, + "bits_per_byte": 1.8951255009474826, + "branch_output_rms_by_sublayer": [ + 0.08229005336761475, + 0.1513136327266693, + 0.08296028524637222, + 0.06130215525627136, + 0.2681557536125183, + 0.023455867543816566, + 0.11001982539892197, + 0.01849420741200447, + 0.047267504036426544, + 0.01671545021235943, + 0.035008542239665985, + 0.04324181750416756, + 0.04728955402970314, + 0.02490479126572609, + 0.04628673940896988, + 0.041959211230278015, + 0.055535849183797836, + 0.03813145309686661, + 0.04947454109787941, + 0.083168163895607, + 0.037920478731393814, + 0.05371610447764397, + 0.03159874305129051, + 0.07563052326440811, + 0.023936081677675247, + 0.05285703018307686, + 0.021295147016644478, + 0.07199715077877045, + 0.018087781965732574, + 0.0706687644124031, + 0.01923547312617302, + 0.13923443853855133 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004973569892103084, + 0.0002771506747073045, + 0.0003595853430528865, + 0.00025621221416812444, + 0.0002736875635381804, + 0.00034154014112545435, + 0.0002723491519532143, + 0.0002954443301970981, + 0.0007531989220686374, + 0.0008991377204894639, + 0.0016884293947290697, + 0.0013942943414501116, + 0.0012489939262520248, + 0.0008593069460315814, + 0.000558781441282585, + 0.00045362324645161216 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034757630528465595, + "first_to_last_ratio": 0.44550989973264, + "imbalance_abs_log_ratio": 0.8085358106960822, + "last_quartile_mean": 0.0007801763900044509, + "mean": 0.0006518182716692285, + "normalized": [ + 0.7630301432585446, + 0.42519623452339683, + 0.5516650248727019, + 0.39307307773376104, + 0.41988323346214174, + 0.523980618479459, + 0.417829882638547, + 0.45326181090398177, + 1.1555351465367567, + 1.3794300644363342, + 2.5903376264755575, + 2.139084468865058, + 1.916168939317244, + 1.3183228874989945, + 0.8572656913277571, + 0.6959351496697651 + ], + "population_cv": 0.6710654904340576 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2534913420677185, + "mean_weights": [ + 0.9232981204986572, + 0.07670190930366516 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5290617346763611, + "mean_weights": [ + 0.7383309602737427, + 0.26166898012161255 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4635298252105713, + "mean_weights": [ + 0.8211046457290649, + 0.17889533936977386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.680541455745697, + "mean_weights": [ + 0.4997203052043915, + 0.5002796649932861 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9984116554260254, + "mean_weights": [ + 0.45704665780067444, + 0.20008087158203125, + 0.3428725004196167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0224952697753906, + "mean_weights": [ + 0.22097927331924438, + 0.2672557532787323, + 0.5117650032043457 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9886391162872314, + "mean_weights": [ + 0.4091557264328003, + 0.17683157324790955, + 0.41401270031929016 + ], + "sources": 3 + }, + { + "entropy_mean": 1.083122730255127, + "mean_weights": [ + 0.2858731150627136, + 0.31187689304351807, + 0.4022499620914459 + ], + "sources": 3 + }, + { + "entropy_mean": 1.262911081314087, + "mean_weights": [ + 0.24318832159042358, + 0.1005454957485199, + 0.3312486708164215, + 0.325017511844635 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3697116374969482, + "mean_weights": [ + 0.2760387659072876, + 0.21072158217430115, + 0.21145658195018768, + 0.30178308486938477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1750009059906006, + "mean_weights": [ + 0.40358343720436096, + 0.10160854458808899, + 0.10028939694166183, + 0.3945186138153076 + ], + "sources": 4 + }, + { + "entropy_mean": 1.355452299118042, + "mean_weights": [ + 0.23967716097831726, + 0.20835967361927032, + 0.20370042324066162, + 0.348262757062912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5273280143737793, + "mean_weights": [ + 0.2132018804550171, + 0.09696971625089645, + 0.14153459668159485, + 0.2435910701751709, + 0.3047027587890625 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5883060693740845, + "mean_weights": [ + 0.2065391093492508, + 0.16525766253471375, + 0.16744214296340942, + 0.19437497854232788, + 0.26638609170913696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4673986434936523, + "mean_weights": [ + 0.19910070300102234, + 0.10532978177070618, + 0.08866854012012482, + 0.23074698448181152, + 0.37615397572517395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6001981496810913, + "mean_weights": [ + 0.22463412582874298, + 0.18846800923347473, + 0.19619622826576233, + 0.2022438496351242, + 0.18845780193805695 + ], + "sources": 5 + }, + { + "entropy_mean": 1.658735752105713, + "mean_weights": [ + 0.16469183564186096, + 0.09157001227140427, + 0.085207998752594, + 0.16954544186592102, + 0.1403937190771103, + 0.34859102964401245 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7217588424682617, + "mean_weights": [ + 0.1685413271188736, + 0.1354280561208725, + 0.15638956427574158, + 0.13712063431739807, + 0.10325069725513458, + 0.29926973581314087 + ], + "sources": 6 + }, + { + "entropy_mean": 1.383854627609253, + "mean_weights": [ + 0.14582018554210663, + 0.09375129640102386, + 0.03237149864435196, + 0.11780930310487747, + 0.08738291263580322, + 0.522864818572998 + ], + "sources": 6 + }, + { + "entropy_mean": 1.758126974105835, + "mean_weights": [ + 0.18006329238414764, + 0.15370121598243713, + 0.18850785493850708, + 0.14673401415348053, + 0.10883604735136032, + 0.2221575826406479 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4768933057785034, + "mean_weights": [ + 0.08639779686927795, + 0.05247560143470764, + 0.02685363218188286, + 0.07686029374599457, + 0.05649450421333313, + 0.2057446390390396, + 0.49517351388931274 + ], + "sources": 7 + }, + { + "entropy_mean": 1.754528522491455, + "mean_weights": [ + 0.1207476407289505, + 0.10044647008180618, + 0.12684977054595947, + 0.0954352393746376, + 0.07363080978393555, + 0.10486453771591187, + 0.37802553176879883 + ], + "sources": 7 + }, + { + "entropy_mean": 1.434983491897583, + "mean_weights": [ + 0.09186187386512756, + 0.054585013538599014, + 0.030214153230190277, + 0.07775386422872543, + 0.05580390244722366, + 0.14904344081878662, + 0.5407377481460571 + ], + "sources": 7 + }, + { + "entropy_mean": 1.814563274383545, + "mean_weights": [ + 0.13872049748897552, + 0.10858511924743652, + 0.09594539552927017, + 0.10938869416713715, + 0.08644531667232513, + 0.13206249475479126, + 0.32885247468948364 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5878477096557617, + "mean_weights": [ + 0.06617997586727142, + 0.04098459705710411, + 0.023611580953001976, + 0.05256733298301697, + 0.03869973123073578, + 0.09328385442495346, + 0.30499327182769775, + 0.37967967987060547 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8095037937164307, + "mean_weights": [ + 0.0948307067155838, + 0.0694318562746048, + 0.0552619993686676, + 0.07500732690095901, + 0.061008021235466, + 0.09275255352258682, + 0.18348734080791473, + 0.3682202100753784 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5925970077514648, + "mean_weights": [ + 0.07132496684789658, + 0.04804965853691101, + 0.030455447733402252, + 0.054331667721271515, + 0.038998860865831375, + 0.07944463938474655, + 0.22967059910297394, + 0.4477241635322571 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8567713499069214, + "mean_weights": [ + 0.09893427789211273, + 0.07625022530555725, + 0.05004327371716499, + 0.08632857352495193, + 0.05865295231342316, + 0.10661771893501282, + 0.2290721982717514, + 0.2941007912158966 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7706098556518555, + "mean_weights": [ + 0.0600857213139534, + 0.04508119821548462, + 0.026483699679374695, + 0.05021750181913376, + 0.03222868591547012, + 0.07324853539466858, + 0.17921699583530426, + 0.23399141430854797, + 0.2994462847709656 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0186803340911865, + "mean_weights": [ + 0.08094437420368195, + 0.07421140372753143, + 0.03662773221731186, + 0.07851459085941315, + 0.05622554570436478, + 0.11645162105560303, + 0.2003173828125, + 0.18591775000095367, + 0.17078962922096252 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8739352226257324, + "mean_weights": [ + 0.07170331478118896, + 0.05929408222436905, + 0.03108443133533001, + 0.05872419476509094, + 0.03831540048122406, + 0.07797900587320328, + 0.1519450843334198, + 0.2740318179130554, + 0.23692265152931213 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.061437346041202545, + 0.05765428766608238, + 0.07246167957782745, + 0.07205748558044434, + 0.1429983228445053, + 0.13230633735656738, + 0.18651185929775238, + 0.18575440347194672, + 0.20683544874191284, + 0.14580051600933075, + 0.1242423728108406, + 0.07512427866458893, + 0.12742595374584198, + 0.08523519337177277, + 0.10536280274391174, + 0.07712213695049286, + 0.12031000852584839, + 0.06152798980474472, + 0.08708149939775467, + 0.05821346119046211, + 0.10479474812746048, + 0.03660533204674721, + 0.06689487397670746, + 0.049551647156476974, + 0.06684304773807526, + 0.04078952595591545, + 0.04357287660241127, + 0.04368526116013527, + 0.05231116712093353, + 0.03767167031764984, + 0.04748373478651047, + 0.0490146204829216 + ], + "loss_nats": 1.3136008977890015, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.685392141342163, + "mean_weights": [ + 0.08037909120321274, + 0.03995431959629059, + 0.0371873639523983, + 0.05237603187561035, + 0.03903363645076752, + 0.04661020264029503, + 0.0628754124045372, + 0.26517099142074585, + 0.37641292810440063 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.08229005336761475, + 0.17702145874500275, + 0.22416120767593384, + 0.2651609778404236, + 0.2681557536125183, + 0.2740711271762848, + 0.37491148710250854, + 0.3786091208457947, + 0.047267504036426544, + 0.05278070271015167, + 0.07517243921756744, + 0.09138669073581696, + 0.04728955402970314, + 0.05598878860473633, + 0.08201033622026443, + 0.10357969254255295, + 0.055535849183797836, + 0.06921031326055527, + 0.0903589129447937, + 0.1325109452009201, + 0.037920478731393814, + 0.06760682165622711, + 0.07624635845422745, + 0.11279971897602081, + 0.023936081677675247, + 0.060408592224121094, + 0.0652509555220604, + 0.10416899621486664, + 0.018087781965732574, + 0.073752261698246, + 0.0771600753068924, + 0.19092032313346863 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.073347956844582, + "cross_entropy_nats": 5.596018373966217, + "step": 0 + }, + { + "bits_per_byte": 5.112499455104693, + "cross_entropy_nats": 3.5437145829200745, + "step": 100 + }, + { + "bits_per_byte": 3.1188867992707414, + "cross_entropy_nats": 2.1618475914001465, + "step": 500 + }, + { + "bits_per_byte": 1.962477669145846, + "cross_entropy_nats": 1.3602858632802963, + "step": 2000 + }, + { + "bits_per_byte": 1.8161008917328916, + "cross_entropy_nats": 1.2588252127170563, + "step": 4000 + }, + { + "bits_per_byte": 1.737390666804078, + "cross_entropy_nats": 1.20426744222641, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "0bff629038a5dd94843a123dbdfc2bdaca027f8a9025a2a7688adc4ec0647b31", + "final_model_state": "efa098925d29c4e5158cd73c7764d7014b6c4ee53df36a146dbd0cff04321b62", + "final_optimizer_state": "b86655a1b88951524b7f3bfca00df30a0958104385aa444bf7b109fbe1d9e51b", + "final_public_parameters": "87107ee0633f1863f2fcaf1d5f91af597c67acce2c9777fec3486aa4fd1463ef", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.63475258274364, + "measured_steps": 7980, + "median_ms": 54.539098986424506, + "p95_ms": 55.062895112496335, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.084436624250287, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.603704452514648, + "step": 1, + "unclipped_grad_norm": 18.78145980834961 + }, + { + "bits_per_byte": 7.484355446790379, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.187759876251221, + "step": 10, + "unclipped_grad_norm": 12.867269515991211 + }, + { + "bits_per_byte": 6.795359238834398, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.710184097290039, + "step": 20, + "unclipped_grad_norm": 4.549929141998291 + }, + { + "bits_per_byte": 6.560912489178218, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.547677993774414, + "step": 30, + "unclipped_grad_norm": 2.9244894981384277 + }, + { + "bits_per_byte": 6.418922236764676, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.449257850646973, + "step": 40, + "unclipped_grad_norm": 2.8415465354919434 + }, + { + "bits_per_byte": 6.252045406840682, + "learning_rate": 3.75e-05, + "loss_nats": 4.333587646484375, + "step": 50, + "unclipped_grad_norm": 2.8266561031341553 + }, + { + "bits_per_byte": 5.950088587414815, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.124287128448486, + "step": 60, + "unclipped_grad_norm": 2.6689200401306152 + }, + { + "bits_per_byte": 5.70102021674758, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.951646089553833, + "step": 70, + "unclipped_grad_norm": 2.5050199031829834 + }, + { + "bits_per_byte": 5.519442653793135, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8257861137390137, + "step": 80, + "unclipped_grad_norm": 2.352375030517578 + }, + { + "bits_per_byte": 5.302846327435307, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.6756529808044434, + "step": 90, + "unclipped_grad_norm": 2.2169463634490967 + }, + { + "bits_per_byte": 5.139077911943008, + "learning_rate": 7.5e-05, + "loss_nats": 3.5621373653411865, + "step": 100, + "unclipped_grad_norm": 2.0478744506835938 + }, + { + "bits_per_byte": 4.944703463903163, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4274072647094727, + "step": 110, + "unclipped_grad_norm": 1.971948504447937 + }, + { + "bits_per_byte": 4.730622899762156, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.279017925262451, + "step": 120, + "unclipped_grad_norm": 1.6964443922042847 + }, + { + "bits_per_byte": 4.549973699157069, + "learning_rate": 9.75e-05, + "loss_nats": 3.153801441192627, + "step": 130, + "unclipped_grad_norm": 1.537685751914978 + }, + { + "bits_per_byte": 4.410896832849143, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.057400703430176, + "step": 140, + "unclipped_grad_norm": 1.355125904083252 + }, + { + "bits_per_byte": 4.261557417798932, + "learning_rate": 0.0001125, + "loss_nats": 2.9538865089416504, + "step": 150, + "unclipped_grad_norm": 1.2107940912246704 + }, + { + "bits_per_byte": 4.165243693662742, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.887126922607422, + "step": 160, + "unclipped_grad_norm": 0.8971752524375916 + }, + { + "bits_per_byte": 4.038897327136983, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7995502948760986, + "step": 170, + "unclipped_grad_norm": 0.8723403215408325 + }, + { + "bits_per_byte": 3.967892225936462, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.750333309173584, + "step": 180, + "unclipped_grad_norm": 0.7337902784347534 + }, + { + "bits_per_byte": 3.8789046187617258, + "learning_rate": 0.0001425, + "loss_nats": 2.6886518001556396, + "step": 190, + "unclipped_grad_norm": 0.8901073336601257 + }, + { + "bits_per_byte": 3.824914105176364, + "learning_rate": 0.00015, + "loss_nats": 2.651228427886963, + "step": 200, + "unclipped_grad_norm": 0.642342746257782 + }, + { + "bits_per_byte": 3.7744440764541642, + "learning_rate": 0.0001575, + "loss_nats": 2.6162452697753906, + "step": 210, + "unclipped_grad_norm": 0.6421008110046387 + }, + { + "bits_per_byte": 3.68308276473222, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5529184341430664, + "step": 220, + "unclipped_grad_norm": 0.7395935654640198 + }, + { + "bits_per_byte": 3.6743831943209755, + "learning_rate": 0.0001725, + "loss_nats": 2.5468883514404297, + "step": 230, + "unclipped_grad_norm": 0.6173842549324036 + }, + { + "bits_per_byte": 3.575872219837362, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4786057472229004, + "step": 240, + "unclipped_grad_norm": 0.8246023058891296 + }, + { + "bits_per_byte": 3.6039487315560175, + "learning_rate": 0.0001875, + "loss_nats": 2.4980669021606445, + "step": 250, + "unclipped_grad_norm": 0.7212417721748352 + }, + { + "bits_per_byte": 3.6017153648519127, + "learning_rate": 0.000195, + "loss_nats": 2.496518850326538, + "step": 260, + "unclipped_grad_norm": 0.6610386967658997 + }, + { + "bits_per_byte": 3.6077832567396544, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.5007247924804688, + "step": 270, + "unclipped_grad_norm": 0.8672931790351868 + }, + { + "bits_per_byte": 3.524929582790591, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4432950019836426, + "step": 280, + "unclipped_grad_norm": 0.7419575452804565 + }, + { + "bits_per_byte": 3.446865969633359, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3891854286193848, + "step": 290, + "unclipped_grad_norm": 0.8831741809844971 + }, + { + "bits_per_byte": 3.491855255237863, + "learning_rate": 0.000225, + "loss_nats": 2.4203696250915527, + "step": 300, + "unclipped_grad_norm": 0.9954257607460022 + }, + { + "bits_per_byte": 3.448469191904702, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.390296697616577, + "step": 310, + "unclipped_grad_norm": 1.0782777070999146 + }, + { + "bits_per_byte": 3.4360496367552953, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.381688117980957, + "step": 320, + "unclipped_grad_norm": 0.9428723454475403 + }, + { + "bits_per_byte": 3.415760155502406, + "learning_rate": 0.0002475, + "loss_nats": 2.367624521255493, + "step": 330, + "unclipped_grad_norm": 1.0516632795333862 + }, + { + "bits_per_byte": 3.5107169365232886, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.433443546295166, + "step": 340, + "unclipped_grad_norm": 1.1474740505218506 + }, + { + "bits_per_byte": 3.4490202243180663, + "learning_rate": 0.0002625, + "loss_nats": 2.390678644180298, + "step": 350, + "unclipped_grad_norm": 1.2037273645401 + }, + { + "bits_per_byte": 3.401095538828636, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.357459783554077, + "step": 360, + "unclipped_grad_norm": 1.0446844100952148 + }, + { + "bits_per_byte": 3.376710806693402, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.34055757522583, + "step": 370, + "unclipped_grad_norm": 0.8781102895736694 + }, + { + "bits_per_byte": 3.422498435763211, + "learning_rate": 0.000285, + "loss_nats": 2.3722951412200928, + "step": 380, + "unclipped_grad_norm": 0.9067438244819641 + }, + { + "bits_per_byte": 3.35829834409203, + "learning_rate": 0.0002925, + "loss_nats": 2.3277950286865234, + "step": 390, + "unclipped_grad_norm": 1.0105773210525513 + }, + { + "bits_per_byte": 3.298973960572809, + "learning_rate": 0.0003, + "loss_nats": 2.2866744995117188, + "step": 400, + "unclipped_grad_norm": 1.1294934749603271 + }, + { + "bits_per_byte": 3.2943352445137375, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.283459186553955, + "step": 410, + "unclipped_grad_norm": 1.2423999309539795 + }, + { + "bits_per_byte": 3.324363071389082, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3042728900909424, + "step": 420, + "unclipped_grad_norm": 1.016541838645935 + }, + { + "bits_per_byte": 3.29597423917646, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.284595251083374, + "step": 430, + "unclipped_grad_norm": 1.1856372356414795 + }, + { + "bits_per_byte": 3.3349637380229638, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.3116207122802734, + "step": 440, + "unclipped_grad_norm": 0.8890857696533203 + }, + { + "bits_per_byte": 3.2500166912475987, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.252739906311035, + "step": 450, + "unclipped_grad_norm": 1.1027717590332031 + }, + { + "bits_per_byte": 3.273506081702441, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.269021511077881, + "step": 460, + "unclipped_grad_norm": 0.9078090190887451 + }, + { + "bits_per_byte": 3.199572459923028, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2177746295928955, + "step": 470, + "unclipped_grad_norm": 1.019633173942566 + }, + { + "bits_per_byte": 3.1991851549932853, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.217506170272827, + "step": 480, + "unclipped_grad_norm": 0.8520210981369019 + }, + { + "bits_per_byte": 3.1319430341936423, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1708974838256836, + "step": 490, + "unclipped_grad_norm": 2.670640707015991 + }, + { + "bits_per_byte": 3.0465694704441373, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1117210388183594, + "step": 500, + "unclipped_grad_norm": 0.961890697479248 + }, + { + "bits_per_byte": 3.080116406321437, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1349740028381348, + "step": 510, + "unclipped_grad_norm": 1.4813798666000366 + }, + { + "bits_per_byte": 2.9958545384271087, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.076568126678467, + "step": 520, + "unclipped_grad_norm": 1.3385756015777588 + }, + { + "bits_per_byte": 3.0340161127923593, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1030197143554688, + "step": 530, + "unclipped_grad_norm": 1.0723164081573486 + }, + { + "bits_per_byte": 2.9825217554016206, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.067326545715332, + "step": 540, + "unclipped_grad_norm": 1.0559418201446533 + }, + { + "bits_per_byte": 2.964598755424604, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.054903268814087, + "step": 550, + "unclipped_grad_norm": 1.3542453050613403 + }, + { + "bits_per_byte": 2.9389846913010023, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.037148952484131, + "step": 560, + "unclipped_grad_norm": 1.3118830919265747 + }, + { + "bits_per_byte": 2.888880297694537, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0024192333221436, + "step": 570, + "unclipped_grad_norm": 1.1606483459472656 + }, + { + "bits_per_byte": 2.9506829512126904, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.045257568359375, + "step": 580, + "unclipped_grad_norm": 1.3442156314849854 + }, + { + "bits_per_byte": 2.8292920927888927, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.961115837097168, + "step": 590, + "unclipped_grad_norm": 0.9916635751724243 + }, + { + "bits_per_byte": 2.904369399196543, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.013155460357666, + "step": 600, + "unclipped_grad_norm": 1.5107635259628296 + }, + { + "bits_per_byte": 2.8103640262002334, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.947995901107788, + "step": 610, + "unclipped_grad_norm": 1.3907541036605835 + }, + { + "bits_per_byte": 2.8376170849865576, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.966886281967163, + "step": 620, + "unclipped_grad_norm": 1.3457273244857788 + }, + { + "bits_per_byte": 2.772177686333259, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9215271472930908, + "step": 630, + "unclipped_grad_norm": 1.2430039644241333 + }, + { + "bits_per_byte": 2.7475067750669893, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9044265747070312, + "step": 640, + "unclipped_grad_norm": 1.7454813718795776 + }, + { + "bits_per_byte": 2.664434339186028, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8468451499938965, + "step": 650, + "unclipped_grad_norm": 1.3669214248657227 + }, + { + "bits_per_byte": 2.607502578305611, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8073830604553223, + "step": 660, + "unclipped_grad_norm": 2.4166994094848633 + }, + { + "bits_per_byte": 2.640132846653812, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.830000638961792, + "step": 670, + "unclipped_grad_norm": 1.6203631162643433 + }, + { + "bits_per_byte": 2.577274563624662, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.7864305973052979, + "step": 680, + "unclipped_grad_norm": 1.443710207939148 + }, + { + "bits_per_byte": 2.648628101675831, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8358891010284424, + "step": 690, + "unclipped_grad_norm": 1.1127547025680542 + }, + { + "bits_per_byte": 2.5587637269469963, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7735998630523682, + "step": 700, + "unclipped_grad_norm": 1.3421220779418945 + }, + { + "bits_per_byte": 2.584466534118443, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.7914156913757324, + "step": 710, + "unclipped_grad_norm": 1.5519825220108032 + }, + { + "bits_per_byte": 2.61796909847182, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8146378993988037, + "step": 720, + "unclipped_grad_norm": 2.2321839332580566 + }, + { + "bits_per_byte": 2.574501859327452, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.7845087051391602, + "step": 730, + "unclipped_grad_norm": 1.835390567779541 + }, + { + "bits_per_byte": 2.509336257054327, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7393393516540527, + "step": 740, + "unclipped_grad_norm": 1.5465850830078125 + }, + { + "bits_per_byte": 2.4645351204698667, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7082855701446533, + "step": 750, + "unclipped_grad_norm": 1.1801629066467285 + }, + { + "bits_per_byte": 2.4095216540771736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.6701531410217285, + "step": 760, + "unclipped_grad_norm": 1.296120524406433 + }, + { + "bits_per_byte": 2.584656058999694, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7915470600128174, + "step": 770, + "unclipped_grad_norm": 2.048929452896118 + }, + { + "bits_per_byte": 2.4472219709728096, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6962850093841553, + "step": 780, + "unclipped_grad_norm": 1.45075261592865 + }, + { + "bits_per_byte": 2.4007569022412523, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.664077877998352, + "step": 790, + "unclipped_grad_norm": 1.5680437088012695 + }, + { + "bits_per_byte": 2.3632249563608103, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6380627155303955, + "step": 800, + "unclipped_grad_norm": 1.358572006225586 + }, + { + "bits_per_byte": 2.4340216145884854, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6871352195739746, + "step": 810, + "unclipped_grad_norm": 1.6542795896530151 + }, + { + "bits_per_byte": 2.3610528154804133, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6365571022033691, + "step": 820, + "unclipped_grad_norm": 1.4101909399032593 + }, + { + "bits_per_byte": 2.429125956449025, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.683741807937622, + "step": 830, + "unclipped_grad_norm": 1.4574730396270752 + }, + { + "bits_per_byte": 2.509321810511654, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7393293380737305, + "step": 840, + "unclipped_grad_norm": 1.71798837184906 + }, + { + "bits_per_byte": 2.3876714302677042, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6550077199935913, + "step": 850, + "unclipped_grad_norm": 1.7788947820663452 + }, + { + "bits_per_byte": 2.402646303643631, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.665387511253357, + "step": 860, + "unclipped_grad_norm": 1.3990782499313354 + }, + { + "bits_per_byte": 2.3188210996998437, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6072843074798584, + "step": 870, + "unclipped_grad_norm": 1.5185577869415283 + }, + { + "bits_per_byte": 2.342377219423221, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6236121654510498, + "step": 880, + "unclipped_grad_norm": 1.4461203813552856 + }, + { + "bits_per_byte": 2.2667829331670766, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.571214199066162, + "step": 890, + "unclipped_grad_norm": 1.2836962938308716 + }, + { + "bits_per_byte": 2.284366783356592, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.583402395248413, + "step": 900, + "unclipped_grad_norm": 1.337744116783142 + }, + { + "bits_per_byte": 2.3096057533186616, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6008967161178589, + "step": 910, + "unclipped_grad_norm": 1.3567826747894287 + }, + { + "bits_per_byte": 2.2867041995644684, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5850225687026978, + "step": 920, + "unclipped_grad_norm": 1.568986177444458 + }, + { + "bits_per_byte": 2.304558234498489, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.597398042678833, + "step": 930, + "unclipped_grad_norm": 1.5530638694763184 + }, + { + "bits_per_byte": 2.297734650842856, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5926682949066162, + "step": 940, + "unclipped_grad_norm": 1.5203993320465088 + }, + { + "bits_per_byte": 2.280084587332759, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5804342031478882, + "step": 950, + "unclipped_grad_norm": 1.4065279960632324 + }, + { + "bits_per_byte": 2.325858113825197, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6121619939804077, + "step": 960, + "unclipped_grad_norm": 1.3770049810409546 + }, + { + "bits_per_byte": 2.231583072032335, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5468155145645142, + "step": 970, + "unclipped_grad_norm": 1.3417105674743652 + }, + { + "bits_per_byte": 2.3223450242160224, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.609726905822754, + "step": 980, + "unclipped_grad_norm": 1.444604754447937 + }, + { + "bits_per_byte": 2.2731906627729233, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5756556987762451, + "step": 990, + "unclipped_grad_norm": 1.3634493350982666 + }, + { + "bits_per_byte": 2.2346108266007745, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5489141941070557, + "step": 1000, + "unclipped_grad_norm": 1.291415810585022 + }, + { + "bits_per_byte": 2.2324534762283514, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5474188327789307, + "step": 1010, + "unclipped_grad_norm": 1.811213493347168 + }, + { + "bits_per_byte": 2.198316295893291, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.523756742477417, + "step": 1020, + "unclipped_grad_norm": 1.5226773023605347 + }, + { + "bits_per_byte": 2.1998723949182977, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5248353481292725, + "step": 1030, + "unclipped_grad_norm": 1.3728047609329224 + }, + { + "bits_per_byte": 2.256233517363152, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5639019012451172, + "step": 1040, + "unclipped_grad_norm": 1.3665989637374878 + }, + { + "bits_per_byte": 2.2423672441546874, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.554290533065796, + "step": 1050, + "unclipped_grad_norm": 1.274046778678894 + }, + { + "bits_per_byte": 2.1489992388624755, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.4895727634429932, + "step": 1060, + "unclipped_grad_norm": 1.2373967170715332 + }, + { + "bits_per_byte": 2.2260787672914706, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5430002212524414, + "step": 1070, + "unclipped_grad_norm": 1.3982683420181274 + }, + { + "bits_per_byte": 2.206438348562938, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5293865203857422, + "step": 1080, + "unclipped_grad_norm": 1.334183692932129 + }, + { + "bits_per_byte": 2.1702291212502822, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5042881965637207, + "step": 1090, + "unclipped_grad_norm": 1.4794831275939941 + }, + { + "bits_per_byte": 2.2378767771406594, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.551177978515625, + "step": 1100, + "unclipped_grad_norm": 1.3411470651626587 + }, + { + "bits_per_byte": 2.087035609583114, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4466228485107422, + "step": 1110, + "unclipped_grad_norm": 1.452073574066162 + }, + { + "bits_per_byte": 2.1415479185311934, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.484407901763916, + "step": 1120, + "unclipped_grad_norm": 1.1817387342453003 + }, + { + "bits_per_byte": 2.079985696758946, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4417362213134766, + "step": 1130, + "unclipped_grad_norm": 1.5416163206100464 + }, + { + "bits_per_byte": 2.2251825656978244, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5423790216445923, + "step": 1140, + "unclipped_grad_norm": 1.5179954767227173 + }, + { + "bits_per_byte": 2.0668831985029628, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4326542615890503, + "step": 1150, + "unclipped_grad_norm": 1.186457633972168 + }, + { + "bits_per_byte": 2.104958265594829, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4590458869934082, + "step": 1160, + "unclipped_grad_norm": 1.5302215814590454 + }, + { + "bits_per_byte": 2.146566716248672, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.487886667251587, + "step": 1170, + "unclipped_grad_norm": 1.187764048576355 + }, + { + "bits_per_byte": 2.1225221657968443, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4712202548980713, + "step": 1180, + "unclipped_grad_norm": 1.1978200674057007 + }, + { + "bits_per_byte": 2.061979457178911, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4292552471160889, + "step": 1190, + "unclipped_grad_norm": 1.333662509918213 + }, + { + "bits_per_byte": 2.04061714814994, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4144480228424072, + "step": 1200, + "unclipped_grad_norm": 1.397094964981079 + }, + { + "bits_per_byte": 2.105093787923709, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4591398239135742, + "step": 1210, + "unclipped_grad_norm": 1.355148196220398 + }, + { + "bits_per_byte": 2.113284805636352, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4648174047470093, + "step": 1220, + "unclipped_grad_norm": 1.2704886198043823 + }, + { + "bits_per_byte": 2.0731149898569767, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4369738101959229, + "step": 1230, + "unclipped_grad_norm": 1.2257503271102905 + }, + { + "bits_per_byte": 2.02175856255223, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4013762474060059, + "step": 1240, + "unclipped_grad_norm": 1.311407208442688 + }, + { + "bits_per_byte": 2.10808387829161, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.461212396621704, + "step": 1250, + "unclipped_grad_norm": 1.183249831199646 + }, + { + "bits_per_byte": 2.0709103442555663, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4354456663131714, + "step": 1260, + "unclipped_grad_norm": 1.2188260555267334 + }, + { + "bits_per_byte": 2.073173119992968, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.437014102935791, + "step": 1270, + "unclipped_grad_norm": 1.2204123735427856 + }, + { + "bits_per_byte": 1.983587529141182, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.3749181032180786, + "step": 1280, + "unclipped_grad_norm": 1.4762879610061646 + }, + { + "bits_per_byte": 2.0899118474361438, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4486165046691895, + "step": 1290, + "unclipped_grad_norm": 1.5250427722930908 + }, + { + "bits_per_byte": 2.0451232655851967, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4175714254379272, + "step": 1300, + "unclipped_grad_norm": 1.2546813488006592 + }, + { + "bits_per_byte": 2.037817958524497, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4125077724456787, + "step": 1310, + "unclipped_grad_norm": 1.2255067825317383 + }, + { + "bits_per_byte": 2.0185500542177333, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.3991522789001465, + "step": 1320, + "unclipped_grad_norm": 1.1987844705581665 + }, + { + "bits_per_byte": 2.097532054730573, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4538984298706055, + "step": 1330, + "unclipped_grad_norm": 1.270599603652954 + }, + { + "bits_per_byte": 2.036339251692379, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.411482810974121, + "step": 1340, + "unclipped_grad_norm": 1.31504225730896 + }, + { + "bits_per_byte": 2.0528777913299496, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4229464530944824, + "step": 1350, + "unclipped_grad_norm": 1.3823131322860718 + }, + { + "bits_per_byte": 2.0989251142025616, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548640251159668, + "step": 1360, + "unclipped_grad_norm": 1.3448033332824707 + }, + { + "bits_per_byte": 2.081831758533307, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4430158138275146, + "step": 1370, + "unclipped_grad_norm": 1.1824978590011597 + }, + { + "bits_per_byte": 2.037543130248418, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4123172760009766, + "step": 1380, + "unclipped_grad_norm": 1.241772174835205 + }, + { + "bits_per_byte": 2.0465340392702243, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4185492992401123, + "step": 1390, + "unclipped_grad_norm": 1.1579062938690186 + }, + { + "bits_per_byte": 1.983804227281269, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3750683069229126, + "step": 1400, + "unclipped_grad_norm": 1.2093477249145508 + }, + { + "bits_per_byte": 2.004750510277803, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.389587163925171, + "step": 1410, + "unclipped_grad_norm": 1.0995537042617798 + }, + { + "bits_per_byte": 2.044002282666873, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4167944192886353, + "step": 1420, + "unclipped_grad_norm": 1.377259612083435 + }, + { + "bits_per_byte": 2.072302027866348, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4364103078842163, + "step": 1430, + "unclipped_grad_norm": 1.2491836547851562 + }, + { + "bits_per_byte": 2.010299358525241, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3934333324432373, + "step": 1440, + "unclipped_grad_norm": 1.1953747272491455 + }, + { + "bits_per_byte": 1.9653831655473537, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3622997999191284, + "step": 1450, + "unclipped_grad_norm": 1.1474745273590088 + }, + { + "bits_per_byte": 2.0178239434657903, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.398648977279663, + "step": 1460, + "unclipped_grad_norm": 1.309095859527588 + }, + { + "bits_per_byte": 1.9352492253764266, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3414125442504883, + "step": 1470, + "unclipped_grad_norm": 1.2327831983566284 + }, + { + "bits_per_byte": 2.02495382822261, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4035910367965698, + "step": 1480, + "unclipped_grad_norm": 1.2556331157684326 + }, + { + "bits_per_byte": 2.0395678819970264, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.413720726966858, + "step": 1490, + "unclipped_grad_norm": 1.1320099830627441 + }, + { + "bits_per_byte": 2.0169458000504847, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3980402946472168, + "step": 1500, + "unclipped_grad_norm": 1.2224371433258057 + }, + { + "bits_per_byte": 2.0172811662196675, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3982727527618408, + "step": 1510, + "unclipped_grad_norm": 1.1516478061676025 + }, + { + "bits_per_byte": 1.959494651564462, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3582181930541992, + "step": 1520, + "unclipped_grad_norm": 0.9697362780570984 + }, + { + "bits_per_byte": 2.05711750763902, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4258852005004883, + "step": 1530, + "unclipped_grad_norm": 1.1719471216201782 + }, + { + "bits_per_byte": 2.0050322178599163, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.389782428741455, + "step": 1540, + "unclipped_grad_norm": 1.2557746171951294 + }, + { + "bits_per_byte": 2.0500259750133427, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209697246551514, + "step": 1550, + "unclipped_grad_norm": 1.1871017217636108 + }, + { + "bits_per_byte": 1.9591589414299777, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.357985496520996, + "step": 1560, + "unclipped_grad_norm": 1.1881691217422485 + }, + { + "bits_per_byte": 2.048241998975943, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4197331666946411, + "step": 1570, + "unclipped_grad_norm": 1.3408379554748535 + }, + { + "bits_per_byte": 2.0003424229535387, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3865317106246948, + "step": 1580, + "unclipped_grad_norm": 1.160094141960144 + }, + { + "bits_per_byte": 1.9972685770346676, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3844010829925537, + "step": 1590, + "unclipped_grad_norm": 1.1695513725280762 + }, + { + "bits_per_byte": 2.0280931995314595, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4057670831680298, + "step": 1600, + "unclipped_grad_norm": 1.1151546239852905 + }, + { + "bits_per_byte": 1.9648931869750457, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3619601726531982, + "step": 1610, + "unclipped_grad_norm": 1.1052403450012207 + }, + { + "bits_per_byte": 1.934925382044852, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3411880731582642, + "step": 1620, + "unclipped_grad_norm": 1.0654687881469727 + }, + { + "bits_per_byte": 1.9529704897039815, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3536959886550903, + "step": 1630, + "unclipped_grad_norm": 1.1012754440307617 + }, + { + "bits_per_byte": 1.9931929321745208, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3815760612487793, + "step": 1640, + "unclipped_grad_norm": 1.143923282623291 + }, + { + "bits_per_byte": 2.059868542122221, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4277920722961426, + "step": 1650, + "unclipped_grad_norm": 1.2584717273712158 + }, + { + "bits_per_byte": 1.992070745377641, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3807982206344604, + "step": 1660, + "unclipped_grad_norm": 1.177032232284546 + }, + { + "bits_per_byte": 2.009216383772758, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.392682671546936, + "step": 1670, + "unclipped_grad_norm": 1.1299933195114136 + }, + { + "bits_per_byte": 2.027195106128654, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4051445722579956, + "step": 1680, + "unclipped_grad_norm": 1.2568069696426392 + }, + { + "bits_per_byte": 2.0647518175108197, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4311769008636475, + "step": 1690, + "unclipped_grad_norm": 1.1674096584320068 + }, + { + "bits_per_byte": 1.9602546428986247, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3587449789047241, + "step": 1700, + "unclipped_grad_norm": 1.178646206855774 + }, + { + "bits_per_byte": 1.8971180919403792, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3149820566177368, + "step": 1710, + "unclipped_grad_norm": 1.0156482458114624 + }, + { + "bits_per_byte": 1.8941140709777584, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3128998279571533, + "step": 1720, + "unclipped_grad_norm": 1.0928010940551758 + }, + { + "bits_per_byte": 1.8690832000232327, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.295549750328064, + "step": 1730, + "unclipped_grad_norm": 1.0898301601409912 + }, + { + "bits_per_byte": 2.0020670649763908, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.38772714138031, + "step": 1740, + "unclipped_grad_norm": 1.1916359663009644 + }, + { + "bits_per_byte": 1.9965376507685006, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.383894443511963, + "step": 1750, + "unclipped_grad_norm": 1.1260547637939453 + }, + { + "bits_per_byte": 2.010018682839033, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3932387828826904, + "step": 1760, + "unclipped_grad_norm": 1.2356810569763184 + }, + { + "bits_per_byte": 2.000046096846102, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3863263130187988, + "step": 1770, + "unclipped_grad_norm": 1.1323615312576294 + }, + { + "bits_per_byte": 1.8857641412957182, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3071120977401733, + "step": 1780, + "unclipped_grad_norm": 1.1074084043502808 + }, + { + "bits_per_byte": 1.98148022572016, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.373457431793213, + "step": 1790, + "unclipped_grad_norm": 1.1702499389648438 + }, + { + "bits_per_byte": 1.8942642118319617, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3130038976669312, + "step": 1800, + "unclipped_grad_norm": 1.2193527221679688 + }, + { + "bits_per_byte": 1.9583697130451683, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3574384450912476, + "step": 1810, + "unclipped_grad_norm": 1.4279322624206543 + }, + { + "bits_per_byte": 1.9039834673801714, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3197407722473145, + "step": 1820, + "unclipped_grad_norm": 1.2399523258209229 + }, + { + "bits_per_byte": 1.902950023631137, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3190244436264038, + "step": 1830, + "unclipped_grad_norm": 1.066636562347412 + }, + { + "bits_per_byte": 1.9139311158887273, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3266359567642212, + "step": 1840, + "unclipped_grad_norm": 1.129875898361206 + }, + { + "bits_per_byte": 1.9166544611651397, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3285236358642578, + "step": 1850, + "unclipped_grad_norm": 1.1898164749145508 + }, + { + "bits_per_byte": 1.9000117000411458, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3169877529144287, + "step": 1860, + "unclipped_grad_norm": 1.1322225332260132 + }, + { + "bits_per_byte": 1.9091849106728662, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3233461380004883, + "step": 1870, + "unclipped_grad_norm": 1.1632918119430542 + }, + { + "bits_per_byte": 1.9066440389890191, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.321584939956665, + "step": 1880, + "unclipped_grad_norm": 1.0685805082321167 + }, + { + "bits_per_byte": 1.915992327959318, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3280646800994873, + "step": 1890, + "unclipped_grad_norm": 1.1415059566497803 + }, + { + "bits_per_byte": 1.9305143710155228, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3381305932998657, + "step": 1900, + "unclipped_grad_norm": 1.0947742462158203 + }, + { + "bits_per_byte": 1.863847876148318, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2919209003448486, + "step": 1910, + "unclipped_grad_norm": 1.0498101711273193 + }, + { + "bits_per_byte": 1.8709641742757193, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2968535423278809, + "step": 1920, + "unclipped_grad_norm": 1.227995753288269 + }, + { + "bits_per_byte": 1.9694514151535134, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3651196956634521, + "step": 1930, + "unclipped_grad_norm": 1.1106438636779785 + }, + { + "bits_per_byte": 1.8401163341210607, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2754714488983154, + "step": 1940, + "unclipped_grad_norm": 1.1273587942123413 + }, + { + "bits_per_byte": 1.9228790292478175, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3328381776809692, + "step": 1950, + "unclipped_grad_norm": 1.0416061878204346 + }, + { + "bits_per_byte": 1.840930843955547, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.276036024093628, + "step": 1960, + "unclipped_grad_norm": 1.0370304584503174 + }, + { + "bits_per_byte": 1.9309775202942963, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.338451623916626, + "step": 1970, + "unclipped_grad_norm": 1.147440791130066 + }, + { + "bits_per_byte": 1.9639140897436835, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3612815141677856, + "step": 1980, + "unclipped_grad_norm": 1.2305821180343628 + }, + { + "bits_per_byte": 1.9535571225260744, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.354102611541748, + "step": 1990, + "unclipped_grad_norm": 1.3296585083007812 + }, + { + "bits_per_byte": 1.92549075778382, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3346484899520874, + "step": 2000, + "unclipped_grad_norm": 1.0916969776153564 + }, + { + "bits_per_byte": 1.880516778635243, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3034749031066895, + "step": 2010, + "unclipped_grad_norm": 1.0219805240631104 + }, + { + "bits_per_byte": 1.9332846675556208, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3400508165359497, + "step": 2020, + "unclipped_grad_norm": 1.1069045066833496 + }, + { + "bits_per_byte": 1.9929170720025369, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3813848495483398, + "step": 2030, + "unclipped_grad_norm": 1.0479846000671387 + }, + { + "bits_per_byte": 1.9184877962233373, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3297944068908691, + "step": 2040, + "unclipped_grad_norm": 1.0204336643218994 + }, + { + "bits_per_byte": 1.9692321372736634, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.364967703819275, + "step": 2050, + "unclipped_grad_norm": 1.0853239297866821 + }, + { + "bits_per_byte": 1.9303957029864274, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3380483388900757, + "step": 2060, + "unclipped_grad_norm": 1.0730464458465576 + }, + { + "bits_per_byte": 1.889977716241857, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3100327253341675, + "step": 2070, + "unclipped_grad_norm": 1.0691337585449219 + }, + { + "bits_per_byte": 1.8844801188243765, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3062220811843872, + "step": 2080, + "unclipped_grad_norm": 1.0554076433181763 + }, + { + "bits_per_byte": 1.8957609768424206, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3140413761138916, + "step": 2090, + "unclipped_grad_norm": 1.1196725368499756 + }, + { + "bits_per_byte": 1.8758003264179814, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3002057075500488, + "step": 2100, + "unclipped_grad_norm": 1.0327236652374268 + }, + { + "bits_per_byte": 1.914194593309849, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.326818585395813, + "step": 2110, + "unclipped_grad_norm": 1.0782309770584106 + }, + { + "bits_per_byte": 1.8761728408397502, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3004639148712158, + "step": 2120, + "unclipped_grad_norm": 1.0318117141723633 + }, + { + "bits_per_byte": 1.8463771905430704, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.279811143875122, + "step": 2130, + "unclipped_grad_norm": 1.1898446083068848 + }, + { + "bits_per_byte": 1.984834747325239, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3757826089859009, + "step": 2140, + "unclipped_grad_norm": 1.1598726511001587 + }, + { + "bits_per_byte": 1.8562880347643413, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.286680817604065, + "step": 2150, + "unclipped_grad_norm": 1.0410785675048828 + }, + { + "bits_per_byte": 1.8990754264898486, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3163387775421143, + "step": 2160, + "unclipped_grad_norm": 1.0785906314849854 + }, + { + "bits_per_byte": 1.9583294691048665, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3574105501174927, + "step": 2170, + "unclipped_grad_norm": 1.1421548128128052 + }, + { + "bits_per_byte": 1.8472800994601, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2804369926452637, + "step": 2180, + "unclipped_grad_norm": 1.0929888486862183 + }, + { + "bits_per_byte": 1.8586782496460328, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.2883375883102417, + "step": 2190, + "unclipped_grad_norm": 1.058180570602417 + }, + { + "bits_per_byte": 1.8910505440179393, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3107763528823853, + "step": 2200, + "unclipped_grad_norm": 0.9802089333534241 + }, + { + "bits_per_byte": 1.8399546704292495, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2753593921661377, + "step": 2210, + "unclipped_grad_norm": 1.060943365097046 + }, + { + "bits_per_byte": 1.8476158095945843, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2806696891784668, + "step": 2220, + "unclipped_grad_norm": 1.0172474384307861 + }, + { + "bits_per_byte": 1.9309195621409554, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3384114503860474, + "step": 2230, + "unclipped_grad_norm": 1.1571786403656006 + }, + { + "bits_per_byte": 1.848552427111183, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2813189029693604, + "step": 2240, + "unclipped_grad_norm": 1.085301160812378 + }, + { + "bits_per_byte": 1.864913308670413, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2926594018936157, + "step": 2250, + "unclipped_grad_norm": 1.0694295167922974 + }, + { + "bits_per_byte": 1.8611869605741684, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.290076494216919, + "step": 2260, + "unclipped_grad_norm": 1.1977293491363525 + }, + { + "bits_per_byte": 1.8337584794839643, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2710645198822021, + "step": 2270, + "unclipped_grad_norm": 1.1360434293746948 + }, + { + "bits_per_byte": 1.8587281246147829, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2883721590042114, + "step": 2280, + "unclipped_grad_norm": 1.113946795463562 + }, + { + "bits_per_byte": 1.8907026231152437, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.310535192489624, + "step": 2290, + "unclipped_grad_norm": 1.1474530696868896 + }, + { + "bits_per_byte": 1.8085284526197394, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.253576397895813, + "step": 2300, + "unclipped_grad_norm": 0.96898353099823 + }, + { + "bits_per_byte": 1.8446171200941404, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2785911560058594, + "step": 2310, + "unclipped_grad_norm": 0.9442379474639893 + }, + { + "bits_per_byte": 1.8195852172437095, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2612403631210327, + "step": 2320, + "unclipped_grad_norm": 0.9633188843727112 + }, + { + "bits_per_byte": 1.8903022475040352, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.3102576732635498, + "step": 2330, + "unclipped_grad_norm": 1.0167381763458252 + }, + { + "bits_per_byte": 1.8286817236131452, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2675455808639526, + "step": 2340, + "unclipped_grad_norm": 1.1636918783187866 + }, + { + "bits_per_byte": 1.8687012265556664, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2952849864959717, + "step": 2350, + "unclipped_grad_norm": 1.0884090662002563 + }, + { + "bits_per_byte": 1.8708572010668825, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2967793941497803, + "step": 2360, + "unclipped_grad_norm": 1.0467369556427002 + }, + { + "bits_per_byte": 1.8979811008824086, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3155802488327026, + "step": 2370, + "unclipped_grad_norm": 1.0231177806854248 + }, + { + "bits_per_byte": 1.853912954355926, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2850345373153687, + "step": 2380, + "unclipped_grad_norm": 1.0531508922576904 + }, + { + "bits_per_byte": 1.909142430958103, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3233166933059692, + "step": 2390, + "unclipped_grad_norm": 1.0615708827972412 + }, + { + "bits_per_byte": 1.8518789155441715, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2836246490478516, + "step": 2400, + "unclipped_grad_norm": 1.0747002363204956 + }, + { + "bits_per_byte": 1.8430525939192415, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.277506709098816, + "step": 2410, + "unclipped_grad_norm": 1.035906434059143 + }, + { + "bits_per_byte": 1.8218237434273397, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2627919912338257, + "step": 2420, + "unclipped_grad_norm": 0.9908418655395508 + }, + { + "bits_per_byte": 1.8454151195941437, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.279144287109375, + "step": 2430, + "unclipped_grad_norm": 1.1172900199890137 + }, + { + "bits_per_byte": 1.9008382486611926, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3175606727600098, + "step": 2440, + "unclipped_grad_norm": 1.0292961597442627 + }, + { + "bits_per_byte": 1.8510478953752025, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2830486297607422, + "step": 2450, + "unclipped_grad_norm": 1.0116122961044312 + }, + { + "bits_per_byte": 1.8498311181203482, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2822052240371704, + "step": 2460, + "unclipped_grad_norm": 0.9883264303207397 + }, + { + "bits_per_byte": 1.849598081628429, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.282043695449829, + "step": 2470, + "unclipped_grad_norm": 1.0902615785598755 + }, + { + "bits_per_byte": 1.8701792454571813, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.296309471130371, + "step": 2480, + "unclipped_grad_norm": 1.4108390808105469 + }, + { + "bits_per_byte": 1.81769770765049, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.259932041168213, + "step": 2490, + "unclipped_grad_norm": 1.0036945343017578 + }, + { + "bits_per_byte": 1.8526576529872785, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2841644287109375, + "step": 2500, + "unclipped_grad_norm": 1.0499781370162964 + }, + { + "bits_per_byte": 1.8174889207123426, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2597873210906982, + "step": 2510, + "unclipped_grad_norm": 1.112528681755066 + }, + { + "bits_per_byte": 1.8559390819657409, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2864389419555664, + "step": 2520, + "unclipped_grad_norm": 1.043527603149414 + }, + { + "bits_per_byte": 1.8664243482408935, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2937067747116089, + "step": 2530, + "unclipped_grad_norm": 1.047589898109436 + }, + { + "bits_per_byte": 1.8385335777851701, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2743743658065796, + "step": 2540, + "unclipped_grad_norm": 1.114875078201294 + }, + { + "bits_per_byte": 1.8303919190933249, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2687309980392456, + "step": 2550, + "unclipped_grad_norm": 1.026410460472107 + }, + { + "bits_per_byte": 1.8414952910156785, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2764272689819336, + "step": 2560, + "unclipped_grad_norm": 1.0519986152648926 + }, + { + "bits_per_byte": 1.8285018297603426, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2674208879470825, + "step": 2570, + "unclipped_grad_norm": 0.9845532178878784 + }, + { + "bits_per_byte": 1.8621820521920607, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2907662391662598, + "step": 2580, + "unclipped_grad_norm": 1.1312247514724731 + }, + { + "bits_per_byte": 1.821957545929711, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2628847360610962, + "step": 2590, + "unclipped_grad_norm": 0.9848816990852356 + }, + { + "bits_per_byte": 1.8336463467956017, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.270986795425415, + "step": 2600, + "unclipped_grad_norm": 1.0540134906768799 + }, + { + "bits_per_byte": 1.762311382870726, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.221541166305542, + "step": 2610, + "unclipped_grad_norm": 0.9237543940544128 + }, + { + "bits_per_byte": 1.8236500271968519, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2640578746795654, + "step": 2620, + "unclipped_grad_norm": 0.9258099794387817 + }, + { + "bits_per_byte": 1.8442078013850869, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2783074378967285, + "step": 2630, + "unclipped_grad_norm": 1.1065376996994019 + }, + { + "bits_per_byte": 1.6990233151970702, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1776732206344604, + "step": 2640, + "unclipped_grad_norm": 0.975441038608551 + }, + { + "bits_per_byte": 1.7897238695744, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2405420541763306, + "step": 2650, + "unclipped_grad_norm": 1.024839997291565 + }, + { + "bits_per_byte": 1.829891621561965, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2683842182159424, + "step": 2660, + "unclipped_grad_norm": 1.0799146890640259 + }, + { + "bits_per_byte": 1.8374299651145836, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2736093997955322, + "step": 2670, + "unclipped_grad_norm": 0.9827171564102173 + }, + { + "bits_per_byte": 1.757754702536116, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.218382716178894, + "step": 2680, + "unclipped_grad_norm": 0.9623468518257141 + }, + { + "bits_per_byte": 1.9223347041578367, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.332460880279541, + "step": 2690, + "unclipped_grad_norm": 1.0313293933868408 + }, + { + "bits_per_byte": 1.8426768118271062, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2772462368011475, + "step": 2700, + "unclipped_grad_norm": 1.154735803604126 + }, + { + "bits_per_byte": 1.7467827253590213, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2107775211334229, + "step": 2710, + "unclipped_grad_norm": 0.9442266225814819 + }, + { + "bits_per_byte": 1.7668878412101854, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2247133255004883, + "step": 2720, + "unclipped_grad_norm": 0.9805414080619812 + }, + { + "bits_per_byte": 1.8017586995338266, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2488839626312256, + "step": 2730, + "unclipped_grad_norm": 1.027875542640686 + }, + { + "bits_per_byte": 1.7992866209103247, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2471704483032227, + "step": 2740, + "unclipped_grad_norm": 1.0912469625473022 + }, + { + "bits_per_byte": 1.8291928560515094, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2678998708724976, + "step": 2750, + "unclipped_grad_norm": 1.0327590703964233 + }, + { + "bits_per_byte": 1.831356913747316, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.269399881362915, + "step": 2760, + "unclipped_grad_norm": 1.0535632371902466 + }, + { + "bits_per_byte": 1.7175185014535075, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.190493106842041, + "step": 2770, + "unclipped_grad_norm": 0.9521624445915222 + }, + { + "bits_per_byte": 1.7419930085324922, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2074575424194336, + "step": 2780, + "unclipped_grad_norm": 0.9253908395767212 + }, + { + "bits_per_byte": 1.829489182158946, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2681052684783936, + "step": 2790, + "unclipped_grad_norm": 1.0019073486328125 + }, + { + "bits_per_byte": 1.900268298156233, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.3171656131744385, + "step": 2800, + "unclipped_grad_norm": 1.1026051044464111 + }, + { + "bits_per_byte": 1.8251356133350047, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.265087604522705, + "step": 2810, + "unclipped_grad_norm": 1.095402479171753 + }, + { + "bits_per_byte": 1.8078448215825598, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2531025409698486, + "step": 2820, + "unclipped_grad_norm": 1.0277965068817139 + }, + { + "bits_per_byte": 1.8903005276775264, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.3102564811706543, + "step": 2830, + "unclipped_grad_norm": 1.0477416515350342 + }, + { + "bits_per_byte": 1.7718302786306808, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2281391620635986, + "step": 2840, + "unclipped_grad_norm": 0.9842959642410278 + }, + { + "bits_per_byte": 1.7997145137456714, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.247467041015625, + "step": 2850, + "unclipped_grad_norm": 1.0457379817962646 + }, + { + "bits_per_byte": 1.7627234533021934, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2218267917633057, + "step": 2860, + "unclipped_grad_norm": 1.0218563079833984 + }, + { + "bits_per_byte": 1.8575106594293251, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2875282764434814, + "step": 2870, + "unclipped_grad_norm": 1.1067496538162231 + }, + { + "bits_per_byte": 1.8563188196588458, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2867021560668945, + "step": 2880, + "unclipped_grad_norm": 1.0256601572036743 + }, + { + "bits_per_byte": 1.76771198207312, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2252845764160156, + "step": 2890, + "unclipped_grad_norm": 1.0549262762069702 + }, + { + "bits_per_byte": 1.7222432088380106, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.19376802444458, + "step": 2900, + "unclipped_grad_norm": 1.0038959980010986 + }, + { + "bits_per_byte": 1.7887707417233183, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2398813962936401, + "step": 2910, + "unclipped_grad_norm": 1.020102620124817 + }, + { + "bits_per_byte": 1.7699561836842288, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2268401384353638, + "step": 2920, + "unclipped_grad_norm": 0.9710083603858948 + }, + { + "bits_per_byte": 1.7845655939270717, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.236966609954834, + "step": 2930, + "unclipped_grad_norm": 1.063584566116333 + }, + { + "bits_per_byte": 1.8044884081683212, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2507760524749756, + "step": 2940, + "unclipped_grad_norm": 1.1001172065734863 + }, + { + "bits_per_byte": 1.788769881810064, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2398808002471924, + "step": 2950, + "unclipped_grad_norm": 1.052484393119812 + }, + { + "bits_per_byte": 1.8266576597951407, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2661426067352295, + "step": 2960, + "unclipped_grad_norm": 1.0507400035858154 + }, + { + "bits_per_byte": 1.7942014378896132, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2436456680297852, + "step": 2970, + "unclipped_grad_norm": 1.016513466835022 + }, + { + "bits_per_byte": 1.776544667056132, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2314069271087646, + "step": 2980, + "unclipped_grad_norm": 1.0853108167648315 + }, + { + "bits_per_byte": 1.797620281006115, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2460154294967651, + "step": 2990, + "unclipped_grad_norm": 1.025924563407898 + }, + { + "bits_per_byte": 1.8018763356670169, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2489655017852783, + "step": 3000, + "unclipped_grad_norm": 1.066794514656067 + }, + { + "bits_per_byte": 1.8036527444677788, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2501968145370483, + "step": 3010, + "unclipped_grad_norm": 0.9892268180847168 + }, + { + "bits_per_byte": 1.7646690930314042, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2231754064559937, + "step": 3020, + "unclipped_grad_norm": 1.1210085153579712 + }, + { + "bits_per_byte": 1.8358253669820335, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2724971771240234, + "step": 3030, + "unclipped_grad_norm": 1.0379256010055542 + }, + { + "bits_per_byte": 1.763304066731506, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.222229242324829, + "step": 3040, + "unclipped_grad_norm": 1.0503100156784058 + }, + { + "bits_per_byte": 1.7804524568490367, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2341156005859375, + "step": 3050, + "unclipped_grad_norm": 1.0255935192108154 + }, + { + "bits_per_byte": 1.779026720673384, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2331273555755615, + "step": 3060, + "unclipped_grad_norm": 1.0667622089385986 + }, + { + "bits_per_byte": 1.7089128335696342, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.184528112411499, + "step": 3070, + "unclipped_grad_norm": 0.9528610110282898 + }, + { + "bits_per_byte": 1.7514905784437398, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.214040756225586, + "step": 3080, + "unclipped_grad_norm": 0.9759883880615234 + }, + { + "bits_per_byte": 1.8129533942437885, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.256643533706665, + "step": 3090, + "unclipped_grad_norm": 1.074378490447998 + }, + { + "bits_per_byte": 1.7348117009630646, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2024798393249512, + "step": 3100, + "unclipped_grad_norm": 1.0595066547393799 + }, + { + "bits_per_byte": 1.8111885082806345, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.255420207977295, + "step": 3110, + "unclipped_grad_norm": 1.0304162502288818 + }, + { + "bits_per_byte": 1.7739528885076294, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2296104431152344, + "step": 3120, + "unclipped_grad_norm": 0.9790826439857483 + }, + { + "bits_per_byte": 1.8333825254091782, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2708039283752441, + "step": 3130, + "unclipped_grad_norm": 1.0250012874603271 + }, + { + "bits_per_byte": 1.807222588351738, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.252671241760254, + "step": 3140, + "unclipped_grad_norm": 1.065406322479248 + }, + { + "bits_per_byte": 1.7667856835155729, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2246425151824951, + "step": 3150, + "unclipped_grad_norm": 1.1064149141311646 + }, + { + "bits_per_byte": 1.77101886448391, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.227576732635498, + "step": 3160, + "unclipped_grad_norm": 0.9769210815429688 + }, + { + "bits_per_byte": 1.6904848925296843, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1717548370361328, + "step": 3170, + "unclipped_grad_norm": 1.003720998764038 + }, + { + "bits_per_byte": 1.7897881910858227, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2405866384506226, + "step": 3180, + "unclipped_grad_norm": 0.9989861249923706 + }, + { + "bits_per_byte": 1.7613923075845153, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2209041118621826, + "step": 3190, + "unclipped_grad_norm": 1.0040642023086548 + }, + { + "bits_per_byte": 1.7417902409871249, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2073169946670532, + "step": 3200, + "unclipped_grad_norm": 0.9894367456436157 + }, + { + "bits_per_byte": 1.7268853645500994, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1969857215881348, + "step": 3210, + "unclipped_grad_norm": 1.0016205310821533 + }, + { + "bits_per_byte": 1.7365394386736324, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2036774158477783, + "step": 3220, + "unclipped_grad_norm": 0.9552999138832092 + }, + { + "bits_per_byte": 1.8281382584364185, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2671688795089722, + "step": 3230, + "unclipped_grad_norm": 1.0519078969955444 + }, + { + "bits_per_byte": 1.7008661093010653, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.178950548171997, + "step": 3240, + "unclipped_grad_norm": 0.956534206867218 + }, + { + "bits_per_byte": 1.8618967329742793, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.290568470954895, + "step": 3250, + "unclipped_grad_norm": 1.118624210357666 + }, + { + "bits_per_byte": 1.7141249397866825, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.188140869140625, + "step": 3260, + "unclipped_grad_norm": 0.9860926270484924 + }, + { + "bits_per_byte": 1.7454612106697915, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2098615169525146, + "step": 3270, + "unclipped_grad_norm": 0.968685507774353 + }, + { + "bits_per_byte": 1.8017138120619516, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2488528490066528, + "step": 3280, + "unclipped_grad_norm": 1.0125019550323486 + }, + { + "bits_per_byte": 1.7954340375483469, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2445000410079956, + "step": 3290, + "unclipped_grad_norm": 1.0358432531356812 + }, + { + "bits_per_byte": 1.7138430602219183, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1879454851150513, + "step": 3300, + "unclipped_grad_norm": 1.0642298460006714 + }, + { + "bits_per_byte": 1.7570346111769533, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.217883586883545, + "step": 3310, + "unclipped_grad_norm": 1.017012596130371 + }, + { + "bits_per_byte": 1.753239298037713, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2152528762817383, + "step": 3320, + "unclipped_grad_norm": 0.9849429130554199 + }, + { + "bits_per_byte": 1.7669095110241941, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2247283458709717, + "step": 3330, + "unclipped_grad_norm": 0.9692639708518982 + }, + { + "bits_per_byte": 1.7337540076602584, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2017467021942139, + "step": 3340, + "unclipped_grad_norm": 0.9712040424346924 + }, + { + "bits_per_byte": 1.7289699662612077, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1984306573867798, + "step": 3350, + "unclipped_grad_norm": 0.9344873428344727 + }, + { + "bits_per_byte": 1.771012673108479, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2275724411010742, + "step": 3360, + "unclipped_grad_norm": 1.0185227394104004 + }, + { + "bits_per_byte": 1.7121330367243894, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1867601871490479, + "step": 3370, + "unclipped_grad_norm": 1.017809271812439 + }, + { + "bits_per_byte": 1.7444269070075025, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2091445922851562, + "step": 3380, + "unclipped_grad_norm": 0.9594104886054993 + }, + { + "bits_per_byte": 1.782493202984175, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.235530138015747, + "step": 3390, + "unclipped_grad_norm": 1.0056157112121582 + }, + { + "bits_per_byte": 1.8303659497130447, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2687129974365234, + "step": 3400, + "unclipped_grad_norm": 0.9542431235313416 + }, + { + "bits_per_byte": 1.8696024156461875, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2959096431732178, + "step": 3410, + "unclipped_grad_norm": 1.0261633396148682 + }, + { + "bits_per_byte": 1.7306273630675726, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1995794773101807, + "step": 3420, + "unclipped_grad_norm": 1.0138797760009766 + }, + { + "bits_per_byte": 1.663148250175041, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1528065204620361, + "step": 3430, + "unclipped_grad_norm": 0.9656930565834045 + }, + { + "bits_per_byte": 1.7509053214828536, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2136350870132446, + "step": 3440, + "unclipped_grad_norm": 1.1262567043304443 + }, + { + "bits_per_byte": 1.8269866626062412, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2663706541061401, + "step": 3450, + "unclipped_grad_norm": 1.0454469919204712 + }, + { + "bits_per_byte": 1.7413009503454202, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2069778442382812, + "step": 3460, + "unclipped_grad_norm": 1.1041505336761475 + }, + { + "bits_per_byte": 1.7427603951206418, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2079894542694092, + "step": 3470, + "unclipped_grad_norm": 1.0516799688339233 + }, + { + "bits_per_byte": 1.6734345325231454, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1599364280700684, + "step": 3480, + "unclipped_grad_norm": 0.9458639621734619 + }, + { + "bits_per_byte": 1.8872512752777288, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.308142900466919, + "step": 3490, + "unclipped_grad_norm": 1.0662349462509155 + }, + { + "bits_per_byte": 1.7517492403506374, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.2142200469970703, + "step": 3500, + "unclipped_grad_norm": 1.0241658687591553 + }, + { + "bits_per_byte": 1.7728004327641977, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2288116216659546, + "step": 3510, + "unclipped_grad_norm": 1.0039124488830566 + }, + { + "bits_per_byte": 1.7731452579791778, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.229050636291504, + "step": 3520, + "unclipped_grad_norm": 0.9908767342567444 + }, + { + "bits_per_byte": 1.743414617124524, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2084429264068604, + "step": 3530, + "unclipped_grad_norm": 0.9787898063659668 + }, + { + "bits_per_byte": 1.7481145590073028, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.211700677871704, + "step": 3540, + "unclipped_grad_norm": 1.0219446420669556 + }, + { + "bits_per_byte": 1.7531405799961175, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2151844501495361, + "step": 3550, + "unclipped_grad_norm": 1.2019073963165283 + }, + { + "bits_per_byte": 1.6952811446969462, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.175079345703125, + "step": 3560, + "unclipped_grad_norm": 0.9977114796638489 + }, + { + "bits_per_byte": 1.7182931113129936, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1910300254821777, + "step": 3570, + "unclipped_grad_norm": 1.0032203197479248 + }, + { + "bits_per_byte": 1.7239327664000867, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.194939136505127, + "step": 3580, + "unclipped_grad_norm": 1.0535842180252075 + }, + { + "bits_per_byte": 1.7655023489748345, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2237529754638672, + "step": 3590, + "unclipped_grad_norm": 1.055069923400879 + }, + { + "bits_per_byte": 1.7734857836278861, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2292866706848145, + "step": 3600, + "unclipped_grad_norm": 1.0577540397644043 + }, + { + "bits_per_byte": 1.7140950148054324, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1881201267242432, + "step": 3610, + "unclipped_grad_norm": 1.0167864561080933 + }, + { + "bits_per_byte": 1.6696412831757155, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1573071479797363, + "step": 3620, + "unclipped_grad_norm": 0.9244848489761353 + }, + { + "bits_per_byte": 1.7374531824976664, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2043107748031616, + "step": 3630, + "unclipped_grad_norm": 0.9926364421844482 + }, + { + "bits_per_byte": 1.7074227758825589, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.183495283126831, + "step": 3640, + "unclipped_grad_norm": 0.9468497633934021 + }, + { + "bits_per_byte": 1.7634653864580154, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2223410606384277, + "step": 3650, + "unclipped_grad_norm": 0.9865193367004395 + }, + { + "bits_per_byte": 1.6666551484087844, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1552373170852661, + "step": 3660, + "unclipped_grad_norm": 1.0027393102645874 + }, + { + "bits_per_byte": 1.749947206134897, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2129709720611572, + "step": 3670, + "unclipped_grad_norm": 1.0161325931549072 + }, + { + "bits_per_byte": 1.700321268263132, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1785728931427002, + "step": 3680, + "unclipped_grad_norm": 1.0996551513671875 + }, + { + "bits_per_byte": 1.6689437215438159, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1568236351013184, + "step": 3690, + "unclipped_grad_norm": 0.9738581776618958 + }, + { + "bits_per_byte": 1.6837737855277157, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1671030521392822, + "step": 3700, + "unclipped_grad_norm": 1.0792776346206665 + }, + { + "bits_per_byte": 1.7280950905162684, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.197824239730835, + "step": 3710, + "unclipped_grad_norm": 0.9678228497505188 + }, + { + "bits_per_byte": 1.705079684247204, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1818711757659912, + "step": 3720, + "unclipped_grad_norm": 0.9917497634887695 + }, + { + "bits_per_byte": 1.649159353336511, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1431101560592651, + "step": 3730, + "unclipped_grad_norm": 0.9426941275596619 + }, + { + "bits_per_byte": 1.6954601786364945, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1752034425735474, + "step": 3740, + "unclipped_grad_norm": 1.0086418390274048 + }, + { + "bits_per_byte": 1.721657263946521, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1933618783950806, + "step": 3750, + "unclipped_grad_norm": 0.9619976878166199 + }, + { + "bits_per_byte": 1.808596729732132, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2536237239837646, + "step": 3760, + "unclipped_grad_norm": 1.012074589729309 + }, + { + "bits_per_byte": 1.7638779728374354, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.22262704372406, + "step": 3770, + "unclipped_grad_norm": 1.050723910331726 + }, + { + "bits_per_byte": 1.589899463311364, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.102034330368042, + "step": 3780, + "unclipped_grad_norm": 0.9615285992622375 + }, + { + "bits_per_byte": 1.6942863970443558, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1743898391723633, + "step": 3790, + "unclipped_grad_norm": 0.9870805144309998 + }, + { + "bits_per_byte": 1.6994689222454558, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1779820919036865, + "step": 3800, + "unclipped_grad_norm": 0.9850614070892334 + }, + { + "bits_per_byte": 1.7199389852817506, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1921708583831787, + "step": 3810, + "unclipped_grad_norm": 0.9774236083030701 + }, + { + "bits_per_byte": 1.718822817877651, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1913971900939941, + "step": 3820, + "unclipped_grad_norm": 1.0395604372024536 + }, + { + "bits_per_byte": 1.762180848038721, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.221450686454773, + "step": 3830, + "unclipped_grad_norm": 1.0331096649169922 + }, + { + "bits_per_byte": 1.68565630762406, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.168407917022705, + "step": 3840, + "unclipped_grad_norm": 1.0491844415664673 + }, + { + "bits_per_byte": 1.6311636046981792, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.13063645362854, + "step": 3850, + "unclipped_grad_norm": 0.9598016738891602 + }, + { + "bits_per_byte": 1.7078809376644575, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1838128566741943, + "step": 3860, + "unclipped_grad_norm": 1.0586371421813965 + }, + { + "bits_per_byte": 1.778100422115837, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.232485294342041, + "step": 3870, + "unclipped_grad_norm": 0.9732621908187866 + }, + { + "bits_per_byte": 1.6634643542873266, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1530256271362305, + "step": 3880, + "unclipped_grad_norm": 0.9620565176010132 + }, + { + "bits_per_byte": 1.6362388127251406, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1341543197631836, + "step": 3890, + "unclipped_grad_norm": 0.9925653338432312 + }, + { + "bits_per_byte": 1.721069255263221, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1929543018341064, + "step": 3900, + "unclipped_grad_norm": 1.121357798576355 + }, + { + "bits_per_byte": 1.7172674067832479, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1903190612792969, + "step": 3910, + "unclipped_grad_norm": 1.0240309238433838 + }, + { + "bits_per_byte": 1.671385531220766, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1585161685943604, + "step": 3920, + "unclipped_grad_norm": 0.9621630311012268 + }, + { + "bits_per_byte": 1.7072636919305109, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1833850145339966, + "step": 3930, + "unclipped_grad_norm": 0.9956491589546204 + }, + { + "bits_per_byte": 1.7254322831329596, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1959785223007202, + "step": 3940, + "unclipped_grad_norm": 0.9730517268180847 + }, + { + "bits_per_byte": 1.7623370082857046, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.221558928489685, + "step": 3950, + "unclipped_grad_norm": 0.9878451228141785 + }, + { + "bits_per_byte": 1.7644881672826966, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2230499982833862, + "step": 3960, + "unclipped_grad_norm": 1.003400444984436 + }, + { + "bits_per_byte": 1.682019734471566, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1658872365951538, + "step": 3970, + "unclipped_grad_norm": 1.0887466669082642 + }, + { + "bits_per_byte": 1.7137744391442242, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1878979206085205, + "step": 3980, + "unclipped_grad_norm": 1.0592182874679565 + }, + { + "bits_per_byte": 1.7370323409510051, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2040190696716309, + "step": 3990, + "unclipped_grad_norm": 1.0668442249298096 + }, + { + "bits_per_byte": 1.620528197568823, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1232645511627197, + "step": 4000, + "unclipped_grad_norm": 0.979034423828125 + }, + { + "bits_per_byte": 1.7224021208074078, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.193878173828125, + "step": 4010, + "unclipped_grad_norm": 1.0592741966247559 + }, + { + "bits_per_byte": 1.6779786581242426, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.163086175918579, + "step": 4020, + "unclipped_grad_norm": 1.0186548233032227 + }, + { + "bits_per_byte": 1.7290010951210137, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1984522342681885, + "step": 4030, + "unclipped_grad_norm": 1.0059210062026978 + }, + { + "bits_per_byte": 1.6595417739864478, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1503067016601562, + "step": 4040, + "unclipped_grad_norm": 1.009366750717163 + }, + { + "bits_per_byte": 1.6963183720643, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1757982969284058, + "step": 4050, + "unclipped_grad_norm": 1.0469119548797607 + }, + { + "bits_per_byte": 1.706078903448717, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1825637817382812, + "step": 4060, + "unclipped_grad_norm": 1.0410232543945312 + }, + { + "bits_per_byte": 1.7226642223673228, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1940598487854004, + "step": 4070, + "unclipped_grad_norm": 1.0424585342407227 + }, + { + "bits_per_byte": 1.7834826191745887, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2362159490585327, + "step": 4080, + "unclipped_grad_norm": 1.0965845584869385 + }, + { + "bits_per_byte": 1.6886807945221334, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1705043315887451, + "step": 4090, + "unclipped_grad_norm": 1.0564258098602295 + }, + { + "bits_per_byte": 1.6871728506393684, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1694591045379639, + "step": 4100, + "unclipped_grad_norm": 1.032038688659668 + }, + { + "bits_per_byte": 1.6672990514536148, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1556836366653442, + "step": 4110, + "unclipped_grad_norm": 1.0212913751602173 + }, + { + "bits_per_byte": 1.7325631997856843, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.2009212970733643, + "step": 4120, + "unclipped_grad_norm": 1.1421489715576172 + }, + { + "bits_per_byte": 1.6623953101295634, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1522846221923828, + "step": 4130, + "unclipped_grad_norm": 1.0443917512893677 + }, + { + "bits_per_byte": 1.7315172013031366, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2001962661743164, + "step": 4140, + "unclipped_grad_norm": 1.0389643907546997 + }, + { + "bits_per_byte": 1.6632394009799982, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.152869701385498, + "step": 4150, + "unclipped_grad_norm": 0.9869729280471802 + }, + { + "bits_per_byte": 1.697900440469587, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1768949031829834, + "step": 4160, + "unclipped_grad_norm": 1.018755316734314 + }, + { + "bits_per_byte": 1.74134876152236, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.2070109844207764, + "step": 4170, + "unclipped_grad_norm": 1.0534698963165283 + }, + { + "bits_per_byte": 1.7750522016119445, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.230372428894043, + "step": 4180, + "unclipped_grad_norm": 1.1320860385894775 + }, + { + "bits_per_byte": 1.6878769476120008, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1699471473693848, + "step": 4190, + "unclipped_grad_norm": 1.0554869174957275 + }, + { + "bits_per_byte": 1.725129765650092, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1957688331604004, + "step": 4200, + "unclipped_grad_norm": 1.0452762842178345 + }, + { + "bits_per_byte": 1.7145272072070508, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1884196996688843, + "step": 4210, + "unclipped_grad_norm": 0.9850034117698669 + }, + { + "bits_per_byte": 1.7283826455085112, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1980235576629639, + "step": 4220, + "unclipped_grad_norm": 1.0109167098999023 + }, + { + "bits_per_byte": 1.7140664656853892, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1881003379821777, + "step": 4230, + "unclipped_grad_norm": 1.0496208667755127 + }, + { + "bits_per_byte": 1.659261958213494, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1501127481460571, + "step": 4240, + "unclipped_grad_norm": 0.9920905232429504 + }, + { + "bits_per_byte": 1.6850357942197471, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1679778099060059, + "step": 4250, + "unclipped_grad_norm": 1.0322012901306152 + }, + { + "bits_per_byte": 1.667805196395104, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1560344696044922, + "step": 4260, + "unclipped_grad_norm": 1.0430845022201538 + }, + { + "bits_per_byte": 1.6625848350108143, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1524159908294678, + "step": 4270, + "unclipped_grad_norm": 1.0265511274337769 + }, + { + "bits_per_byte": 1.6648945620119018, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1540169715881348, + "step": 4280, + "unclipped_grad_norm": 0.9929917454719543 + }, + { + "bits_per_byte": 1.72171487813456, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.19340181350708, + "step": 4290, + "unclipped_grad_norm": 1.0577914714813232 + }, + { + "bits_per_byte": 1.6424783432984433, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.138479232788086, + "step": 4300, + "unclipped_grad_norm": 1.0359195470809937 + }, + { + "bits_per_byte": 1.7237528725472844, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1948144435882568, + "step": 4310, + "unclipped_grad_norm": 1.124787449836731 + }, + { + "bits_per_byte": 1.7081155220002342, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1839754581451416, + "step": 4320, + "unclipped_grad_norm": 1.0496878623962402 + }, + { + "bits_per_byte": 1.6609448084521865, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1512792110443115, + "step": 4330, + "unclipped_grad_norm": 1.0365654230117798 + }, + { + "bits_per_byte": 1.6981388084236828, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1770601272583008, + "step": 4340, + "unclipped_grad_norm": 1.0712857246398926 + }, + { + "bits_per_byte": 1.679871155214337, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.164397954940796, + "step": 4350, + "unclipped_grad_norm": 1.0368938446044922 + }, + { + "bits_per_byte": 1.658068570599157, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1492855548858643, + "step": 4360, + "unclipped_grad_norm": 1.054489254951477 + }, + { + "bits_per_byte": 1.6791636185886873, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.163907527923584, + "step": 4370, + "unclipped_grad_norm": 1.05704927444458 + }, + { + "bits_per_byte": 1.733114060216398, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.2013031244277954, + "step": 4380, + "unclipped_grad_norm": 1.066345453262329 + }, + { + "bits_per_byte": 1.6866214742607022, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.169076919555664, + "step": 4390, + "unclipped_grad_norm": 1.039746880531311 + }, + { + "bits_per_byte": 1.656244522604106, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1480212211608887, + "step": 4400, + "unclipped_grad_norm": 1.0477714538574219 + }, + { + "bits_per_byte": 1.7190543065257124, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1915576457977295, + "step": 4410, + "unclipped_grad_norm": 1.0500718355178833 + }, + { + "bits_per_byte": 1.6606173534849438, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1510522365570068, + "step": 4420, + "unclipped_grad_norm": 1.0457862615585327 + }, + { + "bits_per_byte": 1.7152531459763425, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1889228820800781, + "step": 4430, + "unclipped_grad_norm": 1.0248607397079468 + }, + { + "bits_per_byte": 1.6480127450032085, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.14231538772583, + "step": 4440, + "unclipped_grad_norm": 1.026747703552246 + }, + { + "bits_per_byte": 1.64553137931656, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1405954360961914, + "step": 4450, + "unclipped_grad_norm": 1.0325950384140015 + }, + { + "bits_per_byte": 1.6368648295742814, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1345882415771484, + "step": 4460, + "unclipped_grad_norm": 1.0030871629714966 + }, + { + "bits_per_byte": 1.686227977955528, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1688041687011719, + "step": 4470, + "unclipped_grad_norm": 0.9844985604286194 + }, + { + "bits_per_byte": 1.7063241507088474, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1827337741851807, + "step": 4480, + "unclipped_grad_norm": 1.1298258304595947 + }, + { + "bits_per_byte": 1.6590796566035795, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1499863862991333, + "step": 4490, + "unclipped_grad_norm": 0.9567995071411133 + }, + { + "bits_per_byte": 1.699403568838128, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1779367923736572, + "step": 4500, + "unclipped_grad_norm": 1.0041329860687256 + }, + { + "bits_per_byte": 1.630450564627702, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1301422119140625, + "step": 4510, + "unclipped_grad_norm": 1.0097919702529907 + }, + { + "bits_per_byte": 1.677602188101504, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1628252267837524, + "step": 4520, + "unclipped_grad_norm": 1.0421855449676514 + }, + { + "bits_per_byte": 1.6955814264053528, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1752874851226807, + "step": 4530, + "unclipped_grad_norm": 1.0619962215423584 + }, + { + "bits_per_byte": 1.738972993183341, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.2053642272949219, + "step": 4540, + "unclipped_grad_norm": 1.0259038209915161 + }, + { + "bits_per_byte": 1.6138776284599583, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.118654727935791, + "step": 4550, + "unclipped_grad_norm": 1.1216750144958496 + }, + { + "bits_per_byte": 1.6847919228208237, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1678087711334229, + "step": 4560, + "unclipped_grad_norm": 1.1666667461395264 + }, + { + "bits_per_byte": 1.6566538413131595, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1483049392700195, + "step": 4570, + "unclipped_grad_norm": 0.9492276310920715 + }, + { + "bits_per_byte": 1.6127442627907724, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1178691387176514, + "step": 4580, + "unclipped_grad_norm": 1.0188902616500854 + }, + { + "bits_per_byte": 1.6539726317862082, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1464464664459229, + "step": 4590, + "unclipped_grad_norm": 1.0299646854400635 + }, + { + "bits_per_byte": 1.6561746976478557, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.147972822189331, + "step": 4600, + "unclipped_grad_norm": 1.0103821754455566 + }, + { + "bits_per_byte": 1.719434732149421, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1918213367462158, + "step": 4610, + "unclipped_grad_norm": 1.0808284282684326 + }, + { + "bits_per_byte": 1.7013596995090414, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1792926788330078, + "step": 4620, + "unclipped_grad_norm": 1.0917437076568604 + }, + { + "bits_per_byte": 1.683035292024911, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1665911674499512, + "step": 4630, + "unclipped_grad_norm": 1.0124032497406006 + }, + { + "bits_per_byte": 1.7275268598378177, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1974303722381592, + "step": 4640, + "unclipped_grad_norm": 1.0515177249908447 + }, + { + "bits_per_byte": 1.6549154406782383, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1470999717712402, + "step": 4650, + "unclipped_grad_norm": 1.1102732419967651 + }, + { + "bits_per_byte": 1.6181718632693518, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1216312646865845, + "step": 4660, + "unclipped_grad_norm": 1.066238522529602 + }, + { + "bits_per_byte": 1.7593243881905407, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.219470739364624, + "step": 4670, + "unclipped_grad_norm": 1.1023285388946533 + }, + { + "bits_per_byte": 1.7532637195741354, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.2152698040008545, + "step": 4680, + "unclipped_grad_norm": 1.0341681241989136 + }, + { + "bits_per_byte": 1.7266088164475122, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.196794033050537, + "step": 4690, + "unclipped_grad_norm": 1.0434808731079102 + }, + { + "bits_per_byte": 1.6483247215318737, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1425316333770752, + "step": 4700, + "unclipped_grad_norm": 1.045780897140503 + }, + { + "bits_per_byte": 1.7162458298371228, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1896109580993652, + "step": 4710, + "unclipped_grad_norm": 1.1512850522994995 + }, + { + "bits_per_byte": 1.6600910865733036, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1506874561309814, + "step": 4720, + "unclipped_grad_norm": 1.0870006084442139 + }, + { + "bits_per_byte": 1.579696248583626, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0949620008468628, + "step": 4730, + "unclipped_grad_norm": 1.066159725189209 + }, + { + "bits_per_byte": 1.6295564268258662, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.129522442817688, + "step": 4740, + "unclipped_grad_norm": 1.0375505685806274 + }, + { + "bits_per_byte": 1.6749376608916864, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1609783172607422, + "step": 4750, + "unclipped_grad_norm": 1.0134141445159912 + }, + { + "bits_per_byte": 1.6434221840863785, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1391334533691406, + "step": 4760, + "unclipped_grad_norm": 1.0448397397994995 + }, + { + "bits_per_byte": 1.6362174868764336, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1341395378112793, + "step": 4770, + "unclipped_grad_norm": 1.055105447769165 + }, + { + "bits_per_byte": 1.666649472981306, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.155233383178711, + "step": 4780, + "unclipped_grad_norm": 1.0775461196899414 + }, + { + "bits_per_byte": 1.6314788488972107, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1308549642562866, + "step": 4790, + "unclipped_grad_norm": 1.065148949623108 + }, + { + "bits_per_byte": 1.6269109896902945, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1276887655258179, + "step": 4800, + "unclipped_grad_norm": 1.1373785734176636 + }, + { + "bits_per_byte": 1.6901620810940148, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.171531081199646, + "step": 4810, + "unclipped_grad_norm": 1.1410114765167236 + }, + { + "bits_per_byte": 1.6112782826748178, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1168529987335205, + "step": 4820, + "unclipped_grad_norm": 1.0628036260604858 + }, + { + "bits_per_byte": 1.649350942009572, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1432429552078247, + "step": 4830, + "unclipped_grad_norm": 1.1659979820251465 + }, + { + "bits_per_byte": 1.6571300612733986, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1486350297927856, + "step": 4840, + "unclipped_grad_norm": 1.1067216396331787 + }, + { + "bits_per_byte": 1.668376866726572, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.156430721282959, + "step": 4850, + "unclipped_grad_norm": 0.9945695996284485 + }, + { + "bits_per_byte": 1.600552412671108, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1094183921813965, + "step": 4860, + "unclipped_grad_norm": 1.0181769132614136 + }, + { + "bits_per_byte": 1.6771526254521485, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1625136137008667, + "step": 4870, + "unclipped_grad_norm": 1.0462807416915894 + }, + { + "bits_per_byte": 1.539689988305221, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0672317743301392, + "step": 4880, + "unclipped_grad_norm": 1.0612690448760986 + }, + { + "bits_per_byte": 1.5967718900398415, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1067979335784912, + "step": 4890, + "unclipped_grad_norm": 1.0629737377166748 + }, + { + "bits_per_byte": 1.6387246499607118, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1358773708343506, + "step": 4900, + "unclipped_grad_norm": 1.0522006750106812 + }, + { + "bits_per_byte": 1.7368791044090865, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.2039128541946411, + "step": 4910, + "unclipped_grad_norm": 1.1116063594818115 + }, + { + "bits_per_byte": 1.6773633041994556, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1626596450805664, + "step": 4920, + "unclipped_grad_norm": 1.0861767530441284 + }, + { + "bits_per_byte": 1.6375162998557495, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1350398063659668, + "step": 4930, + "unclipped_grad_norm": 1.0190138816833496 + }, + { + "bits_per_byte": 1.6159926711002692, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1201207637786865, + "step": 4940, + "unclipped_grad_norm": 1.0380074977874756 + }, + { + "bits_per_byte": 1.5895295286293578, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1017779111862183, + "step": 4950, + "unclipped_grad_norm": 1.0411629676818848 + }, + { + "bits_per_byte": 1.5717379233976003, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.08944571018219, + "step": 4960, + "unclipped_grad_norm": 1.0460548400878906 + }, + { + "bits_per_byte": 1.6461665112461963, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1410356760025024, + "step": 4970, + "unclipped_grad_norm": 1.112047553062439 + }, + { + "bits_per_byte": 1.7016685803499911, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.179506778717041, + "step": 4980, + "unclipped_grad_norm": 1.062510371208191 + }, + { + "bits_per_byte": 1.636959248049605, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1346536874771118, + "step": 4990, + "unclipped_grad_norm": 1.1910533905029297 + }, + { + "bits_per_byte": 1.6019551031715449, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1103906631469727, + "step": 5000, + "unclipped_grad_norm": 1.0508692264556885 + }, + { + "bits_per_byte": 1.5837816964548719, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0977938175201416, + "step": 5010, + "unclipped_grad_norm": 1.0227967500686646 + }, + { + "bits_per_byte": 1.639255904369227, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.136245608329773, + "step": 5020, + "unclipped_grad_norm": 1.0390269756317139 + }, + { + "bits_per_byte": 1.6173587292960725, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1210676431655884, + "step": 5030, + "unclipped_grad_norm": 1.083649754524231 + }, + { + "bits_per_byte": 1.591486175248224, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1031341552734375, + "step": 5040, + "unclipped_grad_norm": 1.0160164833068848 + }, + { + "bits_per_byte": 1.5612364907532663, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0821666717529297, + "step": 5050, + "unclipped_grad_norm": 0.9697359800338745 + }, + { + "bits_per_byte": 1.5996653261579574, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1088035106658936, + "step": 5060, + "unclipped_grad_norm": 1.0825191736221313 + }, + { + "bits_per_byte": 1.6612003746713688, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.151456356048584, + "step": 5070, + "unclipped_grad_norm": 1.051671028137207 + }, + { + "bits_per_byte": 1.6135078657606032, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1183984279632568, + "step": 5080, + "unclipped_grad_norm": 1.0758769512176514 + }, + { + "bits_per_byte": 1.6053440213067967, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1127396821975708, + "step": 5090, + "unclipped_grad_norm": 1.0736517906188965 + }, + { + "bits_per_byte": 1.620226196033908, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1230552196502686, + "step": 5100, + "unclipped_grad_norm": 1.0084763765335083 + }, + { + "bits_per_byte": 1.5469340695422136, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0722529888153076, + "step": 5110, + "unclipped_grad_norm": 1.001043438911438 + }, + { + "bits_per_byte": 1.627192525289757, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1278839111328125, + "step": 5120, + "unclipped_grad_norm": 1.0516539812088013 + }, + { + "bits_per_byte": 1.642070228467946, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.138196349143982, + "step": 5130, + "unclipped_grad_norm": 1.082858681678772 + }, + { + "bits_per_byte": 1.6022958008029042, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1106268167495728, + "step": 5140, + "unclipped_grad_norm": 1.0696102380752563 + }, + { + "bits_per_byte": 1.6803482350878305, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1647286415100098, + "step": 5150, + "unclipped_grad_norm": 1.045096755027771 + }, + { + "bits_per_byte": 1.5608273440268638, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0818830728530884, + "step": 5160, + "unclipped_grad_norm": 1.0852984189987183 + }, + { + "bits_per_byte": 1.7284662290768305, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1980814933776855, + "step": 5170, + "unclipped_grad_norm": 1.2249319553375244 + }, + { + "bits_per_byte": 1.6274811121779047, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1280839443206787, + "step": 5180, + "unclipped_grad_norm": 1.1194853782653809 + }, + { + "bits_per_byte": 1.567900990456851, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.086786150932312, + "step": 5190, + "unclipped_grad_norm": 1.0312306880950928 + }, + { + "bits_per_byte": 1.5985697966719614, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.108044147491455, + "step": 5200, + "unclipped_grad_norm": 1.137611985206604 + }, + { + "bits_per_byte": 1.6023676895509649, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.110676646232605, + "step": 5210, + "unclipped_grad_norm": 1.031370759010315 + }, + { + "bits_per_byte": 1.6946726700781938, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1746575832366943, + "step": 5220, + "unclipped_grad_norm": 1.113584280014038 + }, + { + "bits_per_byte": 1.5914005278880945, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1030747890472412, + "step": 5230, + "unclipped_grad_norm": 1.1469851732254028 + }, + { + "bits_per_byte": 1.6053940682581977, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.11277437210083, + "step": 5240, + "unclipped_grad_norm": 1.0413556098937988 + }, + { + "bits_per_byte": 1.5800591319769466, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.095213532447815, + "step": 5250, + "unclipped_grad_norm": 1.0926955938339233 + }, + { + "bits_per_byte": 1.5918442431273205, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.1033823490142822, + "step": 5260, + "unclipped_grad_norm": 1.0704180002212524 + }, + { + "bits_per_byte": 1.589662299235824, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1018699407577515, + "step": 5270, + "unclipped_grad_norm": 1.0476908683776855 + }, + { + "bits_per_byte": 1.6108980290337602, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1165894269943237, + "step": 5280, + "unclipped_grad_norm": 1.1081280708312988 + }, + { + "bits_per_byte": 1.608212003992585, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1147276163101196, + "step": 5290, + "unclipped_grad_norm": 1.1043791770935059 + }, + { + "bits_per_byte": 1.614489714714379, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.1190789937973022, + "step": 5300, + "unclipped_grad_norm": 1.0849814414978027 + }, + { + "bits_per_byte": 1.6119760162893682, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.117336630821228, + "step": 5310, + "unclipped_grad_norm": 1.1325145959854126 + }, + { + "bits_per_byte": 1.5916898027068456, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1032752990722656, + "step": 5320, + "unclipped_grad_norm": 1.072629451751709 + }, + { + "bits_per_byte": 1.6015805249579658, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.110131025314331, + "step": 5330, + "unclipped_grad_norm": 1.1142079830169678 + }, + { + "bits_per_byte": 1.553969879789011, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.07712984085083, + "step": 5340, + "unclipped_grad_norm": 1.0742131471633911 + }, + { + "bits_per_byte": 1.626766180298268, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1275883913040161, + "step": 5350, + "unclipped_grad_norm": 1.127328634262085 + }, + { + "bits_per_byte": 1.5508274127924457, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0749516487121582, + "step": 5360, + "unclipped_grad_norm": 1.1100437641143799 + }, + { + "bits_per_byte": 1.5306884163590617, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0609923601150513, + "step": 5370, + "unclipped_grad_norm": 1.0785573720932007 + }, + { + "bits_per_byte": 1.633070548330946, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.131958246231079, + "step": 5380, + "unclipped_grad_norm": 1.0940150022506714 + }, + { + "bits_per_byte": 1.5790724675089465, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.094529628753662, + "step": 5390, + "unclipped_grad_norm": 1.1135237216949463 + }, + { + "bits_per_byte": 1.6121983898569339, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1174907684326172, + "step": 5400, + "unclipped_grad_norm": 1.1351183652877808 + }, + { + "bits_per_byte": 1.6233261833157102, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1252039670944214, + "step": 5410, + "unclipped_grad_norm": 1.1545859575271606 + }, + { + "bits_per_byte": 1.5857118577455054, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.09913170337677, + "step": 5420, + "unclipped_grad_norm": 1.1009039878845215 + }, + { + "bits_per_byte": 1.6290494219711227, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1291710138320923, + "step": 5430, + "unclipped_grad_norm": 1.1000012159347534 + }, + { + "bits_per_byte": 1.5753132707263873, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0919239521026611, + "step": 5440, + "unclipped_grad_norm": 1.0844659805297852 + }, + { + "bits_per_byte": 1.5681787424379945, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0869786739349365, + "step": 5450, + "unclipped_grad_norm": 1.1008447408676147 + }, + { + "bits_per_byte": 1.5540281819076536, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0771702527999878, + "step": 5460, + "unclipped_grad_norm": 1.077485203742981 + }, + { + "bits_per_byte": 1.5997471898997682, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1088602542877197, + "step": 5470, + "unclipped_grad_norm": 1.0731958150863647 + }, + { + "bits_per_byte": 1.6110182449067132, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.1166727542877197, + "step": 5480, + "unclipped_grad_norm": 1.098453164100647 + }, + { + "bits_per_byte": 1.627262694211309, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1279325485229492, + "step": 5490, + "unclipped_grad_norm": 1.2330873012542725 + }, + { + "bits_per_byte": 1.5907870657724668, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1026495695114136, + "step": 5500, + "unclipped_grad_norm": 1.1288633346557617 + }, + { + "bits_per_byte": 1.677719824234694, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1629067659378052, + "step": 5510, + "unclipped_grad_norm": 1.1779037714004517 + }, + { + "bits_per_byte": 1.6037174093949362, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1116122007369995, + "step": 5520, + "unclipped_grad_norm": 1.07362699508667 + }, + { + "bits_per_byte": 1.5749782485225066, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0916917324066162, + "step": 5530, + "unclipped_grad_norm": 1.0827255249023438 + }, + { + "bits_per_byte": 1.6475040203219564, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1419627666473389, + "step": 5540, + "unclipped_grad_norm": 1.2362996339797974 + }, + { + "bits_per_byte": 1.5801301608117528, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0952627658843994, + "step": 5550, + "unclipped_grad_norm": 1.0816401243209839 + }, + { + "bits_per_byte": 1.566390638816974, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.085739254951477, + "step": 5560, + "unclipped_grad_norm": 1.1727906465530396 + }, + { + "bits_per_byte": 1.545297310653952, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0711184740066528, + "step": 5570, + "unclipped_grad_norm": 1.096352458000183 + }, + { + "bits_per_byte": 1.6197793851069664, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1227455139160156, + "step": 5580, + "unclipped_grad_norm": 1.098443865776062 + }, + { + "bits_per_byte": 1.5488499562728253, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0735809803009033, + "step": 5590, + "unclipped_grad_norm": 1.106048822402954 + }, + { + "bits_per_byte": 1.6060310919969936, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1132159233093262, + "step": 5600, + "unclipped_grad_norm": 1.1421784162521362 + }, + { + "bits_per_byte": 1.6012358717256368, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1098921298980713, + "step": 5610, + "unclipped_grad_norm": 1.0477343797683716 + }, + { + "bits_per_byte": 1.612595669780427, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1177661418914795, + "step": 5620, + "unclipped_grad_norm": 1.0961613655090332 + }, + { + "bits_per_byte": 1.5887812321154537, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.1012592315673828, + "step": 5630, + "unclipped_grad_norm": 1.0803630352020264 + }, + { + "bits_per_byte": 1.5675833385007074, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0865659713745117, + "step": 5640, + "unclipped_grad_norm": 1.0926858186721802 + }, + { + "bits_per_byte": 1.598304599424331, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1078603267669678, + "step": 5650, + "unclipped_grad_norm": 1.1493103504180908 + }, + { + "bits_per_byte": 1.6562716958629424, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1480400562286377, + "step": 5660, + "unclipped_grad_norm": 1.1589452028274536 + }, + { + "bits_per_byte": 1.5621947780838739, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0828309059143066, + "step": 5670, + "unclipped_grad_norm": 1.118128776550293 + }, + { + "bits_per_byte": 1.5797031278896605, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0949667692184448, + "step": 5680, + "unclipped_grad_norm": 1.079695701599121 + }, + { + "bits_per_byte": 1.650425317629512, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1439876556396484, + "step": 5690, + "unclipped_grad_norm": 1.1217325925827026 + }, + { + "bits_per_byte": 1.5713879387030945, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.089203119277954, + "step": 5700, + "unclipped_grad_norm": 1.0890491008758545 + }, + { + "bits_per_byte": 1.5792038622542057, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.094620704650879, + "step": 5710, + "unclipped_grad_norm": 1.0754131078720093 + }, + { + "bits_per_byte": 1.6025209260928837, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1107828617095947, + "step": 5720, + "unclipped_grad_norm": 1.0796459913253784 + }, + { + "bits_per_byte": 1.5944050647986676, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.1051573753356934, + "step": 5730, + "unclipped_grad_norm": 1.0518229007720947 + }, + { + "bits_per_byte": 1.599884088089855, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1089551448822021, + "step": 5740, + "unclipped_grad_norm": 1.2038764953613281 + }, + { + "bits_per_byte": 1.5972181850188307, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.1071072816848755, + "step": 5750, + "unclipped_grad_norm": 1.1661580801010132 + }, + { + "bits_per_byte": 1.6206038699352028, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.123317003250122, + "step": 5760, + "unclipped_grad_norm": 1.1536771059036255 + }, + { + "bits_per_byte": 1.569340313261922, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0877838134765625, + "step": 5770, + "unclipped_grad_norm": 1.1234960556030273 + }, + { + "bits_per_byte": 1.548520437513772, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.073352575302124, + "step": 5780, + "unclipped_grad_norm": 1.0824602842330933 + }, + { + "bits_per_byte": 1.575600653735979, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0921231508255005, + "step": 5790, + "unclipped_grad_norm": 1.0697625875473022 + }, + { + "bits_per_byte": 1.5474534571478191, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.072613000869751, + "step": 5800, + "unclipped_grad_norm": 1.121078372001648 + }, + { + "bits_per_byte": 1.6281222635003212, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.128528356552124, + "step": 5810, + "unclipped_grad_norm": 1.163888931274414 + }, + { + "bits_per_byte": 1.5571622217543264, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0793426036834717, + "step": 5820, + "unclipped_grad_norm": 1.0646225214004517 + }, + { + "bits_per_byte": 1.5715148618994312, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0892910957336426, + "step": 5830, + "unclipped_grad_norm": 1.1465011835098267 + }, + { + "bits_per_byte": 1.5656843060698804, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.085249662399292, + "step": 5840, + "unclipped_grad_norm": 1.0994694232940674 + }, + { + "bits_per_byte": 1.529561070082656, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.060210943222046, + "step": 5850, + "unclipped_grad_norm": 1.123828649520874 + }, + { + "bits_per_byte": 1.5867750544931392, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.099868655204773, + "step": 5860, + "unclipped_grad_norm": 1.1375004053115845 + }, + { + "bits_per_byte": 1.5937161022993112, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.104679822921753, + "step": 5870, + "unclipped_grad_norm": 1.0862858295440674 + }, + { + "bits_per_byte": 1.6139739387444414, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1187214851379395, + "step": 5880, + "unclipped_grad_norm": 1.1279133558273315 + }, + { + "bits_per_byte": 1.6231863614205586, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1251070499420166, + "step": 5890, + "unclipped_grad_norm": 1.1251599788665771 + }, + { + "bits_per_byte": 1.6124319422968056, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1176526546478271, + "step": 5900, + "unclipped_grad_norm": 1.1988507509231567 + }, + { + "bits_per_byte": 1.6244132856518139, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.1259574890136719, + "step": 5910, + "unclipped_grad_norm": 1.1206265687942505 + }, + { + "bits_per_byte": 1.5302899325570125, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.060716152191162, + "step": 5920, + "unclipped_grad_norm": 1.1038360595703125 + }, + { + "bits_per_byte": 1.6585005910181243, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1495850086212158, + "step": 5930, + "unclipped_grad_norm": 1.2028963565826416 + }, + { + "bits_per_byte": 1.5036174872000014, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0422282218933105, + "step": 5940, + "unclipped_grad_norm": 1.0828943252563477 + }, + { + "bits_per_byte": 1.564500033536039, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0844287872314453, + "step": 5950, + "unclipped_grad_norm": 1.1633635759353638 + }, + { + "bits_per_byte": 1.6026038217305996, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1108403205871582, + "step": 5960, + "unclipped_grad_norm": 1.1734113693237305 + }, + { + "bits_per_byte": 1.582973034030515, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0972332954406738, + "step": 5970, + "unclipped_grad_norm": 1.255017638206482 + }, + { + "bits_per_byte": 1.6250308753510623, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.1263855695724487, + "step": 5980, + "unclipped_grad_norm": 1.1516860723495483 + }, + { + "bits_per_byte": 1.5794843659577629, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0948151350021362, + "step": 5990, + "unclipped_grad_norm": 1.1278661489486694 + }, + { + "bits_per_byte": 1.5522483334538744, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0759365558624268, + "step": 6000, + "unclipped_grad_norm": 1.067203402519226 + }, + { + "bits_per_byte": 1.577093635128119, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0931580066680908, + "step": 6010, + "unclipped_grad_norm": 1.1185256242752075 + }, + { + "bits_per_byte": 1.534402725669746, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0635669231414795, + "step": 6020, + "unclipped_grad_norm": 1.1662800312042236 + }, + { + "bits_per_byte": 1.5661140907143865, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0855475664138794, + "step": 6030, + "unclipped_grad_norm": 1.1327592134475708 + }, + { + "bits_per_byte": 1.6326177180112242, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.131644368171692, + "step": 6040, + "unclipped_grad_norm": 1.1918420791625977 + }, + { + "bits_per_byte": 1.5457630396724886, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0714412927627563, + "step": 6050, + "unclipped_grad_norm": 1.147464394569397 + }, + { + "bits_per_byte": 1.5419047808830324, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0687669515609741, + "step": 6060, + "unclipped_grad_norm": 1.1692534685134888 + }, + { + "bits_per_byte": 1.6009338701907216, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.1096827983856201, + "step": 6070, + "unclipped_grad_norm": 1.1858196258544922 + }, + { + "bits_per_byte": 1.5324048032146727, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.062182068824768, + "step": 6080, + "unclipped_grad_norm": 1.1669044494628906 + }, + { + "bits_per_byte": 1.5495642002218584, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0740760564804077, + "step": 6090, + "unclipped_grad_norm": 1.0681768655776978 + }, + { + "bits_per_byte": 1.5432428059067451, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0696943998336792, + "step": 6100, + "unclipped_grad_norm": 1.086612582206726 + }, + { + "bits_per_byte": 1.6119146184830102, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1172940731048584, + "step": 6110, + "unclipped_grad_norm": 1.1307899951934814 + }, + { + "bits_per_byte": 1.6174417969164392, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.1211252212524414, + "step": 6120, + "unclipped_grad_norm": 1.2087455987930298 + }, + { + "bits_per_byte": 1.5350297744147916, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0640015602111816, + "step": 6130, + "unclipped_grad_norm": 1.1170084476470947 + }, + { + "bits_per_byte": 1.5441861307467275, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0703482627868652, + "step": 6140, + "unclipped_grad_norm": 1.191177487373352 + }, + { + "bits_per_byte": 1.6015168913771465, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.1100869178771973, + "step": 6150, + "unclipped_grad_norm": 1.1631611585617065 + }, + { + "bits_per_byte": 1.4840940166740553, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0286955833435059, + "step": 6160, + "unclipped_grad_norm": 1.0670503377914429 + }, + { + "bits_per_byte": 1.5765931656141083, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.092811107635498, + "step": 6170, + "unclipped_grad_norm": 1.12460458278656 + }, + { + "bits_per_byte": 1.531391309453138, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0614795684814453, + "step": 6180, + "unclipped_grad_norm": 1.1588267087936401 + }, + { + "bits_per_byte": 1.5676526475090051, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0866140127182007, + "step": 6190, + "unclipped_grad_norm": 1.2090874910354614 + }, + { + "bits_per_byte": 1.5723472579296074, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0898680686950684, + "step": 6200, + "unclipped_grad_norm": 1.1369835138320923 + }, + { + "bits_per_byte": 1.5728810920778855, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.090238094329834, + "step": 6210, + "unclipped_grad_norm": 1.2529284954071045 + }, + { + "bits_per_byte": 1.5681563846933824, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.086963176727295, + "step": 6220, + "unclipped_grad_norm": 1.0888316631317139 + }, + { + "bits_per_byte": 1.5207944264375752, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0541343688964844, + "step": 6230, + "unclipped_grad_norm": 1.15282142162323 + }, + { + "bits_per_byte": 1.5617125387308546, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0824966430664062, + "step": 6240, + "unclipped_grad_norm": 1.227396845817566 + }, + { + "bits_per_byte": 1.595514008931431, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.1059260368347168, + "step": 6250, + "unclipped_grad_norm": 1.1354495286941528 + }, + { + "bits_per_byte": 1.4806825688115406, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0263309478759766, + "step": 6260, + "unclipped_grad_norm": 1.1806098222732544 + }, + { + "bits_per_byte": 1.561176984756068, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0821254253387451, + "step": 6270, + "unclipped_grad_norm": 1.1072839498519897 + }, + { + "bits_per_byte": 1.557884892853252, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.079843521118164, + "step": 6280, + "unclipped_grad_norm": 1.16572904586792 + }, + { + "bits_per_byte": 1.5084749651909706, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0455951690673828, + "step": 6290, + "unclipped_grad_norm": 1.1536295413970947 + }, + { + "bits_per_byte": 1.5632667459467018, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0835739374160767, + "step": 6300, + "unclipped_grad_norm": 1.1973258256912231 + }, + { + "bits_per_byte": 1.5420322200273218, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0688552856445312, + "step": 6310, + "unclipped_grad_norm": 1.146715760231018 + }, + { + "bits_per_byte": 1.4920311879940247, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.034197211265564, + "step": 6320, + "unclipped_grad_norm": 1.1094690561294556 + }, + { + "bits_per_byte": 1.5741831727275677, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.091140627861023, + "step": 6330, + "unclipped_grad_norm": 1.2368261814117432 + }, + { + "bits_per_byte": 1.5350222071781536, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0639963150024414, + "step": 6340, + "unclipped_grad_norm": 1.105275273323059 + }, + { + "bits_per_byte": 1.6462146663884378, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1410690546035767, + "step": 6350, + "unclipped_grad_norm": 1.3086775541305542 + }, + { + "bits_per_byte": 1.52795922367252, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.05910062789917, + "step": 6360, + "unclipped_grad_norm": 1.1954522132873535 + }, + { + "bits_per_byte": 1.5079418189732958, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0452256202697754, + "step": 6370, + "unclipped_grad_norm": 1.1107361316680908 + }, + { + "bits_per_byte": 1.5318044117805105, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0617659091949463, + "step": 6380, + "unclipped_grad_norm": 1.0837970972061157 + }, + { + "bits_per_byte": 1.5828646849604715, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0971581935882568, + "step": 6390, + "unclipped_grad_norm": 1.1240276098251343 + }, + { + "bits_per_byte": 1.5799186221511916, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.095116138458252, + "step": 6400, + "unclipped_grad_norm": 1.1799321174621582 + }, + { + "bits_per_byte": 1.552409309215082, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0760481357574463, + "step": 6410, + "unclipped_grad_norm": 1.1518590450286865 + }, + { + "bits_per_byte": 1.5442817531006072, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0704145431518555, + "step": 6420, + "unclipped_grad_norm": 1.1955360174179077 + }, + { + "bits_per_byte": 1.5805626971786728, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0955625772476196, + "step": 6430, + "unclipped_grad_norm": 1.156053066253662 + }, + { + "bits_per_byte": 1.5364046037257888, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0649545192718506, + "step": 6440, + "unclipped_grad_norm": 1.1582752466201782 + }, + { + "bits_per_byte": 1.5944274225432797, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.105172872543335, + "step": 6450, + "unclipped_grad_norm": 1.2264126539230347 + }, + { + "bits_per_byte": 1.645741886081216, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.1407413482666016, + "step": 6460, + "unclipped_grad_norm": 1.2444238662719727 + }, + { + "bits_per_byte": 1.6345733327341851, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.1329998970031738, + "step": 6470, + "unclipped_grad_norm": 1.2292388677597046 + }, + { + "bits_per_byte": 1.5111716531564994, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.047464370727539, + "step": 6480, + "unclipped_grad_norm": 1.2222721576690674 + }, + { + "bits_per_byte": 1.6197842006211907, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.122748851776123, + "step": 6490, + "unclipped_grad_norm": 1.2203607559204102 + }, + { + "bits_per_byte": 1.5564085937782457, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0788202285766602, + "step": 6500, + "unclipped_grad_norm": 1.1637120246887207 + }, + { + "bits_per_byte": 1.5288421826020495, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0597126483917236, + "step": 6510, + "unclipped_grad_norm": 1.1861555576324463 + }, + { + "bits_per_byte": 1.5068566084463515, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.04447340965271, + "step": 6520, + "unclipped_grad_norm": 1.1399540901184082 + }, + { + "bits_per_byte": 1.5271775625243484, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0585588216781616, + "step": 6530, + "unclipped_grad_norm": 1.200762152671814 + }, + { + "bits_per_byte": 1.5654438743239742, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0850830078125, + "step": 6540, + "unclipped_grad_norm": 1.1505160331726074 + }, + { + "bits_per_byte": 1.505103417303456, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0432581901550293, + "step": 6550, + "unclipped_grad_norm": 1.1259007453918457 + }, + { + "bits_per_byte": 1.5822820077393482, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0967543125152588, + "step": 6560, + "unclipped_grad_norm": 1.1489042043685913 + }, + { + "bits_per_byte": 1.5307443107205922, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0610311031341553, + "step": 6570, + "unclipped_grad_norm": 1.2110720872879028 + }, + { + "bits_per_byte": 1.5820749406277095, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0966107845306396, + "step": 6580, + "unclipped_grad_norm": 1.2832127809524536 + }, + { + "bits_per_byte": 1.555234640203456, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0780065059661865, + "step": 6590, + "unclipped_grad_norm": 1.1875557899475098 + }, + { + "bits_per_byte": 1.4893780116391642, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.032358169555664, + "step": 6600, + "unclipped_grad_norm": 1.1434928178787231 + }, + { + "bits_per_byte": 1.4980550523231455, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0383726358413696, + "step": 6610, + "unclipped_grad_norm": 1.1379001140594482 + }, + { + "bits_per_byte": 1.4935792038344409, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0352702140808105, + "step": 6620, + "unclipped_grad_norm": 1.1951154470443726 + }, + { + "bits_per_byte": 1.5329076804857955, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0625306367874146, + "step": 6630, + "unclipped_grad_norm": 1.144928216934204 + }, + { + "bits_per_byte": 1.5874349519244997, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.1003260612487793, + "step": 6640, + "unclipped_grad_norm": 1.2067164182662964 + }, + { + "bits_per_byte": 1.5760562357781147, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0924389362335205, + "step": 6650, + "unclipped_grad_norm": 1.1507362127304077 + }, + { + "bits_per_byte": 1.5807757836830918, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.095710277557373, + "step": 6660, + "unclipped_grad_norm": 1.134096622467041 + }, + { + "bits_per_byte": 1.5557022610311522, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.078330636024475, + "step": 6670, + "unclipped_grad_norm": 1.2659887075424194 + }, + { + "bits_per_byte": 1.5265064862206816, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0580936670303345, + "step": 6680, + "unclipped_grad_norm": 1.196817398071289 + }, + { + "bits_per_byte": 1.558342366704547, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0801606178283691, + "step": 6690, + "unclipped_grad_norm": 1.191389799118042 + }, + { + "bits_per_byte": 1.6202516494662358, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.123072862625122, + "step": 6700, + "unclipped_grad_norm": 1.2432118654251099 + }, + { + "bits_per_byte": 1.5747254340257382, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0915164947509766, + "step": 6710, + "unclipped_grad_norm": 1.1514227390289307 + }, + { + "bits_per_byte": 1.5896156919374402, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1018376350402832, + "step": 6720, + "unclipped_grad_norm": 1.1839321851730347 + }, + { + "bits_per_byte": 1.5521088555240246, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.075839877128601, + "step": 6730, + "unclipped_grad_norm": 1.2353639602661133 + }, + { + "bits_per_byte": 1.4976266435398462, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0380756855010986, + "step": 6740, + "unclipped_grad_norm": 1.1758193969726562 + }, + { + "bits_per_byte": 1.5596673210467942, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0810790061950684, + "step": 6750, + "unclipped_grad_norm": 1.1156134605407715 + }, + { + "bits_per_byte": 1.536689063030316, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0651516914367676, + "step": 6760, + "unclipped_grad_norm": 1.2122220993041992 + }, + { + "bits_per_byte": 1.443431470599958, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0005104541778564, + "step": 6770, + "unclipped_grad_norm": 1.1533452272415161 + }, + { + "bits_per_byte": 1.5402759331967106, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0676379203796387, + "step": 6780, + "unclipped_grad_norm": 1.1691333055496216 + }, + { + "bits_per_byte": 1.523325667092974, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0558888912200928, + "step": 6790, + "unclipped_grad_norm": 1.2405608892440796 + }, + { + "bits_per_byte": 1.57861671348416, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0942137241363525, + "step": 6800, + "unclipped_grad_norm": 1.2281029224395752 + }, + { + "bits_per_byte": 1.5738859867068768, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0909346342086792, + "step": 6810, + "unclipped_grad_norm": 1.2371455430984497 + }, + { + "bits_per_byte": 1.582858837550342, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.097154140472412, + "step": 6820, + "unclipped_grad_norm": 1.3370100259780884 + }, + { + "bits_per_byte": 1.5273318309621724, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0586657524108887, + "step": 6830, + "unclipped_grad_norm": 1.196358561515808 + }, + { + "bits_per_byte": 1.4954078953610652, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.036537766456604, + "step": 6840, + "unclipped_grad_norm": 1.1750952005386353 + }, + { + "bits_per_byte": 1.5476113372213112, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0727224349975586, + "step": 6850, + "unclipped_grad_norm": 1.1924985647201538 + }, + { + "bits_per_byte": 1.526412067745358, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.058028221130371, + "step": 6860, + "unclipped_grad_norm": 1.1877104043960571 + }, + { + "bits_per_byte": 1.4832949852781467, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.028141736984253, + "step": 6870, + "unclipped_grad_norm": 1.175843596458435 + }, + { + "bits_per_byte": 1.4908706490660026, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0333927869796753, + "step": 6880, + "unclipped_grad_norm": 1.1246671676635742 + }, + { + "bits_per_byte": 1.5010700801754213, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0404624938964844, + "step": 6890, + "unclipped_grad_norm": 1.1924952268600464 + }, + { + "bits_per_byte": 1.6045640799851337, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.112199068069458, + "step": 6900, + "unclipped_grad_norm": 1.2239227294921875 + }, + { + "bits_per_byte": 1.5736668808096776, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0907827615737915, + "step": 6910, + "unclipped_grad_norm": 1.2664575576782227 + }, + { + "bits_per_byte": 1.5080825007817016, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.045323133468628, + "step": 6920, + "unclipped_grad_norm": 1.1531251668930054 + }, + { + "bits_per_byte": 1.5085007625886, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0456130504608154, + "step": 6930, + "unclipped_grad_norm": 1.2472591400146484 + }, + { + "bits_per_byte": 1.484484417291514, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0289661884307861, + "step": 6940, + "unclipped_grad_norm": 1.187600016593933 + }, + { + "bits_per_byte": 1.4929882714460763, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.034860610961914, + "step": 6950, + "unclipped_grad_norm": 1.228901743888855 + }, + { + "bits_per_byte": 1.5665772399931601, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0858685970306396, + "step": 6960, + "unclipped_grad_norm": 1.205407738685608 + }, + { + "bits_per_byte": 1.5512240047853354, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0752265453338623, + "step": 6970, + "unclipped_grad_norm": 1.2078452110290527 + }, + { + "bits_per_byte": 1.5256931802647515, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0575299263000488, + "step": 6980, + "unclipped_grad_norm": 1.216407060623169 + }, + { + "bits_per_byte": 1.5495781308165784, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0740857124328613, + "step": 6990, + "unclipped_grad_norm": 1.2360340356826782 + }, + { + "bits_per_byte": 1.579722561929208, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.094980239868164, + "step": 7000, + "unclipped_grad_norm": 1.2281304597854614 + }, + { + "bits_per_byte": 1.5775139607268278, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.093449354171753, + "step": 7010, + "unclipped_grad_norm": 1.238048791885376 + }, + { + "bits_per_byte": 1.5315163408403152, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0615662336349487, + "step": 7020, + "unclipped_grad_norm": 1.2856208086013794 + }, + { + "bits_per_byte": 1.5716388613907033, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0893770456314087, + "step": 7030, + "unclipped_grad_norm": 1.228158950805664 + }, + { + "bits_per_byte": 1.5737668027298288, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0908520221710205, + "step": 7040, + "unclipped_grad_norm": 1.227662444114685 + }, + { + "bits_per_byte": 1.5448513596402649, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0708093643188477, + "step": 7050, + "unclipped_grad_norm": 1.2028948068618774 + }, + { + "bits_per_byte": 1.5624737339435735, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.083024263381958, + "step": 7060, + "unclipped_grad_norm": 1.259883999824524 + }, + { + "bits_per_byte": 1.539107139101447, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0668277740478516, + "step": 7070, + "unclipped_grad_norm": 1.2382662296295166 + }, + { + "bits_per_byte": 1.5001651074665814, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0398352146148682, + "step": 7080, + "unclipped_grad_norm": 1.1878225803375244 + }, + { + "bits_per_byte": 1.5461551601164558, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0717130899429321, + "step": 7090, + "unclipped_grad_norm": 1.3277226686477661 + }, + { + "bits_per_byte": 1.525824231044709, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0576207637786865, + "step": 7100, + "unclipped_grad_norm": 1.2164647579193115 + }, + { + "bits_per_byte": 1.5404795606553323, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0677790641784668, + "step": 7110, + "unclipped_grad_norm": 1.1725497245788574 + }, + { + "bits_per_byte": 1.5736082347257334, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0907421112060547, + "step": 7120, + "unclipped_grad_norm": 1.218058705329895 + }, + { + "bits_per_byte": 1.5565896915096042, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0789457559585571, + "step": 7130, + "unclipped_grad_norm": 1.3638907670974731 + }, + { + "bits_per_byte": 1.5273908210114182, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0587066411972046, + "step": 7140, + "unclipped_grad_norm": 1.2031904458999634 + }, + { + "bits_per_byte": 1.5130964829849558, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0487985610961914, + "step": 7150, + "unclipped_grad_norm": 1.1664750576019287 + }, + { + "bits_per_byte": 1.5860308855628558, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0993528366088867, + "step": 7160, + "unclipped_grad_norm": 1.2491129636764526 + }, + { + "bits_per_byte": 1.5206325907631133, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.054022192955017, + "step": 7170, + "unclipped_grad_norm": 1.20036780834198 + }, + { + "bits_per_byte": 1.5424085180674096, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0691161155700684, + "step": 7180, + "unclipped_grad_norm": 1.2460989952087402 + }, + { + "bits_per_byte": 1.5104255924170564, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0469472408294678, + "step": 7190, + "unclipped_grad_norm": 1.2477872371673584 + }, + { + "bits_per_byte": 1.550646659026389, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0748263597488403, + "step": 7200, + "unclipped_grad_norm": 1.26192045211792 + }, + { + "bits_per_byte": 1.5958559104413463, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.1061630249023438, + "step": 7210, + "unclipped_grad_norm": 1.2875406742095947 + }, + { + "bits_per_byte": 1.5889428958072649, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.1013712882995605, + "step": 7220, + "unclipped_grad_norm": 1.2469474077224731 + }, + { + "bits_per_byte": 1.512987102019007, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.048722743988037, + "step": 7230, + "unclipped_grad_norm": 1.2010514736175537 + }, + { + "bits_per_byte": 1.5378745394427134, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0659734010696411, + "step": 7240, + "unclipped_grad_norm": 1.201562762260437 + }, + { + "bits_per_byte": 1.5587365509403246, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0804338455200195, + "step": 7250, + "unclipped_grad_norm": 1.305864691734314 + }, + { + "bits_per_byte": 1.457526824699372, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0102806091308594, + "step": 7260, + "unclipped_grad_norm": 1.1730221509933472 + }, + { + "bits_per_byte": 1.4923226985872373, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0343992710113525, + "step": 7270, + "unclipped_grad_norm": 1.222469687461853 + }, + { + "bits_per_byte": 1.56038208094378, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0815744400024414, + "step": 7280, + "unclipped_grad_norm": 1.2304381132125854 + }, + { + "bits_per_byte": 1.4970130094415677, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0376503467559814, + "step": 7290, + "unclipped_grad_norm": 1.1906161308288574 + }, + { + "bits_per_byte": 1.5193676583660174, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.053145408630371, + "step": 7300, + "unclipped_grad_norm": 1.2887614965438843 + }, + { + "bits_per_byte": 1.5227811700203424, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.055511474609375, + "step": 7310, + "unclipped_grad_norm": 1.1584672927856445 + }, + { + "bits_per_byte": 1.4581731355013146, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0107285976409912, + "step": 7320, + "unclipped_grad_norm": 1.2041478157043457 + }, + { + "bits_per_byte": 1.4976147767369368, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0380674600601196, + "step": 7330, + "unclipped_grad_norm": 1.2121145725250244 + }, + { + "bits_per_byte": 1.456979231939025, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0099010467529297, + "step": 7340, + "unclipped_grad_norm": 1.236539363861084 + }, + { + "bits_per_byte": 1.5469846324415673, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0722880363464355, + "step": 7350, + "unclipped_grad_norm": 1.240252137184143 + }, + { + "bits_per_byte": 1.4978240796230369, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.038212537765503, + "step": 7360, + "unclipped_grad_norm": 1.2412164211273193 + }, + { + "bits_per_byte": 1.5397654886889498, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.067284107208252, + "step": 7370, + "unclipped_grad_norm": 1.2414754629135132 + }, + { + "bits_per_byte": 1.4664532402271044, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0164679288864136, + "step": 7380, + "unclipped_grad_norm": 1.196345567703247 + }, + { + "bits_per_byte": 1.5561741814251195, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0786577463150024, + "step": 7390, + "unclipped_grad_norm": 1.2357971668243408 + }, + { + "bits_per_byte": 1.4878109057245024, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0312719345092773, + "step": 7400, + "unclipped_grad_norm": 1.2395251989364624 + }, + { + "bits_per_byte": 1.4828184213526059, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0278114080429077, + "step": 7410, + "unclipped_grad_norm": 1.1855907440185547 + }, + { + "bits_per_byte": 1.5432890692398271, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0697264671325684, + "step": 7420, + "unclipped_grad_norm": 1.2337415218353271 + }, + { + "bits_per_byte": 1.5564147851536767, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.078824520111084, + "step": 7430, + "unclipped_grad_norm": 1.271388053894043 + }, + { + "bits_per_byte": 1.5798298791033463, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0950546264648438, + "step": 7440, + "unclipped_grad_norm": 1.3756663799285889 + }, + { + "bits_per_byte": 1.5198117175705452, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0534532070159912, + "step": 7450, + "unclipped_grad_norm": 1.2638351917266846 + }, + { + "bits_per_byte": 1.5153886677556554, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0503873825073242, + "step": 7460, + "unclipped_grad_norm": 1.2390333414077759 + }, + { + "bits_per_byte": 1.500396252149341, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0399954319000244, + "step": 7470, + "unclipped_grad_norm": 1.1914620399475098 + }, + { + "bits_per_byte": 1.5462138062004, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.071753740310669, + "step": 7480, + "unclipped_grad_norm": 1.2698674201965332 + }, + { + "bits_per_byte": 1.5915580639962847, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.1031839847564697, + "step": 7490, + "unclipped_grad_norm": 1.3026750087738037 + }, + { + "bits_per_byte": 1.5533068866699349, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0766702890396118, + "step": 7500, + "unclipped_grad_norm": 1.1910096406936646 + }, + { + "bits_per_byte": 1.5004994417398587, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0400669574737549, + "step": 7510, + "unclipped_grad_norm": 1.2149840593338013 + }, + { + "bits_per_byte": 1.5185469571561, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0525765419006348, + "step": 7520, + "unclipped_grad_norm": 1.25714111328125 + }, + { + "bits_per_byte": 1.5776331447038756, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0935319662094116, + "step": 7530, + "unclipped_grad_norm": 1.2656311988830566 + }, + { + "bits_per_byte": 1.5129728274589855, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0487128496170044, + "step": 7540, + "unclipped_grad_norm": 1.2008068561553955 + }, + { + "bits_per_byte": 1.542592195538531, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0692434310913086, + "step": 7550, + "unclipped_grad_norm": 1.3036167621612549 + }, + { + "bits_per_byte": 1.5036082001368547, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0422217845916748, + "step": 7560, + "unclipped_grad_norm": 1.2302430868148804 + }, + { + "bits_per_byte": 1.4793622580008667, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0254157781600952, + "step": 7570, + "unclipped_grad_norm": 1.2421420812606812 + }, + { + "bits_per_byte": 1.6626436530774094, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1524567604064941, + "step": 7580, + "unclipped_grad_norm": 1.2984310388565063 + }, + { + "bits_per_byte": 1.510733441362101, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0471606254577637, + "step": 7590, + "unclipped_grad_norm": 1.2202924489974976 + }, + { + "bits_per_byte": 1.5062574208907455, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.044058084487915, + "step": 7600, + "unclipped_grad_norm": 1.2830675840377808 + }, + { + "bits_per_byte": 1.5167008953817387, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0512969493865967, + "step": 7610, + "unclipped_grad_norm": 1.2556214332580566 + }, + { + "bits_per_byte": 1.4693692060724834, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.018489122390747, + "step": 7620, + "unclipped_grad_norm": 1.2434303760528564 + }, + { + "bits_per_byte": 1.482038652013594, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0272709131240845, + "step": 7630, + "unclipped_grad_norm": 1.2516939640045166 + }, + { + "bits_per_byte": 1.4886369383965965, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0318444967269897, + "step": 7640, + "unclipped_grad_norm": 1.2500675916671753 + }, + { + "bits_per_byte": 1.61461749782397, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1191675662994385, + "step": 7650, + "unclipped_grad_norm": 1.2970722913742065 + }, + { + "bits_per_byte": 1.5327549598918293, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0624247789382935, + "step": 7660, + "unclipped_grad_norm": 1.313043236732483 + }, + { + "bits_per_byte": 1.560077327686451, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0813632011413574, + "step": 7670, + "unclipped_grad_norm": 1.2718157768249512 + }, + { + "bits_per_byte": 1.506822211916179, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0444495677947998, + "step": 7680, + "unclipped_grad_norm": 1.2939503192901611 + }, + { + "bits_per_byte": 1.5160353225228995, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0508356094360352, + "step": 7690, + "unclipped_grad_norm": 1.2968342304229736 + }, + { + "bits_per_byte": 1.4732078588397413, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0211498737335205, + "step": 7700, + "unclipped_grad_norm": 1.2705638408660889 + }, + { + "bits_per_byte": 1.539139815805111, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0668504238128662, + "step": 7710, + "unclipped_grad_norm": 1.2731654644012451 + }, + { + "bits_per_byte": 1.5645254869683667, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0844464302062988, + "step": 7720, + "unclipped_grad_norm": 1.281749963760376 + }, + { + "bits_per_byte": 1.5594863952980864, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.080953598022461, + "step": 7730, + "unclipped_grad_norm": 1.2457833290100098 + }, + { + "bits_per_byte": 1.5044648457208023, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0428155660629272, + "step": 7740, + "unclipped_grad_norm": 1.313395619392395 + }, + { + "bits_per_byte": 1.5058419108062608, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0437700748443604, + "step": 7750, + "unclipped_grad_norm": 1.252753496170044 + }, + { + "bits_per_byte": 1.5365542286320395, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0650582313537598, + "step": 7760, + "unclipped_grad_norm": 1.208566665649414 + }, + { + "bits_per_byte": 1.4627389309164203, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0138933658599854, + "step": 7770, + "unclipped_grad_norm": 1.2469086647033691 + }, + { + "bits_per_byte": 1.539320397588517, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0669755935668945, + "step": 7780, + "unclipped_grad_norm": 1.2808659076690674 + }, + { + "bits_per_byte": 1.4840185162903266, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.028643250465393, + "step": 7790, + "unclipped_grad_norm": 1.1823458671569824 + }, + { + "bits_per_byte": 1.5017442521668036, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0409297943115234, + "step": 7800, + "unclipped_grad_norm": 1.2036527395248413 + }, + { + "bits_per_byte": 1.4928728710873473, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0347806215286255, + "step": 7810, + "unclipped_grad_norm": 1.2650182247161865 + }, + { + "bits_per_byte": 1.4653472197994057, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0157012939453125, + "step": 7820, + "unclipped_grad_norm": 1.2730145454406738 + }, + { + "bits_per_byte": 1.5153749091435864, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0503778457641602, + "step": 7830, + "unclipped_grad_norm": 1.3102401494979858 + }, + { + "bits_per_byte": 1.5122207473267624, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0481915473937988, + "step": 7840, + "unclipped_grad_norm": 1.2572261095046997 + }, + { + "bits_per_byte": 1.458128591994741, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0106977224349976, + "step": 7850, + "unclipped_grad_norm": 1.253402829170227 + }, + { + "bits_per_byte": 1.5241797329371587, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.056480884552002, + "step": 7860, + "unclipped_grad_norm": 1.2006174325942993 + }, + { + "bits_per_byte": 1.539945898489705, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0674091577529907, + "step": 7870, + "unclipped_grad_norm": 1.2778089046478271 + }, + { + "bits_per_byte": 1.453407496245906, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.007425308227539, + "step": 7880, + "unclipped_grad_norm": 1.2744390964508057 + }, + { + "bits_per_byte": 1.5524588402185304, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.076082468032837, + "step": 7890, + "unclipped_grad_norm": 1.2452044486999512 + }, + { + "bits_per_byte": 1.5187442212566398, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0527132749557495, + "step": 7900, + "unclipped_grad_norm": 1.270271897315979 + }, + { + "bits_per_byte": 1.5143311464355, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0496543645858765, + "step": 7910, + "unclipped_grad_norm": 1.2356430292129517 + }, + { + "bits_per_byte": 1.5350039770171622, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.063983678817749, + "step": 7920, + "unclipped_grad_norm": 1.2489451169967651 + }, + { + "bits_per_byte": 1.4882054339255817, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0315454006195068, + "step": 7930, + "unclipped_grad_norm": 1.2472636699676514 + }, + { + "bits_per_byte": 1.4964919020094534, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0372891426086426, + "step": 7940, + "unclipped_grad_norm": 1.3003731966018677 + }, + { + "bits_per_byte": 1.5605986071012163, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0817245244979858, + "step": 7950, + "unclipped_grad_norm": 1.2693594694137573 + }, + { + "bits_per_byte": 1.4811777068633747, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0266741514205933, + "step": 7960, + "unclipped_grad_norm": 1.3084068298339844 + }, + { + "bits_per_byte": 1.5064964767754447, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0442237854003906, + "step": 7970, + "unclipped_grad_norm": 1.2679471969604492 + }, + { + "bits_per_byte": 1.5642893547887322, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0842827558517456, + "step": 7980, + "unclipped_grad_norm": 1.3388662338256836 + }, + { + "bits_per_byte": 1.598760697414419, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.1081764698028564, + "step": 7990, + "unclipped_grad_norm": 1.3723206520080566 + }, + { + "bits_per_byte": 1.4810289418703784, + "learning_rate": 3e-05, + "loss_nats": 1.0265710353851318, + "step": 8000, + "unclipped_grad_norm": 1.1791528463363647 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073002.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073002.json new file mode 100644 index 0000000..fe09b46 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073002.json @@ -0,0 +1,9616 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "6db370af4c7fc867e59305adb946f528981bb02fea76beb10dfc1fa42360d383", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00026696090935729444, + 0.00023596938990522176, + 0.00020828393462579697, + 0.00018593396816868335, + 0.00016940363275352865, + 0.00015857137623243034, + 0.00015041495498735458, + 0.00014485203428193927, + 0.00013822126493323594, + 0.00013424422650132328, + 0.00013012201816309243, + 0.00012678767961915582, + 0.00012444572348613292, + 0.00012238229101058096, + 0.00012059078289894387, + 0.00011903457198059186 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022428705051424913, + "first_to_last_ratio": 1.84426351740015, + "imbalance_abs_log_ratio": 0.6120800202390253, + "last_quartile_mean": 0.0001216133423440624, + "mean": 0.00015851367243158165, + "normalized": [ + 1.6841506808980231, + 1.488637454961949, + 1.3139808789408836, + 1.1729837894515815, + 1.0687004480742655, + 1.0003640304332333, + 0.9489083981211609, + 0.9138141338846396, + 0.8719832353445448, + 0.8468936744826624, + 0.8208882941580717, + 0.7998532724291052, + 0.785078798422557, + 0.772061419896786, + 0.7607595045215659, + 0.7509419859789702 + ], + "population_cv": 0.27150931413441437 + }, + "activation_output_rms_by_block": [ + 0.0048858714289963245, + 0.006843581795692444, + 0.004983869381248951, + 0.007334618363529444, + 0.005787155125290155, + 0.007929109036922455, + 0.005876626819372177, + 0.008892860263586044, + 0.006918553728610277, + 0.009880779311060905, + 0.007473294623196125, + 0.010543949902057648, + 0.008116732351481915, + 0.011967957951128483, + 0.008228716440498829, + 0.012017112225294113 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006011985242366791, + "first_to_last_ratio": 0.5962715478148781, + "imbalance_abs_log_ratio": 0.5170590985397028, + "last_quartile_mean": 0.010082629742100835, + "mean": 0.007980049296747893, + "normalized": [ + 0.612260807835794, + 0.8575864059488212, + 0.6245411771177937, + 0.9191194303171183, + 0.725202929216063, + 0.9936165482278163, + 0.736414851693595, + 1.114386632575083, + 0.866981326973731, + 1.2381852471873345, + 0.9364973003665171, + 1.321288817896729, + 1.017128096538166, + 1.499734839483483, + 1.031161103710509, + 1.5058944849114457 + ], + "population_cv": 0.2698221312597705 + }, + "bits_per_byte": 8.029409055280222, + "branch_output_rms_by_sublayer": [ + 0.0029694242402911186, + 0.0038970725145190954, + 0.003062367672100663, + 0.0038208949845284224, + 0.003184171626344323, + 0.0038178148679435253, + 0.003822639584541321, + 0.0037665588315576315, + 0.004263176582753658, + 0.0038892142474651337, + 0.004122398793697357, + 0.003805827582255006, + 0.004464023746550083, + 0.0037649995647370815, + 0.005492920521646738, + 0.0037115272134542465, + 0.005817505065351725, + 0.003727257251739502, + 0.006086217239499092, + 0.00368275074288249, + 0.0065266890451312065, + 0.0036771725863218307, + 0.0064827753230929375, + 0.003755199955776334, + 0.007209060713648796, + 0.0037714343052357435, + 0.007561341859400272, + 0.0037067290395498276, + 0.007512317039072514, + 0.0035454558674246073, + 0.007687372621148825, + 0.0036056456156075 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006887647168374071, + 0.005982369825085796, + 0.005174924477158116, + 0.004946754810206586, + 0.004967935165757302, + 0.004123664448940147, + 0.00409438714185196, + 0.004247982317789705, + 0.0044010147653324285, + 0.004095379628913903, + 0.00386485833459634, + 0.0038097011691932423, + 0.0037333099925226194, + 0.004058250822051008, + 0.003851730608686184, + 0.00381805442027398 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005747924070206143, + "first_to_last_ratio": 1.487043658003427, + "imbalance_abs_log_ratio": 0.39679002683413417, + "last_quartile_mean": 0.003865336460883448, + "mean": 0.004503622818545837, + "normalized": [ + 1.5293570189783356, + 1.3283461040410636, + 1.1490581440008412, + 1.0983945613375836, + 1.10309752080024, + 0.9156327283801325, + 0.909131893770355, + 0.9432366983079914, + 0.9772165526849029, + 0.9093522690331003, + 0.858166522889289, + 0.8459192349556811, + 0.8289570736583262, + 0.9011080602352411, + 0.8552515971863425, + 0.8477740197405745 + ], + "population_cv": 0.191198476656264 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02822229079902172, + 0.014168336987495422, + 0.01429277565330267, + 0.014353902079164982, + 0.014476437121629715, + 0.009725152514874935, + 0.00981744285672903, + 0.009922475554049015, + 0.010004729963839054, + 0.0075368694961071014, + 0.007599710486829281, + 0.007675632368773222, + 0.007730280980467796, + 0.006251458544284105, + 0.0062932223081588745, + 0.006393968593329191, + 0.006436701864004135, + 0.005461040884256363, + 0.005505258683115244, + 0.005599391181021929, + 0.005641042720526457, + 0.004922671243548393, + 0.004954405128955841, + 0.005046301055699587, + 0.005088083446025848, + 0.00459245452657342, + 0.0046256110072135925, + 0.0047518787905573845, + 0.004772109445184469, + 0.004340849816799164, + 0.004358765669167042, + 0.004422077443450689 + ], + "loss_nats": 5.5655622482299805, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0029694242402911186, + 0.0048858714289963245, + 0.005668045952916145, + 0.006843581795692444, + 0.003184171626344323, + 0.004983869381248951, + 0.006258165929466486, + 0.007334618363529444, + 0.004263176582753658, + 0.005787155125290155, + 0.006935684010386467, + 0.007929109036922455, + 0.004464023746550083, + 0.005876626819372177, + 0.008052977733314037, + 0.008892860263586044, + 0.005817505065351725, + 0.006918553728610277, + 0.009139667265117168, + 0.009880779311060905, + 0.0065266890451312065, + 0.007473294623196125, + 0.00979399960488081, + 0.010543949902057648, + 0.007209060713648796, + 0.008116732351481915, + 0.0113961361348629, + 0.011967957951128483, + 0.007512317039072514, + 0.008228716440498829, + 0.011409307830035686, + 0.012017112225294113 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.836213237373158e-05, + 8.102934953058138e-05, + 6.889242649776861e-05, + 6.090942770242691e-05, + 5.060216426500119e-05, + 4.3459844164317474e-05, + 3.709086377057247e-05, + 3.2571624615229666e-05, + 2.8896074582007714e-05, + 2.6415440515847877e-05, + 2.491714622010477e-05, + 2.3614533347426914e-05, + 2.2646127035841346e-05, + 2.2026059014024213e-05, + 2.118862721545156e-05, + 2.081490310956724e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.479833402612712e-05, + "first_to_last_ratio": 3.4518703579034327, + "imbalance_abs_log_ratio": 1.2389162168680456, + "last_quartile_mean": 2.166892909372109e-05, + "mean": 4.083979649749381e-05, + "normalized": [ + 2.1636281262849595, + 1.984078190388482, + 1.6868944609456193, + 1.4914233890995197, + 1.2390405585910906, + 1.0641542782169457, + 0.9082039322318514, + 0.7975462026905171, + 0.7075469777079071, + 0.6468063697983631, + 0.6101192551641105, + 0.578223580249134, + 0.5545112605355675, + 0.5393283244047475, + 0.518823036171393, + 0.5096720575197914 + ], + "population_cv": 0.5343944992100478 + }, + "activation_output_rms_by_block": [ + 0.009077239781618118, + 0.016407418996095657, + 0.010880433954298496, + 0.01978805474936962, + 0.012077964842319489, + 0.02216547541320324, + 0.013358108699321747, + 0.02644582837820053, + 0.017588114365935326, + 0.03464006260037422, + 0.023338599130511284, + 0.0432722382247448, + 0.023765092715620995, + 0.04745635390281677, + 0.02874108962714672, + 0.05685042589902878 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014038286870345473, + "first_to_last_ratio": 0.3580899608898232, + "imbalance_abs_log_ratio": 1.0269710367470868, + "last_quartile_mean": 0.039203240536153316, + "mean": 0.025365781330037862, + "normalized": [ + 0.35785374254838964, + 0.6468327855789793, + 0.4289414078254319, + 0.7801082289523973, + 0.4761518947581916, + 0.8738337339112493, + 0.5266192484086122, + 1.042578899270242, + 0.6933795626909267, + 1.3656217464649458, + 0.9200820123318648, + 1.7059296404760187, + 0.9368957496877358, + 1.8708808250514843, + 1.133065418060348, + 2.2412251039831825 + ], + "population_cv": 0.5292067698975893 + }, + "bits_per_byte": 5.216988460540972, + "branch_output_rms_by_sublayer": [ + 0.004302801564335823, + 0.007946929894387722, + 0.004027083050459623, + 0.008643880486488342, + 0.004623814020305872, + 0.0097366226837039, + 0.005764479748904705, + 0.009610678069293499, + 0.0056845624931156635, + 0.010435172356665134, + 0.006739616859704256, + 0.010610987432301044, + 0.006770467385649681, + 0.010895426385104656, + 0.009306530468165874, + 0.011636263690888882, + 0.010160733014345169, + 0.012227990664541721, + 0.012430583126842976, + 0.012838989496231079, + 0.015689926221966743, + 0.013102410361170769, + 0.015432302840054035, + 0.013170000165700912, + 0.015551909804344177, + 0.012761367484927177, + 0.018755123019218445, + 0.013125414028763771, + 0.019000543281435966, + 0.01451444998383522, + 0.022471774369478226, + 0.013329307548701763 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00035471167919676877, + 0.00030778414793465906, + 0.0002905596100869004, + 0.0002669019974562734, + 0.0002519467909729747, + 0.0002484952213280207, + 0.00022573236044592232, + 0.00021694664140763627, + 0.00020872399951088957, + 0.00018641204799172754, + 0.0001823569017298164, + 0.0001572320679108608, + 0.00014731276753985554, + 0.00014544881518036755, + 0.00014011786536140868, + 0.0001387938590477502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003049893586686504, + "first_to_last_ratio": 2.1340115402633257, + "imbalance_abs_log_ratio": 0.7580035606734332, + "last_quartile_mean": 0.0001429183267823455, + "mean": 0.00021684229831886448, + "normalized": [ + 1.6358048311919693, + 1.4193916515405378, + 1.339958173933553, + 1.230857630294052, + 1.1618895064582344, + 1.1459720878009276, + 1.0409978228232257, + 1.0004811934275772, + 0.9625612766931799, + 0.8596664462466195, + 0.840965545668873, + 0.7250986967480513, + 0.6793543911033149, + 0.6707585019528177, + 0.6461740465200508, + 0.6400681975970168 + ], + "population_cv": 0.294333589312772 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6924017667770386, + "mean_weights": [ + 0.4816230535507202, + 0.5183769464492798 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925060749053955, + "mean_weights": [ + 0.5170555114746094, + 0.4829444885253906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922789812088013, + "mean_weights": [ + 0.5104873776435852, + 0.4895126223564148 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927056312561035, + "mean_weights": [ + 0.5133966207504272, + 0.48660340905189514 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097900390625, + "mean_weights": [ + 0.3491937518119812, + 0.32761698961257935, + 0.32318928837776184 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982853174209595, + "mean_weights": [ + 0.3437661826610565, + 0.3295633792877197, + 0.32667046785354614 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097551703453064, + "mean_weights": [ + 0.3531244695186615, + 0.3277324438095093, + 0.3191430866718292 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098252296447754, + "mean_weights": [ + 0.3423447012901306, + 0.3336608111858368, + 0.323994517326355 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3856292963027954, + "mean_weights": [ + 0.2598890960216522, + 0.25528842210769653, + 0.2465861737728119, + 0.23823627829551697 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862417936325073, + "mean_weights": [ + 0.2525826096534729, + 0.2512264847755432, + 0.24808365106582642, + 0.24810725450515747 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854787349700928, + "mean_weights": [ + 0.2609134912490845, + 0.2559056282043457, + 0.24643474817276, + 0.23674613237380981 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386183738708496, + "mean_weights": [ + 0.25091099739074707, + 0.2541857361793518, + 0.24937650561332703, + 0.24552680552005768 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6086668968200684, + "mean_weights": [ + 0.20840778946876526, + 0.2072581648826599, + 0.20017966628074646, + 0.19401825964450836, + 0.1901361048221588 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093640327453613, + "mean_weights": [ + 0.20082800090312958, + 0.20345230400562286, + 0.19957157969474792, + 0.1978151500225067, + 0.19833296537399292 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085686683654785, + "mean_weights": [ + 0.2064795196056366, + 0.20873931050300598, + 0.20255593955516815, + 0.1941128820180893, + 0.18811233341693878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093556880950928, + "mean_weights": [ + 0.2007957398891449, + 0.20384852588176727, + 0.2003384232521057, + 0.19736894965171814, + 0.19764836132526398 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7904106378555298, + "mean_weights": [ + 0.17261356115341187, + 0.1785532385110855, + 0.16990220546722412, + 0.16298523545265198, + 0.1574374884366989, + 0.1585082709789276 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791741132736206, + "mean_weights": [ + 0.16599833965301514, + 0.16723501682281494, + 0.16713106632232666, + 0.1660757064819336, + 0.16713276505470276, + 0.1664271056652069 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893474102020264, + "mean_weights": [ + 0.17279052734375, + 0.18296070396900177, + 0.1714734584093094, + 0.16615013778209686, + 0.1573297083377838, + 0.14929547905921936 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916802167892456, + "mean_weights": [ + 0.1656593233346939, + 0.16598697006702423, + 0.1664597988128662, + 0.16510574519634247, + 0.16595518589019775, + 0.17083296179771423 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9424470663070679, + "mean_weights": [ + 0.14923666417598724, + 0.16057875752449036, + 0.15159368515014648, + 0.14453238248825073, + 0.13740569353103638, + 0.1286274939775467, + 0.12802532315254211 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9457793235778809, + "mean_weights": [ + 0.1414237767457962, + 0.14120322465896606, + 0.14217954874038696, + 0.14177344739437103, + 0.14084486663341522, + 0.145671084523201, + 0.14690405130386353 + ], + "sources": 7 + }, + { + "entropy_mean": 1.940324306488037, + "mean_weights": [ + 0.15125462412834167, + 0.16445259749889374, + 0.15269356966018677, + 0.1458570957183838, + 0.1393839716911316, + 0.12596309185028076, + 0.12039506435394287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458446502685547, + "mean_weights": [ + 0.1414341926574707, + 0.14160503447055817, + 0.14392146468162537, + 0.1416645050048828, + 0.14250624179840088, + 0.1432597041130066, + 0.1456088423728943 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073972225189209, + "mean_weights": [ + 0.13393926620483398, + 0.1460031270980835, + 0.13552871346473694, + 0.12954440712928772, + 0.12087934464216232, + 0.11472145467996597, + 0.10819212347269058, + 0.11119155585765839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0792579650878906, + "mean_weights": [ + 0.12344250082969666, + 0.1223982647061348, + 0.12281741201877594, + 0.12404577434062958, + 0.1245848536491394, + 0.1274431347846985, + 0.1261020004749298, + 0.12916606664657593 + ], + "sources": 8 + }, + { + "entropy_mean": 2.070988178253174, + "mean_weights": [ + 0.13450230658054352, + 0.1483522206544876, + 0.13837148249149323, + 0.13302284479141235, + 0.12319637835025787, + 0.11490452289581299, + 0.10592599213123322, + 0.10172426700592041 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0791521072387695, + "mean_weights": [ + 0.12388300150632858, + 0.12212474644184113, + 0.12333755195140839, + 0.12223256379365921, + 0.12343418598175049, + 0.12649230659008026, + 0.12949144840240479, + 0.12900421023368835 + ], + "sources": 8 + }, + { + "entropy_mean": 2.18794584274292, + "mean_weights": [ + 0.12259161472320557, + 0.13473162055015564, + 0.1250339150428772, + 0.11940491199493408, + 0.11259052157402039, + 0.10314525663852692, + 0.09605535864830017, + 0.09219000488519669, + 0.09425680339336395 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197138786315918, + "mean_weights": [ + 0.11026664823293686, + 0.11047966778278351, + 0.11075460910797119, + 0.10965213924646378, + 0.10944131761789322, + 0.111859992146492, + 0.11317223310470581, + 0.1125045120716095, + 0.11186887323856354 + ], + "sources": 9 + }, + { + "entropy_mean": 2.189896583557129, + "mean_weights": [ + 0.12157853692770004, + 0.13215823471546173, + 0.1242145225405693, + 0.11826580762863159, + 0.11163709312677383, + 0.1045699343085289, + 0.09787097573280334, + 0.09515326470136642, + 0.09455163776874542 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028616169467568398, + 0.013977974653244019, + 0.015653911978006363, + 0.015701081603765488, + 0.01704590953886509, + 0.011780167929828167, + 0.012960037216544151, + 0.013451050966978073, + 0.014845823869109154, + 0.011557077057659626, + 0.012834185734391212, + 0.013193685561418533, + 0.014761172235012054, + 0.012218841351568699, + 0.013451457023620605, + 0.01399444043636322, + 0.015514986589550972, + 0.013508559204638004, + 0.014893320389091969, + 0.015537911094725132, + 0.017302624881267548, + 0.015652775764465332, + 0.017217284068465233, + 0.017610477283596992, + 0.019684365019202232, + 0.017625300213694572, + 0.019488012418150902, + 0.019592516124248505, + 0.022200310602784157, + 0.019619757309556007, + 0.022100279107689857, + 0.021992051973938942 + ], + "loss_nats": 3.616140842437744, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1926791667938232, + "mean_weights": [ + 0.09771968424320221, + 0.09518586099147797, + 0.10263510048389435, + 0.10697779804468155, + 0.11385782063007355, + 0.11843808740377426, + 0.1225852519273758, + 0.12233239412307739, + 0.12026800215244293 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004302801564335823, + 0.009077239781618118, + 0.010217261500656605, + 0.016407418996095657, + 0.004623814020305872, + 0.010880433954298496, + 0.012616241350769997, + 0.01978805474936962, + 0.0056845624931156635, + 0.012077964842319489, + 0.014198417775332928, + 0.02216547541320324, + 0.006770467385649681, + 0.013358108699321747, + 0.017934804782271385, + 0.02644582837820053, + 0.010160733014345169, + 0.017588114365935326, + 0.02522732876241207, + 0.03464006260037422, + 0.015689926221966743, + 0.023338599130511284, + 0.03383738547563553, + 0.0432722382247448, + 0.015551909804344177, + 0.023765092715620995, + 0.03784027695655823, + 0.04745635390281677, + 0.019000543281435966, + 0.02874108962714672, + 0.04672664403915405, + 0.05685042589902878 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015875036478973925, + 0.00016262078133877367, + 0.00014673685654997826, + 0.00013248690811451524, + 0.00010898245091084391, + 9.717086504679173e-05, + 8.911665645428002e-05, + 7.969579746713862e-05, + 7.858985918574035e-05, + 6.784927973058075e-05, + 6.598868640139699e-05, + 5.512806092156097e-05, + 4.8474703362444416e-05, + 4.3001487938454375e-05, + 3.610895510064438e-05, + 3.5184959415346384e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001501487276982516, + "first_to_last_ratio": 3.689835475493601, + "imbalance_abs_log_ratio": 1.3055818704713245, + "last_quartile_mean": 4.069252645422239e-05, + "mean": 8.786791704551433e-05, + "normalized": [ + 1.806693160912291, + 1.8507412808538344, + 1.6699708094135275, + 1.5077961623454543, + 1.2402985591930278, + 1.1058742293442396, + 1.0142115512777987, + 0.9069954102344013, + 0.8944090383413996, + 0.7721735305895072, + 0.7509986422827776, + 0.6273969245567232, + 0.5516769372982315, + 0.489387815079083, + 0.4109458413807676, + 0.4004301068969357 + ], + "population_cv": 0.4737558240032668 + }, + "activation_output_rms_by_block": [ + 0.02468283288180828, + 0.04329179599881172, + 0.023283371701836586, + 0.03508218377828598, + 0.025414658710360527, + 0.03976430371403694, + 0.02767801284790039, + 0.03781749680638313, + 0.03550800308585167, + 0.04360951855778694, + 0.040389809757471085, + 0.05188606306910515, + 0.03616104647517204, + 0.06982707977294922, + 0.04418032616376877, + 0.09354322403669357 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03158504609018564, + "first_to_last_ratio": 0.5184002104527677, + "imbalance_abs_log_ratio": 0.6570077279781673, + "last_quartile_mean": 0.0609279191121459, + "mean": 0.042007482959888875, + "normalized": [ + 0.5875818102545408, + 1.0305734347413573, + 0.5542672414833535, + 0.8351412964157943, + 0.6050031308618963, + 0.9466004842995747, + 0.6588829155588524, + 0.9002561958423794, + 0.8452780453367492, + 1.038136910022145, + 0.9614908323842579, + 1.2351623904400295, + 0.8608239277202269, + 1.662253361850413, + 1.0517251463496313, + 2.2268228764387987 + ], + "population_cv": 0.4130647858768022 + }, + "bits_per_byte": 3.137961051112634, + "branch_output_rms_by_sublayer": [ + 0.013650949113070965, + 0.01843169331550598, + 0.019044609740376472, + 0.014011893421411514, + 0.018698271363973618, + 0.012105143629014492, + 0.021805183961987495, + 0.010238726623356342, + 0.020464032888412476, + 0.011298555880784988, + 0.021483227610588074, + 0.011180058121681213, + 0.022897671908140182, + 0.01260110642760992, + 0.023534495383501053, + 0.012234007939696312, + 0.028167055919766426, + 0.01504397764801979, + 0.03861295431852341, + 0.015801267698407173, + 0.035523753613233566, + 0.0176701582968235, + 0.020830485969781876, + 0.024693600833415985, + 0.022914253175258636, + 0.028566768392920494, + 0.02199499122798443, + 0.03558572381734848, + 0.013380669057369232, + 0.04152822494506836, + 0.013844908215105534, + 0.04695267602801323 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0008395536207241178, + 0.0006692968987216388, + 0.0005333404576068953, + 0.0004419910350210091, + 0.0004333787114417237, + 0.0004609433225170186, + 0.0004705681902150354, + 0.0005107414594492524, + 0.0005339389183678795, + 0.0004946080122888242, + 0.000448918054550765, + 0.00035668617352665593, + 0.00032572588316331664, + 0.000267052972482006, + 0.0002153120435976381, + 0.0002101978579413297 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006210455030184152, + "first_to_last_ratio": 2.439565392966498, + "imbalance_abs_log_ratio": 0.8918199058035992, + "last_quartile_mean": 0.00025457218929607263, + "mean": 0.0004507658507259441, + "normalized": [ + 1.8625049332642285, + 1.484799475478804, + 1.183187361571351, + 0.980533539329114, + 0.9614275587730993, + 1.022578178392804, + 1.04393043407614, + 1.1330526893878927, + 1.1845150148530281, + 1.0972614972768537, + 0.9959007627303548, + 0.7912892535053935, + 0.7226055004804499, + 0.5924427772244185, + 0.47765828589473863, + 0.46631273776132937 + ], + "population_cv": 0.34661834917809303 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6546831130981445, + "mean_weights": [ + 0.3866303563117981, + 0.6133696436882019 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6832369565963745, + "mean_weights": [ + 0.48925086855888367, + 0.510749101638794 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6867785453796387, + "mean_weights": [ + 0.5347544550895691, + 0.4652456045150757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6725893020629883, + "mean_weights": [ + 0.4844614565372467, + 0.5155385732650757 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0954453945159912, + "mean_weights": [ + 0.332688570022583, + 0.3308390974998474, + 0.3364723324775696 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0907697677612305, + "mean_weights": [ + 0.3209903836250305, + 0.33586689829826355, + 0.34314271807670593 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0933208465576172, + "mean_weights": [ + 0.33219069242477417, + 0.312181293964386, + 0.35562801361083984 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0861705541610718, + "mean_weights": [ + 0.319164514541626, + 0.32808834314346313, + 0.3527471721172333 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3827753067016602, + "mean_weights": [ + 0.24703660607337952, + 0.23925676941871643, + 0.2577625513076782, + 0.25594407320022583 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3791232109069824, + "mean_weights": [ + 0.24712958931922913, + 0.2454403042793274, + 0.24896711111068726, + 0.25846296548843384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3816200494766235, + "mean_weights": [ + 0.2438572347164154, + 0.2418915331363678, + 0.2618292272090912, + 0.252422034740448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765439987182617, + "mean_weights": [ + 0.24315571784973145, + 0.25071609020233154, + 0.23800383508205414, + 0.26812440156936646 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6069204807281494, + "mean_weights": [ + 0.1977769434452057, + 0.19419988989830017, + 0.19346478581428528, + 0.20617546141147614, + 0.20838290452957153 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6016991138458252, + "mean_weights": [ + 0.19928976893424988, + 0.19587931036949158, + 0.19414740800857544, + 0.18995173275470734, + 0.22073179483413696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6063685417175293, + "mean_weights": [ + 0.19720253348350525, + 0.19060829281806946, + 0.18910765647888184, + 0.20065464079380035, + 0.2224268764257431 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6036955118179321, + "mean_weights": [ + 0.19818633794784546, + 0.20014071464538574, + 0.19487890601158142, + 0.18811310827732086, + 0.2186809629201889 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7846078872680664, + "mean_weights": [ + 0.15616348385810852, + 0.1693512499332428, + 0.1684132218360901, + 0.14385344088077545, + 0.16338428854942322, + 0.19883427023887634 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7881443500518799, + "mean_weights": [ + 0.16932255029678345, + 0.1670200079679489, + 0.16782812774181366, + 0.15219610929489136, + 0.1653819978237152, + 0.1782512068748474 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7859947681427002, + "mean_weights": [ + 0.15620459616184235, + 0.18192027509212494, + 0.16651901602745056, + 0.14659695327281952, + 0.16306468844413757, + 0.18569447100162506 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7886135578155518, + "mean_weights": [ + 0.17137612402439117, + 0.16813044250011444, + 0.16593416035175323, + 0.15192663669586182, + 0.16282179951667786, + 0.17981085181236267 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937737226486206, + "mean_weights": [ + 0.12751945853233337, + 0.15645764768123627, + 0.14123380184173584, + 0.1275305449962616, + 0.13446006178855896, + 0.15007954835891724, + 0.16271892189979553 + ], + "sources": 7 + }, + { + "entropy_mean": 1.943496823310852, + "mean_weights": [ + 0.14288359880447388, + 0.1535882204771042, + 0.14250940084457397, + 0.1380811333656311, + 0.1316976547241211, + 0.13936954736709595, + 0.15187042951583862 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273982048034668, + "mean_weights": [ + 0.12311284244060516, + 0.16783711314201355, + 0.141703262925148, + 0.11705678701400757, + 0.12322613596916199, + 0.14240297675132751, + 0.184660866856575 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937564730644226, + "mean_weights": [ + 0.13963916897773743, + 0.16193091869354248, + 0.14834126830101013, + 0.12877905368804932, + 0.1304946392774582, + 0.13670659065246582, + 0.15410837531089783 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0582637786865234, + "mean_weights": [ + 0.10422320663928986, + 0.14683584868907928, + 0.12450192868709564, + 0.10290674865245819, + 0.1043052226305008, + 0.1136581152677536, + 0.13712583482265472, + 0.1664431095123291 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073244333267212, + "mean_weights": [ + 0.11922207474708557, + 0.13852310180664062, + 0.1310172826051712, + 0.1193084791302681, + 0.1166592389345169, + 0.11741948127746582, + 0.12622521817684174, + 0.13162514567375183 + ], + "sources": 8 + }, + { + "entropy_mean": 2.05503249168396, + "mean_weights": [ + 0.10400840640068054, + 0.1620255410671234, + 0.13802370429039001, + 0.10328710824251175, + 0.10247690975666046, + 0.11085900664329529, + 0.13948379456996918, + 0.13983553647994995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0654377937316895, + "mean_weights": [ + 0.12162524461746216, + 0.14755912125110626, + 0.1359378546476364, + 0.11907999217510223, + 0.10864339768886566, + 0.11126887798309326, + 0.12606029212474823, + 0.12982521951198578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.158114194869995, + "mean_weights": [ + 0.08706305176019669, + 0.1523345708847046, + 0.1240597814321518, + 0.09008736908435822, + 0.08798909187316895, + 0.09299354255199432, + 0.11455917358398438, + 0.09811963140964508, + 0.15279380977153778 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1800241470336914, + "mean_weights": [ + 0.10917439311742783, + 0.13041174411773682, + 0.1256810426712036, + 0.10147396475076675, + 0.10038554668426514, + 0.10393467545509338, + 0.11099927127361298, + 0.11076852679252625, + 0.10717083513736725 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1717891693115234, + "mean_weights": [ + 0.09730195999145508, + 0.15644855797290802, + 0.1323733627796173, + 0.10013202577829361, + 0.09525948762893677, + 0.09674295783042908, + 0.12173393368721008, + 0.10994285345077515, + 0.09006483852863312 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04337252303957939, + 0.018801197409629822, + 0.025790294632315636, + 0.029389970004558563, + 0.031753405928611755, + 0.022941799834370613, + 0.02458134852349758, + 0.02529502473771572, + 0.026913003996014595, + 0.020798955112695694, + 0.022398613393306732, + 0.022611256688833237, + 0.02451939694583416, + 0.01929222047328949, + 0.020593678578734398, + 0.020593887194991112, + 0.021762654185295105, + 0.01771371066570282, + 0.019139636307954788, + 0.0189701896160841, + 0.02020873688161373, + 0.016272971406579018, + 0.017277248203754425, + 0.01622883230447769, + 0.018368225544691086, + 0.014602466486394405, + 0.017397841438651085, + 0.017420755699276924, + 0.02048002928495407, + 0.016635844483971596, + 0.021542873233556747, + 0.021214226260781288 + ], + "loss_nats": 2.1750688552856445, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.173111915588379, + "mean_weights": [ + 0.1217261254787445, + 0.09291160106658936, + 0.09640607982873917, + 0.10597392916679382, + 0.09402626007795334, + 0.09312883019447327, + 0.11353092640638351, + 0.147413432598114, + 0.13488279283046722 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.013650949113070965, + 0.02468283288180828, + 0.0356261320412159, + 0.04329179599881172, + 0.018698271363973618, + 0.023283371701836586, + 0.03009101189672947, + 0.03508218377828598, + 0.020464032888412476, + 0.025414658710360527, + 0.034230802208185196, + 0.03976430371403694, + 0.022897671908140182, + 0.02767801284790039, + 0.031892068684101105, + 0.03781749680638313, + 0.028167055919766426, + 0.03550800308585167, + 0.035372670739889145, + 0.04360951855778694, + 0.035523753613233566, + 0.040389809757471085, + 0.03893519565463066, + 0.05188606306910515, + 0.022914253175258636, + 0.03616104647517204, + 0.04390736669301987, + 0.06982707977294922, + 0.013380669057369232, + 0.04418032616376877, + 0.051131755113601685, + 0.09354322403669357 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.532809104304761e-05, + 7.798708247719333e-05, + 7.679135887883604e-05, + 7.304617611225694e-05, + 7.868888496886939e-05, + 7.320931035792455e-05, + 8.902617992134765e-05, + 7.941434887470677e-05, + 8.246234938269481e-05, + 7.884583465056494e-05, + 0.0001634695363463834, + 0.00012072100798832253, + 0.000141287746373564, + 8.843849354889244e-05, + 5.323962977854535e-05, + 4.3788353650597855e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.578817712783348e-05, + "first_to_last_ratio": 0.9277698246768501, + "imbalance_abs_log_ratio": 0.07497161071478954, + "last_quartile_mean": 8.168855583789991e-05, + "mean": 8.723402402210922e-05, + "normalized": [ + 0.8635173246617158, + 0.8939984524550617, + 0.8802913741474712, + 0.8373587820933674, + 0.9020435071174282, + 0.8392288579897432, + 1.0205442305262016, + 0.9103598024387758, + 0.9453003034893238, + 0.9038426867776004, + 1.8739194732659872, + 1.383875263598136, + 1.6196403634636298, + 1.0138073365327953, + 0.6103080807673381, + 0.5019641606754243 + ], + "population_cv": 0.3374467083690701 + }, + "activation_output_rms_by_block": [ + 0.09266375005245209, + 0.20113667845726013, + 0.14358533918857574, + 0.17041823267936707, + 0.05199930816888809, + 0.07249397784471512, + 0.03721517324447632, + 0.07351206243038177, + 0.05156216770410538, + 0.08937478810548782, + 0.062482453882694244, + 0.09533937275409698, + 0.05472978577017784, + 0.08922836184501648, + 0.08201298862695694, + 0.20069743692874908 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15195100009441376, + "first_to_last_ratio": 1.4245342605399758, + "imbalance_abs_log_ratio": 0.353844925589721, + "last_quartile_mean": 0.10666714329272509, + "mean": 0.09802824235521257, + "normalized": [ + 0.9452760533712127, + 2.0518237767482, + 1.4647344044819621, + 1.7384605557023458, + 0.5304523157771692, + 0.739521347144304, + 0.37963725912400215, + 0.7499069723601224, + 0.5259929839123919, + 0.9117248861979151, + 0.6373923716420872, + 0.9725704599356962, + 0.5583063049509795, + 0.9102311711526044, + 0.8366261258645934, + 2.0473430116344136 + ], + "population_cv": 0.5182803671925184 + }, + "bits_per_byte": 2.0687232408845437, + "branch_output_rms_by_sublayer": [ + 0.04482239484786987, + 0.07626598328351974, + 0.10980217158794403, + 0.05728650838136673, + 0.14111557602882385, + 0.014705370180308819, + 0.04088159278035164, + 0.014925715513527393, + 0.04827577620744705, + 0.015019712969660759, + 0.04055904969573021, + 0.02146836742758751, + 0.028491884469985962, + 0.0222648773342371, + 0.0389801524579525, + 0.027360456064343452, + 0.036527831107378006, + 0.03173159062862396, + 0.03612164035439491, + 0.0401885062456131, + 0.04509544372558594, + 0.040587812662124634, + 0.02791302651166916, + 0.05573032796382904, + 0.02739257924258709, + 0.044091735035181046, + 0.017023129388689995, + 0.05625532940030098, + 0.02216578833758831, + 0.07780560851097107, + 0.0160520039498806, + 0.13296744227409363 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003765937729119843, + 0.0003149609092301355, + 0.00027693435492303714, + 0.0001635333627570268, + 0.000180917699642147, + 0.00017837059454152616, + 0.000200147742296221, + 0.00024611271555124715, + 0.0002957900398296376, + 0.0002986151161329657, + 0.0009593337211615513, + 0.0007552262449127961, + 0.0007605792150238727, + 0.0005098371211048817, + 0.00029447700210242914, + 0.0002817503499702034 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00028300559995554593, + "first_to_last_ratio": 0.6130161476493403, + "imbalance_abs_log_ratio": 0.48936400138742103, + "last_quartile_mean": 0.00046166092205034674, + "mean": 0.00038082374763072894, + "normalized": [ + 0.9888925657996352, + 0.827051651031862, + 0.727198229222749, + 0.4294200762805349, + 0.47506937465878935, + 0.46838096534485474, + 0.5255652872002538, + 0.6462640974530133, + 0.7767111207477916, + 0.7841294509357175, + 2.5191016241239836, + 1.9831385243472572, + 1.997194817171378, + 1.3387744968027897, + 0.7732632324914067, + 0.7398444863879827 + ], + "population_cv": 0.6091669692594756 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.573702335357666, + "mean_weights": [ + 0.7102901935577393, + 0.28970980644226074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6441590189933777, + "mean_weights": [ + 0.564466118812561, + 0.43553388118743896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3999374806880951, + "mean_weights": [ + 0.8575316667556763, + 0.14246836304664612 + ], + "sources": 2 + }, + { + "entropy_mean": 0.676960825920105, + "mean_weights": [ + 0.46661365032196045, + 0.5333863496780396 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9920796155929565, + "mean_weights": [ + 0.5238763093948364, + 0.2048875242471695, + 0.2712361812591553 + ], + "sources": 3 + }, + { + "entropy_mean": 1.092700719833374, + "mean_weights": [ + 0.2899855673313141, + 0.34371256828308105, + 0.36630189418792725 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9979273080825806, + "mean_weights": [ + 0.49421679973602295, + 0.18891796469688416, + 0.3168652057647705 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091429591178894, + "mean_weights": [ + 0.3115330636501312, + 0.3673863708972931, + 0.3210805654525757 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2927134037017822, + "mean_weights": [ + 0.35746461153030396, + 0.11402665078639984, + 0.22846516966819763, + 0.3000435531139374 + ], + "sources": 4 + }, + { + "entropy_mean": 1.378767490386963, + "mean_weights": [ + 0.2549898624420166, + 0.23654252290725708, + 0.21668663620948792, + 0.2917810082435608 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2672696113586426, + "mean_weights": [ + 0.3839718699455261, + 0.12064628303050995, + 0.1715085655450821, + 0.32387328147888184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.375225305557251, + "mean_weights": [ + 0.23853997886180878, + 0.24968335032463074, + 0.2089165896177292, + 0.3028600811958313 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5798927545547485, + "mean_weights": [ + 0.22726407647132874, + 0.12221817672252655, + 0.18549029529094696, + 0.23680329322814941, + 0.22822415828704834 + ], + "sources": 5 + }, + { + "entropy_mean": 1.594498634338379, + "mean_weights": [ + 0.21032124757766724, + 0.19536492228507996, + 0.1600019335746765, + 0.1794220358133316, + 0.2548898458480835 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5424928665161133, + "mean_weights": [ + 0.23003876209259033, + 0.11985675245523453, + 0.1379462629556656, + 0.19240085780620575, + 0.319757342338562 + ], + "sources": 5 + }, + { + "entropy_mean": 1.595255732536316, + "mean_weights": [ + 0.2046268880367279, + 0.19130583107471466, + 0.17092418670654297, + 0.17953911423683167, + 0.253603994846344 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7457678318023682, + "mean_weights": [ + 0.17887650430202484, + 0.11389653384685516, + 0.11488981544971466, + 0.1474636197090149, + 0.20503896474838257, + 0.23983454704284668 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7712774276733398, + "mean_weights": [ + 0.16810789704322815, + 0.16247668862342834, + 0.1383969932794571, + 0.13383737206459045, + 0.1661752164363861, + 0.23100584745407104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.675686001777649, + "mean_weights": [ + 0.16882894933223724, + 0.11588791012763977, + 0.08043210208415985, + 0.1250714659690857, + 0.19613179564476013, + 0.3136477470397949 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7837879657745361, + "mean_weights": [ + 0.17464125156402588, + 0.18035069108009338, + 0.1619444340467453, + 0.15330034494400024, + 0.15676669776439667, + 0.17299658060073853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6526988744735718, + "mean_weights": [ + 0.09642067551612854, + 0.07173497974872589, + 0.05561239272356033, + 0.09171071648597717, + 0.11456114053726196, + 0.12951162457466125, + 0.44044846296310425 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8849880695343018, + "mean_weights": [ + 0.13606244325637817, + 0.13500544428825378, + 0.12342768162488937, + 0.11247340589761734, + 0.11233137547969818, + 0.11334770917892456, + 0.2673519253730774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.644601583480835, + "mean_weights": [ + 0.10291393101215363, + 0.08116044104099274, + 0.05550362914800644, + 0.08702410012483597, + 0.11089185625314713, + 0.11528079956769943, + 0.4472252428531647 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9121408462524414, + "mean_weights": [ + 0.14125728607177734, + 0.1494411677122116, + 0.12786200642585754, + 0.11922307312488556, + 0.1195257231593132, + 0.11599892377853394, + 0.2266918122768402 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7500674724578857, + "mean_weights": [ + 0.07664570212364197, + 0.0604618638753891, + 0.041497331112623215, + 0.06405743956565857, + 0.0716431736946106, + 0.07514288276433945, + 0.270142525434494, + 0.3404091000556946 + ], + "sources": 8 + }, + { + "entropy_mean": 1.952926754951477, + "mean_weights": [ + 0.10562807321548462, + 0.10351885855197906, + 0.07600137591362, + 0.08402621746063232, + 0.08850346505641937, + 0.09127135574817657, + 0.15880608558654785, + 0.2922445833683014 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7527103424072266, + "mean_weights": [ + 0.08054207265377045, + 0.06648321449756622, + 0.04810335487127304, + 0.06408451497554779, + 0.07014857977628708, + 0.07217346131801605, + 0.20459653437137604, + 0.39386826753616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9524379968643188, + "mean_weights": [ + 0.10777557641267776, + 0.09555133432149887, + 0.06806209683418274, + 0.08129536360502243, + 0.0918373167514801, + 0.10075562447309494, + 0.2000727653503418, + 0.25464993715286255 + ], + "sources": 8 + }, + { + "entropy_mean": 2.000408887863159, + "mean_weights": [ + 0.07838036119937897, + 0.07984089851379395, + 0.04941036179661751, + 0.06521518528461456, + 0.06765615195035934, + 0.06713232398033142, + 0.18892306089401245, + 0.23950761556625366, + 0.16393405199050903 + ], + "sources": 9 + }, + { + "entropy_mean": 2.096604347229004, + "mean_weights": [ + 0.09755367040634155, + 0.08867941796779633, + 0.060870569199323654, + 0.06980546563863754, + 0.08883253484964371, + 0.09541568160057068, + 0.17317476868629456, + 0.19020704925060272, + 0.13546085357666016 + ], + "sources": 9 + }, + { + "entropy_mean": 2.027402877807617, + "mean_weights": [ + 0.08629745244979858, + 0.08307671546936035, + 0.05121873319149017, + 0.0663638487458229, + 0.07480055093765259, + 0.08018200099468231, + 0.15043827891349792, + 0.25634968280792236, + 0.1512727439403534 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.055510263890028, + 0.04334520921111107, + 0.05630175396800041, + 0.058260008692741394, + 0.11489111185073853, + 0.08273965120315552, + 0.11550594866275787, + 0.09142354130744934, + 0.12066955119371414, + 0.0711931437253952, + 0.08880246430635452, + 0.06823744624853134, + 0.09300445020198822, + 0.06510575115680695, + 0.07136023044586182, + 0.05894701927900314, + 0.07342072576284409, + 0.05002941936254501, + 0.059311795979738235, + 0.04801420122385025, + 0.06730704009532928, + 0.02817549556493759, + 0.046817973256111145, + 0.03746112436056137, + 0.05246857553720474, + 0.03106791339814663, + 0.03552677109837532, + 0.03383694216609001, + 0.03949129208922386, + 0.03412646800279617, + 0.039247769862413406, + 0.04047730192542076 + ], + "loss_nats": 1.433929681777954, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.913235068321228, + "mean_weights": [ + 0.09498223662376404, + 0.06303347647190094, + 0.06038350611925125, + 0.06479769200086594, + 0.05589054524898529, + 0.06028280779719353, + 0.07675591856241226, + 0.2476816177368164, + 0.27619218826293945 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04482239484786987, + 0.09266375005245209, + 0.17245197296142578, + 0.20113667845726013, + 0.14111557602882385, + 0.14358533918857574, + 0.16786204278469086, + 0.17041823267936707, + 0.04827577620744705, + 0.05199930816888809, + 0.06626670062541962, + 0.07249397784471512, + 0.028491884469985962, + 0.03721517324447632, + 0.05938867852091789, + 0.07351206243038177, + 0.036527831107378006, + 0.05156216770410538, + 0.066385917365551, + 0.08937478810548782, + 0.04509544372558594, + 0.062482453882694244, + 0.07032289355993271, + 0.09533937275409698, + 0.02739257924258709, + 0.05472978577017784, + 0.058526892215013504, + 0.08922836184501648, + 0.02216578833758831, + 0.08201298862695694, + 0.0856289193034172, + 0.20069743692874908 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.492530908668414e-05, + 5.6656335800653324e-05, + 6.325786671368405e-05, + 5.9555983170866966e-05, + 6.814395601395518e-05, + 6.255660991882905e-05, + 7.046976679703221e-05, + 6.272071914281696e-05, + 7.296914554899558e-05, + 6.464503530878574e-05, + 0.00014864247350487858, + 0.00010719253623392433, + 0.0001445752423023805, + 9.124213829636574e-05, + 6.016199040459469e-05, + 4.548503056867048e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.859887369297212e-05, + "first_to_last_ratio": 0.6864419649392262, + "imbalance_abs_log_ratio": 0.37623359494524666, + "last_quartile_mean": 8.536610039300285e-05, + "mean": 7.707500867581984e-05, + "normalized": [ + 0.7126215102706234, + 0.7350804985173837, + 0.8207312305308824, + 0.772701608395031, + 0.8841251812318441, + 0.8116328625007921, + 0.9143011205283218, + 0.8137620769739057, + 0.9467289955931933, + 0.8387288748897194, + 1.9285430654970663, + 1.3907560709435642, + 1.8757732861305145, + 1.18380964029642, + 0.7805641729816484, + 0.5901398047190899 + ], + "population_cv": 0.3863401678553132 + }, + "activation_output_rms_by_block": [ + 0.14486487209796906, + 0.30995267629623413, + 0.2574450671672821, + 0.28713259100914, + 0.05978552997112274, + 0.08792455494403839, + 0.04697949066758156, + 0.09852420538663864, + 0.07244286686182022, + 0.13140366971492767, + 0.08212646842002869, + 0.12985831499099731, + 0.06840891391038895, + 0.11060241609811783, + 0.0924442708492279, + 0.23933660984039307 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24984880164265633, + "first_to_last_ratio": 1.9565592145673032, + "imbalance_abs_log_ratio": 0.6711874277699096, + "last_quartile_mean": 0.12769805267453194, + "mean": 0.13870203238911927, + "normalized": [ + 1.0444322235420485, + 2.234665714390418, + 1.8561016210998063, + 2.0701397525568246, + 0.43103571693454673, + 0.6339096365752733, + 0.3387080193301259, + 0.7103299330916477, + 0.5222913147990983, + 0.9473809968860692, + 0.5921071739571081, + 0.9362394534110972, + 0.4932077254533107, + 0.7974102051210767, + 0.6664954309384717, + 1.725545081913077 + ], + "population_cv": 0.5979328649415788 + }, + "bits_per_byte": 1.9415087059372296, + "branch_output_rms_by_sublayer": [ + 0.06868160516023636, + 0.12181960046291351, + 0.16538235545158386, + 0.0929274708032608, + 0.2537953853607178, + 0.01320737786591053, + 0.04932497814297676, + 0.013867277652025223, + 0.055594079196453094, + 0.0172379482537508, + 0.04591496288776398, + 0.035032741725444794, + 0.03727728873491287, + 0.025992874056100845, + 0.0513935461640358, + 0.04214895889163017, + 0.05297093465924263, + 0.04459983855485916, + 0.05040198564529419, + 0.06986947357654572, + 0.054249413311481476, + 0.05820845440030098, + 0.03775457665324211, + 0.08249213546514511, + 0.03362919017672539, + 0.05606101080775261, + 0.02498369663953781, + 0.07096737623214722, + 0.02770768664777279, + 0.08668208867311478, + 0.022235019132494926, + 0.16932900249958038 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003274625120219605, + 0.0002950920627246982, + 0.0002675536202763891, + 0.00011636112532418193, + 0.00013949143839672265, + 0.0001705676155207084, + 0.00017763939173179324, + 0.00023071980568273914, + 0.00033685005198124334, + 0.0003711494248343089, + 0.0010020275600187679, + 0.0008489747430264115, + 0.0009447094037588545, + 0.0006663788253529681, + 0.0003910019531118643, + 0.00034755869426738895 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00025161733008680743, + "first_to_last_ratio": 0.4283488185904071, + "imbalance_abs_log_ratio": 0.8478174186232621, + "last_quartile_mean": 0.000587412219122769, + "mean": 0.00041459613925193755, + "normalized": [ + 0.789834928547107, + 0.7117578645501561, + 0.6453355324512677, + 0.2806613817826046, + 0.3364513684290676, + 0.4114066663246461, + 0.4284636901161473, + 0.5564928947457894, + 0.8124775416120065, + 0.895207141832, + 2.4168762444984226, + 2.047715023488232, + 2.278625665601732, + 1.607296263190791, + 0.9430911581023291, + 0.8383066347277007 + ], + "population_cv": 0.6738437321539833 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4194117784500122, + "mean_weights": [ + 0.8254866600036621, + 0.1745133101940155 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5809611082077026, + "mean_weights": [ + 0.6678193211555481, + 0.3321806788444519 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31704896688461304, + "mean_weights": [ + 0.8999109268188477, + 0.10008908808231354 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6732906103134155, + "mean_weights": [ + 0.4609490633010864, + 0.5390509366989136 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9927794337272644, + "mean_weights": [ + 0.4832823872566223, + 0.17606672644615173, + 0.34065088629722595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832945108413696, + "mean_weights": [ + 0.2762538492679596, + 0.3156396150588989, + 0.40810656547546387 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9905548095703125, + "mean_weights": [ + 0.4605642259120941, + 0.16346418857574463, + 0.37597161531448364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.09372079372406, + "mean_weights": [ + 0.32242417335510254, + 0.34579548239707947, + 0.3317803740501404 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2746227979660034, + "mean_weights": [ + 0.37224799394607544, + 0.09596331417560577, + 0.2427763193845749, + 0.2890123426914215 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3707606792449951, + "mean_weights": [ + 0.27432072162628174, + 0.2153838574886322, + 0.20746882259845734, + 0.30282658338546753 + ], + "sources": 4 + }, + { + "entropy_mean": 1.198214054107666, + "mean_weights": [ + 0.44253695011138916, + 0.10026887059211731, + 0.13153521716594696, + 0.325658917427063 + ], + "sources": 4 + }, + { + "entropy_mean": 1.372360110282898, + "mean_weights": [ + 0.24982473254203796, + 0.2343549281358719, + 0.2051662802696228, + 0.31065404415130615 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5764262676239014, + "mean_weights": [ + 0.2228686511516571, + 0.11472160369157791, + 0.20744341611862183, + 0.22938773036003113, + 0.22557863593101501 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5863620042800903, + "mean_weights": [ + 0.2179623544216156, + 0.17305074632167816, + 0.15634828805923462, + 0.18247520923614502, + 0.2701634168624878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4903430938720703, + "mean_weights": [ + 0.2547823488712311, + 0.09995707869529724, + 0.10266982018947601, + 0.1966245472431183, + 0.3459661602973938 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5923073291778564, + "mean_weights": [ + 0.22257764637470245, + 0.17832019925117493, + 0.1619993895292282, + 0.1890055239200592, + 0.2480972558259964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.691253662109375, + "mean_weights": [ + 0.18423792719841003, + 0.09657466411590576, + 0.08509969711303711, + 0.15257084369659424, + 0.18859976530075073, + 0.29291707277297974 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7595478296279907, + "mean_weights": [ + 0.1724337935447693, + 0.14752474427223206, + 0.13264453411102295, + 0.13973915576934814, + 0.15476800501346588, + 0.2528897523880005 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5512144565582275, + "mean_weights": [ + 0.16704371571540833, + 0.08849328011274338, + 0.04574323818087578, + 0.12566779553890228, + 0.1718127727508545, + 0.40123921632766724 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7799327373504639, + "mean_weights": [ + 0.18437626957893372, + 0.17012780904769897, + 0.17119532823562622, + 0.15378496050834656, + 0.1488233208656311, + 0.17169234156608582 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5764116048812866, + "mean_weights": [ + 0.09582588821649551, + 0.05901939421892166, + 0.040849827229976654, + 0.09081389009952545, + 0.10021352022886276, + 0.13485172390937805, + 0.4784257411956787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8431057929992676, + "mean_weights": [ + 0.13672879338264465, + 0.12035658955574036, + 0.11983893811702728, + 0.10482897609472275, + 0.10095320641994476, + 0.10664592683315277, + 0.3106476068496704 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5526533126831055, + "mean_weights": [ + 0.10306400060653687, + 0.06465981900691986, + 0.04082629829645157, + 0.08607284724712372, + 0.09878605604171753, + 0.11178258061408997, + 0.4948084056377411 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8949611186981201, + "mean_weights": [ + 0.1451701521873474, + 0.13929708302021027, + 0.1330569088459015, + 0.11840799450874329, + 0.10893036425113678, + 0.10698798298835754, + 0.24814949929714203 + ], + "sources": 7 + }, + { + "entropy_mean": 1.676889419555664, + "mean_weights": [ + 0.07736113667488098, + 0.04877929389476776, + 0.032506294548511505, + 0.06437233090400696, + 0.06250663101673126, + 0.06869478523731232, + 0.2701185345649719, + 0.3756609857082367 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8798964023590088, + "mean_weights": [ + 0.10042576491832733, + 0.08721707761287689, + 0.06748545169830322, + 0.0845440924167633, + 0.07678259164094925, + 0.08073122054338455, + 0.1554347574710846, + 0.34737905859947205 + ], + "sources": 8 + }, + { + "entropy_mean": 1.663856029510498, + "mean_weights": [ + 0.07920834422111511, + 0.05299394577741623, + 0.03748705983161926, + 0.06209969520568848, + 0.060945168137550354, + 0.06926221400499344, + 0.19488413631916046, + 0.44311946630477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9090958833694458, + "mean_weights": [ + 0.1065661758184433, + 0.0803120881319046, + 0.06028372794389725, + 0.08741366118192673, + 0.08287163078784943, + 0.09473372995853424, + 0.1887454390525818, + 0.29907354712486267 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8632590770721436, + "mean_weights": [ + 0.0691547840833664, + 0.05765335261821747, + 0.03405343368649483, + 0.05519368499517441, + 0.05223262310028076, + 0.057386673986911774, + 0.1615200936794281, + 0.25146448612213135, + 0.2613408863544464 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0701708793640137, + "mean_weights": [ + 0.09429951012134552, + 0.07856893539428711, + 0.04940951615571976, + 0.076503224670887, + 0.08159374445676804, + 0.0895766168832779, + 0.16711056232452393, + 0.2026878148317337, + 0.16025006771087646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9422186613082886, + "mean_weights": [ + 0.07884211093187332, + 0.06442099809646606, + 0.037228696048259735, + 0.06615173816680908, + 0.06501315534114838, + 0.0695233941078186, + 0.13319489359855652, + 0.2805612087249756, + 0.20506379008293152 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05865398049354553, + 0.05170911177992821, + 0.06799201667308807, + 0.06424286961555481, + 0.17415167391300201, + 0.13562268018722534, + 0.1871119737625122, + 0.1505064219236374, + 0.18585707247257233, + 0.1044568344950676, + 0.12573961913585663, + 0.08286763727664948, + 0.13412274420261383, + 0.0987003743648529, + 0.09964247792959213, + 0.07276491075754166, + 0.1041010394692421, + 0.06199917569756508, + 0.0827898308634758, + 0.06073001027107239, + 0.10040723532438278, + 0.036688607186079025, + 0.0660671666264534, + 0.05293381214141846, + 0.07866564393043518, + 0.04279600828886032, + 0.04821427911520004, + 0.04622510448098183, + 0.053889211267232895, + 0.04172775521874428, + 0.05102568119764328, + 0.053032174706459045 + ], + "loss_nats": 1.3457512855529785, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7949464321136475, + "mean_weights": [ + 0.0834353119134903, + 0.05303390324115753, + 0.048570070415735245, + 0.05988207459449768, + 0.04784438759088516, + 0.05165069177746773, + 0.06509792059659958, + 0.2334408313035965, + 0.3570447862148285 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06868160516023636, + 0.14486487209796906, + 0.2633455693721771, + 0.30995267629623413, + 0.2537953853607178, + 0.2574450671672821, + 0.28461524844169617, + 0.28713259100914, + 0.055594079196453094, + 0.05978552997112274, + 0.07737205922603607, + 0.08792455494403839, + 0.03727728873491287, + 0.04697949066758156, + 0.07717122882604599, + 0.09852420538663864, + 0.05297093465924263, + 0.07244286686182022, + 0.09201814234256744, + 0.13140366971492767, + 0.054249413311481476, + 0.08212646842002869, + 0.09215818345546722, + 0.12985831499099731, + 0.03362919017672539, + 0.06840891391038895, + 0.07437368482351303, + 0.11060241609811783, + 0.02770768664777279, + 0.0924442708492279, + 0.09714528173208237, + 0.23933660984039307 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.899879372213036e-05, + 7.050983549561352e-05, + 8.400688238907605e-05, + 8.067955786827952e-05, + 9.6201452834066e-05, + 8.90634983079508e-05, + 0.00010480848868610337, + 9.241609222954139e-05, + 0.00012084992340533063, + 0.00010282631410518661, + 0.00023664520995225757, + 0.0001637248496990651, + 0.00023947370937094092, + 0.0001411983830621466, + 0.00010364947956986725, + 6.421437865355983e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.604876736877486e-05, + "first_to_last_ratio": 0.5545581271583456, + "imbalance_abs_log_ratio": 0.5895836496274593, + "last_quartile_mean": 0.00013713398766412865, + "mean": 0.00011620417808444472, + "normalized": [ + 0.5937720558721171, + 0.6067753901617422, + 0.7229248016196876, + 0.6942913688494942, + 0.8278656965686476, + 0.7664397251123708, + 0.9019339098972828, + 0.7952906147865294, + 1.0399791590755874, + 0.8848762221825061, + 2.0364604255476015, + 1.4089411619958057, + 2.060801197672231, + 1.2150886946555293, + 0.891960007621636, + 0.5525995683812308 + ], + "population_cv": 0.4517546138899352 + }, + "activation_output_rms_by_block": [ + 0.15090593695640564, + 0.3367030918598175, + 0.29793402552604675, + 0.3255155086517334, + 0.05378299206495285, + 0.08782409131526947, + 0.044081781059503555, + 0.09709836542606354, + 0.0669482871890068, + 0.12247904390096664, + 0.07360157370567322, + 0.12167256325483322, + 0.06349270045757294, + 0.10058031976222992, + 0.07551935315132141, + 0.19356463849544525 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2777646407485008, + "first_to_last_ratio": 2.565024996839381, + "imbalance_abs_log_ratio": 0.941968223931731, + "last_quartile_mean": 0.10828925296664238, + "mean": 0.13823151704855263, + "normalized": [ + 1.0916897982346618, + 2.4357910485895444, + 2.155326309711265, + 2.3548573842056504, + 0.38907908422984344, + 0.6353405734845695, + 0.3188981934128684, + 0.7024329002477675, + 0.4843199916954669, + 0.8860428252259355, + 0.5324514645948746, + 0.8802085505008006, + 0.45932144718683565, + 0.7276221943430017, + 0.5463251417894509, + 1.400293092547464 + ], + "population_cv": 0.6866316887805146 + }, + "bits_per_byte": 1.8676321823979032, + "branch_output_rms_by_sublayer": [ + 0.07169820368289948, + 0.12625248730182648, + 0.19025787711143494, + 0.09543261677026749, + 0.2934682369232178, + 0.013759235851466656, + 0.04595690965652466, + 0.014787186868488789, + 0.048878561705350876, + 0.017967192456126213, + 0.039782870560884476, + 0.03611106425523758, + 0.03420666605234146, + 0.025706099346280098, + 0.046541497111320496, + 0.04672740399837494, + 0.04840917885303497, + 0.04226908087730408, + 0.0452793650329113, + 0.07270561158657074, + 0.04624715819954872, + 0.05458534136414528, + 0.0338163822889328, + 0.0825076773762703, + 0.03031807579100132, + 0.0527048334479332, + 0.022609181702136993, + 0.06718777120113373, + 0.02396680787205696, + 0.07009968906641006, + 0.018549330532550812, + 0.1417386531829834 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00046247293295990515, + 0.0003824975682884986, + 0.00041690386457328876, + 0.00021732383526524047, + 0.00022655054548791778, + 0.0002917516444309179, + 0.000280629308661355, + 0.0003847878626551635, + 0.0005123546145807317, + 0.0005899763687075028, + 0.0015400377759773427, + 0.0012824907104132966, + 0.001537981149104749, + 0.001057571910144345, + 0.0006563117026664665, + 0.0005021677227557722 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036979955027173326, + "first_to_last_ratio": 0.394029142562531, + "imbalance_abs_log_ratio": 0.9313304065247964, + "last_quartile_mean": 0.0009385081211678332, + "mean": 0.0006463630947920308, + "normalized": [ + 0.7155002144865761, + 0.5917688855852271, + 0.6449994870258314, + 0.3362256246006202, + 0.3505004343739815, + 0.45137423033842894, + 0.434166664097133, + 0.595312241301416, + 0.792673063652533, + 0.9127630792369563, + 2.3826202151483513, + 1.9841645055956383, + 2.3794383706260316, + 1.6361885736756394, + 1.0153916706485797, + 0.7769127396070564 + ], + "population_cv": 0.675897465620864 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.43482720851898193, + "mean_weights": [ + 0.8150075674057007, + 0.18499243259429932 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611247420310974, + "mean_weights": [ + 0.6986296772956848, + 0.3013703525066376 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31192272901535034, + "mean_weights": [ + 0.9031083583831787, + 0.09689167141914368 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6764674186706543, + "mean_weights": [ + 0.4619978070259094, + 0.5380021929740906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9888706207275391, + "mean_weights": [ + 0.48646074533462524, + 0.1758481115102768, + 0.33769115805625916 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0781972408294678, + "mean_weights": [ + 0.276906281709671, + 0.2993447184562683, + 0.42374899983406067 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9839773178100586, + "mean_weights": [ + 0.46118396520614624, + 0.15739500522613525, + 0.3814210295677185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0957449674606323, + "mean_weights": [ + 0.3292894959449768, + 0.32970887422561646, + 0.3410016596317291 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2506439685821533, + "mean_weights": [ + 0.3648650050163269, + 0.0835169106721878, + 0.212466761469841, + 0.3391513228416443 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655824661254883, + "mean_weights": [ + 0.2861315608024597, + 0.20811524987220764, + 0.20031729340553284, + 0.3054359257221222 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1652674674987793, + "mean_weights": [ + 0.43905186653137207, + 0.08880466222763062, + 0.10791443288326263, + 0.3642290234565735 + ], + "sources": 4 + }, + { + "entropy_mean": 1.363007664680481, + "mean_weights": [ + 0.2524264454841614, + 0.21885012090206146, + 0.19597390294075012, + 0.33274954557418823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5649902820587158, + "mean_weights": [ + 0.22229184210300446, + 0.10384112596511841, + 0.1934116780757904, + 0.2539443373680115, + 0.22651101648807526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5797945261001587, + "mean_weights": [ + 0.21991753578186035, + 0.1605391800403595, + 0.15625253319740295, + 0.18265283107757568, + 0.2806379497051239 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4139833450317383, + "mean_weights": [ + 0.24990326166152954, + 0.08050692081451416, + 0.06835947930812836, + 0.20686665177345276, + 0.394363671541214 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5895863771438599, + "mean_weights": [ + 0.2244223803281784, + 0.16505929827690125, + 0.1643591672182083, + 0.19251328706741333, + 0.2536458969116211 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6177464723587036, + "mean_weights": [ + 0.1738494634628296, + 0.07661910355091095, + 0.06195354461669922, + 0.14467598497867584, + 0.19003543257713318, + 0.35286644101142883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7444759607315063, + "mean_weights": [ + 0.172578826546669, + 0.1303727626800537, + 0.13036483526229858, + 0.13626831769943237, + 0.1541597843170166, + 0.27625542879104614 + ], + "sources": 6 + }, + { + "entropy_mean": 1.470452904701233, + "mean_weights": [ + 0.15901926159858704, + 0.07292591035366058, + 0.03418249264359474, + 0.11740352213382721, + 0.16174395382404327, + 0.45472484827041626 + ], + "sources": 6 + }, + { + "entropy_mean": 1.778969645500183, + "mean_weights": [ + 0.18775978684425354, + 0.15501554310321808, + 0.17432880401611328, + 0.15116485953330994, + 0.14906319975852966, + 0.1826678216457367 + ], + "sources": 6 + }, + { + "entropy_mean": 1.531482219696045, + "mean_weights": [ + 0.09255115687847137, + 0.04949861764907837, + 0.03270748257637024, + 0.08605319261550903, + 0.09672325849533081, + 0.1471216380596161, + 0.4953446388244629 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8260389566421509, + "mean_weights": [ + 0.13673919439315796, + 0.10493747889995575, + 0.11646746844053268, + 0.09983028471469879, + 0.10169579833745956, + 0.1137751042842865, + 0.32655465602874756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4906136989593506, + "mean_weights": [ + 0.10088150203227997, + 0.05257141590118408, + 0.03239753842353821, + 0.08061188459396362, + 0.09571854770183563, + 0.11411523073911667, + 0.523703932762146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8863525390625, + "mean_weights": [ + 0.14810651540756226, + 0.12629631161689758, + 0.13384857773780823, + 0.11124427616596222, + 0.1082591712474823, + 0.11263860762119293, + 0.2596065402030945 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5897703170776367, + "mean_weights": [ + 0.07248121500015259, + 0.03783262521028519, + 0.02471475675702095, + 0.053565122187137604, + 0.058412034064531326, + 0.06658737361431122, + 0.26026394963264465, + 0.42614293098449707 + ], + "sources": 8 + }, + { + "entropy_mean": 1.814478874206543, + "mean_weights": [ + 0.09743618965148926, + 0.07339455187320709, + 0.06428516656160355, + 0.07672879844903946, + 0.07112542539834976, + 0.07841956615447998, + 0.14423531293869019, + 0.3943749666213989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5432119369506836, + "mean_weights": [ + 0.07432855665683746, + 0.04178062081336975, + 0.029931148514151573, + 0.0517733134329319, + 0.053720228374004364, + 0.06251072138547897, + 0.17899465560913086, + 0.5069607496261597 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8694422245025635, + "mean_weights": [ + 0.10448876023292542, + 0.07235300540924072, + 0.05752243846654892, + 0.08492527902126312, + 0.07930260896682739, + 0.08949960768222809, + 0.1757010817527771, + 0.33620721101760864 + ], + "sources": 8 + }, + { + "entropy_mean": 1.746185302734375, + "mean_weights": [ + 0.06162667274475098, + 0.04293470457196236, + 0.02593991532921791, + 0.04546971619129181, + 0.044149380177259445, + 0.052234895527362823, + 0.14164786040782928, + 0.2587880492210388, + 0.3272087574005127 + ], + "sources": 9 + }, + { + "entropy_mean": 2.061832904815674, + "mean_weights": [ + 0.09089995175600052, + 0.06807379424571991, + 0.04443599283695221, + 0.08077649772167206, + 0.08381937444210052, + 0.09205536544322968, + 0.15836367011070251, + 0.20683875679969788, + 0.17473658919334412 + ], + "sources": 9 + }, + { + "entropy_mean": 1.849208116531372, + "mean_weights": [ + 0.07359056174755096, + 0.050991274416446686, + 0.029770921915769577, + 0.0592363178730011, + 0.055861424654722214, + 0.06031768023967743, + 0.11290363222360611, + 0.2822529673576355, + 0.275075227022171 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.06120532751083374, + 0.05362993851304054, + 0.06824863702058792, + 0.06727377325296402, + 0.1878596842288971, + 0.1534913182258606, + 0.21103329956531525, + 0.1696806252002716, + 0.20562554895877838, + 0.10344218462705612, + 0.13388557732105255, + 0.08117230981588364, + 0.14070458710193634, + 0.10246162861585617, + 0.10507974773645401, + 0.06554406136274338, + 0.11050253361463547, + 0.05469200015068054, + 0.08388946205377579, + 0.05699421092867851, + 0.10618287324905396, + 0.03451571613550186, + 0.06762637197971344, + 0.050157804042100906, + 0.08278355002403259, + 0.03850048407912254, + 0.04651740565896034, + 0.04368532821536064, + 0.05266447737812996, + 0.036846842616796494, + 0.04717044159770012, + 0.048093993216753006 + ], + "loss_nats": 1.294543981552124, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.636631727218628, + "mean_weights": [ + 0.07672396302223206, + 0.03940216451883316, + 0.03699969872832298, + 0.05207572132349014, + 0.04068697616457939, + 0.04208337888121605, + 0.051315970718860626, + 0.2188674658536911, + 0.4418446719646454 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07169820368289948, + 0.15090593695640564, + 0.29125523567199707, + 0.3367030918598175, + 0.2934682369232178, + 0.29793402552604675, + 0.3226769268512726, + 0.3255155086517334, + 0.048878561705350876, + 0.05378299206495285, + 0.0762643963098526, + 0.08782409131526947, + 0.03420666605234146, + 0.044081781059503555, + 0.07245275378227234, + 0.09709836542606354, + 0.04840917885303497, + 0.0669482871890068, + 0.08269722014665604, + 0.12247904390096664, + 0.04624715819954872, + 0.07360157370567322, + 0.08215674012899399, + 0.12167256325483322, + 0.03031807579100132, + 0.06349270045757294, + 0.06805067509412766, + 0.10058031976222992, + 0.02396680787205696, + 0.07551935315132141, + 0.07906875014305115, + 0.19356463849544525 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.030247040746552, + "cross_entropy_nats": 5.566143095493317, + "step": 0 + }, + { + "bits_per_byte": 5.230785295754151, + "cross_entropy_nats": 3.6257040798664093, + "step": 100 + }, + { + "bits_per_byte": 3.16099769723908, + "cross_entropy_nats": 2.191036641597748, + "step": 500 + }, + { + "bits_per_byte": 1.983385019569791, + "cross_entropy_nats": 1.3747777342796326, + "step": 2000 + }, + { + "bits_per_byte": 1.818564586202164, + "cross_entropy_nats": 1.2605329155921936, + "step": 4000 + }, + { + "bits_per_byte": 1.7311772990915377, + "cross_entropy_nats": 1.1999606639146805, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "7517e8197003ce4ba5267d82e82d6a41d49e7ce8744b4095b5a5160f1d4d01c2", + "final_model_state": "9867cf3c0d2372bb731808e8fd98ce5f38db0a40bd3cc08683bc5f66c9472428", + "final_optimizer_state": "fa42da28c86335ac8bab241bdce833e1a77d2508025d738b471764f2dd16cf93", + "final_public_parameters": "8c73d93d872090c7e7d7cdebca989ac8c083cf6d1cb5fb16509c6e6993d3c9a0", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.937352295237, + "measured_steps": 7980, + "median_ms": 54.89947448950261, + "p95_ms": 55.28610254696105, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.033595800932828, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.568464279174805, + "step": 1, + "unclipped_grad_norm": 13.626673698425293 + }, + { + "bits_per_byte": 7.5882983213188435, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.259807586669922, + "step": 10, + "unclipped_grad_norm": 10.73694896697998 + }, + { + "bits_per_byte": 6.986202883052029, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.842466831207275, + "step": 20, + "unclipped_grad_norm": 4.4299492835998535 + }, + { + "bits_per_byte": 6.739012218966915, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.6711273193359375, + "step": 30, + "unclipped_grad_norm": 3.205341339111328 + }, + { + "bits_per_byte": 6.623832685961678, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.5912909507751465, + "step": 40, + "unclipped_grad_norm": 3.10516619682312 + }, + { + "bits_per_byte": 6.3865585415253125, + "learning_rate": 3.75e-05, + "loss_nats": 4.426825046539307, + "step": 50, + "unclipped_grad_norm": 3.1714048385620117 + }, + { + "bits_per_byte": 6.103358837933146, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.230525970458984, + "step": 60, + "unclipped_grad_norm": 2.932384490966797 + }, + { + "bits_per_byte": 5.8558104499342365, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 4.058938503265381, + "step": 70, + "unclipped_grad_norm": 2.6460657119750977 + }, + { + "bits_per_byte": 5.640669096802708, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.90981388092041, + "step": 80, + "unclipped_grad_norm": 2.5319507122039795 + }, + { + "bits_per_byte": 5.451491620506636, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.778686046600342, + "step": 90, + "unclipped_grad_norm": 2.4583022594451904 + }, + { + "bits_per_byte": 5.270375658987107, + "learning_rate": 7.5e-05, + "loss_nats": 3.6531460285186768, + "step": 100, + "unclipped_grad_norm": 2.433866262435913 + }, + { + "bits_per_byte": 5.10413206918359, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.537914752960205, + "step": 110, + "unclipped_grad_norm": 2.285804271697998 + }, + { + "bits_per_byte": 4.823540998439711, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.343423843383789, + "step": 120, + "unclipped_grad_norm": 2.0681240558624268 + }, + { + "bits_per_byte": 4.663901886394628, + "learning_rate": 9.75e-05, + "loss_nats": 3.2327704429626465, + "step": 130, + "unclipped_grad_norm": 1.8398888111114502 + }, + { + "bits_per_byte": 4.463271741412299, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0937042236328125, + "step": 140, + "unclipped_grad_norm": 1.681030035018921 + }, + { + "bits_per_byte": 4.327779681478958, + "learning_rate": 0.0001125, + "loss_nats": 2.999788284301758, + "step": 150, + "unclipped_grad_norm": 1.4562726020812988 + }, + { + "bits_per_byte": 4.208912840439223, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.917396068572998, + "step": 160, + "unclipped_grad_norm": 1.1446112394332886 + }, + { + "bits_per_byte": 4.0496410833363825, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.806997299194336, + "step": 170, + "unclipped_grad_norm": 1.193884253501892 + }, + { + "bits_per_byte": 3.9053046435997705, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7069509029388428, + "step": 180, + "unclipped_grad_norm": 1.1651051044464111 + }, + { + "bits_per_byte": 3.9164302012840855, + "learning_rate": 0.0001425, + "loss_nats": 2.714662551879883, + "step": 190, + "unclipped_grad_norm": 1.1523211002349854 + }, + { + "bits_per_byte": 3.7885273917680182, + "learning_rate": 0.00015, + "loss_nats": 2.626007080078125, + "step": 200, + "unclipped_grad_norm": 0.7202922701835632 + }, + { + "bits_per_byte": 3.72972377171561, + "learning_rate": 0.0001575, + "loss_nats": 2.58524751663208, + "step": 210, + "unclipped_grad_norm": 0.606606662273407 + }, + { + "bits_per_byte": 3.7010143638417796, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.565347671508789, + "step": 220, + "unclipped_grad_norm": 0.6799013018608093 + }, + { + "bits_per_byte": 3.715506997864687, + "learning_rate": 0.0001725, + "loss_nats": 2.5753931999206543, + "step": 230, + "unclipped_grad_norm": 0.6519911289215088 + }, + { + "bits_per_byte": 3.7068112110717615, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5693657398223877, + "step": 240, + "unclipped_grad_norm": 0.912699818611145 + }, + { + "bits_per_byte": 3.5753026132977044, + "learning_rate": 0.0001875, + "loss_nats": 2.478210926055908, + "step": 250, + "unclipped_grad_norm": 0.6829943060874939 + }, + { + "bits_per_byte": 3.5693214006659977, + "learning_rate": 0.000195, + "loss_nats": 2.474065065383911, + "step": 260, + "unclipped_grad_norm": 0.640023410320282 + }, + { + "bits_per_byte": 3.539509240034835, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4534008502960205, + "step": 270, + "unclipped_grad_norm": 0.6751542687416077 + }, + { + "bits_per_byte": 3.5394008909647914, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4533257484436035, + "step": 280, + "unclipped_grad_norm": 0.7841543555259705 + }, + { + "bits_per_byte": 3.4452169999768865, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.388042449951172, + "step": 290, + "unclipped_grad_norm": 0.8627198934555054 + }, + { + "bits_per_byte": 3.5355663657811545, + "learning_rate": 0.000225, + "loss_nats": 2.4506678581237793, + "step": 300, + "unclipped_grad_norm": 1.1167978048324585 + }, + { + "bits_per_byte": 3.4429038333227817, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.386439085006714, + "step": 310, + "unclipped_grad_norm": 0.8460172414779663 + }, + { + "bits_per_byte": 3.4221950583670893, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.372084856033325, + "step": 320, + "unclipped_grad_norm": 0.821152925491333 + }, + { + "bits_per_byte": 3.429671832130699, + "learning_rate": 0.0002475, + "loss_nats": 2.377267360687256, + "step": 330, + "unclipped_grad_norm": 0.9970561265945435 + }, + { + "bits_per_byte": 3.425132178078524, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3741207122802734, + "step": 340, + "unclipped_grad_norm": 1.0991977453231812 + }, + { + "bits_per_byte": 3.393254505810499, + "learning_rate": 0.0002625, + "loss_nats": 2.352024793624878, + "step": 350, + "unclipped_grad_norm": 0.9725445508956909 + }, + { + "bits_per_byte": 3.382284076477262, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3444206714630127, + "step": 360, + "unclipped_grad_norm": 1.0039488077163696 + }, + { + "bits_per_byte": 3.4119693139120884, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.364996910095215, + "step": 370, + "unclipped_grad_norm": 1.7546980381011963 + }, + { + "bits_per_byte": 3.446912748914394, + "learning_rate": 0.000285, + "loss_nats": 2.3892178535461426, + "step": 380, + "unclipped_grad_norm": 0.9759950041770935 + }, + { + "bits_per_byte": 3.3737516732026567, + "learning_rate": 0.0002925, + "loss_nats": 2.3385064601898193, + "step": 390, + "unclipped_grad_norm": 0.9621731638908386 + }, + { + "bits_per_byte": 3.3279967208365115, + "learning_rate": 0.0003, + "loss_nats": 2.3067915439605713, + "step": 400, + "unclipped_grad_norm": 0.9455732107162476 + }, + { + "bits_per_byte": 3.307004174506895, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.292240619659424, + "step": 410, + "unclipped_grad_norm": 1.4261541366577148 + }, + { + "bits_per_byte": 3.3082469211420302, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2931020259857178, + "step": 420, + "unclipped_grad_norm": 0.9587261080741882 + }, + { + "bits_per_byte": 3.315426508884949, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2980785369873047, + "step": 430, + "unclipped_grad_norm": 2.1751644611358643 + }, + { + "bits_per_byte": 3.299565580891777, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2870845794677734, + "step": 440, + "unclipped_grad_norm": 1.11757493019104 + }, + { + "bits_per_byte": 3.305290883339, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.291053056716919, + "step": 450, + "unclipped_grad_norm": 1.8777321577072144 + }, + { + "bits_per_byte": 3.227380334741036, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2370495796203613, + "step": 460, + "unclipped_grad_norm": 1.1160534620285034 + }, + { + "bits_per_byte": 3.2510836716135514, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.253479480743408, + "step": 470, + "unclipped_grad_norm": 1.6702216863632202 + }, + { + "bits_per_byte": 3.198324209843066, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.216909408569336, + "step": 480, + "unclipped_grad_norm": 1.0277327299118042 + }, + { + "bits_per_byte": 3.1569424323230604, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.188225746154785, + "step": 490, + "unclipped_grad_norm": 1.0115255117416382 + }, + { + "bits_per_byte": 3.1652801512368893, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.194005012512207, + "step": 500, + "unclipped_grad_norm": 1.3175867795944214 + }, + { + "bits_per_byte": 3.134954794375552, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.172985076904297, + "step": 510, + "unclipped_grad_norm": 0.9790136814117432 + }, + { + "bits_per_byte": 3.0134504274021845, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0887646675109863, + "step": 520, + "unclipped_grad_norm": 1.0963760614395142 + }, + { + "bits_per_byte": 3.0906245462891544, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1422576904296875, + "step": 530, + "unclipped_grad_norm": 1.0304492712020874 + }, + { + "bits_per_byte": 3.024995622754605, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0967671871185303, + "step": 540, + "unclipped_grad_norm": 1.323768973350525 + }, + { + "bits_per_byte": 2.9805828229957934, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0659825801849365, + "step": 550, + "unclipped_grad_norm": 1.1477609872817993 + }, + { + "bits_per_byte": 2.9872017472968992, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.070570468902588, + "step": 560, + "unclipped_grad_norm": 1.223684310913086 + }, + { + "bits_per_byte": 2.9579880862907393, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.050321102142334, + "step": 570, + "unclipped_grad_norm": 1.308420181274414 + }, + { + "bits_per_byte": 2.8546603937044113, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.978699803352356, + "step": 580, + "unclipped_grad_norm": 1.0794485807418823 + }, + { + "bits_per_byte": 2.911534540396789, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0181219577789307, + "step": 590, + "unclipped_grad_norm": 1.4487859010696411 + }, + { + "bits_per_byte": 2.825803768681442, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9586979150772095, + "step": 600, + "unclipped_grad_norm": 1.2767362594604492 + }, + { + "bits_per_byte": 2.883143472409706, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9984427690505981, + "step": 610, + "unclipped_grad_norm": 1.1309611797332764 + }, + { + "bits_per_byte": 2.7721532647968363, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9215102195739746, + "step": 620, + "unclipped_grad_norm": 1.533046007156372 + }, + { + "bits_per_byte": 2.7733690101557857, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.922352910041809, + "step": 630, + "unclipped_grad_norm": 0.9842169880867004 + }, + { + "bits_per_byte": 2.7997499449195846, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.940638780593872, + "step": 640, + "unclipped_grad_norm": 1.2905892133712769 + }, + { + "bits_per_byte": 2.715926288819655, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8825366497039795, + "step": 650, + "unclipped_grad_norm": 1.1535414457321167 + }, + { + "bits_per_byte": 2.6764600540649592, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8551807403564453, + "step": 660, + "unclipped_grad_norm": 1.1525585651397705 + }, + { + "bits_per_byte": 2.696502224265908, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8690729141235352, + "step": 670, + "unclipped_grad_norm": 1.264070749282837 + }, + { + "bits_per_byte": 2.7320018511611033, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8936793804168701, + "step": 680, + "unclipped_grad_norm": 1.6425000429153442 + }, + { + "bits_per_byte": 2.602245068668735, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8037388324737549, + "step": 690, + "unclipped_grad_norm": 1.3727850914001465 + }, + { + "bits_per_byte": 2.5536510267021466, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7700560092926025, + "step": 700, + "unclipped_grad_norm": 2.108612298965454 + }, + { + "bits_per_byte": 2.623769213372168, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8186582326889038, + "step": 710, + "unclipped_grad_norm": 1.441254734992981 + }, + { + "bits_per_byte": 2.6835147824033516, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8600707054138184, + "step": 720, + "unclipped_grad_norm": 1.4497488737106323 + }, + { + "bits_per_byte": 2.4942898388956416, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.728909969329834, + "step": 730, + "unclipped_grad_norm": 1.7889822721481323 + }, + { + "bits_per_byte": 2.518159998939494, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7454555034637451, + "step": 740, + "unclipped_grad_norm": 1.1689488887786865 + }, + { + "bits_per_byte": 2.5159042744907776, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743891954421997, + "step": 750, + "unclipped_grad_norm": 1.466252088546753 + }, + { + "bits_per_byte": 2.5144026939660944, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7428511381149292, + "step": 760, + "unclipped_grad_norm": 1.454863429069519 + }, + { + "bits_per_byte": 2.485389736713491, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.722740888595581, + "step": 770, + "unclipped_grad_norm": 1.2171542644500732 + }, + { + "bits_per_byte": 2.4154798210336645, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6742830276489258, + "step": 780, + "unclipped_grad_norm": 1.3978826999664307 + }, + { + "bits_per_byte": 2.492438617641754, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7276268005371094, + "step": 790, + "unclipped_grad_norm": 1.4353379011154175 + }, + { + "bits_per_byte": 2.4424274386320564, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6929616928100586, + "step": 800, + "unclipped_grad_norm": 1.3198370933532715 + }, + { + "bits_per_byte": 2.473405641636069, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7144341468811035, + "step": 810, + "unclipped_grad_norm": 1.3822002410888672 + }, + { + "bits_per_byte": 2.441873482513627, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6925777196884155, + "step": 820, + "unclipped_grad_norm": 1.3033113479614258 + }, + { + "bits_per_byte": 2.4253691674235776, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6811378002166748, + "step": 830, + "unclipped_grad_norm": 1.450317621231079 + }, + { + "bits_per_byte": 2.4107564895103684, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6710090637207031, + "step": 840, + "unclipped_grad_norm": 1.322706699371338 + }, + { + "bits_per_byte": 2.347782118191886, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6273585557937622, + "step": 850, + "unclipped_grad_norm": 1.2753863334655762 + }, + { + "bits_per_byte": 2.3967822111971615, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.661322832107544, + "step": 860, + "unclipped_grad_norm": 2.4916832447052 + }, + { + "bits_per_byte": 2.3462526764777634, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.626298427581787, + "step": 870, + "unclipped_grad_norm": 1.3712787628173828 + }, + { + "bits_per_byte": 2.4080783756711326, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6691527366638184, + "step": 880, + "unclipped_grad_norm": 1.4296884536743164 + }, + { + "bits_per_byte": 2.363901364126654, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6385315656661987, + "step": 890, + "unclipped_grad_norm": 1.3781569004058838 + }, + { + "bits_per_byte": 2.2957977822288393, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.5913257598876953, + "step": 900, + "unclipped_grad_norm": 1.3345599174499512 + }, + { + "bits_per_byte": 2.3445476404771095, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6251165866851807, + "step": 910, + "unclipped_grad_norm": 1.6203302145004272 + }, + { + "bits_per_byte": 2.297230053745225, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5923185348510742, + "step": 920, + "unclipped_grad_norm": 1.685173511505127 + }, + { + "bits_per_byte": 2.297947909329926, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5928161144256592, + "step": 930, + "unclipped_grad_norm": 1.4890943765640259 + }, + { + "bits_per_byte": 2.2609902135207154, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5671989917755127, + "step": 940, + "unclipped_grad_norm": 1.3654035329818726 + }, + { + "bits_per_byte": 2.2430326450308753, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5547517538070679, + "step": 950, + "unclipped_grad_norm": 1.1998120546340942 + }, + { + "bits_per_byte": 2.2479709548677502, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.558174729347229, + "step": 960, + "unclipped_grad_norm": 1.40827214717865 + }, + { + "bits_per_byte": 2.2631277858882894, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5686806440353394, + "step": 970, + "unclipped_grad_norm": 1.4645981788635254 + }, + { + "bits_per_byte": 2.2528444272452495, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5615527629852295, + "step": 980, + "unclipped_grad_norm": 1.594014048576355 + }, + { + "bits_per_byte": 2.177915369882643, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5096158981323242, + "step": 990, + "unclipped_grad_norm": 1.2862826585769653 + }, + { + "bits_per_byte": 2.2127330855671676, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5337496995925903, + "step": 1000, + "unclipped_grad_norm": 1.4609711170196533 + }, + { + "bits_per_byte": 2.208716086790965, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5309653282165527, + "step": 1010, + "unclipped_grad_norm": 1.2298572063446045 + }, + { + "bits_per_byte": 2.233356385145381, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5480446815490723, + "step": 1020, + "unclipped_grad_norm": 1.469051718711853 + }, + { + "bits_per_byte": 2.1593615375422615, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4967553615570068, + "step": 1030, + "unclipped_grad_norm": 1.3269134759902954 + }, + { + "bits_per_byte": 2.236164345886018, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5499910116195679, + "step": 1040, + "unclipped_grad_norm": 1.3589463233947754 + }, + { + "bits_per_byte": 2.2039167389359875, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5276386737823486, + "step": 1050, + "unclipped_grad_norm": 1.4478572607040405 + }, + { + "bits_per_byte": 2.1642855728191144, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5001684427261353, + "step": 1060, + "unclipped_grad_norm": 1.4022979736328125 + }, + { + "bits_per_byte": 2.2672321518511303, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5715255737304688, + "step": 1070, + "unclipped_grad_norm": 1.4223567247390747 + }, + { + "bits_per_byte": 2.187999916581285, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5166059732437134, + "step": 1080, + "unclipped_grad_norm": 1.188625454902649 + }, + { + "bits_per_byte": 2.1147346193831256, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.4658223390579224, + "step": 1090, + "unclipped_grad_norm": 1.1601507663726807 + }, + { + "bits_per_byte": 2.1969277079702247, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.522794246673584, + "step": 1100, + "unclipped_grad_norm": 1.1728529930114746 + }, + { + "bits_per_byte": 2.2198211785398274, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5386627912521362, + "step": 1110, + "unclipped_grad_norm": 1.3392254114151 + }, + { + "bits_per_byte": 2.126032503683605, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4736534357070923, + "step": 1120, + "unclipped_grad_norm": 1.3545775413513184 + }, + { + "bits_per_byte": 2.1170070261489755, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4673974514007568, + "step": 1130, + "unclipped_grad_norm": 1.2578480243682861 + }, + { + "bits_per_byte": 2.127665650936198, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4747854471206665, + "step": 1140, + "unclipped_grad_norm": 1.6450976133346558 + }, + { + "bits_per_byte": 2.1180404698980104, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4681137800216675, + "step": 1150, + "unclipped_grad_norm": 1.3040683269500732 + }, + { + "bits_per_byte": 2.0764574726764953, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4392906427383423, + "step": 1160, + "unclipped_grad_norm": 1.3104010820388794 + }, + { + "bits_per_byte": 2.1839928927988326, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5138285160064697, + "step": 1170, + "unclipped_grad_norm": 1.3359254598617554 + }, + { + "bits_per_byte": 2.091760488950268, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4498978853225708, + "step": 1180, + "unclipped_grad_norm": 1.3903101682662964 + }, + { + "bits_per_byte": 2.1040587963308166, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4584224224090576, + "step": 1190, + "unclipped_grad_norm": 1.4786474704742432 + }, + { + "bits_per_byte": 2.081396986391926, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4427144527435303, + "step": 1200, + "unclipped_grad_norm": 1.1696821451187134 + }, + { + "bits_per_byte": 2.2277741722636764, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.544175386428833, + "step": 1210, + "unclipped_grad_norm": 1.6089588403701782 + }, + { + "bits_per_byte": 2.071474963298349, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4358370304107666, + "step": 1220, + "unclipped_grad_norm": 1.219665765762329 + }, + { + "bits_per_byte": 2.020156544159443, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4002658128738403, + "step": 1230, + "unclipped_grad_norm": 1.1330406665802002 + }, + { + "bits_per_byte": 2.0202031514578267, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4002981185913086, + "step": 1240, + "unclipped_grad_norm": 1.4843499660491943 + }, + { + "bits_per_byte": 2.1912978558942307, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5188919305801392, + "step": 1250, + "unclipped_grad_norm": 1.2303693294525146 + }, + { + "bits_per_byte": 2.124901717754182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.472869634628296, + "step": 1260, + "unclipped_grad_norm": 1.2569689750671387 + }, + { + "bits_per_byte": 2.0056606424661694, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3902180194854736, + "step": 1270, + "unclipped_grad_norm": 1.3274110555648804 + }, + { + "bits_per_byte": 2.135876962601643, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4804770946502686, + "step": 1280, + "unclipped_grad_norm": 1.203521966934204 + }, + { + "bits_per_byte": 2.0353620462701763, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4108054637908936, + "step": 1290, + "unclipped_grad_norm": 1.1280517578125 + }, + { + "bits_per_byte": 2.0816712987200523, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4429045915603638, + "step": 1300, + "unclipped_grad_norm": 1.2584275007247925 + }, + { + "bits_per_byte": 2.144767433755345, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4866394996643066, + "step": 1310, + "unclipped_grad_norm": 1.2562772035598755 + }, + { + "bits_per_byte": 2.0818640912716693, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4430382251739502, + "step": 1320, + "unclipped_grad_norm": 1.1640011072158813 + }, + { + "bits_per_byte": 1.9798470784675668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3723254203796387, + "step": 1330, + "unclipped_grad_norm": 1.20253324508667 + }, + { + "bits_per_byte": 2.0020558861040847, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3877193927764893, + "step": 1340, + "unclipped_grad_norm": 1.1452504396438599 + }, + { + "bits_per_byte": 2.176502532405805, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.5086365938186646, + "step": 1350, + "unclipped_grad_norm": 1.2560323476791382 + }, + { + "bits_per_byte": 2.009120589436227, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3926162719726562, + "step": 1360, + "unclipped_grad_norm": 1.1015381813049316 + }, + { + "bits_per_byte": 2.0924102394052277, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4503482580184937, + "step": 1370, + "unclipped_grad_norm": 1.1648008823394775 + }, + { + "bits_per_byte": 2.0633229856474515, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4301865100860596, + "step": 1380, + "unclipped_grad_norm": 1.124997854232788 + }, + { + "bits_per_byte": 2.0840274610368725, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4445377588272095, + "step": 1390, + "unclipped_grad_norm": 1.2330824136734009 + }, + { + "bits_per_byte": 2.084845582507027, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4451048374176025, + "step": 1400, + "unclipped_grad_norm": 1.2093770503997803 + }, + { + "bits_per_byte": 2.0117663705371007, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.3944501876831055, + "step": 1410, + "unclipped_grad_norm": 1.1717233657836914 + }, + { + "bits_per_byte": 2.0551273244032355, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4245057106018066, + "step": 1420, + "unclipped_grad_norm": 1.118058204650879 + }, + { + "bits_per_byte": 2.04396186674392, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4167664051055908, + "step": 1430, + "unclipped_grad_norm": 1.282050371170044 + }, + { + "bits_per_byte": 2.01288047414939, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3952224254608154, + "step": 1440, + "unclipped_grad_norm": 1.2009295225143433 + }, + { + "bits_per_byte": 2.0606357567277205, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4283238649368286, + "step": 1450, + "unclipped_grad_norm": 1.1897740364074707 + }, + { + "bits_per_byte": 2.0059322030718816, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3904062509536743, + "step": 1460, + "unclipped_grad_norm": 1.0764319896697998 + }, + { + "bits_per_byte": 2.00337052148728, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3886306285858154, + "step": 1470, + "unclipped_grad_norm": 1.102418065071106 + }, + { + "bits_per_byte": 1.9153828214446602, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3276422023773193, + "step": 1480, + "unclipped_grad_norm": 1.2223329544067383 + }, + { + "bits_per_byte": 2.0269326606034372, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4049626588821411, + "step": 1490, + "unclipped_grad_norm": 1.1502348184585571 + }, + { + "bits_per_byte": 1.9137665284918515, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.326521873474121, + "step": 1500, + "unclipped_grad_norm": 1.104032039642334 + }, + { + "bits_per_byte": 1.9998878728073082, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.386216640472412, + "step": 1510, + "unclipped_grad_norm": 1.2259501218795776 + }, + { + "bits_per_byte": 1.9900984483375461, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.379431128501892, + "step": 1520, + "unclipped_grad_norm": 1.192402958869934 + }, + { + "bits_per_byte": 1.9577958069392392, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3570406436920166, + "step": 1530, + "unclipped_grad_norm": 1.1483442783355713 + }, + { + "bits_per_byte": 2.0275844748502077, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4054144620895386, + "step": 1540, + "unclipped_grad_norm": 1.132032871246338 + }, + { + "bits_per_byte": 2.0123937632474482, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3948850631713867, + "step": 1550, + "unclipped_grad_norm": 1.1288913488388062 + }, + { + "bits_per_byte": 2.071048790289511, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4355416297912598, + "step": 1560, + "unclipped_grad_norm": 1.146075963973999 + }, + { + "bits_per_byte": 2.0218080935556784, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4014105796813965, + "step": 1570, + "unclipped_grad_norm": 1.1695427894592285 + }, + { + "bits_per_byte": 2.0099278759993773, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3931758403778076, + "step": 1580, + "unclipped_grad_norm": 1.1285966634750366 + }, + { + "bits_per_byte": 1.999800161655368, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3861558437347412, + "step": 1590, + "unclipped_grad_norm": 1.136389136314392 + }, + { + "bits_per_byte": 2.019440580383901, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3997695446014404, + "step": 1600, + "unclipped_grad_norm": 1.4346710443496704 + }, + { + "bits_per_byte": 2.0397723693689023, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4138624668121338, + "step": 1610, + "unclipped_grad_norm": 1.3147592544555664 + }, + { + "bits_per_byte": 2.0494484572717453, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4205694198608398, + "step": 1620, + "unclipped_grad_norm": 1.2143282890319824 + }, + { + "bits_per_byte": 1.921803965697274, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3320930004119873, + "step": 1630, + "unclipped_grad_norm": 1.3924999237060547 + }, + { + "bits_per_byte": 2.0487386848716347, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4200774431228638, + "step": 1640, + "unclipped_grad_norm": 1.1033058166503906 + }, + { + "bits_per_byte": 1.9803865880433233, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3726993799209595, + "step": 1650, + "unclipped_grad_norm": 1.1198365688323975 + }, + { + "bits_per_byte": 1.9604881953384965, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.358906865119934, + "step": 1660, + "unclipped_grad_norm": 1.0561511516571045 + }, + { + "bits_per_byte": 2.1180337625746266, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.468109130859375, + "step": 1670, + "unclipped_grad_norm": 1.1442371606826782 + }, + { + "bits_per_byte": 2.035602821981384, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4109723567962646, + "step": 1680, + "unclipped_grad_norm": 1.0968410968780518 + }, + { + "bits_per_byte": 1.9074798746722124, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3221642971038818, + "step": 1690, + "unclipped_grad_norm": 1.049187183380127 + }, + { + "bits_per_byte": 1.9460256582794904, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3488821983337402, + "step": 1700, + "unclipped_grad_norm": 1.1202492713928223 + }, + { + "bits_per_byte": 2.030805537918217, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.4076471328735352, + "step": 1710, + "unclipped_grad_norm": 1.272226095199585 + }, + { + "bits_per_byte": 2.0120939974869945, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3946772813796997, + "step": 1720, + "unclipped_grad_norm": 1.1422768831253052 + }, + { + "bits_per_byte": 1.9049754633103482, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3204283714294434, + "step": 1730, + "unclipped_grad_norm": 1.1759874820709229 + }, + { + "bits_per_byte": 1.9628171843964806, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3605211973190308, + "step": 1740, + "unclipped_grad_norm": 1.0964117050170898 + }, + { + "bits_per_byte": 2.0712307479341234, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4356677532196045, + "step": 1750, + "unclipped_grad_norm": 1.1146889925003052 + }, + { + "bits_per_byte": 1.9217773083863903, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.332074522972107, + "step": 1760, + "unclipped_grad_norm": 1.044190764427185 + }, + { + "bits_per_byte": 1.9951038314082574, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.382900595664978, + "step": 1770, + "unclipped_grad_norm": 1.0358567237854004 + }, + { + "bits_per_byte": 1.9346029145744843, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3409645557403564, + "step": 1780, + "unclipped_grad_norm": 1.1725130081176758 + }, + { + "bits_per_byte": 1.9131714685198662, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3261094093322754, + "step": 1790, + "unclipped_grad_norm": 1.2142103910446167 + }, + { + "bits_per_byte": 1.9953318804033016, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3830586671829224, + "step": 1800, + "unclipped_grad_norm": 1.2420480251312256 + }, + { + "bits_per_byte": 1.9723242133535257, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3671109676361084, + "step": 1810, + "unclipped_grad_norm": 1.1430673599243164 + }, + { + "bits_per_byte": 2.0342909383206025, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4100630283355713, + "step": 1820, + "unclipped_grad_norm": 1.1687225103378296 + }, + { + "bits_per_byte": 1.8908906001526369, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.310665488243103, + "step": 1830, + "unclipped_grad_norm": 1.146242380142212 + }, + { + "bits_per_byte": 1.978836336428446, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3716248273849487, + "step": 1840, + "unclipped_grad_norm": 1.0820521116256714 + }, + { + "bits_per_byte": 1.8676517884201014, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2945575714111328, + "step": 1850, + "unclipped_grad_norm": 1.0944465398788452 + }, + { + "bits_per_byte": 2.0069171477133727, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3910889625549316, + "step": 1860, + "unclipped_grad_norm": 1.2028480768203735 + }, + { + "bits_per_byte": 2.0278259384920188, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4055818319320679, + "step": 1870, + "unclipped_grad_norm": 1.1793227195739746 + }, + { + "bits_per_byte": 1.8922862393643884, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3116328716278076, + "step": 1880, + "unclipped_grad_norm": 1.123720645904541 + }, + { + "bits_per_byte": 1.9751340659033223, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.369058609008789, + "step": 1890, + "unclipped_grad_norm": 1.006156086921692 + }, + { + "bits_per_byte": 1.8741764662285347, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2990801334381104, + "step": 1900, + "unclipped_grad_norm": 1.0260387659072876 + }, + { + "bits_per_byte": 1.8761183223394267, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3004261255264282, + "step": 1910, + "unclipped_grad_norm": 1.11003839969635 + }, + { + "bits_per_byte": 1.9401261374069434, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3447929620742798, + "step": 1920, + "unclipped_grad_norm": 1.1032140254974365 + }, + { + "bits_per_byte": 1.9031926911515042, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.31919264793396, + "step": 1930, + "unclipped_grad_norm": 1.1424028873443604 + }, + { + "bits_per_byte": 1.8781860697507502, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3018593788146973, + "step": 1940, + "unclipped_grad_norm": 1.0988123416900635 + }, + { + "bits_per_byte": 1.9390512458390508, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3440479040145874, + "step": 1950, + "unclipped_grad_norm": 1.28093421459198 + }, + { + "bits_per_byte": 1.9095305958011006, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3235857486724854, + "step": 1960, + "unclipped_grad_norm": 1.052490234375 + }, + { + "bits_per_byte": 1.8582639434401043, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2880504131317139, + "step": 1970, + "unclipped_grad_norm": 1.0047657489776611 + }, + { + "bits_per_byte": 1.8448327863383223, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.278740644454956, + "step": 1980, + "unclipped_grad_norm": 1.101292610168457 + }, + { + "bits_per_byte": 1.9462698736437154, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3490514755249023, + "step": 1990, + "unclipped_grad_norm": 1.17987859249115 + }, + { + "bits_per_byte": 1.896080176642422, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3142626285552979, + "step": 2000, + "unclipped_grad_norm": 1.1739213466644287 + }, + { + "bits_per_byte": 1.9557370026257606, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3556135892868042, + "step": 2010, + "unclipped_grad_norm": 1.0457618236541748 + }, + { + "bits_per_byte": 1.9291909645171335, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3372132778167725, + "step": 2020, + "unclipped_grad_norm": 1.1008185148239136 + }, + { + "bits_per_byte": 1.9305355248815788, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3381452560424805, + "step": 2030, + "unclipped_grad_norm": 1.1080964803695679 + }, + { + "bits_per_byte": 1.9343679862734056, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.34080171585083, + "step": 2040, + "unclipped_grad_norm": 1.104992389678955 + }, + { + "bits_per_byte": 1.8835350741578853, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3055670261383057, + "step": 2050, + "unclipped_grad_norm": 1.1497511863708496 + }, + { + "bits_per_byte": 1.7856179557677012, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2376960515975952, + "step": 2060, + "unclipped_grad_norm": 1.0708798170089722 + }, + { + "bits_per_byte": 1.8782170266279055, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3018808364868164, + "step": 2070, + "unclipped_grad_norm": 1.0489474534988403 + }, + { + "bits_per_byte": 1.9968320850667778, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3840985298156738, + "step": 2080, + "unclipped_grad_norm": 1.0985883474349976 + }, + { + "bits_per_byte": 1.8512350124993413, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2831783294677734, + "step": 2090, + "unclipped_grad_norm": 0.9845348000526428 + }, + { + "bits_per_byte": 1.9251153196569866, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.334388256072998, + "step": 2100, + "unclipped_grad_norm": 1.009676456451416 + }, + { + "bits_per_byte": 1.8359577936231979, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2725889682769775, + "step": 2110, + "unclipped_grad_norm": 1.0760974884033203 + }, + { + "bits_per_byte": 1.8868116876221235, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3078382015228271, + "step": 2120, + "unclipped_grad_norm": 0.9828676581382751 + }, + { + "bits_per_byte": 1.910013867050025, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.323920726776123, + "step": 2130, + "unclipped_grad_norm": 1.0712695121765137 + }, + { + "bits_per_byte": 1.8022548694815659, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2492278814315796, + "step": 2140, + "unclipped_grad_norm": 1.0451561212539673 + }, + { + "bits_per_byte": 1.8551622363317934, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2859004735946655, + "step": 2150, + "unclipped_grad_norm": 1.0957838296890259 + }, + { + "bits_per_byte": 1.9088151479735114, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.323089838027954, + "step": 2160, + "unclipped_grad_norm": 1.0826293230056763 + }, + { + "bits_per_byte": 1.871177432762789, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2970013618469238, + "step": 2170, + "unclipped_grad_norm": 1.0311518907546997 + }, + { + "bits_per_byte": 1.8055650195627224, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2515223026275635, + "step": 2180, + "unclipped_grad_norm": 1.040400505065918 + }, + { + "bits_per_byte": 1.9270800494604432, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3357501029968262, + "step": 2190, + "unclipped_grad_norm": 1.10823392868042 + }, + { + "bits_per_byte": 1.980437322925328, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.372734546661377, + "step": 2200, + "unclipped_grad_norm": 1.11663019657135 + }, + { + "bits_per_byte": 1.9995734885215308, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3859987258911133, + "step": 2210, + "unclipped_grad_norm": 1.0498319864273071 + }, + { + "bits_per_byte": 1.8802302555389057, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3032763004302979, + "step": 2220, + "unclipped_grad_norm": 1.0160832405090332 + }, + { + "bits_per_byte": 1.8937652901618087, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3126580715179443, + "step": 2230, + "unclipped_grad_norm": 1.0416548252105713 + }, + { + "bits_per_byte": 1.8469913405893015, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.280236840248108, + "step": 2240, + "unclipped_grad_norm": 1.0407441854476929 + }, + { + "bits_per_byte": 1.8529231942002105, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.284348487854004, + "step": 2250, + "unclipped_grad_norm": 0.9867598414421082 + }, + { + "bits_per_byte": 1.8825788506190881, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3049042224884033, + "step": 2260, + "unclipped_grad_norm": 0.9744513034820557 + }, + { + "bits_per_byte": 1.9268210435882438, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.3355705738067627, + "step": 2270, + "unclipped_grad_norm": 1.0475958585739136 + }, + { + "bits_per_byte": 1.8280555347813536, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2671115398406982, + "step": 2280, + "unclipped_grad_norm": 1.0294238328933716 + }, + { + "bits_per_byte": 1.8388550133596329, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.27459716796875, + "step": 2290, + "unclipped_grad_norm": 1.0671623945236206 + }, + { + "bits_per_byte": 1.8867002428643644, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3077609539031982, + "step": 2300, + "unclipped_grad_norm": 0.9974833726882935 + }, + { + "bits_per_byte": 1.8601865374940996, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.289383053779602, + "step": 2310, + "unclipped_grad_norm": 1.0144871473312378 + }, + { + "bits_per_byte": 1.874428248829398, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2992546558380127, + "step": 2320, + "unclipped_grad_norm": 1.0221670866012573 + }, + { + "bits_per_byte": 1.8178058847378828, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2600070238113403, + "step": 2330, + "unclipped_grad_norm": 1.0877128839492798 + }, + { + "bits_per_byte": 1.8362632348111303, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2728006839752197, + "step": 2340, + "unclipped_grad_norm": 1.050228238105774 + }, + { + "bits_per_byte": 1.780467247357011, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2341258525848389, + "step": 2350, + "unclipped_grad_norm": 0.9972993731498718 + }, + { + "bits_per_byte": 1.8791837411084054, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.3025509119033813, + "step": 2360, + "unclipped_grad_norm": 1.0690455436706543 + }, + { + "bits_per_byte": 1.8768631792003134, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3009424209594727, + "step": 2370, + "unclipped_grad_norm": 1.0405195951461792 + }, + { + "bits_per_byte": 1.8106722163627444, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2550623416900635, + "step": 2380, + "unclipped_grad_norm": 1.000722050666809 + }, + { + "bits_per_byte": 1.7959209204329396, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2448375225067139, + "step": 2390, + "unclipped_grad_norm": 1.0424110889434814 + }, + { + "bits_per_byte": 1.9177011475782908, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3292491436004639, + "step": 2400, + "unclipped_grad_norm": 1.0584142208099365 + }, + { + "bits_per_byte": 1.8792028311826512, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.3025641441345215, + "step": 2410, + "unclipped_grad_norm": 1.109039068222046 + }, + { + "bits_per_byte": 1.9615544877738458, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.359645962715149, + "step": 2420, + "unclipped_grad_norm": 1.0733245611190796 + }, + { + "bits_per_byte": 1.8745142401548294, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.299314260482788, + "step": 2430, + "unclipped_grad_norm": 1.0562880039215088 + }, + { + "bits_per_byte": 1.7973332419618249, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2458164691925049, + "step": 2440, + "unclipped_grad_norm": 1.0554269552230835 + }, + { + "bits_per_byte": 1.8932479663480133, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3122994899749756, + "step": 2450, + "unclipped_grad_norm": 1.1328672170639038 + }, + { + "bits_per_byte": 1.9065160838967772, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3214962482452393, + "step": 2460, + "unclipped_grad_norm": 1.1163077354431152 + }, + { + "bits_per_byte": 1.8460499075584784, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.279584288597107, + "step": 2470, + "unclipped_grad_norm": 1.0860918760299683 + }, + { + "bits_per_byte": 1.8485799443353212, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2813379764556885, + "step": 2480, + "unclipped_grad_norm": 0.99190354347229 + }, + { + "bits_per_byte": 1.7724624868552525, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2285773754119873, + "step": 2490, + "unclipped_grad_norm": 1.025232195854187 + }, + { + "bits_per_byte": 1.8512375922391042, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2831801176071167, + "step": 2500, + "unclipped_grad_norm": 1.0759034156799316 + }, + { + "bits_per_byte": 1.8193396260182773, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2610701322555542, + "step": 2510, + "unclipped_grad_norm": 1.0067415237426758 + }, + { + "bits_per_byte": 1.8146814759196581, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2578413486480713, + "step": 2520, + "unclipped_grad_norm": 0.990001916885376 + }, + { + "bits_per_byte": 1.8138671380678228, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2572768926620483, + "step": 2530, + "unclipped_grad_norm": 0.9393356442451477 + }, + { + "bits_per_byte": 1.8368911434694304, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2732359170913696, + "step": 2540, + "unclipped_grad_norm": 1.1141552925109863 + }, + { + "bits_per_byte": 1.8603834576293374, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2895195484161377, + "step": 2550, + "unclipped_grad_norm": 1.0569089651107788 + }, + { + "bits_per_byte": 1.816235855118156, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2589187622070312, + "step": 2560, + "unclipped_grad_norm": 0.9373295903205872 + }, + { + "bits_per_byte": 1.8059492288047498, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.25178861618042, + "step": 2570, + "unclipped_grad_norm": 1.143198847770691 + }, + { + "bits_per_byte": 1.8450266107858446, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2788749933242798, + "step": 2580, + "unclipped_grad_norm": 1.057027816772461 + }, + { + "bits_per_byte": 1.8154227211448768, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2583551406860352, + "step": 2590, + "unclipped_grad_norm": 1.0370285511016846 + }, + { + "bits_per_byte": 1.8561616275159571, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2865931987762451, + "step": 2600, + "unclipped_grad_norm": 0.9653204083442688 + }, + { + "bits_per_byte": 1.7605767658541238, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2203388214111328, + "step": 2610, + "unclipped_grad_norm": 1.0564771890640259 + }, + { + "bits_per_byte": 1.812449141111459, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2562940120697021, + "step": 2620, + "unclipped_grad_norm": 1.0267692804336548 + }, + { + "bits_per_byte": 1.8349191903946374, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2718690633773804, + "step": 2630, + "unclipped_grad_norm": 0.9602619409561157 + }, + { + "bits_per_byte": 1.8363758834474455, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2728787660598755, + "step": 2640, + "unclipped_grad_norm": 1.0029752254486084 + }, + { + "bits_per_byte": 1.771385015547597, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2278305292129517, + "step": 2650, + "unclipped_grad_norm": 0.9146384000778198 + }, + { + "bits_per_byte": 1.779632271587072, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2335470914840698, + "step": 2660, + "unclipped_grad_norm": 0.9666176438331604 + }, + { + "bits_per_byte": 1.8136729696549987, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2571423053741455, + "step": 2670, + "unclipped_grad_norm": 1.0540763139724731 + }, + { + "bits_per_byte": 1.7953530337197905, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2444438934326172, + "step": 2680, + "unclipped_grad_norm": 1.0946050882339478 + }, + { + "bits_per_byte": 1.8769845989518226, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3010265827178955, + "step": 2690, + "unclipped_grad_norm": 1.068537712097168 + }, + { + "bits_per_byte": 1.8642107595416384, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2921724319458008, + "step": 2700, + "unclipped_grad_norm": 1.1353117227554321 + }, + { + "bits_per_byte": 1.7364708175959382, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2036298513412476, + "step": 2710, + "unclipped_grad_norm": 0.9047647714614868 + }, + { + "bits_per_byte": 1.8594279220211436, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2888572216033936, + "step": 2720, + "unclipped_grad_norm": 1.0444105863571167 + }, + { + "bits_per_byte": 1.7814004252205926, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2347726821899414, + "step": 2730, + "unclipped_grad_norm": 0.9630917310714722 + }, + { + "bits_per_byte": 1.7840532576101513, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2366114854812622, + "step": 2740, + "unclipped_grad_norm": 1.0223243236541748 + }, + { + "bits_per_byte": 1.8198583256932797, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2614296674728394, + "step": 2750, + "unclipped_grad_norm": 1.0687127113342285 + }, + { + "bits_per_byte": 1.9021738659277931, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3184864521026611, + "step": 2760, + "unclipped_grad_norm": 1.0979900360107422 + }, + { + "bits_per_byte": 1.7971648709466301, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2456997632980347, + "step": 2770, + "unclipped_grad_norm": 0.965108335018158 + }, + { + "bits_per_byte": 1.8086140999798692, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2536357641220093, + "step": 2780, + "unclipped_grad_norm": 1.0402190685272217 + }, + { + "bits_per_byte": 1.7938860217079309, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.243427038192749, + "step": 2790, + "unclipped_grad_norm": 1.0398236513137817 + }, + { + "bits_per_byte": 1.7550451158717724, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2165045738220215, + "step": 2800, + "unclipped_grad_norm": 1.0063706636428833 + }, + { + "bits_per_byte": 1.828329675126829, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2673015594482422, + "step": 2810, + "unclipped_grad_norm": 0.9668177366256714 + }, + { + "bits_per_byte": 1.8254919613875926, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2653346061706543, + "step": 2820, + "unclipped_grad_norm": 0.9527478814125061 + }, + { + "bits_per_byte": 1.7516809632382448, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2141727209091187, + "step": 2830, + "unclipped_grad_norm": 0.9219958186149597 + }, + { + "bits_per_byte": 1.8254823303591443, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2653279304504395, + "step": 2840, + "unclipped_grad_norm": 1.0127615928649902 + }, + { + "bits_per_byte": 1.7530225998976257, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2151026725769043, + "step": 2850, + "unclipped_grad_norm": 1.0182650089263916 + }, + { + "bits_per_byte": 1.7287747659524784, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1982953548431396, + "step": 2860, + "unclipped_grad_norm": 0.978410542011261 + }, + { + "bits_per_byte": 1.855343334063152, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2860260009765625, + "step": 2870, + "unclipped_grad_norm": 1.018862009048462 + }, + { + "bits_per_byte": 1.768852571013642, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2260751724243164, + "step": 2880, + "unclipped_grad_norm": 0.9639438390731812 + }, + { + "bits_per_byte": 1.8105554401428086, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2549813985824585, + "step": 2890, + "unclipped_grad_norm": 0.9698073267936707 + }, + { + "bits_per_byte": 1.7403103302764504, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2062911987304688, + "step": 2900, + "unclipped_grad_norm": 0.9920547604560852 + }, + { + "bits_per_byte": 1.7643693272709504, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2229676246643066, + "step": 2910, + "unclipped_grad_norm": 1.0128531455993652 + }, + { + "bits_per_byte": 1.8473500963990013, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2804855108261108, + "step": 2920, + "unclipped_grad_norm": 1.0619620084762573 + }, + { + "bits_per_byte": 1.8003603085996611, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2479146718978882, + "step": 2930, + "unclipped_grad_norm": 1.0764249563217163 + }, + { + "bits_per_byte": 1.8426398355571707, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.277220606803894, + "step": 2940, + "unclipped_grad_norm": 1.0425148010253906 + }, + { + "bits_per_byte": 1.7862247105599451, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2381166219711304, + "step": 2950, + "unclipped_grad_norm": 1.004768967628479 + }, + { + "bits_per_byte": 1.7811238771180051, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2345809936523438, + "step": 2960, + "unclipped_grad_norm": 0.9997458457946777 + }, + { + "bits_per_byte": 1.8373286673332254, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2735391855239868, + "step": 2970, + "unclipped_grad_norm": 1.039349913597107 + }, + { + "bits_per_byte": 1.7631166056420657, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2220993041992188, + "step": 2980, + "unclipped_grad_norm": 0.9808847904205322 + }, + { + "bits_per_byte": 1.9032267437163752, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.319216251373291, + "step": 2990, + "unclipped_grad_norm": 1.1331971883773804 + }, + { + "bits_per_byte": 1.81958332543455, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2612390518188477, + "step": 3000, + "unclipped_grad_norm": 0.944084107875824 + }, + { + "bits_per_byte": 1.7894560925870064, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2403564453125, + "step": 3010, + "unclipped_grad_norm": 1.057902216911316 + }, + { + "bits_per_byte": 1.7654951257034983, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.223747968673706, + "step": 3020, + "unclipped_grad_norm": 0.971477746963501 + }, + { + "bits_per_byte": 1.791092679492617, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2414908409118652, + "step": 3030, + "unclipped_grad_norm": 1.017361044883728 + }, + { + "bits_per_byte": 1.760655361925568, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2203933000564575, + "step": 3040, + "unclipped_grad_norm": 0.9738672971725464 + }, + { + "bits_per_byte": 1.7859287284178103, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2379114627838135, + "step": 3050, + "unclipped_grad_norm": 1.0311107635498047 + }, + { + "bits_per_byte": 1.7871233199107035, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2387394905090332, + "step": 3060, + "unclipped_grad_norm": 1.097463607788086 + }, + { + "bits_per_byte": 1.7530494291911602, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.2151212692260742, + "step": 3070, + "unclipped_grad_norm": 0.95722895860672 + }, + { + "bits_per_byte": 1.7753578147825277, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2305842638015747, + "step": 3080, + "unclipped_grad_norm": 1.0311440229415894 + }, + { + "bits_per_byte": 1.7406597990230035, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.206533432006836, + "step": 3090, + "unclipped_grad_norm": 0.9659063220024109 + }, + { + "bits_per_byte": 1.8326863396384856, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2703213691711426, + "step": 3100, + "unclipped_grad_norm": 1.0265647172927856 + }, + { + "bits_per_byte": 1.8128345542320425, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2565611600875854, + "step": 3110, + "unclipped_grad_norm": 1.0288156270980835 + }, + { + "bits_per_byte": 1.6999466900495526, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1783132553100586, + "step": 3120, + "unclipped_grad_norm": 0.9754294753074646 + }, + { + "bits_per_byte": 1.7980240962703409, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2462953329086304, + "step": 3130, + "unclipped_grad_norm": 1.0256381034851074 + }, + { + "bits_per_byte": 1.7337763654048706, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2017621994018555, + "step": 3140, + "unclipped_grad_norm": 0.9683999419212341 + }, + { + "bits_per_byte": 1.7391683654747212, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2054996490478516, + "step": 3150, + "unclipped_grad_norm": 0.9884417057037354 + }, + { + "bits_per_byte": 1.7859572775378536, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.237931251525879, + "step": 3160, + "unclipped_grad_norm": 1.0163674354553223 + }, + { + "bits_per_byte": 1.8529599984874952, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2843739986419678, + "step": 3170, + "unclipped_grad_norm": 1.0605453252792358 + }, + { + "bits_per_byte": 1.7071207743476438, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1832859516143799, + "step": 3180, + "unclipped_grad_norm": 1.0476000308990479 + }, + { + "bits_per_byte": 1.8011364663030052, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2484526634216309, + "step": 3190, + "unclipped_grad_norm": 0.9400752782821655 + }, + { + "bits_per_byte": 1.7046130953154133, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.18154776096344, + "step": 3200, + "unclipped_grad_norm": 0.9428834915161133 + }, + { + "bits_per_byte": 1.7343896555378473, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2021872997283936, + "step": 3210, + "unclipped_grad_norm": 0.982049822807312 + }, + { + "bits_per_byte": 1.7352428614687776, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.202778697013855, + "step": 3220, + "unclipped_grad_norm": 0.9146644473075867 + }, + { + "bits_per_byte": 1.7184315573469382, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1911259889602661, + "step": 3230, + "unclipped_grad_norm": 0.9842981696128845 + }, + { + "bits_per_byte": 1.7760342225483712, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.231053113937378, + "step": 3240, + "unclipped_grad_norm": 1.0283819437026978 + }, + { + "bits_per_byte": 1.780553754630395, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.234185814857483, + "step": 3250, + "unclipped_grad_norm": 0.9948496222496033 + }, + { + "bits_per_byte": 1.8669033199235465, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2940387725830078, + "step": 3260, + "unclipped_grad_norm": 1.0557905435562134 + }, + { + "bits_per_byte": 1.7253210103578513, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1959013938903809, + "step": 3270, + "unclipped_grad_norm": 1.0562386512756348 + }, + { + "bits_per_byte": 1.6636741731213793, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1531710624694824, + "step": 3280, + "unclipped_grad_norm": 0.9674620628356934 + }, + { + "bits_per_byte": 1.7215127985197962, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.193261742591858, + "step": 3290, + "unclipped_grad_norm": 1.081762671470642 + }, + { + "bits_per_byte": 1.784075099406811, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2366266250610352, + "step": 3300, + "unclipped_grad_norm": 0.9871881604194641 + }, + { + "bits_per_byte": 1.7790666206483843, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2331550121307373, + "step": 3310, + "unclipped_grad_norm": 1.018639326095581 + }, + { + "bits_per_byte": 1.7964894950766919, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2452316284179688, + "step": 3320, + "unclipped_grad_norm": 1.0376429557800293 + }, + { + "bits_per_byte": 1.7442502808250664, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2090221643447876, + "step": 3330, + "unclipped_grad_norm": 0.9751812815666199 + }, + { + "bits_per_byte": 1.740043757167613, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.206106424331665, + "step": 3340, + "unclipped_grad_norm": 1.0885974168777466 + }, + { + "bits_per_byte": 1.7459623681144056, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2102088928222656, + "step": 3350, + "unclipped_grad_norm": 1.028318166732788 + }, + { + "bits_per_byte": 1.7533509147781228, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2153302431106567, + "step": 3360, + "unclipped_grad_norm": 1.038756251335144 + }, + { + "bits_per_byte": 1.6755667734285427, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.161414384841919, + "step": 3370, + "unclipped_grad_norm": 1.0451469421386719 + }, + { + "bits_per_byte": 1.794013632834871, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2435154914855957, + "step": 3380, + "unclipped_grad_norm": 1.1703273057937622 + }, + { + "bits_per_byte": 1.7554886591283476, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.216812014579773, + "step": 3390, + "unclipped_grad_norm": 1.036725401878357 + }, + { + "bits_per_byte": 1.7445490146896152, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2092292308807373, + "step": 3400, + "unclipped_grad_norm": 1.0196164846420288 + }, + { + "bits_per_byte": 1.794997717563108, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2441976070404053, + "step": 3410, + "unclipped_grad_norm": 1.202675700187683 + }, + { + "bits_per_byte": 1.7549782146205868, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2164582014083862, + "step": 3420, + "unclipped_grad_norm": 0.9739375710487366 + }, + { + "bits_per_byte": 1.7093242160704982, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1848132610321045, + "step": 3430, + "unclipped_grad_norm": 1.0012179613113403 + }, + { + "bits_per_byte": 1.7178566193451037, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1907274723052979, + "step": 3440, + "unclipped_grad_norm": 1.0250310897827148 + }, + { + "bits_per_byte": 1.7050728049411694, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1818664073944092, + "step": 3450, + "unclipped_grad_norm": 1.0344855785369873 + }, + { + "bits_per_byte": 1.706558735044624, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.182896375656128, + "step": 3460, + "unclipped_grad_norm": 0.9980728626251221 + }, + { + "bits_per_byte": 1.7653320861504804, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.223634958267212, + "step": 3470, + "unclipped_grad_norm": 1.0550833940505981 + }, + { + "bits_per_byte": 1.8114688400015408, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2556145191192627, + "step": 3480, + "unclipped_grad_norm": 1.0545322895050049 + }, + { + "bits_per_byte": 1.740959048835505, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2067408561706543, + "step": 3490, + "unclipped_grad_norm": 0.9756637811660767 + }, + { + "bits_per_byte": 1.704705106033625, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1816115379333496, + "step": 3500, + "unclipped_grad_norm": 0.9728481769561768 + }, + { + "bits_per_byte": 1.7154590092094253, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.1890655755996704, + "step": 3510, + "unclipped_grad_norm": 0.9984562993049622 + }, + { + "bits_per_byte": 1.7629102264610303, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2219562530517578, + "step": 3520, + "unclipped_grad_norm": 1.0547406673431396 + }, + { + "bits_per_byte": 1.7070648799861134, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1832472085952759, + "step": 3530, + "unclipped_grad_norm": 0.9733150601387024 + }, + { + "bits_per_byte": 1.7245068444886669, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1953370571136475, + "step": 3540, + "unclipped_grad_norm": 1.0125547647476196 + }, + { + "bits_per_byte": 1.7698511022845516, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2267673015594482, + "step": 3550, + "unclipped_grad_norm": 1.0299363136291504 + }, + { + "bits_per_byte": 1.7181577609667646, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1909362077713013, + "step": 3560, + "unclipped_grad_norm": 0.9789594411849976 + }, + { + "bits_per_byte": 1.7459088815099875, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2101718187332153, + "step": 3570, + "unclipped_grad_norm": 1.0010499954223633 + }, + { + "bits_per_byte": 1.7705113436812139, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2272249460220337, + "step": 3580, + "unclipped_grad_norm": 1.082968831062317 + }, + { + "bits_per_byte": 1.7425258107848651, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.207826852798462, + "step": 3590, + "unclipped_grad_norm": 1.0055863857269287 + }, + { + "bits_per_byte": 1.7755588625013865, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2307236194610596, + "step": 3600, + "unclipped_grad_norm": 1.0473418235778809 + }, + { + "bits_per_byte": 1.7693408297594415, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.226413607597351, + "step": 3610, + "unclipped_grad_norm": 1.1090856790542603 + }, + { + "bits_per_byte": 1.7112244523798812, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.186130404472351, + "step": 3620, + "unclipped_grad_norm": 0.9742531776428223 + }, + { + "bits_per_byte": 1.7137644641504741, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1878910064697266, + "step": 3630, + "unclipped_grad_norm": 1.0429282188415527 + }, + { + "bits_per_byte": 1.6520834023664803, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1451369524002075, + "step": 3640, + "unclipped_grad_norm": 0.9745917320251465 + }, + { + "bits_per_byte": 1.7256807980634563, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.196150779724121, + "step": 3650, + "unclipped_grad_norm": 1.0385748147964478 + }, + { + "bits_per_byte": 1.7126971398192194, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1871511936187744, + "step": 3660, + "unclipped_grad_norm": 0.9310101270675659 + }, + { + "bits_per_byte": 1.7674923602279682, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2251323461532593, + "step": 3670, + "unclipped_grad_norm": 1.0780647993087769 + }, + { + "bits_per_byte": 1.6993364956042913, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1778903007507324, + "step": 3680, + "unclipped_grad_norm": 0.9666725397109985 + }, + { + "bits_per_byte": 1.7697671747509305, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.2267091274261475, + "step": 3690, + "unclipped_grad_norm": 1.0202223062515259 + }, + { + "bits_per_byte": 1.6641650116069417, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1535112857818604, + "step": 3700, + "unclipped_grad_norm": 1.0115612745285034 + }, + { + "bits_per_byte": 1.726182815421325, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1964987516403198, + "step": 3710, + "unclipped_grad_norm": 0.9808560013771057 + }, + { + "bits_per_byte": 1.7616059100368868, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.2210521697998047, + "step": 3720, + "unclipped_grad_norm": 1.0511683225631714 + }, + { + "bits_per_byte": 1.7329405297216773, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2011828422546387, + "step": 3730, + "unclipped_grad_norm": 1.0690499544143677 + }, + { + "bits_per_byte": 1.7000292417219667, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.178370475769043, + "step": 3740, + "unclipped_grad_norm": 0.9935287237167358 + }, + { + "bits_per_byte": 1.7278594742845865, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1976609230041504, + "step": 3750, + "unclipped_grad_norm": 1.010900855064392 + }, + { + "bits_per_byte": 1.743004610484867, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2081587314605713, + "step": 3760, + "unclipped_grad_norm": 1.0119491815567017 + }, + { + "bits_per_byte": 1.751619221466585, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.21412992477417, + "step": 3770, + "unclipped_grad_norm": 1.011305570602417 + }, + { + "bits_per_byte": 1.8012363882231563, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2485219240188599, + "step": 3780, + "unclipped_grad_norm": 1.0745822191238403 + }, + { + "bits_per_byte": 1.698407961272283, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1772466897964478, + "step": 3790, + "unclipped_grad_norm": 1.0157029628753662 + }, + { + "bits_per_byte": 1.7299170747195092, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.199087142944336, + "step": 3800, + "unclipped_grad_norm": 1.0643014907836914 + }, + { + "bits_per_byte": 1.6600432753963636, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1506543159484863, + "step": 3810, + "unclipped_grad_norm": 0.9955329895019531 + }, + { + "bits_per_byte": 1.661928377232471, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1519609689712524, + "step": 3820, + "unclipped_grad_norm": 1.0222501754760742 + }, + { + "bits_per_byte": 1.7482129330835965, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.2117688655853271, + "step": 3830, + "unclipped_grad_norm": 1.0880961418151855 + }, + { + "bits_per_byte": 1.6885180989344173, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.17039155960083, + "step": 3840, + "unclipped_grad_norm": 0.9980700016021729 + }, + { + "bits_per_byte": 1.661674014891845, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1517846584320068, + "step": 3850, + "unclipped_grad_norm": 0.961553156375885 + }, + { + "bits_per_byte": 1.6409110654011305, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1373928785324097, + "step": 3860, + "unclipped_grad_norm": 1.019968032836914 + }, + { + "bits_per_byte": 1.6069470715954888, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1138508319854736, + "step": 3870, + "unclipped_grad_norm": 0.9610836505889893 + }, + { + "bits_per_byte": 1.7034871249002144, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.180767297744751, + "step": 3880, + "unclipped_grad_norm": 0.9531551003456116 + }, + { + "bits_per_byte": 1.6934515932570677, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1738111972808838, + "step": 3890, + "unclipped_grad_norm": 1.0675544738769531 + }, + { + "bits_per_byte": 1.6830007235120874, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1665672063827515, + "step": 3900, + "unclipped_grad_norm": 1.0058032274246216 + }, + { + "bits_per_byte": 1.7034874688655162, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.18076753616333, + "step": 3910, + "unclipped_grad_norm": 0.9943428039550781 + }, + { + "bits_per_byte": 1.743468619676895, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.2084803581237793, + "step": 3920, + "unclipped_grad_norm": 1.2279266119003296 + }, + { + "bits_per_byte": 1.721944302990811, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1935608386993408, + "step": 3930, + "unclipped_grad_norm": 1.008062720298767 + }, + { + "bits_per_byte": 1.6512047430032224, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1445279121398926, + "step": 3940, + "unclipped_grad_norm": 0.9797955751419067 + }, + { + "bits_per_byte": 1.750776850442659, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2135460376739502, + "step": 3950, + "unclipped_grad_norm": 1.0508832931518555 + }, + { + "bits_per_byte": 1.7346926889686676, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.202397346496582, + "step": 3960, + "unclipped_grad_norm": 1.0876473188400269 + }, + { + "bits_per_byte": 1.6890646597988592, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1707704067230225, + "step": 3970, + "unclipped_grad_norm": 1.0190049409866333 + }, + { + "bits_per_byte": 1.6854375456921626, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1682562828063965, + "step": 3980, + "unclipped_grad_norm": 1.0094109773635864 + }, + { + "bits_per_byte": 1.710786068602832, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1858265399932861, + "step": 3990, + "unclipped_grad_norm": 1.082020878791809 + }, + { + "bits_per_byte": 1.6440261871562087, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.139552116394043, + "step": 4000, + "unclipped_grad_norm": 0.9749544262886047 + }, + { + "bits_per_byte": 1.706837174956371, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1830893754959106, + "step": 4010, + "unclipped_grad_norm": 1.1478910446166992 + }, + { + "bits_per_byte": 1.7296654641012967, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1989127397537231, + "step": 4020, + "unclipped_grad_norm": 1.0259206295013428 + }, + { + "bits_per_byte": 1.6863480218458304, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1688873767852783, + "step": 4030, + "unclipped_grad_norm": 1.0554102659225464 + }, + { + "bits_per_byte": 1.6602591136231966, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1508039236068726, + "step": 4040, + "unclipped_grad_norm": 1.00386643409729 + }, + { + "bits_per_byte": 1.6813582891963477, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1654287576675415, + "step": 4050, + "unclipped_grad_norm": 0.9927709698677063 + }, + { + "bits_per_byte": 1.6720621109692604, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1589851379394531, + "step": 4060, + "unclipped_grad_norm": 1.0185434818267822 + }, + { + "bits_per_byte": 1.7963911210003984, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2451634407043457, + "step": 4070, + "unclipped_grad_norm": 1.0195153951644897 + }, + { + "bits_per_byte": 1.6278679011596953, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1283520460128784, + "step": 4080, + "unclipped_grad_norm": 0.9588671326637268 + }, + { + "bits_per_byte": 1.6697190193339053, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1573610305786133, + "step": 4090, + "unclipped_grad_norm": 1.0382872819900513 + }, + { + "bits_per_byte": 1.786499366853373, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.238306999206543, + "step": 4100, + "unclipped_grad_norm": 1.0833542346954346 + }, + { + "bits_per_byte": 1.6685632959201073, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.156559944152832, + "step": 4110, + "unclipped_grad_norm": 1.0042530298233032 + }, + { + "bits_per_byte": 1.6813281922324468, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1654078960418701, + "step": 4120, + "unclipped_grad_norm": 1.0854146480560303 + }, + { + "bits_per_byte": 1.7532864212840493, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.2152855396270752, + "step": 4130, + "unclipped_grad_norm": 1.0470608472824097 + }, + { + "bits_per_byte": 1.7313932018118645, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2001103162765503, + "step": 4140, + "unclipped_grad_norm": 1.0418859720230103 + }, + { + "bits_per_byte": 1.6694201134867057, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.157153844833374, + "step": 4150, + "unclipped_grad_norm": 1.0297493934631348 + }, + { + "bits_per_byte": 1.6896791538103921, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1711963415145874, + "step": 4160, + "unclipped_grad_norm": 1.0267698764801025 + }, + { + "bits_per_byte": 1.7146273011098527, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1884890794754028, + "step": 4170, + "unclipped_grad_norm": 1.1191542148590088 + }, + { + "bits_per_byte": 1.6875347021367837, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1697099208831787, + "step": 4180, + "unclipped_grad_norm": 1.0307927131652832 + }, + { + "bits_per_byte": 1.7039927538937512, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1811177730560303, + "step": 4190, + "unclipped_grad_norm": 1.089546799659729 + }, + { + "bits_per_byte": 1.7112641803722306, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1861579418182373, + "step": 4200, + "unclipped_grad_norm": 1.0752745866775513 + }, + { + "bits_per_byte": 1.7401058429045744, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.2061494588851929, + "step": 4210, + "unclipped_grad_norm": 1.0523611307144165 + }, + { + "bits_per_byte": 1.693671731050172, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1739637851715088, + "step": 4220, + "unclipped_grad_norm": 0.997297465801239 + }, + { + "bits_per_byte": 1.7078159282224312, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1837677955627441, + "step": 4230, + "unclipped_grad_norm": 1.0999088287353516 + }, + { + "bits_per_byte": 1.7184320732948908, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1911263465881348, + "step": 4240, + "unclipped_grad_norm": 0.9742965698242188 + }, + { + "bits_per_byte": 1.6537339198668108, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1462810039520264, + "step": 4250, + "unclipped_grad_norm": 0.9919567108154297 + }, + { + "bits_per_byte": 1.6890791063415318, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1707804203033447, + "step": 4260, + "unclipped_grad_norm": 1.0240992307662964 + }, + { + "bits_per_byte": 1.6052090149258693, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1126461029052734, + "step": 4270, + "unclipped_grad_norm": 0.9614320993423462 + }, + { + "bits_per_byte": 1.677325124050964, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1626331806182861, + "step": 4280, + "unclipped_grad_norm": 1.0358916521072388 + }, + { + "bits_per_byte": 1.6570518091672561, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.14858078956604, + "step": 4290, + "unclipped_grad_norm": 1.0068124532699585 + }, + { + "bits_per_byte": 1.6268774530733763, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1276655197143555, + "step": 4300, + "unclipped_grad_norm": 1.0985108613967896 + }, + { + "bits_per_byte": 1.6144661530912108, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1190626621246338, + "step": 4310, + "unclipped_grad_norm": 0.9936105608940125 + }, + { + "bits_per_byte": 1.7397008237617926, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.2058687210083008, + "step": 4320, + "unclipped_grad_norm": 1.1129765510559082 + }, + { + "bits_per_byte": 1.6818943591190871, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1658003330230713, + "step": 4330, + "unclipped_grad_norm": 1.0247257947921753 + }, + { + "bits_per_byte": 1.6456326770979182, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1406656503677368, + "step": 4340, + "unclipped_grad_norm": 1.0460948944091797 + }, + { + "bits_per_byte": 1.6091236840248087, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1153595447540283, + "step": 4350, + "unclipped_grad_norm": 1.0163395404815674 + }, + { + "bits_per_byte": 1.686970427059303, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1693187952041626, + "step": 4360, + "unclipped_grad_norm": 1.063033938407898 + }, + { + "bits_per_byte": 1.65933092325649, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.150160551071167, + "step": 4370, + "unclipped_grad_norm": 0.9919126033782959 + }, + { + "bits_per_byte": 1.6024796502566765, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1107542514801025, + "step": 4380, + "unclipped_grad_norm": 1.0177253484725952 + }, + { + "bits_per_byte": 1.7322169987094975, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.2006813287734985, + "step": 4390, + "unclipped_grad_norm": 0.9770697951316833 + }, + { + "bits_per_byte": 1.6952770171133256, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1750764846801758, + "step": 4400, + "unclipped_grad_norm": 1.0351864099502563 + }, + { + "bits_per_byte": 1.6960017520040616, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1755788326263428, + "step": 4410, + "unclipped_grad_norm": 1.1030160188674927 + }, + { + "bits_per_byte": 1.7127384156554266, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1871798038482666, + "step": 4420, + "unclipped_grad_norm": 1.0353890657424927 + }, + { + "bits_per_byte": 1.6726172709662455, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.159369945526123, + "step": 4430, + "unclipped_grad_norm": 1.0875210762023926 + }, + { + "bits_per_byte": 1.680560461678995, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1648757457733154, + "step": 4440, + "unclipped_grad_norm": 0.9928231239318848 + }, + { + "bits_per_byte": 1.6836972532480818, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1670500040054321, + "step": 4450, + "unclipped_grad_norm": 0.9873722791671753 + }, + { + "bits_per_byte": 1.6829482688035744, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1665308475494385, + "step": 4460, + "unclipped_grad_norm": 1.0991053581237793 + }, + { + "bits_per_byte": 1.6608082542274016, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1511845588684082, + "step": 4470, + "unclipped_grad_norm": 1.0593136548995972 + }, + { + "bits_per_byte": 1.698860447626703, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1775603294372559, + "step": 4480, + "unclipped_grad_norm": 1.0394423007965088 + }, + { + "bits_per_byte": 1.740063707155113, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.206120252609253, + "step": 4490, + "unclipped_grad_norm": 1.0491414070129395 + }, + { + "bits_per_byte": 1.6564844384020596, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.148187518119812, + "step": 4500, + "unclipped_grad_norm": 1.0240522623062134 + }, + { + "bits_per_byte": 1.6424668204608355, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.138471245765686, + "step": 4510, + "unclipped_grad_norm": 1.025274634361267 + }, + { + "bits_per_byte": 1.71951625192593, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.191877841949463, + "step": 4520, + "unclipped_grad_norm": 1.063368320465088 + }, + { + "bits_per_byte": 1.680697359869082, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1649706363677979, + "step": 4530, + "unclipped_grad_norm": 1.0728412866592407 + }, + { + "bits_per_byte": 1.6916681331676204, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1725749969482422, + "step": 4540, + "unclipped_grad_norm": 1.1416125297546387 + }, + { + "bits_per_byte": 1.6884355472620032, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1703343391418457, + "step": 4550, + "unclipped_grad_norm": 1.0188941955566406 + }, + { + "bits_per_byte": 1.6889325771229966, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1706788539886475, + "step": 4560, + "unclipped_grad_norm": 1.0978926420211792 + }, + { + "bits_per_byte": 1.6867642198609183, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1691758632659912, + "step": 4570, + "unclipped_grad_norm": 1.043805480003357 + }, + { + "bits_per_byte": 1.6647294586670736, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.153902530670166, + "step": 4580, + "unclipped_grad_norm": 1.061089277267456 + }, + { + "bits_per_byte": 1.6460370083100966, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1409459114074707, + "step": 4590, + "unclipped_grad_norm": 1.0722659826278687 + }, + { + "bits_per_byte": 1.6445638049228057, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1399247646331787, + "step": 4600, + "unclipped_grad_norm": 1.0390986204147339 + }, + { + "bits_per_byte": 1.6450002968906956, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1402273178100586, + "step": 4610, + "unclipped_grad_norm": 1.0198664665222168 + }, + { + "bits_per_byte": 1.7089066421942032, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1845238208770752, + "step": 4620, + "unclipped_grad_norm": 1.0599243640899658 + }, + { + "bits_per_byte": 1.69811834248823, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1770459413528442, + "step": 4630, + "unclipped_grad_norm": 1.1346435546875 + }, + { + "bits_per_byte": 1.7044230544862098, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1814160346984863, + "step": 4640, + "unclipped_grad_norm": 1.0656381845474243 + }, + { + "bits_per_byte": 1.7338580571640303, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.201818823814392, + "step": 4650, + "unclipped_grad_norm": 1.0118787288665771 + }, + { + "bits_per_byte": 1.652913906587497, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1457126140594482, + "step": 4660, + "unclipped_grad_norm": 1.0275835990905762 + }, + { + "bits_per_byte": 1.66807348933045, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1562204360961914, + "step": 4670, + "unclipped_grad_norm": 1.0584690570831299 + }, + { + "bits_per_byte": 1.6481818039490068, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1424325704574585, + "step": 4680, + "unclipped_grad_norm": 1.1420236825942993 + }, + { + "bits_per_byte": 1.658637833173513, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1496801376342773, + "step": 4690, + "unclipped_grad_norm": 1.080548644065857 + }, + { + "bits_per_byte": 1.6754130209386713, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1613078117370605, + "step": 4700, + "unclipped_grad_norm": 1.063712239265442 + }, + { + "bits_per_byte": 1.6541807307937524, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1465907096862793, + "step": 4710, + "unclipped_grad_norm": 1.065613865852356 + }, + { + "bits_per_byte": 1.6715305125954434, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1586166620254517, + "step": 4720, + "unclipped_grad_norm": 1.041472315788269 + }, + { + "bits_per_byte": 1.7163307892666488, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1896698474884033, + "step": 4730, + "unclipped_grad_norm": 1.0371977090835571 + }, + { + "bits_per_byte": 1.6468645888260485, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.141519546508789, + "step": 4740, + "unclipped_grad_norm": 1.118260383605957 + }, + { + "bits_per_byte": 1.6331114802018514, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1319866180419922, + "step": 4750, + "unclipped_grad_norm": 1.1252490282058716 + }, + { + "bits_per_byte": 1.6043413624522664, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1120446920394897, + "step": 4760, + "unclipped_grad_norm": 1.0504546165466309 + }, + { + "bits_per_byte": 1.6005166402797284, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.10939359664917, + "step": 4770, + "unclipped_grad_norm": 0.984404981136322 + }, + { + "bits_per_byte": 1.6404936635074865, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.13710355758667, + "step": 4780, + "unclipped_grad_norm": 1.0672088861465454 + }, + { + "bits_per_byte": 1.6438352864137509, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1394197940826416, + "step": 4790, + "unclipped_grad_norm": 1.0523202419281006 + }, + { + "bits_per_byte": 1.6719128300283113, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.158881664276123, + "step": 4800, + "unclipped_grad_norm": 1.1081420183181763 + }, + { + "bits_per_byte": 1.6392536685947656, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1362440586090088, + "step": 4810, + "unclipped_grad_norm": 1.0507419109344482 + }, + { + "bits_per_byte": 1.648858383697501, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1429015398025513, + "step": 4820, + "unclipped_grad_norm": 1.1054445505142212 + }, + { + "bits_per_byte": 1.6256828615804833, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1268374919891357, + "step": 4830, + "unclipped_grad_norm": 1.0212544202804565 + }, + { + "bits_per_byte": 1.5951660880287355, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1056848764419556, + "step": 4840, + "unclipped_grad_norm": 1.0648442506790161 + }, + { + "bits_per_byte": 1.6564994008926848, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.148197889328003, + "step": 4850, + "unclipped_grad_norm": 1.047589898109436 + }, + { + "bits_per_byte": 1.641553764567405, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.137838363647461, + "step": 4860, + "unclipped_grad_norm": 1.0177857875823975 + }, + { + "bits_per_byte": 1.6551041056462348, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1472307443618774, + "step": 4870, + "unclipped_grad_norm": 1.0283644199371338 + }, + { + "bits_per_byte": 1.6180980827121316, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1215801239013672, + "step": 4880, + "unclipped_grad_norm": 1.0304375886917114 + }, + { + "bits_per_byte": 1.5903827345602883, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1023693084716797, + "step": 4890, + "unclipped_grad_norm": 1.0921995639801025 + }, + { + "bits_per_byte": 1.567706994026678, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0866516828536987, + "step": 4900, + "unclipped_grad_norm": 1.0604755878448486 + }, + { + "bits_per_byte": 1.6439632415059928, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1395084857940674, + "step": 4910, + "unclipped_grad_norm": 1.0503017902374268 + }, + { + "bits_per_byte": 1.642120275419347, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1382310390472412, + "step": 4920, + "unclipped_grad_norm": 1.1088591814041138 + }, + { + "bits_per_byte": 1.622944209848144, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.124939203262329, + "step": 4930, + "unclipped_grad_norm": 1.099297046661377 + }, + { + "bits_per_byte": 1.6933755769253864, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1737585067749023, + "step": 4940, + "unclipped_grad_norm": 1.08061945438385 + }, + { + "bits_per_byte": 1.5670388414280758, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.086188554763794, + "step": 4950, + "unclipped_grad_norm": 1.1187851428985596 + }, + { + "bits_per_byte": 1.6577792957804058, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1490850448608398, + "step": 4960, + "unclipped_grad_norm": 1.095150351524353 + }, + { + "bits_per_byte": 1.7221861105979241, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1937284469604492, + "step": 4970, + "unclipped_grad_norm": 1.2121388912200928 + }, + { + "bits_per_byte": 1.5930121773093298, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1041918992996216, + "step": 4980, + "unclipped_grad_norm": 1.1045044660568237 + }, + { + "bits_per_byte": 1.665033523993799, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1541132926940918, + "step": 4990, + "unclipped_grad_norm": 1.1089035272598267 + }, + { + "bits_per_byte": 1.6784994215910551, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1634471416473389, + "step": 5000, + "unclipped_grad_norm": 1.0390889644622803 + }, + { + "bits_per_byte": 1.642436207548982, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.138450026512146, + "step": 5010, + "unclipped_grad_norm": 1.1114050149917603 + }, + { + "bits_per_byte": 1.6556807634745778, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1476304531097412, + "step": 5020, + "unclipped_grad_norm": 1.0220290422439575 + }, + { + "bits_per_byte": 1.636998288111351, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1346807479858398, + "step": 5030, + "unclipped_grad_norm": 1.1270323991775513 + }, + { + "bits_per_byte": 1.6052772920382619, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.112693428993225, + "step": 5040, + "unclipped_grad_norm": 1.009588360786438 + }, + { + "bits_per_byte": 1.6583182894082098, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.149458646774292, + "step": 5050, + "unclipped_grad_norm": 1.060039758682251 + }, + { + "bits_per_byte": 1.609498090255737, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1156190633773804, + "step": 5060, + "unclipped_grad_norm": 1.0725975036621094 + }, + { + "bits_per_byte": 1.6927082842400387, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1732959747314453, + "step": 5070, + "unclipped_grad_norm": 1.120245099067688 + }, + { + "bits_per_byte": 1.690623166580978, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1718506813049316, + "step": 5080, + "unclipped_grad_norm": 1.1778441667556763 + }, + { + "bits_per_byte": 1.67411111227164, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1604053974151611, + "step": 5090, + "unclipped_grad_norm": 1.3122999668121338 + }, + { + "bits_per_byte": 1.5883206625764432, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1009399890899658, + "step": 5100, + "unclipped_grad_norm": 1.0812361240386963 + }, + { + "bits_per_byte": 1.6343635139001327, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1328544616699219, + "step": 5110, + "unclipped_grad_norm": 1.054042935371399 + }, + { + "bits_per_byte": 1.6096652573923755, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1157349348068237, + "step": 5120, + "unclipped_grad_norm": 1.183487892150879 + }, + { + "bits_per_byte": 1.6704048861455463, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1578364372253418, + "step": 5130, + "unclipped_grad_norm": 1.1154927015304565 + }, + { + "bits_per_byte": 1.6086259662332116, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1150145530700684, + "step": 5140, + "unclipped_grad_norm": 1.1280518770217896 + }, + { + "bits_per_byte": 1.607841897327928, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1144710779190063, + "step": 5150, + "unclipped_grad_norm": 1.1053173542022705 + }, + { + "bits_per_byte": 1.6377722100402334, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1352171897888184, + "step": 5160, + "unclipped_grad_norm": 1.0772578716278076 + }, + { + "bits_per_byte": 1.6387738369988585, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.135911464691162, + "step": 5170, + "unclipped_grad_norm": 1.08511221408844 + }, + { + "bits_per_byte": 1.6731652076918944, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1597497463226318, + "step": 5180, + "unclipped_grad_norm": 1.0842925310134888 + }, + { + "bits_per_byte": 1.6308702022958073, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.1304330825805664, + "step": 5190, + "unclipped_grad_norm": 1.1379977464675903 + }, + { + "bits_per_byte": 1.5988843529403896, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1082621812820435, + "step": 5200, + "unclipped_grad_norm": 1.1327811479568481 + }, + { + "bits_per_byte": 1.6538405491103456, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1463549137115479, + "step": 5210, + "unclipped_grad_norm": 1.131410837173462 + }, + { + "bits_per_byte": 1.6579122383695226, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1491771936416626, + "step": 5220, + "unclipped_grad_norm": 1.111109972000122 + }, + { + "bits_per_byte": 1.6745703059494434, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1607236862182617, + "step": 5230, + "unclipped_grad_norm": 1.1052273511886597 + }, + { + "bits_per_byte": 1.6223493218588094, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.124526858329773, + "step": 5240, + "unclipped_grad_norm": 1.0814346075057983 + }, + { + "bits_per_byte": 1.5820224859191963, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0965744256973267, + "step": 5250, + "unclipped_grad_norm": 1.0832712650299072 + }, + { + "bits_per_byte": 1.5670176875620196, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0861738920211792, + "step": 5260, + "unclipped_grad_norm": 1.0310815572738647 + }, + { + "bits_per_byte": 1.6681894056371316, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1563007831573486, + "step": 5270, + "unclipped_grad_norm": 1.1380984783172607 + }, + { + "bits_per_byte": 1.6634579909292448, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.153021216392517, + "step": 5280, + "unclipped_grad_norm": 1.1410281658172607 + }, + { + "bits_per_byte": 1.627283848077365, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.127947211265564, + "step": 5290, + "unclipped_grad_norm": 1.0713688135147095 + }, + { + "bits_per_byte": 1.57530983107337, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0919215679168701, + "step": 5300, + "unclipped_grad_norm": 1.103183388710022 + }, + { + "bits_per_byte": 1.6444668067077193, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.139857530593872, + "step": 5310, + "unclipped_grad_norm": 1.1310855150222778 + }, + { + "bits_per_byte": 1.6034007893346978, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1113927364349365, + "step": 5320, + "unclipped_grad_norm": 1.0775246620178223 + }, + { + "bits_per_byte": 1.6848084331553064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1678202152252197, + "step": 5330, + "unclipped_grad_norm": 1.1329472064971924 + }, + { + "bits_per_byte": 1.5891140185448733, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1014899015426636, + "step": 5340, + "unclipped_grad_norm": 1.0905873775482178 + }, + { + "bits_per_byte": 1.6324792719772796, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1315484046936035, + "step": 5350, + "unclipped_grad_norm": 1.13417649269104 + }, + { + "bits_per_byte": 1.556942083961222, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0791900157928467, + "step": 5360, + "unclipped_grad_norm": 1.11483633518219 + }, + { + "bits_per_byte": 1.585282417066301, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0988340377807617, + "step": 5370, + "unclipped_grad_norm": 1.0698448419570923 + }, + { + "bits_per_byte": 1.6547109533063624, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1469582319259644, + "step": 5380, + "unclipped_grad_norm": 1.1219947338104248 + }, + { + "bits_per_byte": 1.5562904416971028, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0787383317947388, + "step": 5390, + "unclipped_grad_norm": 1.115169644355774 + }, + { + "bits_per_byte": 1.6265400231123834, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1274316310882568, + "step": 5400, + "unclipped_grad_norm": 1.079226016998291 + }, + { + "bits_per_byte": 1.5897746039068374, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1019477844238281, + "step": 5410, + "unclipped_grad_norm": 1.0649707317352295 + }, + { + "bits_per_byte": 1.5705084194265821, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0885934829711914, + "step": 5420, + "unclipped_grad_norm": 1.115608811378479 + }, + { + "bits_per_byte": 1.6107165873370999, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1164636611938477, + "step": 5430, + "unclipped_grad_norm": 1.1144897937774658 + }, + { + "bits_per_byte": 1.5986757379848928, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1081175804138184, + "step": 5440, + "unclipped_grad_norm": 1.0857055187225342 + }, + { + "bits_per_byte": 1.5965995634236771, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1066784858703613, + "step": 5450, + "unclipped_grad_norm": 1.0530359745025635 + }, + { + "bits_per_byte": 1.5899876904112564, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.1020954847335815, + "step": 5460, + "unclipped_grad_norm": 1.1017006635665894 + }, + { + "bits_per_byte": 1.611714258694755, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1171551942825317, + "step": 5470, + "unclipped_grad_norm": 1.11431884765625 + }, + { + "bits_per_byte": 1.5834313677950644, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0975509881973267, + "step": 5480, + "unclipped_grad_norm": 1.0543001890182495 + }, + { + "bits_per_byte": 1.6635541292310771, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.153087854385376, + "step": 5490, + "unclipped_grad_norm": 1.154997706413269 + }, + { + "bits_per_byte": 1.62576197359988, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.126892328262329, + "step": 5500, + "unclipped_grad_norm": 1.1008871793746948 + }, + { + "bits_per_byte": 1.5806631350467766, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0956321954727173, + "step": 5510, + "unclipped_grad_norm": 1.0707919597625732 + }, + { + "bits_per_byte": 1.5578388015028208, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0798115730285645, + "step": 5520, + "unclipped_grad_norm": 1.0223692655563354 + }, + { + "bits_per_byte": 1.573020741990386, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0903348922729492, + "step": 5530, + "unclipped_grad_norm": 1.1420265436172485 + }, + { + "bits_per_byte": 1.6833484724321321, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1668082475662231, + "step": 5540, + "unclipped_grad_norm": 1.1607446670532227 + }, + { + "bits_per_byte": 1.6178791487975832, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.121428370475769, + "step": 5550, + "unclipped_grad_norm": 1.1443098783493042 + }, + { + "bits_per_byte": 1.6447739677221602, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1400704383850098, + "step": 5560, + "unclipped_grad_norm": 1.1718472242355347 + }, + { + "bits_per_byte": 1.59615963180277, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1063735485076904, + "step": 5570, + "unclipped_grad_norm": 1.093506097793579 + }, + { + "bits_per_byte": 1.624393163681663, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1259435415267944, + "step": 5580, + "unclipped_grad_norm": 1.0647783279418945 + }, + { + "bits_per_byte": 1.6292150412639035, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1292858123779297, + "step": 5590, + "unclipped_grad_norm": 1.1650937795639038 + }, + { + "bits_per_byte": 1.6101528282075717, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1160728931427002, + "step": 5600, + "unclipped_grad_norm": 1.1434698104858398 + }, + { + "bits_per_byte": 1.5665055232277503, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.085818886756897, + "step": 5610, + "unclipped_grad_norm": 1.1304930448532104 + }, + { + "bits_per_byte": 1.6202220684502873, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1230523586273193, + "step": 5620, + "unclipped_grad_norm": 1.1116455793380737 + }, + { + "bits_per_byte": 1.538340096478599, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.066296100616455, + "step": 5630, + "unclipped_grad_norm": 1.058684229850769 + }, + { + "bits_per_byte": 1.5357409226761094, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0644944906234741, + "step": 5640, + "unclipped_grad_norm": 1.033333420753479 + }, + { + "bits_per_byte": 1.6143326945541414, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1189701557159424, + "step": 5650, + "unclipped_grad_norm": 1.1008256673812866 + }, + { + "bits_per_byte": 1.5570710709493691, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0792794227600098, + "step": 5660, + "unclipped_grad_norm": 1.1397353410720825 + }, + { + "bits_per_byte": 1.6224767610030988, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.12461519241333, + "step": 5670, + "unclipped_grad_norm": 1.1256861686706543 + }, + { + "bits_per_byte": 1.6278986860541997, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.128373384475708, + "step": 5680, + "unclipped_grad_norm": 1.2497283220291138 + }, + { + "bits_per_byte": 1.6402167714395974, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1369116306304932, + "step": 5690, + "unclipped_grad_norm": 1.099176049232483 + }, + { + "bits_per_byte": 1.6346018818542285, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1330196857452393, + "step": 5700, + "unclipped_grad_norm": 1.132989525794983 + }, + { + "bits_per_byte": 1.6142821316547877, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1189351081848145, + "step": 5710, + "unclipped_grad_norm": 1.169878363609314 + }, + { + "bits_per_byte": 1.6389509791292471, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1360342502593994, + "step": 5720, + "unclipped_grad_norm": 1.1713305711746216 + }, + { + "bits_per_byte": 1.5731861893005161, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.090449571609497, + "step": 5730, + "unclipped_grad_norm": 1.0991312265396118 + }, + { + "bits_per_byte": 1.642317195554585, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1383675336837769, + "step": 5740, + "unclipped_grad_norm": 1.074776530265808 + }, + { + "bits_per_byte": 1.5380594207923908, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0661015510559082, + "step": 5750, + "unclipped_grad_norm": 1.1045373678207397 + }, + { + "bits_per_byte": 1.574528857855802, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.09138023853302, + "step": 5760, + "unclipped_grad_norm": 1.1376982927322388 + }, + { + "bits_per_byte": 1.5630001728378644, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.083389163017273, + "step": 5770, + "unclipped_grad_norm": 1.1196208000183105 + }, + { + "bits_per_byte": 1.658872245526639, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.149842619895935, + "step": 5780, + "unclipped_grad_norm": 1.1231145858764648 + }, + { + "bits_per_byte": 1.4960170579104213, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0369600057601929, + "step": 5790, + "unclipped_grad_norm": 1.1306540966033936 + }, + { + "bits_per_byte": 1.632525363327711, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.1315803527832031, + "step": 5800, + "unclipped_grad_norm": 1.1228681802749634 + }, + { + "bits_per_byte": 1.5271961366506415, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.058571696281433, + "step": 5810, + "unclipped_grad_norm": 1.0998179912567139 + }, + { + "bits_per_byte": 1.544303422914616, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0704295635223389, + "step": 5820, + "unclipped_grad_norm": 1.1354378461837769 + }, + { + "bits_per_byte": 1.5973088198758354, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.1071701049804688, + "step": 5830, + "unclipped_grad_norm": 1.112725853919983 + }, + { + "bits_per_byte": 1.5755839714188453, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.092111587524414, + "step": 5840, + "unclipped_grad_norm": 1.105390191078186 + }, + { + "bits_per_byte": 1.551858620767019, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0756664276123047, + "step": 5850, + "unclipped_grad_norm": 1.0705221891403198 + }, + { + "bits_per_byte": 1.58110633433805, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0959393978118896, + "step": 5860, + "unclipped_grad_norm": 1.1176000833511353 + }, + { + "bits_per_byte": 1.6256357383341469, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1268048286437988, + "step": 5870, + "unclipped_grad_norm": 1.1718623638153076 + }, + { + "bits_per_byte": 1.5672180473502748, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0863127708435059, + "step": 5880, + "unclipped_grad_norm": 1.1114801168441772 + }, + { + "bits_per_byte": 1.6374564498932493, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1349983215332031, + "step": 5890, + "unclipped_grad_norm": 1.2559460401535034 + }, + { + "bits_per_byte": 1.6402117839427224, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1369081735610962, + "step": 5900, + "unclipped_grad_norm": 1.1818214654922485 + }, + { + "bits_per_byte": 1.5692250848858438, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0877039432525635, + "step": 5910, + "unclipped_grad_norm": 1.0690149068832397 + }, + { + "bits_per_byte": 1.5417845650100792, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0686836242675781, + "step": 5920, + "unclipped_grad_norm": 1.1236122846603394 + }, + { + "bits_per_byte": 1.6662781624380931, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1549760103225708, + "step": 5930, + "unclipped_grad_norm": 1.2468876838684082 + }, + { + "bits_per_byte": 1.596861664983592, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.1068601608276367, + "step": 5940, + "unclipped_grad_norm": 1.2251354455947876 + }, + { + "bits_per_byte": 1.5910101272706358, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.102804183959961, + "step": 5950, + "unclipped_grad_norm": 1.11275053024292 + }, + { + "bits_per_byte": 1.5416916943786134, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0686192512512207, + "step": 5960, + "unclipped_grad_norm": 1.1575579643249512 + }, + { + "bits_per_byte": 1.5659371205666486, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0854249000549316, + "step": 5970, + "unclipped_grad_norm": 1.102811336517334 + }, + { + "bits_per_byte": 1.5505245513442762, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0747417211532593, + "step": 5980, + "unclipped_grad_norm": 1.1679552793502808 + }, + { + "bits_per_byte": 1.5609080038901184, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0819389820098877, + "step": 5990, + "unclipped_grad_norm": 1.1159348487854004 + }, + { + "bits_per_byte": 1.6522259759840456, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1452357769012451, + "step": 6000, + "unclipped_grad_norm": 1.1946688890457153 + }, + { + "bits_per_byte": 1.610257393659296, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.116145372390747, + "step": 6010, + "unclipped_grad_norm": 1.2001700401306152 + }, + { + "bits_per_byte": 1.5550891428808262, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0779056549072266, + "step": 6020, + "unclipped_grad_norm": 1.11931574344635 + }, + { + "bits_per_byte": 1.5522581364649735, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0759433507919312, + "step": 6030, + "unclipped_grad_norm": 1.0901384353637695 + }, + { + "bits_per_byte": 1.5778765001548465, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.093700647354126, + "step": 6040, + "unclipped_grad_norm": 1.1465646028518677 + }, + { + "bits_per_byte": 1.585459043248737, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0989564657211304, + "step": 6050, + "unclipped_grad_norm": 1.093497395515442 + }, + { + "bits_per_byte": 1.580860743112618, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0957691669464111, + "step": 6060, + "unclipped_grad_norm": 1.1618996858596802 + }, + { + "bits_per_byte": 1.4692863104347675, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0184316635131836, + "step": 6070, + "unclipped_grad_norm": 1.095646619796753 + }, + { + "bits_per_byte": 1.5593341906520728, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0808480978012085, + "step": 6080, + "unclipped_grad_norm": 1.2226048707962036 + }, + { + "bits_per_byte": 1.5548665973306097, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0777513980865479, + "step": 6090, + "unclipped_grad_norm": 1.160715103149414 + }, + { + "bits_per_byte": 1.6437169623499572, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.1393377780914307, + "step": 6100, + "unclipped_grad_norm": 1.216336965560913 + }, + { + "bits_per_byte": 1.5953482176559992, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1058111190795898, + "step": 6110, + "unclipped_grad_norm": 1.1638948917388916 + }, + { + "bits_per_byte": 1.5315409343593887, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0615832805633545, + "step": 6120, + "unclipped_grad_norm": 1.1416045427322388 + }, + { + "bits_per_byte": 1.6181004904692438, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.121581792831421, + "step": 6130, + "unclipped_grad_norm": 1.1373443603515625 + }, + { + "bits_per_byte": 1.5591868015202834, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0807459354400635, + "step": 6140, + "unclipped_grad_norm": 1.0817177295684814 + }, + { + "bits_per_byte": 1.5489426549216403, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0736452341079712, + "step": 6150, + "unclipped_grad_norm": 1.156706690788269 + }, + { + "bits_per_byte": 1.6171114182441317, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1208962202072144, + "step": 6160, + "unclipped_grad_norm": 1.2064051628112793 + }, + { + "bits_per_byte": 1.5388670513208427, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0666613578796387, + "step": 6170, + "unclipped_grad_norm": 1.0822783708572388 + }, + { + "bits_per_byte": 1.5490690621700245, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.073732852935791, + "step": 6180, + "unclipped_grad_norm": 1.109446406364441 + }, + { + "bits_per_byte": 1.5625440748477764, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0830730199813843, + "step": 6190, + "unclipped_grad_norm": 1.1676242351531982 + }, + { + "bits_per_byte": 1.5465116801516945, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.071960210800171, + "step": 6200, + "unclipped_grad_norm": 1.094281792640686 + }, + { + "bits_per_byte": 1.5807004552820139, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0956580638885498, + "step": 6210, + "unclipped_grad_norm": 1.2013437747955322 + }, + { + "bits_per_byte": 1.592342648849521, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.1037278175354004, + "step": 6220, + "unclipped_grad_norm": 1.1159899234771729 + }, + { + "bits_per_byte": 1.550343453612918, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0746161937713623, + "step": 6230, + "unclipped_grad_norm": 1.156030535697937 + }, + { + "bits_per_byte": 1.5936101609863798, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.1046063899993896, + "step": 6240, + "unclipped_grad_norm": 1.2167954444885254 + }, + { + "bits_per_byte": 1.5477754086702344, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.07283616065979, + "step": 6250, + "unclipped_grad_norm": 1.1424623727798462 + }, + { + "bits_per_byte": 1.5145149958892723, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0497817993164062, + "step": 6260, + "unclipped_grad_norm": 1.159928798675537 + }, + { + "bits_per_byte": 1.558105374611658, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0799963474273682, + "step": 6270, + "unclipped_grad_norm": 1.1427968740463257 + }, + { + "bits_per_byte": 1.5868821996846267, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.099942922592163, + "step": 6280, + "unclipped_grad_norm": 1.2086946964263916 + }, + { + "bits_per_byte": 1.5572024656946284, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0793704986572266, + "step": 6290, + "unclipped_grad_norm": 1.26358163356781 + }, + { + "bits_per_byte": 1.525056844456559, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.057088851928711, + "step": 6300, + "unclipped_grad_norm": 1.1142038106918335 + }, + { + "bits_per_byte": 1.585295831713068, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0988433361053467, + "step": 6310, + "unclipped_grad_norm": 1.202506422996521 + }, + { + "bits_per_byte": 1.5589988244828903, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0806156396865845, + "step": 6320, + "unclipped_grad_norm": 1.1080162525177002 + }, + { + "bits_per_byte": 1.5721742433828394, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0897481441497803, + "step": 6330, + "unclipped_grad_norm": 1.1463266611099243 + }, + { + "bits_per_byte": 1.4897746036320538, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0326330661773682, + "step": 6340, + "unclipped_grad_norm": 1.0811859369277954 + }, + { + "bits_per_byte": 1.5979289893148465, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1075999736785889, + "step": 6350, + "unclipped_grad_norm": 1.1469974517822266 + }, + { + "bits_per_byte": 1.6121533304024078, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.1174595355987549, + "step": 6360, + "unclipped_grad_norm": 1.2663604021072388 + }, + { + "bits_per_byte": 1.522211219515383, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0551164150238037, + "step": 6370, + "unclipped_grad_norm": 1.136039137840271 + }, + { + "bits_per_byte": 1.5209657211578345, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.054253101348877, + "step": 6380, + "unclipped_grad_norm": 1.1138842105865479 + }, + { + "bits_per_byte": 1.6340888576067047, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1326640844345093, + "step": 6390, + "unclipped_grad_norm": 1.227125883102417 + }, + { + "bits_per_byte": 1.5932569086215076, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1043615341186523, + "step": 6400, + "unclipped_grad_norm": 1.1935430765151978 + }, + { + "bits_per_byte": 1.6221133616618257, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.1243633031845093, + "step": 6410, + "unclipped_grad_norm": 1.289966106414795 + }, + { + "bits_per_byte": 1.5341647016809516, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0634019374847412, + "step": 6420, + "unclipped_grad_norm": 1.2202966213226318 + }, + { + "bits_per_byte": 1.5828500664351481, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.097148060798645, + "step": 6430, + "unclipped_grad_norm": 1.1407660245895386 + }, + { + "bits_per_byte": 1.518648426920109, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0526468753814697, + "step": 6440, + "unclipped_grad_norm": 1.1754008531570435 + }, + { + "bits_per_byte": 1.5227140967865058, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0554649829864502, + "step": 6450, + "unclipped_grad_norm": 1.1674573421478271 + }, + { + "bits_per_byte": 1.5506523344538674, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0748302936553955, + "step": 6460, + "unclipped_grad_norm": 1.1624079942703247 + }, + { + "bits_per_byte": 1.5647150118496176, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0845777988433838, + "step": 6470, + "unclipped_grad_norm": 1.1915463209152222 + }, + { + "bits_per_byte": 1.5303090226312583, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0607293844223022, + "step": 6480, + "unclipped_grad_norm": 1.1169558763504028 + }, + { + "bits_per_byte": 1.610437975442702, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1162705421447754, + "step": 6490, + "unclipped_grad_norm": 1.212812900543213 + }, + { + "bits_per_byte": 1.6629645727039195, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.152679204940796, + "step": 6500, + "unclipped_grad_norm": 1.249900221824646 + }, + { + "bits_per_byte": 1.6085310318099353, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.1149487495422363, + "step": 6510, + "unclipped_grad_norm": 1.2940583229064941 + }, + { + "bits_per_byte": 1.5857588090091908, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0991642475128174, + "step": 6520, + "unclipped_grad_norm": 1.1899638175964355 + }, + { + "bits_per_byte": 1.557970024265429, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0799025297164917, + "step": 6530, + "unclipped_grad_norm": 1.18255615234375 + }, + { + "bits_per_byte": 1.5442463246745295, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.070389986038208, + "step": 6540, + "unclipped_grad_norm": 1.1805294752120972 + }, + { + "bits_per_byte": 1.5888496812104973, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.101306676864624, + "step": 6550, + "unclipped_grad_norm": 1.1853868961334229 + }, + { + "bits_per_byte": 1.598041465968511, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.107677936553955, + "step": 6560, + "unclipped_grad_norm": 1.167456865310669 + }, + { + "bits_per_byte": 1.5763221209563485, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.092623233795166, + "step": 6570, + "unclipped_grad_norm": 1.2579692602157593 + }, + { + "bits_per_byte": 1.5598688847136053, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0812187194824219, + "step": 6580, + "unclipped_grad_norm": 1.218462347984314 + }, + { + "bits_per_byte": 1.5745766690327418, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0914133787155151, + "step": 6590, + "unclipped_grad_norm": 1.1964205503463745 + }, + { + "bits_per_byte": 1.483007602268555, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0279425382614136, + "step": 6600, + "unclipped_grad_norm": 1.1824510097503662 + }, + { + "bits_per_byte": 1.5325055850480782, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0622519254684448, + "step": 6610, + "unclipped_grad_norm": 1.1718802452087402 + }, + { + "bits_per_byte": 1.648197798335537, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1424436569213867, + "step": 6620, + "unclipped_grad_norm": 1.2758471965789795 + }, + { + "bits_per_byte": 1.5936395700196773, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.1046267747879028, + "step": 6630, + "unclipped_grad_norm": 1.1752488613128662 + }, + { + "bits_per_byte": 1.550369594975849, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.074634313583374, + "step": 6640, + "unclipped_grad_norm": 1.1907187700271606 + }, + { + "bits_per_byte": 1.539030778804464, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.066774845123291, + "step": 6650, + "unclipped_grad_norm": 1.172040343284607 + }, + { + "bits_per_byte": 1.5824082430050816, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.096841812133789, + "step": 6660, + "unclipped_grad_norm": 1.2395251989364624 + }, + { + "bits_per_byte": 1.55066678099654, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0748403072357178, + "step": 6670, + "unclipped_grad_norm": 1.1819844245910645 + }, + { + "bits_per_byte": 1.567913545190364, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0867948532104492, + "step": 6680, + "unclipped_grad_norm": 1.207129955291748 + }, + { + "bits_per_byte": 1.5601953077849429, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0814449787139893, + "step": 6690, + "unclipped_grad_norm": 1.284845232963562 + }, + { + "bits_per_byte": 1.5747911313983678, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.091562032699585, + "step": 6700, + "unclipped_grad_norm": 1.1885749101638794 + }, + { + "bits_per_byte": 1.5878793550943293, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.1006340980529785, + "step": 6710, + "unclipped_grad_norm": 1.178633689880371 + }, + { + "bits_per_byte": 1.6366173465396898, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1344166994094849, + "step": 6720, + "unclipped_grad_norm": 1.265533685684204 + }, + { + "bits_per_byte": 1.5154492056487592, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.050429344177246, + "step": 6730, + "unclipped_grad_norm": 1.138245940208435 + }, + { + "bits_per_byte": 1.547962697777024, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0729659795761108, + "step": 6740, + "unclipped_grad_norm": 1.167292833328247 + }, + { + "bits_per_byte": 1.5245414124519232, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0567315816879272, + "step": 6750, + "unclipped_grad_norm": 1.1478474140167236 + }, + { + "bits_per_byte": 1.5232747602283185, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0558536052703857, + "step": 6760, + "unclipped_grad_norm": 1.185871958732605 + }, + { + "bits_per_byte": 1.5451932611501802, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0710463523864746, + "step": 6770, + "unclipped_grad_norm": 1.2135790586471558 + }, + { + "bits_per_byte": 1.5261805790972967, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0578677654266357, + "step": 6780, + "unclipped_grad_norm": 1.2295221090316772 + }, + { + "bits_per_byte": 1.5602444948230898, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0814790725708008, + "step": 6790, + "unclipped_grad_norm": 1.174469232559204 + }, + { + "bits_per_byte": 1.5335944072106906, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0630066394805908, + "step": 6800, + "unclipped_grad_norm": 1.1970314979553223 + }, + { + "bits_per_byte": 1.5348973477736272, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0639097690582275, + "step": 6810, + "unclipped_grad_norm": 1.162747859954834 + }, + { + "bits_per_byte": 1.5590328770477613, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0806392431259155, + "step": 6820, + "unclipped_grad_norm": 1.2186238765716553 + }, + { + "bits_per_byte": 1.5711721004762618, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0890535116195679, + "step": 6830, + "unclipped_grad_norm": 1.1893293857574463 + }, + { + "bits_per_byte": 1.526197777362383, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0578796863555908, + "step": 6840, + "unclipped_grad_norm": 1.2071653604507446 + }, + { + "bits_per_byte": 1.624858204769596, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.1262658834457397, + "step": 6850, + "unclipped_grad_norm": 1.2998149394989014 + }, + { + "bits_per_byte": 1.574837222748799, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0915939807891846, + "step": 6860, + "unclipped_grad_norm": 1.2379318475723267 + }, + { + "bits_per_byte": 1.5010148737444944, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0404242277145386, + "step": 6870, + "unclipped_grad_norm": 1.1776995658874512 + }, + { + "bits_per_byte": 1.5287346934452601, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0596381425857544, + "step": 6880, + "unclipped_grad_norm": 1.2149008512496948 + }, + { + "bits_per_byte": 1.5161942344922967, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.05094575881958, + "step": 6890, + "unclipped_grad_norm": 1.1354871988296509 + }, + { + "bits_per_byte": 1.5760503883679853, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0924348831176758, + "step": 6900, + "unclipped_grad_norm": 1.2098510265350342 + }, + { + "bits_per_byte": 1.5705858116194704, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0886471271514893, + "step": 6910, + "unclipped_grad_norm": 1.2706652879714966 + }, + { + "bits_per_byte": 1.523291786510754, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0558654069900513, + "step": 6920, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 1.5043128130574397, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0427101850509644, + "step": 6930, + "unclipped_grad_norm": 1.1997718811035156 + }, + { + "bits_per_byte": 1.5789452003473081, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0944414138793945, + "step": 6940, + "unclipped_grad_norm": 1.2278647422790527 + }, + { + "bits_per_byte": 1.4992432804579567, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.039196252822876, + "step": 6950, + "unclipped_grad_norm": 1.2732465267181396 + }, + { + "bits_per_byte": 1.4942526878952196, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0357370376586914, + "step": 6960, + "unclipped_grad_norm": 1.1388111114501953 + }, + { + "bits_per_byte": 1.4775614276636824, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.024167537689209, + "step": 6970, + "unclipped_grad_norm": 1.2496774196624756 + }, + { + "bits_per_byte": 1.5972561931846714, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1071336269378662, + "step": 6980, + "unclipped_grad_norm": 1.2329373359680176 + }, + { + "bits_per_byte": 1.5648880263963856, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0846977233886719, + "step": 6990, + "unclipped_grad_norm": 1.225949764251709 + }, + { + "bits_per_byte": 1.5748815942727217, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0916247367858887, + "step": 7000, + "unclipped_grad_norm": 1.2655893564224243 + }, + { + "bits_per_byte": 1.4699690815586928, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0189049243927002, + "step": 7010, + "unclipped_grad_norm": 1.1935946941375732 + }, + { + "bits_per_byte": 1.5527095909234883, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.076256275177002, + "step": 7020, + "unclipped_grad_norm": 1.2986186742782593 + }, + { + "bits_per_byte": 1.5324563980099315, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0622178316116333, + "step": 7030, + "unclipped_grad_norm": 1.2135052680969238 + }, + { + "bits_per_byte": 1.5215922539549278, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0546873807907104, + "step": 7040, + "unclipped_grad_norm": 1.1919881105422974 + }, + { + "bits_per_byte": 1.5129705916845244, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0487112998962402, + "step": 7050, + "unclipped_grad_norm": 1.177833914756775 + }, + { + "bits_per_byte": 1.5002927185935215, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0399236679077148, + "step": 7060, + "unclipped_grad_norm": 1.2103830575942993 + }, + { + "bits_per_byte": 1.5379128915738558, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.065999984741211, + "step": 7070, + "unclipped_grad_norm": 1.2223129272460938 + }, + { + "bits_per_byte": 1.5494862920810175, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0740220546722412, + "step": 7080, + "unclipped_grad_norm": 1.2125953435897827 + }, + { + "bits_per_byte": 1.5050088268454815, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0431926250457764, + "step": 7090, + "unclipped_grad_norm": 1.274321436882019 + }, + { + "bits_per_byte": 1.5086218383748073, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0456969738006592, + "step": 7100, + "unclipped_grad_norm": 1.244695782661438 + }, + { + "bits_per_byte": 1.480645764524256, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0263054370880127, + "step": 7110, + "unclipped_grad_norm": 1.1544733047485352 + }, + { + "bits_per_byte": 1.5449419944972695, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.070872187614441, + "step": 7120, + "unclipped_grad_norm": 1.25104558467865 + }, + { + "bits_per_byte": 1.5693447848108444, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0877869129180908, + "step": 7130, + "unclipped_grad_norm": 1.2125219106674194 + }, + { + "bits_per_byte": 1.5401648324042534, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0675609111785889, + "step": 7140, + "unclipped_grad_norm": 1.2392476797103882 + }, + { + "bits_per_byte": 1.581065058501843, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0959107875823975, + "step": 7150, + "unclipped_grad_norm": 1.2786449193954468 + }, + { + "bits_per_byte": 1.568793580414829, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0874048471450806, + "step": 7160, + "unclipped_grad_norm": 1.2588245868682861 + }, + { + "bits_per_byte": 1.5794673396753276, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0948033332824707, + "step": 7170, + "unclipped_grad_norm": 1.2507553100585938 + }, + { + "bits_per_byte": 1.519351835962138, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0531344413757324, + "step": 7180, + "unclipped_grad_norm": 1.194468379020691 + }, + { + "bits_per_byte": 1.5739864245749806, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0910042524337769, + "step": 7190, + "unclipped_grad_norm": 1.2753334045410156 + }, + { + "bits_per_byte": 1.5095894127685614, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0463676452636719, + "step": 7200, + "unclipped_grad_norm": 1.1532927751541138 + }, + { + "bits_per_byte": 1.5104264523303108, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0469478368759155, + "step": 7210, + "unclipped_grad_norm": 1.1723830699920654 + }, + { + "bits_per_byte": 1.5078061246617651, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0451315641403198, + "step": 7220, + "unclipped_grad_norm": 1.254270076751709 + }, + { + "bits_per_byte": 1.5518125294165879, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.075634479522705, + "step": 7230, + "unclipped_grad_norm": 1.2554386854171753 + }, + { + "bits_per_byte": 1.5582962753541159, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0801286697387695, + "step": 7240, + "unclipped_grad_norm": 1.2507786750793457 + }, + { + "bits_per_byte": 1.571778683285855, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0894739627838135, + "step": 7250, + "unclipped_grad_norm": 1.2601356506347656 + }, + { + "bits_per_byte": 1.5335579468887077, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.062981367111206, + "step": 7260, + "unclipped_grad_norm": 1.203548789024353 + }, + { + "bits_per_byte": 1.5322409037484004, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0620684623718262, + "step": 7270, + "unclipped_grad_norm": 1.2632365226745605 + }, + { + "bits_per_byte": 1.522663877852454, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0554301738739014, + "step": 7280, + "unclipped_grad_norm": 1.226701259613037 + }, + { + "bits_per_byte": 1.5219511817472784, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.054936170578003, + "step": 7290, + "unclipped_grad_norm": 1.2358945608139038 + }, + { + "bits_per_byte": 1.5307491262348163, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0610344409942627, + "step": 7300, + "unclipped_grad_norm": 1.1983592510223389 + }, + { + "bits_per_byte": 1.5145411372522035, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.049799919128418, + "step": 7310, + "unclipped_grad_norm": 1.224237084388733 + }, + { + "bits_per_byte": 1.4925153191562035, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0345327854156494, + "step": 7320, + "unclipped_grad_norm": 1.2309412956237793 + }, + { + "bits_per_byte": 1.55153787312316, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0754441022872925, + "step": 7330, + "unclipped_grad_norm": 1.2790515422821045 + }, + { + "bits_per_byte": 1.5015385609163716, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0407872200012207, + "step": 7340, + "unclipped_grad_norm": 1.2610660791397095 + }, + { + "bits_per_byte": 1.5181352306899345, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0522911548614502, + "step": 7350, + "unclipped_grad_norm": 1.2389053106307983 + }, + { + "bits_per_byte": 1.5177651240252776, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.052034616470337, + "step": 7360, + "unclipped_grad_norm": 1.2198708057403564 + }, + { + "bits_per_byte": 1.4782792832483838, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.024665117263794, + "step": 7370, + "unclipped_grad_norm": 1.2335501909255981 + }, + { + "bits_per_byte": 1.4776752801785535, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0242464542388916, + "step": 7380, + "unclipped_grad_norm": 1.1802135705947876 + }, + { + "bits_per_byte": 1.4876678161589845, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.031172752380371, + "step": 7390, + "unclipped_grad_norm": 1.2134222984313965 + }, + { + "bits_per_byte": 1.508026778402822, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0452845096588135, + "step": 7400, + "unclipped_grad_norm": 1.23512601852417 + }, + { + "bits_per_byte": 1.5449191208047048, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0708563327789307, + "step": 7410, + "unclipped_grad_norm": 1.250113606452942 + }, + { + "bits_per_byte": 1.5275736385692853, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.058833360671997, + "step": 7420, + "unclipped_grad_norm": 1.2959712743759155 + }, + { + "bits_per_byte": 1.4779425412179943, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0244317054748535, + "step": 7430, + "unclipped_grad_norm": 1.2527908086776733 + }, + { + "bits_per_byte": 1.468023269846831, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0175561904907227, + "step": 7440, + "unclipped_grad_norm": 1.2249972820281982 + }, + { + "bits_per_byte": 1.5643605556061893, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0843321084976196, + "step": 7450, + "unclipped_grad_norm": 1.2696515321731567 + }, + { + "bits_per_byte": 1.5085103936170483, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0456197261810303, + "step": 7460, + "unclipped_grad_norm": 1.2498137950897217 + }, + { + "bits_per_byte": 1.4827952036947394, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0277953147888184, + "step": 7470, + "unclipped_grad_norm": 1.2631312608718872 + }, + { + "bits_per_byte": 1.5577199614910746, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0797291994094849, + "step": 7480, + "unclipped_grad_norm": 1.1877262592315674 + }, + { + "bits_per_byte": 1.530344623039987, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0607540607452393, + "step": 7490, + "unclipped_grad_norm": 1.268275499343872 + }, + { + "bits_per_byte": 1.5391859071555423, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0668823719024658, + "step": 7500, + "unclipped_grad_norm": 1.3035756349563599 + }, + { + "bits_per_byte": 1.5412744644676202, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0683300495147705, + "step": 7510, + "unclipped_grad_norm": 1.2108540534973145 + }, + { + "bits_per_byte": 1.5426088778556648, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.069254994392395, + "step": 7520, + "unclipped_grad_norm": 1.265081524848938 + }, + { + "bits_per_byte": 1.5294915890917073, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0601627826690674, + "step": 7530, + "unclipped_grad_norm": 1.2194392681121826 + }, + { + "bits_per_byte": 1.5287204188852386, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0596282482147217, + "step": 7540, + "unclipped_grad_norm": 1.2046699523925781 + }, + { + "bits_per_byte": 1.5311642923539992, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0613222122192383, + "step": 7550, + "unclipped_grad_norm": 1.2151964902877808 + }, + { + "bits_per_byte": 1.4454386801181778, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0019017457962036, + "step": 7560, + "unclipped_grad_norm": 1.2129703760147095 + }, + { + "bits_per_byte": 1.4692137337561033, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0183813571929932, + "step": 7570, + "unclipped_grad_norm": 1.2061578035354614 + }, + { + "bits_per_byte": 1.4652694836412157, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0156474113464355, + "step": 7580, + "unclipped_grad_norm": 1.1768887042999268 + }, + { + "bits_per_byte": 1.5398244787381958, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0673249959945679, + "step": 7590, + "unclipped_grad_norm": 1.2310807704925537 + }, + { + "bits_per_byte": 1.5671884663343265, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0862922668457031, + "step": 7600, + "unclipped_grad_norm": 1.2284492254257202 + }, + { + "bits_per_byte": 1.5395315922837765, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.067121982574463, + "step": 7610, + "unclipped_grad_norm": 1.2651493549346924 + }, + { + "bits_per_byte": 1.5411196800818436, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0682227611541748, + "step": 7620, + "unclipped_grad_norm": 1.2471308708190918 + }, + { + "bits_per_byte": 1.5165897945892812, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0512199401855469, + "step": 7630, + "unclipped_grad_norm": 1.253797173500061 + }, + { + "bits_per_byte": 1.497296092884888, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.037846565246582, + "step": 7640, + "unclipped_grad_norm": 1.318769931793213 + }, + { + "bits_per_byte": 1.4637429656321574, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0145893096923828, + "step": 7650, + "unclipped_grad_norm": 1.2420055866241455 + }, + { + "bits_per_byte": 1.4840704550508872, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0286792516708374, + "step": 7660, + "unclipped_grad_norm": 1.299368977546692 + }, + { + "bits_per_byte": 1.5379041204586616, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0659939050674438, + "step": 7670, + "unclipped_grad_norm": 1.2423371076583862 + }, + { + "bits_per_byte": 1.5171365274363742, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0515989065170288, + "step": 7680, + "unclipped_grad_norm": 1.2243461608886719 + }, + { + "bits_per_byte": 1.5051288707357837, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0432758331298828, + "step": 7690, + "unclipped_grad_norm": 1.245628833770752 + }, + { + "bits_per_byte": 1.544223278999314, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0703740119934082, + "step": 7700, + "unclipped_grad_norm": 1.2915515899658203 + }, + { + "bits_per_byte": 1.4916550619365878, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0339365005493164, + "step": 7710, + "unclipped_grad_norm": 1.201813817024231 + }, + { + "bits_per_byte": 1.4940050328779773, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0355653762817383, + "step": 7720, + "unclipped_grad_norm": 1.2110096216201782 + }, + { + "bits_per_byte": 1.4993191248069873, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0392488241195679, + "step": 7730, + "unclipped_grad_norm": 1.2087814807891846 + }, + { + "bits_per_byte": 1.5356127956012164, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0644056797027588, + "step": 7740, + "unclipped_grad_norm": 1.3031466007232666 + }, + { + "bits_per_byte": 1.4696118735928507, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0186573266983032, + "step": 7750, + "unclipped_grad_norm": 1.249332070350647 + }, + { + "bits_per_byte": 1.4759148657643217, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0230262279510498, + "step": 7760, + "unclipped_grad_norm": 1.2120082378387451 + }, + { + "bits_per_byte": 1.5360676897127488, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0647209882736206, + "step": 7770, + "unclipped_grad_norm": 1.2807056903839111 + }, + { + "bits_per_byte": 1.529660132089553, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0602796077728271, + "step": 7780, + "unclipped_grad_norm": 1.2968212366104126 + }, + { + "bits_per_byte": 1.5315347429839576, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0615789890289307, + "step": 7790, + "unclipped_grad_norm": 1.2608845233917236 + }, + { + "bits_per_byte": 1.5176234103209667, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.051936388015747, + "step": 7800, + "unclipped_grad_norm": 1.2756648063659668 + }, + { + "bits_per_byte": 1.5054119541791038, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0434720516204834, + "step": 7810, + "unclipped_grad_norm": 1.2882802486419678 + }, + { + "bits_per_byte": 1.5431284374439214, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.069615125656128, + "step": 7820, + "unclipped_grad_norm": 1.3333698511123657 + }, + { + "bits_per_byte": 1.5271158207526887, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.058516025543213, + "step": 7830, + "unclipped_grad_norm": 1.229974389076233 + }, + { + "bits_per_byte": 1.501238623173267, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0405793190002441, + "step": 7840, + "unclipped_grad_norm": 1.2474827766418457 + }, + { + "bits_per_byte": 1.4945333635814277, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0359315872192383, + "step": 7850, + "unclipped_grad_norm": 1.2343677282333374 + }, + { + "bits_per_byte": 1.534035198744852, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0633121728897095, + "step": 7860, + "unclipped_grad_norm": 1.256583571434021 + }, + { + "bits_per_byte": 1.5241123157380205, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.056434154510498, + "step": 7870, + "unclipped_grad_norm": 1.3308851718902588 + }, + { + "bits_per_byte": 1.498756569556015, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0388588905334473, + "step": 7880, + "unclipped_grad_norm": 1.301889181137085 + }, + { + "bits_per_byte": 1.516417124007815, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.051100254058838, + "step": 7890, + "unclipped_grad_norm": 1.2672597169876099 + }, + { + "bits_per_byte": 1.4738032627770283, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0215625762939453, + "step": 7900, + "unclipped_grad_norm": 1.2230888605117798 + }, + { + "bits_per_byte": 1.5374244608454053, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0656614303588867, + "step": 7910, + "unclipped_grad_norm": 1.3816039562225342 + }, + { + "bits_per_byte": 1.5497416863175488, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0741990804672241, + "step": 7920, + "unclipped_grad_norm": 1.2244949340820312 + }, + { + "bits_per_byte": 1.5130077399371107, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0487370491027832, + "step": 7930, + "unclipped_grad_norm": 1.2602943181991577 + }, + { + "bits_per_byte": 1.5360826522033737, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0647313594818115, + "step": 7940, + "unclipped_grad_norm": 1.278623342514038 + }, + { + "bits_per_byte": 1.479992574416279, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0258526802062988, + "step": 7950, + "unclipped_grad_norm": 1.2290197610855103 + }, + { + "bits_per_byte": 1.5513340736818877, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0753028392791748, + "step": 7960, + "unclipped_grad_norm": 1.3293349742889404 + }, + { + "bits_per_byte": 1.5048769161522697, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.043101191520691, + "step": 7970, + "unclipped_grad_norm": 1.2828478813171387 + }, + { + "bits_per_byte": 1.5449772509406965, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0708966255187988, + "step": 7980, + "unclipped_grad_norm": 1.325577974319458 + }, + { + "bits_per_byte": 1.5031080745881458, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0418751239776611, + "step": 7990, + "unclipped_grad_norm": 1.3532280921936035 + }, + { + "bits_per_byte": 1.5250520289423348, + "learning_rate": 3e-05, + "loss_nats": 1.0570855140686035, + "step": 8000, + "unclipped_grad_norm": 1.3144081830978394 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073003.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073003.json new file mode 100644 index 0000000..5b96cb6 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-16-block-seed-2026073003.json @@ -0,0 +1,9616 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "a4c4ddb026a74a7ed70ffc58516bae9bdba5550dc232a1cb003063f2c7f9ca1d", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0003044046170543879, + 0.0002627764770295471, + 0.0002327302354387939, + 0.0002064981235889718, + 0.0001888095575850457, + 0.00017376731557305902, + 0.00016044666699599475, + 0.0001515327749075368, + 0.000144502759212628, + 0.00014044572890270501, + 0.00013478762411978096, + 0.00013080857752356678, + 0.00012769536988344043, + 0.00012482800229918212, + 0.00012209055421408266, + 0.0001202532512252219 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002516023632779252, + "first_to_last_ratio": 2.0336961080101905, + "imbalance_abs_log_ratio": 0.7098548803728145, + "last_quartile_mean": 0.00012371679440548178, + "mean": 0.00017039860222212155, + "normalized": [ + 1.7864267258341946, + 1.5421281254819637, + 1.3657989702017657, + 1.2118533890306977, + 1.1080463989893807, + 1.0197696067162934, + 0.941596145177564, + 0.8892841427772256, + 0.8480278437041564, + 0.8242187850791655, + 0.7910136724248463, + 0.7676622684560079, + 0.7493921206993488, + 0.7325647081098923, + 0.7164997401500549, + 0.7057173571674424 + ], + "population_cv": 0.31342321744292045 + }, + "activation_output_rms_by_block": [ + 0.005092422943562269, + 0.007266214583069086, + 0.00560376700013876, + 0.007757550571113825, + 0.005751485005021095, + 0.008497623726725578, + 0.006417509168386459, + 0.009013078175485134, + 0.006685016211122274, + 0.010579346679151058, + 0.007517023477703333, + 0.010370824486017227, + 0.008143824525177479, + 0.011447666212916374, + 0.009210918098688126, + 0.013363312929868698 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006429988774470985, + "first_to_last_ratio": 0.6099730781372781, + "imbalance_abs_log_ratio": 0.4943404569899061, + "last_quartile_mean": 0.01054143044166267, + "mean": 0.008294848987134174, + "normalized": [ + 0.6139259378273111, + 0.8759911837261217, + 0.675571913223562, + 0.93522505149235, + 0.6933803151741528, + 1.0244458627162376, + 0.7736740208701105, + 1.0865873736176486, + 0.8059237993954019, + 1.275411607319227, + 0.9062278878569946, + 1.250272850307826, + 0.9817929823447126, + 1.3800933845417096, + 1.1104383109294493, + 1.6110375186571844 + ], + "population_cv": 0.2672227042610828 + }, + "bits_per_byte": 8.019773211317682, + "branch_output_rms_by_sublayer": [ + 0.0034366261679679155, + 0.003824356012046337, + 0.0035537455696612597, + 0.003800130682066083, + 0.004073208663612604, + 0.0038295022677630186, + 0.004064237233251333, + 0.0037886372301727533, + 0.004280691966414452, + 0.0038349912501871586, + 0.0050888159312307835, + 0.0038198749534785748, + 0.005177160259336233, + 0.0037978272885084152, + 0.005646859761327505, + 0.003781275823712349, + 0.0056319693103432655, + 0.003643249161541462, + 0.006857539992779493, + 0.0037886397913098335, + 0.006555364932864904, + 0.003700864966958761, + 0.0065165311098098755, + 0.0036778782960027456, + 0.0071639628149569035, + 0.003695491934195161, + 0.007599730510264635, + 0.0037103460635989904, + 0.00840203370898962, + 0.0037034295964986086, + 0.008103356696665287, + 0.0036014309152960777 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006980641236819168, + 0.006790815718474037, + 0.006855853249106197, + 0.00601204909014858, + 0.005899153963056259, + 0.006300783956032697, + 0.005704077288608638, + 0.005731973180528048, + 0.005353243010845193, + 0.00629832195028187, + 0.005348066742481173, + 0.005021425359765903, + 0.004977505888369926, + 0.004996266682376707, + 0.0052721592596541, + 0.004713786845169383 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006659839823636995, + "first_to_last_ratio": 1.3346560503958143, + "imbalance_abs_log_ratio": 0.2886736185043388, + "last_quartile_mean": 0.004989929668892529, + "mean": 0.005766007713857367, + "normalized": [ + 1.2106541619850226, + 1.1777326801269727, + 1.189012153526888, + 1.0426710106023453, + 1.0230915836062624, + 1.092746362598529, + 0.9892593925776594, + 0.9940973833164453, + 0.9284141257702132, + 1.0923193763936871, + 0.9275164043967956, + 0.8708669167573224, + 0.8632499530667561, + 0.8665036417431853, + 0.914351752770568, + 0.8175131007613471 + ], + "population_cv": 0.12132893681311732 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028142472729086876, + 0.014130397699773312, + 0.014246499165892601, + 0.01435180939733982, + 0.01446386519819498, + 0.009747256524860859, + 0.009819178842008114, + 0.00989843625575304, + 0.009975779801607132, + 0.007541276980191469, + 0.007608718704432249, + 0.007703919429332018, + 0.0077462210319936275, + 0.00631322804838419, + 0.006355294957756996, + 0.006443085614591837, + 0.006486334837973118, + 0.005488897208124399, + 0.00551867438480258, + 0.005632160697132349, + 0.0056700571440160275, + 0.004926848225295544, + 0.004952999297529459, + 0.005005215294659138, + 0.005028663668781519, + 0.004523467272520065, + 0.004554289858788252, + 0.004648423288017511, + 0.004657648038119078, + 0.004240449517965317, + 0.0042638350278139114, + 0.004358324687927961 + ], + "loss_nats": 5.558883190155029, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0034366261679679155, + 0.005092422943562269, + 0.006235890556126833, + 0.007266214583069086, + 0.004073208663612604, + 0.00560376700013876, + 0.006768420804291964, + 0.007757550571113825, + 0.004280691966414452, + 0.005751485005021095, + 0.00762521754950285, + 0.008497623726725578, + 0.005177160259336233, + 0.006417509168386459, + 0.008189847692847252, + 0.009013078175485134, + 0.0056319693103432655, + 0.006685016211122274, + 0.009837556630373001, + 0.010579346679151058, + 0.006555364932864904, + 0.007517023477703333, + 0.009742370806634426, + 0.010370824486017227, + 0.0071639628149569035, + 0.008143824525177479, + 0.01089458167552948, + 0.011447666212916374, + 0.00840203370898962, + 0.009210918098688126, + 0.012791652232408524, + 0.013363312929868698 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.495960016967729e-05, + 6.749085150659084e-05, + 5.70677402720321e-05, + 5.0705322792055085e-05, + 4.370361421024427e-05, + 3.821287100436166e-05, + 3.268615546403453e-05, + 2.8849746740888804e-05, + 2.5781664589885622e-05, + 2.37614458455937e-05, + 2.279552791151218e-05, + 2.1788309823023155e-05, + 2.0940320609952323e-05, + 2.0449986550374888e-05, + 1.9821660316665657e-05, + 1.9541532310540788e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.255587868508883e-05, + "first_to_last_ratio": 3.098608919721194, + "imbalance_abs_log_ratio": 1.1309532752053548, + "last_quartile_mean": 2.0188374946883414e-05, + "mean": 3.5534771882339555e-05, + "normalized": [ + 2.1094718271409953, + 1.8992904113768394, + 1.605968949540217, + 1.4269212972563121, + 1.229883066506037, + 1.0753655920710468, + 0.9198358039911673, + 0.811873700397296, + 0.7255334204832441, + 0.6686815360535046, + 0.6414992049756577, + 0.6131546276747519, + 0.5892909817822544, + 0.575492270446749, + 0.5578102592665534, + 0.549927051037374 + ], + "population_cv": 0.4936925451995582 + }, + "activation_output_rms_by_block": [ + 0.009570118971168995, + 0.01747945137321949, + 0.011283415369689465, + 0.020727796480059624, + 0.012305145151913166, + 0.023130038753151894, + 0.015205477364361286, + 0.028237100690603256, + 0.017268717288970947, + 0.03324708342552185, + 0.021605173125863075, + 0.04313390702009201, + 0.02657386288046837, + 0.05039682611823082, + 0.03066091425716877, + 0.0630461722612381 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014765195548534393, + "first_to_last_ratio": 0.34603674681838265, + "imbalance_abs_log_ratio": 1.0612103048863237, + "last_quartile_mean": 0.042669443879276514, + "mean": 0.02649195003323257, + "normalized": [ + 0.3612463015808142, + 0.6598023683153774, + 0.42591864153205383, + 0.7824186763925585, + 0.46448619812724606, + 0.8730968737347248, + 0.5739659536307038, + 1.0658747527147492, + 0.6518477223197376, + 1.2549881523940432, + 0.8155372895827101, + 1.628189203361138, + 1.0030919901001265, + 1.9023449030747457, + 1.1573672084805566, + 2.3798237646587146 + ], + "population_cv": 0.544699846184418 + }, + "bits_per_byte": 5.090461856197109, + "branch_output_rms_by_sublayer": [ + 0.004518123809248209, + 0.008520865812897682, + 0.004510615952312946, + 0.009082728065550327, + 0.005324283614754677, + 0.0099419467151165, + 0.004656728357076645, + 0.010662632063031197, + 0.00614966033026576, + 0.010214399546384811, + 0.0060766469687223434, + 0.011413711123168468, + 0.0087440712377429, + 0.011133860796689987, + 0.009014265611767769, + 0.011885596439242363, + 0.010757530108094215, + 0.011522865854203701, + 0.011306028813123703, + 0.012577465735375881, + 0.014357471838593483, + 0.01213979721069336, + 0.01672077551484108, + 0.013535024598240852, + 0.01905498467385769, + 0.012188552878797054, + 0.018409499898552895, + 0.013168000616133213, + 0.021850280463695526, + 0.012649565935134888, + 0.02609730139374733, + 0.013150178827345371 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00025280350140969286, + 0.00022135664689195577, + 0.00022214031151724322, + 0.0001810293778015987, + 0.00018338803861517842, + 0.00016050318916432303, + 0.0001626860438404638, + 0.00015310192441761306, + 0.00014075054270255738, + 0.00013076915091262195, + 0.00013639133469956174, + 0.0001377615870880476, + 0.00013681686670328103, + 0.0001370125878547912, + 0.0001387461931789955, + 0.0001464231652858264 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021933245940512265, + "first_to_last_ratio": 1.5694663694832551, + "imbalance_abs_log_ratio": 0.45073566953494, + "last_quartile_mean": 0.00013974970325572353, + "mean": 0.0001651050288802345, + "normalized": [ + 1.531167785283354, + 1.3407020270262369, + 1.3454484882976008, + 1.0964498115494445, + 1.1107356320939579, + 0.9721278040582907, + 0.9853488106560013, + 0.9273001885888748, + 0.8524909486836794, + 0.7920361469273026, + 0.8260883125401264, + 0.8343875896595401, + 0.8286656538034743, + 0.8298510880257848, + 0.8403511032946221, + 0.8868486095117084 + ], + "population_cv": 0.21861370307121916 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6929093599319458, + "mean_weights": [ + 0.4913179874420166, + 0.5086820125579834 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6919252872467041, + "mean_weights": [ + 0.5235024690628052, + 0.4764975309371948 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930201053619385, + "mean_weights": [ + 0.4991857707500458, + 0.5008142590522766 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6926943063735962, + "mean_weights": [ + 0.5129889249801636, + 0.48701101541519165 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097741961479187, + "mean_weights": [ + 0.3480756878852844, + 0.33060726523399353, + 0.32131701707839966 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982271432876587, + "mean_weights": [ + 0.3435906767845154, + 0.327891081571579, + 0.328518271446228 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097235918045044, + "mean_weights": [ + 0.35132721066474915, + 0.33742234110832214, + 0.3112504482269287 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098421573638916, + "mean_weights": [ + 0.340750515460968, + 0.3319299519062042, + 0.32731953263282776 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3854012489318848, + "mean_weights": [ + 0.26307183504104614, + 0.2547699213027954, + 0.2451326996088028, + 0.23702551424503326 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862006664276123, + "mean_weights": [ + 0.25451934337615967, + 0.25035756826400757, + 0.24778643250465393, + 0.24733665585517883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854570388793945, + "mean_weights": [ + 0.26253488659858704, + 0.2546469569206238, + 0.2452046275138855, + 0.2376135289669037 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861933946609497, + "mean_weights": [ + 0.2505362629890442, + 0.254558801651001, + 0.2493266463279724, + 0.24557828903198242 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6085797548294067, + "mean_weights": [ + 0.2100008726119995, + 0.20838744938373566, + 0.19692982733249664, + 0.19323000311851501, + 0.19145184755325317 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093010902404785, + "mean_weights": [ + 0.19938474893569946, + 0.20374906063079834, + 0.20229801535606384, + 0.19861868023872375, + 0.1959494650363922 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085379123687744, + "mean_weights": [ + 0.20589761435985565, + 0.21123698353767395, + 0.20005744695663452, + 0.1941205859184265, + 0.18868738412857056 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093740463256836, + "mean_weights": [ + 0.19956088066101074, + 0.2027459442615509, + 0.20118315517902374, + 0.1998024433851242, + 0.1967075765132904 + ], + "sources": 5 + }, + { + "entropy_mean": 1.789860725402832, + "mean_weights": [ + 0.17310243844985962, + 0.1810513734817505, + 0.17217648029327393, + 0.1649792194366455, + 0.1559385061264038, + 0.15275199711322784 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791734218597412, + "mean_weights": [ + 0.16535353660583496, + 0.16847901046276093, + 0.1662793755531311, + 0.16693073511123657, + 0.16613470017910004, + 0.1668226420879364 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890782356262207, + "mean_weights": [ + 0.17188119888305664, + 0.18350960314273834, + 0.17409591376781464, + 0.16590778529644012, + 0.15644267201423645, + 0.148162841796875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791684627532959, + "mean_weights": [ + 0.16539111733436584, + 0.16517111659049988, + 0.16458021104335785, + 0.16693061590194702, + 0.16811248660087585, + 0.16981446743011475 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9417986869812012, + "mean_weights": [ + 0.14831508696079254, + 0.16164575517177582, + 0.15476199984550476, + 0.14618372917175293, + 0.13434207439422607, + 0.1278611123561859, + 0.12689024209976196 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456918239593506, + "mean_weights": [ + 0.14120136201381683, + 0.1386352777481079, + 0.14049293100833893, + 0.1421712040901184, + 0.14506074786186218, + 0.1458473801612854, + 0.14659109711647034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9406665563583374, + "mean_weights": [ + 0.14782890677452087, + 0.1637636125087738, + 0.15569329261779785, + 0.14784488081932068, + 0.13445764780044556, + 0.12887068092823029, + 0.12154098600149155 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9455351829528809, + "mean_weights": [ + 0.14214086532592773, + 0.1382882297039032, + 0.1401882916688919, + 0.1396511197090149, + 0.14402973651885986, + 0.14592906832695007, + 0.14977267384529114 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0713937282562256, + "mean_weights": [ + 0.1321096122264862, + 0.1488187313079834, + 0.13977529108524323, + 0.13292929530143738, + 0.12093928456306458, + 0.11470195651054382, + 0.10486428439617157, + 0.10586154460906982 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0790562629699707, + "mean_weights": [ + 0.12411445379257202, + 0.1212267130613327, + 0.12224400043487549, + 0.12172792106866837, + 0.12399555742740631, + 0.12670958042144775, + 0.1301731914281845, + 0.12980858981609344 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068070888519287, + "mean_weights": [ + 0.13267414271831512, + 0.15216735005378723, + 0.1437777280807495, + 0.13539820909500122, + 0.1199566125869751, + 0.11211173236370087, + 0.10534092038869858, + 0.09857328981161118 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079178810119629, + "mean_weights": [ + 0.12415283918380737, + 0.12161671370267868, + 0.12301617860794067, + 0.12307952344417572, + 0.12411122024059296, + 0.12585631012916565, + 0.12885767221450806, + 0.1293095499277115 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1886847019195557, + "mean_weights": [ + 0.11892519891262054, + 0.13333868980407715, + 0.12725916504859924, + 0.12129361927509308, + 0.10894724726676941, + 0.10413059592247009, + 0.09661625325679779, + 0.09465864300727844, + 0.09483058750629425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969594955444336, + "mean_weights": [ + 0.1101110577583313, + 0.10734203457832336, + 0.10833269357681274, + 0.10946638882160187, + 0.11091698706150055, + 0.11324633657932281, + 0.11330728977918625, + 0.11495929211378098, + 0.11231791973114014 + ], + "sources": 9 + }, + { + "entropy_mean": 2.186551570892334, + "mean_weights": [ + 0.11992302536964417, + 0.13721907138824463, + 0.12899263203144073, + 0.12160105258226395, + 0.10788539052009583, + 0.10353681445121765, + 0.09616796672344208, + 0.09295656532049179, + 0.09171747416257858 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02856776863336563, + 0.014094225130975246, + 0.01573058031499386, + 0.015321774408221245, + 0.017058780416846275, + 0.011773223988711834, + 0.013085381127893925, + 0.013459180481731892, + 0.015175317414104939, + 0.011872655712068081, + 0.013221408240497112, + 0.013565259985625744, + 0.015276176854968071, + 0.012822598218917847, + 0.014198821038007736, + 0.014604844152927399, + 0.0162027720361948, + 0.014113145880401134, + 0.015384308062493801, + 0.015927067026495934, + 0.017611440271139145, + 0.01576142944395542, + 0.017285119742155075, + 0.01772329770028591, + 0.019923174753785133, + 0.017852960154414177, + 0.019824320450425148, + 0.01966804265975952, + 0.022481828927993774, + 0.020090021193027496, + 0.02255004271864891, + 0.022322168573737144 + ], + "loss_nats": 3.5284392833709717, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.193685531616211, + "mean_weights": [ + 0.10035791993141174, + 0.09579961001873016, + 0.10334210842847824, + 0.108812615275383, + 0.11718772351741791, + 0.11711248755455017, + 0.12047697603702545, + 0.11920724809169769, + 0.11770330369472504 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004518123809248209, + 0.009570118971168995, + 0.010795549489557743, + 0.01747945137321949, + 0.005324283614754677, + 0.011283415369689465, + 0.012478979304432869, + 0.020727796480059624, + 0.00614966033026576, + 0.012305145151913166, + 0.014650273136794567, + 0.023130038753151894, + 0.0087440712377429, + 0.015205477364361286, + 0.019528985023498535, + 0.028237100690603256, + 0.010757530108094215, + 0.017268717288970947, + 0.02412688545882702, + 0.03324708342552185, + 0.014357471838593483, + 0.021605173125863075, + 0.03339937701821327, + 0.04313390702009201, + 0.01905498467385769, + 0.02657386288046837, + 0.04070562124252319, + 0.05039682611823082, + 0.021850280463695526, + 0.03066091425716877, + 0.052838243544101715, + 0.0630461722612381 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015504666953347623, + 0.00015570323739666492, + 0.00013642148405779153, + 0.00012308824807405472, + 0.00010802906763274223, + 9.803942521102726e-05, + 8.508376777172089e-05, + 7.592589827254415e-05, + 7.549385918537155e-05, + 6.5492422436364e-05, + 6.254414620343596e-05, + 5.1708771934499964e-05, + 4.589578747982159e-05, + 4.085309410584159e-05, + 3.8330261304508895e-05, + 3.7662506656488404e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014256490976549685, + "first_to_last_ratio": 3.504079260905398, + "imbalance_abs_log_ratio": 1.2539277929397175, + "last_quartile_mean": 4.068541238666512e-05, + "mean": 8.470741545352212e-05, + "normalized": [ + 1.8303789426623265, + 1.8381299507608906, + 1.610502260367562, + 1.4530988510869118, + 1.2753200774023898, + 1.1573889332607517, + 1.0044429677872213, + 0.8963311873705286, + 0.8912308182368528, + 0.7731604380291695, + 0.7383550291149319, + 0.6104397313700582, + 0.5418154624845569, + 0.4822847431610892, + 0.45250183941145294, + 0.44461876749330576 + ], + "population_cv": 0.46442228143345987 + }, + "activation_output_rms_by_block": [ + 0.02604864165186882, + 0.04358876869082451, + 0.022780675441026688, + 0.03670342266559601, + 0.027025330811738968, + 0.05019363388419151, + 0.040244098752737045, + 0.04879867285490036, + 0.029798155650496483, + 0.04518449306488037, + 0.02838377095758915, + 0.051337067037820816, + 0.03289651870727539, + 0.0653979703783989, + 0.04346369579434395, + 0.08895192295312881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.032280377112329006, + "first_to_last_ratio": 0.5596699237065876, + "imbalance_abs_log_ratio": 0.5804080909828722, + "last_quartile_mean": 0.05767752695828676, + "mean": 0.04254980245605111, + "normalized": [ + 0.6121918351741814, + 1.0244176511946566, + 0.5353885124274412, + 0.8625991320055194, + 0.6351458585419503, + 1.1796443458470947, + 0.9458116472880084, + 1.1468601506506073, + 0.7003124322674456, + 1.0619201607704427, + 0.6670717446198772, + 1.2065171651700595, + 0.7731297634402319, + 1.5369747120670472, + 1.0214782040230805, + 2.0905366845123567 + ], + "population_cv": 0.38343105399742256 + }, + "bits_per_byte": 3.1146838872489564, + "branch_output_rms_by_sublayer": [ + 0.01352347619831562, + 0.020455393940210342, + 0.016984812915325165, + 0.0142062958329916, + 0.01867547444999218, + 0.011138904839754105, + 0.019149528816342354, + 0.0099713783711195, + 0.023481043055653572, + 0.011000796221196651, + 0.02560650371015072, + 0.010696547105908394, + 0.0353584885597229, + 0.012510747648775578, + 0.023770872503519058, + 0.012255984358489513, + 0.023732667788863182, + 0.014518127776682377, + 0.021041536703705788, + 0.015058746561408043, + 0.021396929398179054, + 0.018767673522233963, + 0.022012433037161827, + 0.023688357323408127, + 0.020202528685331345, + 0.02677428349852562, + 0.015811076387763023, + 0.03653915971517563, + 0.014067054726183414, + 0.0391332171857357, + 0.010850721038877964, + 0.043844256550073624 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006567095640112879, + 0.00045868421249221203, + 0.00039439577454117915, + 0.0003702348543320087, + 0.0003691077163774564, + 0.00041151853265381964, + 0.0004306774133910072, + 0.00035317417015185023, + 0.00038561746855580054, + 0.0003833308661148803, + 0.00032325470855348767, + 0.0002967607909467712, + 0.00021968458301266372, + 0.00017220390788738176, + 0.00016255203232502606, + 0.0001597780110258732 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00047000610134417196, + "first_to_last_ratio": 2.6322817390176136, + "imbalance_abs_log_ratio": 0.967851051434004, + "last_quartile_mean": 0.0001785546335627362, + "mean": 0.00034673028789829414, + "normalized": [ + 1.8940069181493584, + 1.3228847565424027, + 1.137471366957324, + 1.0677891930819992, + 1.0645384301867686, + 1.1868548754371575, + 1.242110736854166, + 1.0185847111673332, + 1.1121539767789572, + 1.105559218487778, + 0.9322944081778846, + 0.8558836689623711, + 0.6335892498583898, + 0.49665089522809175, + 0.46881405518489694, + 0.46081353894511995 + ], + "population_cv": 0.3565995806724142 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6422993540763855, + "mean_weights": [ + 0.370745986700058, + 0.6292540431022644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6814000606536865, + "mean_weights": [ + 0.4577118158340454, + 0.5422881841659546 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6871446371078491, + "mean_weights": [ + 0.49052417278289795, + 0.5094758868217468 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6793373227119446, + "mean_weights": [ + 0.4676857590675354, + 0.5323142409324646 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0915337800979614, + "mean_weights": [ + 0.29751554131507874, + 0.351987361907959, + 0.3504970669746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0892891883850098, + "mean_weights": [ + 0.3054355978965759, + 0.34222692251205444, + 0.35233747959136963 + ], + "sources": 3 + }, + { + "entropy_mean": 1.093106746673584, + "mean_weights": [ + 0.30644628405570984, + 0.34674984216690063, + 0.3468038737773895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0885257720947266, + "mean_weights": [ + 0.3052448034286499, + 0.3325393795967102, + 0.3622158169746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.381313443183899, + "mean_weights": [ + 0.2415536344051361, + 0.24482353031635284, + 0.25347256660461426, + 0.2601502537727356 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3802430629730225, + "mean_weights": [ + 0.24572397768497467, + 0.2491092085838318, + 0.25083041191101074, + 0.254336416721344 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3813848495483398, + "mean_weights": [ + 0.2464662790298462, + 0.23372246325016022, + 0.2595708668231964, + 0.260240375995636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772881031036377, + "mean_weights": [ + 0.24203142523765564, + 0.2443695217370987, + 0.2359469085931778, + 0.27765214443206787 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6055738925933838, + "mean_weights": [ + 0.20364060997962952, + 0.18423235416412354, + 0.18416231870651245, + 0.20781230926513672, + 0.22015240788459778 + ], + "sources": 5 + }, + { + "entropy_mean": 1.602186679840088, + "mean_weights": [ + 0.19954648613929749, + 0.1888481080532074, + 0.18637022376060486, + 0.21049518883228302, + 0.21474000811576843 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6062731742858887, + "mean_weights": [ + 0.19444647431373596, + 0.1968603879213333, + 0.18490161001682281, + 0.20200669765472412, + 0.2217848002910614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6028468608856201, + "mean_weights": [ + 0.20196445286273956, + 0.19006873667240143, + 0.1860634982585907, + 0.1993759572505951, + 0.22252735495567322 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7881723642349243, + "mean_weights": [ + 0.15337401628494263, + 0.17425060272216797, + 0.1601826399564743, + 0.1535373479127884, + 0.17563557624816895, + 0.18301980197429657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7856398820877075, + "mean_weights": [ + 0.1621149629354477, + 0.15582670271396637, + 0.15492847561836243, + 0.1618887484073639, + 0.1784553974866867, + 0.1867857277393341 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7870616912841797, + "mean_weights": [ + 0.1564463973045349, + 0.18363508582115173, + 0.15811879932880402, + 0.15924012660980225, + 0.15980733931064606, + 0.18275225162506104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7877882719039917, + "mean_weights": [ + 0.17034202814102173, + 0.15881556272506714, + 0.1534808874130249, + 0.16709929704666138, + 0.16602207720279694, + 0.1842401623725891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9344044923782349, + "mean_weights": [ + 0.12371677905321121, + 0.1564711332321167, + 0.13837161660194397, + 0.1230623871088028, + 0.12980599701404572, + 0.152274027466774, + 0.17629806697368622 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9417431354522705, + "mean_weights": [ + 0.14328667521476746, + 0.1450044810771942, + 0.14678549766540527, + 0.13414078950881958, + 0.1328907310962677, + 0.1347782164812088, + 0.16311360895633698 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9288355112075806, + "mean_weights": [ + 0.12207633256912231, + 0.16755375266075134, + 0.13929706811904907, + 0.12109649926424026, + 0.12036697566509247, + 0.15116539597511292, + 0.1784439980983734 + ], + "sources": 7 + }, + { + "entropy_mean": 1.938415288925171, + "mean_weights": [ + 0.1377446949481964, + 0.1613207459449768, + 0.156096950173378, + 0.13170479238033295, + 0.12767548859119415, + 0.13129547238349915, + 0.15416184067726135 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0616660118103027, + "mean_weights": [ + 0.10379506647586823, + 0.1403447985649109, + 0.12122530490159988, + 0.10852085053920746, + 0.10464349389076233, + 0.1186104342341423, + 0.13291975855827332, + 0.1699402928352356 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0691847801208496, + "mean_weights": [ + 0.11521603912115097, + 0.14423911273479462, + 0.13052678108215332, + 0.12005230039358139, + 0.10937757790088654, + 0.12122473865747452, + 0.12272199243307114, + 0.1366414725780487 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054232597351074, + "mean_weights": [ + 0.10057584941387177, + 0.16718831658363342, + 0.1352493166923523, + 0.11277885735034943, + 0.09778623282909393, + 0.11311028897762299, + 0.13487182557582855, + 0.1384393274784088 + ], + "sources": 8 + }, + { + "entropy_mean": 2.060153007507324, + "mean_weights": [ + 0.11611463129520416, + 0.15179042518138885, + 0.14188171923160553, + 0.1159345731139183, + 0.10906632244586945, + 0.12251802533864975, + 0.12206383049488068, + 0.12063045799732208 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1772594451904297, + "mean_weights": [ + 0.09167024493217468, + 0.14425000548362732, + 0.1316833794116974, + 0.09630900621414185, + 0.09528810530900955, + 0.1022176742553711, + 0.12638677656650543, + 0.10320886224508286, + 0.10898594558238983 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191457986831665, + "mean_weights": [ + 0.10587693750858307, + 0.12391996383666992, + 0.11278913170099258, + 0.11095453053712845, + 0.10569566488265991, + 0.11266011744737625, + 0.11442379653453827, + 0.11124123632907867, + 0.10243862122297287 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1550254821777344, + "mean_weights": [ + 0.08980270475149155, + 0.17290523648262024, + 0.13377593457698822, + 0.10608047246932983, + 0.09299036115407944, + 0.1071130782365799, + 0.12428411841392517, + 0.09728670865297318, + 0.07576138526201248 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04350041598081589, + 0.017946822568774223, + 0.025110680609941483, + 0.028106264770030975, + 0.031104078516364098, + 0.02212802693247795, + 0.02379882149398327, + 0.02537565305829048, + 0.026546044275164604, + 0.02055767923593521, + 0.022140897810459137, + 0.02340756356716156, + 0.02546003647148609, + 0.020307816565036774, + 0.02165406011044979, + 0.021843569353222847, + 0.023044398054480553, + 0.018649185076355934, + 0.019831404089927673, + 0.019260063767433167, + 0.02048473060131073, + 0.01540753897279501, + 0.01695212349295616, + 0.01553493645042181, + 0.017758244648575783, + 0.013987960293889046, + 0.016796667128801346, + 0.01665738970041275, + 0.01972099393606186, + 0.01747863180935383, + 0.020752927288413048, + 0.019990377128124237 + ], + "loss_nats": 2.1589343547821045, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1796395778656006, + "mean_weights": [ + 0.12411575019359589, + 0.09159804880619049, + 0.10466736555099487, + 0.10315616428852081, + 0.09333926439285278, + 0.09817550331354141, + 0.11351706087589264, + 0.1361408531665802, + 0.1352899968624115 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01352347619831562, + 0.02604864165186882, + 0.03525395691394806, + 0.04358876869082451, + 0.01867547444999218, + 0.022780675441026688, + 0.03279132395982742, + 0.03670342266559601, + 0.023481043055653572, + 0.027025330811738968, + 0.04530210420489311, + 0.05019363388419151, + 0.0353584885597229, + 0.040244098752737045, + 0.04290904477238655, + 0.04879867285490036, + 0.023732667788863182, + 0.029798155650496483, + 0.03790274262428284, + 0.04518449306488037, + 0.021396929398179054, + 0.02838377095758915, + 0.03953101858496666, + 0.051337067037820816, + 0.020202528685331345, + 0.03289651870727539, + 0.039050742983818054, + 0.0653979703783989, + 0.014067054726183414, + 0.04346369579434395, + 0.048713527619838715, + 0.08895192295312881 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.91841812315397e-05, + 8.04664014140144e-05, + 6.887180643389001e-05, + 6.264931289479136e-05, + 6.203571683727205e-05, + 6.256609776755795e-05, + 7.726731564616784e-05, + 7.684857700951397e-05, + 0.0001297412090934813, + 0.00010362039029132575, + 0.0001698000414762646, + 0.00012187293032184243, + 0.00013834609126206487, + 8.384700049646199e-05, + 4.917434125673026e-05, + 4.311763041187078e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.279292549355887e-05, + "first_to_last_ratio": 0.9258681439467011, + "imbalance_abs_log_ratio": 0.07702344762093526, + "last_quartile_mean": 7.862126585678197e-05, + "mean": 8.808806524029933e-05, + "normalized": [ + 0.8989206541831708, + 0.91347677116652, + 0.7818517326497246, + 0.7112122706281212, + 0.7042465590320558, + 0.7102675895637067, + 0.8771598676323168, + 0.8724062311946045, + 1.4728579716168646, + 1.1763272358026606, + 1.9276168799151117, + 1.3835351019389504, + 1.570542966117651, + 0.9518542638861694, + 0.5582406779236824, + 0.4894832267486894 + ], + "population_cv": 0.3870844002085187 + }, + "activation_output_rms_by_block": [ + 0.10205306112766266, + 0.1637924462556839, + 0.1347782015800476, + 0.18643726408481598, + 0.08823998272418976, + 0.14544272422790527, + 0.051831819117069244, + 0.08038940280675888, + 0.045988935977220535, + 0.08743251860141754, + 0.060310039669275284, + 0.09679152816534042, + 0.05415637046098709, + 0.0906868502497673, + 0.09274918586015701, + 0.19966518878936768 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14676524326205254, + "first_to_last_ratio": 1.3425975426784762, + "imbalance_abs_log_ratio": 0.2946062022500781, + "last_quartile_mean": 0.10931439884006977, + "mean": 0.10504659498110414, + "normalized": [ + 0.9715027997434856, + 1.5592361302634037, + 1.2830325590685887, + 1.7748054005782146, + 0.840008024439654, + 1.3845543899263715, + 0.49341741278137374, + 0.7652737608603057, + 0.4377955895238019, + 0.8323213010107083, + 0.5741265547933648, + 0.9214151889716308, + 0.5155461771105363, + 0.8633011880687815, + 0.8829337674090323, + 1.9007297554507465 + ], + "population_cv": 0.43941183630186603 + }, + "bits_per_byte": 2.0898630043632984, + "branch_output_rms_by_sublayer": [ + 0.04098131135106087, + 0.09075035154819489, + 0.05343177169561386, + 0.04652838781476021, + 0.13196392357349396, + 0.018610643222928047, + 0.06760989129543304, + 0.01781350187957287, + 0.0839577317237854, + 0.0164200272411108, + 0.07349737733602524, + 0.016965076327323914, + 0.04464862495660782, + 0.016964683309197426, + 0.0346895232796669, + 0.021388454362750053, + 0.03376077860593796, + 0.026654686778783798, + 0.04212235286831856, + 0.042033009231090546, + 0.0419253334403038, + 0.040410157293081284, + 0.028147248551249504, + 0.05647630989551544, + 0.025301706045866013, + 0.04489820450544357, + 0.015376480296254158, + 0.058749374002218246, + 0.025178087875247, + 0.0875716358423233, + 0.01239082682877779, + 0.1223064437508583 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040335839120789034, + 0.00023897821552144955, + 0.00022638555923562337, + 0.00017534070905568056, + 0.00017358160175308344, + 0.00015895432307312814, + 0.00019058997613466408, + 0.00021161798453125184, + 0.00037807554491806804, + 0.0004740713531015684, + 0.0009894587672613622, + 0.0007545475239678793, + 0.0006624146206615605, + 0.0004575991184432586, + 0.0002840295434450018, + 0.00025831777406547436 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00026101571875516093, + "first_to_last_ratio": 0.6280602344874717, + "imbalance_abs_log_ratio": 0.46511920232449994, + "last_quartile_mean": 0.00041559026415382384, + "mean": 0.00037733256289855906, + "normalized": [ + 1.0689731840512475, + 0.633335786568984, + 0.5999629544203535, + 0.464684806709403, + 0.460022852042453, + 0.4212579000658921, + 0.5050981412009801, + 0.5608261924326488, + 1.001969037640965, + 1.2563754091613246, + 2.6222459033501577, + 1.9996883337384523, + 1.7555193635372572, + 1.2127209878949095, + 0.7527300089426935, + 0.6845891382422771 + ], + "population_cv": 0.6186471183066214 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4914427399635315, + "mean_weights": [ + 0.7962706685066223, + 0.2037293016910553 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6456852555274963, + "mean_weights": [ + 0.6051734089851379, + 0.39482662081718445 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5276303291320801, + "mean_weights": [ + 0.7698293924331665, + 0.2301705777645111 + ], + "sources": 2 + }, + { + "entropy_mean": 0.667390763759613, + "mean_weights": [ + 0.5093090534210205, + 0.4906909465789795 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9880077838897705, + "mean_weights": [ + 0.5180364847183228, + 0.2465059459209442, + 0.23545756936073303 + ], + "sources": 3 + }, + { + "entropy_mean": 1.08414626121521, + "mean_weights": [ + 0.2851155400276184, + 0.3203214704990387, + 0.3945629596710205 + ], + "sources": 3 + }, + { + "entropy_mean": 0.995106041431427, + "mean_weights": [ + 0.5032885074615479, + 0.23357318341732025, + 0.2631382942199707 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0889081954956055, + "mean_weights": [ + 0.2929039001464844, + 0.3457888960838318, + 0.36130720376968384 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3091163635253906, + "mean_weights": [ + 0.3480933904647827, + 0.15371239185333252, + 0.20259280502796173, + 0.29560136795043945 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786191940307617, + "mean_weights": [ + 0.22527548670768738, + 0.26118987798690796, + 0.22951865196228027, + 0.284015953540802 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308091163635254, + "mean_weights": [ + 0.28280675411224365, + 0.12650004029273987, + 0.24999231100082397, + 0.3407008647918701 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3780848979949951, + "mean_weights": [ + 0.23603014647960663, + 0.2631712555885315, + 0.21569286286830902, + 0.28510570526123047 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5637545585632324, + "mean_weights": [ + 0.21651385724544525, + 0.1117275133728981, + 0.18637539446353912, + 0.22181779146194458, + 0.2635654807090759 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585724115371704, + "mean_weights": [ + 0.18968582153320312, + 0.1962759643793106, + 0.1558789312839508, + 0.1798120141029358, + 0.2783472537994385 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5276048183441162, + "mean_weights": [ + 0.2346682846546173, + 0.15222865343093872, + 0.12689977884292603, + 0.14891111850738525, + 0.3372921645641327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.59938383102417, + "mean_weights": [ + 0.20912373065948486, + 0.21754522621631622, + 0.17016828060150146, + 0.17554700374603271, + 0.22761575877666473 + ], + "sources": 5 + }, + { + "entropy_mean": 1.762988805770874, + "mean_weights": [ + 0.15954776108264923, + 0.12860070168972015, + 0.13993129134178162, + 0.14435188472270966, + 0.1894117146730423, + 0.23815666139125824 + ], + "sources": 6 + }, + { + "entropy_mean": 1.730720043182373, + "mean_weights": [ + 0.17182672023773193, + 0.16310763359069824, + 0.12390270084142685, + 0.11688461899757385, + 0.1326894462108612, + 0.2915888726711273 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4424301385879517, + "mean_weights": [ + 0.1671508103609085, + 0.1365666687488556, + 0.058271970599889755, + 0.05150005966424942, + 0.10222888737916946, + 0.48428159952163696 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7727878093719482, + "mean_weights": [ + 0.1808200478553772, + 0.17958280444145203, + 0.1469326913356781, + 0.13797307014465332, + 0.14092227816581726, + 0.2137690931558609 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5771737098693848, + "mean_weights": [ + 0.09523316472768784, + 0.08390435576438904, + 0.04673727601766586, + 0.04574791342020035, + 0.07810044288635254, + 0.19550111889839172, + 0.45477572083473206 + ], + "sources": 7 + }, + { + "entropy_mean": 1.866133689880371, + "mean_weights": [ + 0.13646650314331055, + 0.12685352563858032, + 0.11049237847328186, + 0.10180146992206573, + 0.1026906818151474, + 0.1334708034992218, + 0.28822463750839233 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755802392959595, + "mean_weights": [ + 0.09840903431177139, + 0.08739377558231354, + 0.04908968508243561, + 0.048939045518636703, + 0.07940414547920227, + 0.16869762539863586, + 0.46806666254997253 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8938008546829224, + "mean_weights": [ + 0.14332836866378784, + 0.14485859870910645, + 0.1094474121928215, + 0.10118386149406433, + 0.10821370780467987, + 0.14364898204803467, + 0.24931909143924713 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7736961841583252, + "mean_weights": [ + 0.08116129040718079, + 0.07111318409442902, + 0.04280973970890045, + 0.041376810520887375, + 0.06107718497514725, + 0.1160595640540123, + 0.31332647800445557, + 0.2730756998062134 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9035927057266235, + "mean_weights": [ + 0.10087604820728302, + 0.09542917460203171, + 0.06220542639493942, + 0.057604290544986725, + 0.0788167417049408, + 0.11659102141857147, + 0.18421337008476257, + 0.3042638897895813 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7131891250610352, + "mean_weights": [ + 0.0740746557712555, + 0.07079474627971649, + 0.04342404007911682, + 0.04123558849096298, + 0.055328819900751114, + 0.09829520434141159, + 0.21916092932224274, + 0.39768603444099426 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9703106880187988, + "mean_weights": [ + 0.11310406029224396, + 0.11765976250171661, + 0.07340675592422485, + 0.0660770833492279, + 0.08407266438007355, + 0.12691649794578552, + 0.20392963290214539, + 0.2148335576057434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9610494375228882, + "mean_weights": [ + 0.07792562991380692, + 0.0730937123298645, + 0.040804408490657806, + 0.03727761656045914, + 0.05315954610705376, + 0.0937754213809967, + 0.19250980019569397, + 0.26475197076797485, + 0.16670189797878265 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9782097339630127, + "mean_weights": [ + 0.08397573232650757, + 0.08872128278017044, + 0.0407399907708168, + 0.036753468215465546, + 0.058452628552913666, + 0.11618129909038544, + 0.22768989205360413, + 0.21488600969314575, + 0.13259968161582947 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0315680503845215, + "mean_weights": [ + 0.08375388383865356, + 0.10381080955266953, + 0.0493716336786747, + 0.043391548097133636, + 0.06576120853424072, + 0.10571172833442688, + 0.18509769439697266, + 0.2181759774684906, + 0.14492549002170563 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05570797994732857, + 0.04578034579753876, + 0.058472853153944016, + 0.058589257299900055, + 0.09159071743488312, + 0.0729803666472435, + 0.0950615182518959, + 0.08442825824022293, + 0.11022263020277023, + 0.07856440544128418, + 0.0938955619931221, + 0.10307082533836365, + 0.10586344450712204, + 0.08232943713665009, + 0.08149705082178116, + 0.07412237673997879, + 0.0869104191660881, + 0.0654435008764267, + 0.0601484477519989, + 0.04590349644422531, + 0.06966738402843475, + 0.028740333393216133, + 0.04708963632583618, + 0.03821079805493355, + 0.0513334795832634, + 0.034749191254377365, + 0.034582361578941345, + 0.034875962883234024, + 0.041408468037843704, + 0.03493491932749748, + 0.0412786528468132, + 0.04175883159041405 + ], + "loss_nats": 1.448582649230957, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.893564224243164, + "mean_weights": [ + 0.09584413468837738, + 0.060246460139751434, + 0.05342919006943703, + 0.050279051065444946, + 0.05668472498655319, + 0.06780324131250381, + 0.0837026908993721, + 0.2589869201183319, + 0.2730236053466797 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04098131135106087, + 0.10205306112766266, + 0.13058459758758545, + 0.1637924462556839, + 0.13196392357349396, + 0.1347782015800476, + 0.18347561359405518, + 0.18643726408481598, + 0.0839577317237854, + 0.08823998272418976, + 0.14121752977371216, + 0.14544272422790527, + 0.04464862495660782, + 0.051831819117069244, + 0.07186394184827805, + 0.08038940280675888, + 0.03376077860593796, + 0.045988935977220535, + 0.06799308955669403, + 0.08743251860141754, + 0.0419253334403038, + 0.060310039669275284, + 0.07043248414993286, + 0.09679152816534042, + 0.025301706045866013, + 0.05415637046098709, + 0.05762659013271332, + 0.0906868502497673, + 0.025178087875247, + 0.09274918586015701, + 0.0948956087231636, + 0.19966518878936768 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.6704131566220894e-05, + 5.7463053963147104e-05, + 5.5973152484511957e-05, + 4.9604332161834463e-05, + 4.9622754886513576e-05, + 5.175438491278328e-05, + 6.153700815048069e-05, + 5.993711238261312e-05, + 0.00010211551852989942, + 8.014418563107029e-05, + 0.00014815112808719277, + 0.00010670819756342098, + 0.00014386174734681845, + 8.734197035664693e-05, + 5.547845285036601e-05, + 4.4348533265292645e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.4936167543928605e-05, + "first_to_last_ratio": 0.6638196023526066, + "imbalance_abs_log_ratio": 0.4097448496221764, + "last_quartile_mean": 8.275767595478101e-05, + "mean": 7.567160400867579e-05, + "normalized": [ + 0.7493449135784109, + 0.7593740705768433, + 0.7396850273993744, + 0.6555210876215509, + 0.6557645438680578, + 0.6839340277080636, + 0.8132113618660104, + 0.792068744515331, + 1.3494562440911366, + 1.0591051515424692, + 1.9578166741411636, + 1.4101484825296797, + 1.9011325216566657, + 1.1542238531990563, + 0.7331475733487212, + 0.5860657223574653 + ], + "population_cv": 0.42524285954419916 + }, + "activation_output_rms_by_block": [ + 0.16565655171871185, + 0.24343863129615784, + 0.21182885766029358, + 0.28418517112731934, + 0.12311498820781708, + 0.18273214995861053, + 0.06521740555763245, + 0.10707039386034012, + 0.06429450213909149, + 0.1264011114835739, + 0.0799982100725174, + 0.12911832332611084, + 0.06644809991121292, + 0.1084718257188797, + 0.10412788391113281, + 0.23786598443984985 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22627730295062065, + "first_to_last_ratio": 1.750986764798193, + "imbalance_abs_log_ratio": 0.5601794946219173, + "last_quartile_mean": 0.12922844849526882, + "mean": 0.14374813064932823, + "normalized": [ + 1.152408389385104, + 1.6935081534383452, + 1.4736112163924233, + 1.976966029705009, + 0.8564632294812553, + 1.2711967044940802, + 0.4536921994257406, + 0.7448472086328344, + 0.4472719182410596, + 0.8793235147657527, + 0.5565165245012613, + 0.8982261038308273, + 0.4622536627854468, + 0.7545964266032466, + 0.7243773080093228, + 1.654741410308291 + ], + "population_cv": 0.4691623243633533 + }, + "bits_per_byte": 1.9440733112268958, + "branch_output_rms_by_sublayer": [ + 0.06033274158835411, + 0.15077131986618042, + 0.0633837878704071, + 0.06536085903644562, + 0.20718541741371155, + 0.025066526606678963, + 0.08910981565713882, + 0.0229792483150959, + 0.11816111207008362, + 0.01873566210269928, + 0.06798963993787766, + 0.025694282725453377, + 0.05406087636947632, + 0.025319905951619148, + 0.05011621490120888, + 0.03549748659133911, + 0.04759044572710991, + 0.038602542132139206, + 0.05848398804664612, + 0.0661846399307251, + 0.052182719111442566, + 0.057186078280210495, + 0.03716399520635605, + 0.08089981228113174, + 0.03087303601205349, + 0.055707309395074844, + 0.02278325706720352, + 0.07142244279384613, + 0.0293489471077919, + 0.09790552407503128, + 0.01774938404560089, + 0.15747468173503876 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00039487297360913916, + 0.00021664788561845028, + 0.0001999236891396226, + 0.000155700993556381, + 0.00014909408964546676, + 0.0001300025003667294, + 0.0001780845905877945, + 0.00021758503566474802, + 0.0003627569164977402, + 0.0004938025952783663, + 0.0010348495424230683, + 0.0008427402552292494, + 0.0008322795274123101, + 0.0005665088497243569, + 0.00036612876328907985, + 0.0003194282437589591 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00024178638548089825, + "first_to_last_ratio": 0.464004453994026, + "imbalance_abs_log_ratio": 0.7678611276768971, + "last_quartile_mean": 0.0005210863460461766, + "mean": 0.00040377540323759136, + "normalized": [ + 0.9779520259107666, + 0.5365554312652605, + 0.49513587884892185, + 0.3856128736617538, + 0.36925005448570164, + 0.3219673593892314, + 0.44104863535485134, + 0.5388763998997622, + 0.8984126164918598, + 1.2229635369537375, + 2.5629335866556917, + 2.087151046031797, + 2.061243752687373, + 1.4030296179061932, + 0.9067634143965939, + 0.7911037700605048 + ], + "population_cv": 0.6734220009206653 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2750788629055023, + "mean_weights": [ + 0.9180313944816589, + 0.08196862041950226 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5785284042358398, + "mean_weights": [ + 0.7040213346481323, + 0.2959786057472229 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4888499975204468, + "mean_weights": [ + 0.7967724800109863, + 0.20322750508785248 + ], + "sources": 2 + }, + { + "entropy_mean": 0.651995062828064, + "mean_weights": [ + 0.5629274845123291, + 0.4370725154876709 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9925327301025391, + "mean_weights": [ + 0.4965394139289856, + 0.22532734274864197, + 0.27813321352005005 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0594394207000732, + "mean_weights": [ + 0.2700359523296356, + 0.27400293946266174, + 0.455961138010025 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9841134548187256, + "mean_weights": [ + 0.49647465348243713, + 0.21238623559474945, + 0.2911390960216522 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0744149684906006, + "mean_weights": [ + 0.26788532733917236, + 0.3177185654640198, + 0.41439610719680786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.307446002960205, + "mean_weights": [ + 0.33086591958999634, + 0.13986191153526306, + 0.23523268103599548, + 0.2940394878387451 + ], + "sources": 4 + }, + { + "entropy_mean": 1.376603603363037, + "mean_weights": [ + 0.21549805998802185, + 0.245200514793396, + 0.24268285930156708, + 0.29661858081817627 + ], + "sources": 4 + }, + { + "entropy_mean": 1.270529866218567, + "mean_weights": [ + 0.2375539094209671, + 0.10673755407333374, + 0.29245367646217346, + 0.3632548451423645 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3762142658233643, + "mean_weights": [ + 0.22679615020751953, + 0.24722415208816528, + 0.2286011129617691, + 0.2973785698413849 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5536365509033203, + "mean_weights": [ + 0.20543108880519867, + 0.10043682903051376, + 0.22657056152820587, + 0.26188796758651733, + 0.20567354559898376 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5861575603485107, + "mean_weights": [ + 0.1991366446018219, + 0.19079142808914185, + 0.16120794415473938, + 0.1751076579093933, + 0.27375632524490356 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4829623699188232, + "mean_weights": [ + 0.2499978393316269, + 0.13764306902885437, + 0.10969669371843338, + 0.13450437784194946, + 0.3681580424308777 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5994092226028442, + "mean_weights": [ + 0.2161637395620346, + 0.20309701561927795, + 0.17752361297607422, + 0.17640036344528198, + 0.22681528329849243 + ], + "sources": 5 + }, + { + "entropy_mean": 1.757448434829712, + "mean_weights": [ + 0.1632193624973297, + 0.11841119825839996, + 0.1406504064798355, + 0.1471654176712036, + 0.19204986095428467, + 0.23850372433662415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.719214677810669, + "mean_weights": [ + 0.17631781101226807, + 0.15250492095947266, + 0.11967972666025162, + 0.11400137841701508, + 0.13607722520828247, + 0.3014189600944519 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3921294212341309, + "mean_weights": [ + 0.18369951844215393, + 0.1281489133834839, + 0.04485193267464638, + 0.042511094361543655, + 0.10842494666576385, + 0.4923636019229889 + ], + "sources": 6 + }, + { + "entropy_mean": 1.769790768623352, + "mean_weights": [ + 0.1860106885433197, + 0.17010365426540375, + 0.1509144902229309, + 0.1396978795528412, + 0.13775232434272766, + 0.215520977973938 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5072013139724731, + "mean_weights": [ + 0.09493429213762283, + 0.07214362919330597, + 0.03675283491611481, + 0.03880048543214798, + 0.07758605480194092, + 0.19188526272773743, + 0.48789745569229126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8300648927688599, + "mean_weights": [ + 0.13595148921012878, + 0.11550325155258179, + 0.1085444986820221, + 0.09914489090442657, + 0.09707978367805481, + 0.12102536857128143, + 0.3227507174015045 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5086650848388672, + "mean_weights": [ + 0.10266698896884918, + 0.07744058966636658, + 0.03928206115961075, + 0.0407833606004715, + 0.07880876958370209, + 0.16221223771572113, + 0.4988059997558594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8771061897277832, + "mean_weights": [ + 0.1470566689968109, + 0.13075211644172668, + 0.10456536710262299, + 0.09934695065021515, + 0.1088101863861084, + 0.1398048996925354, + 0.26966381072998047 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6780993938446045, + "mean_weights": [ + 0.08037982881069183, + 0.05826176330447197, + 0.03183364123106003, + 0.03276165574789047, + 0.053941305726766586, + 0.10113498568534851, + 0.30644336342811584, + 0.33524346351623535 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294007778167725, + "mean_weights": [ + 0.09819990396499634, + 0.08131108433008194, + 0.05535554885864258, + 0.052511729300022125, + 0.07309524714946747, + 0.1031290739774704, + 0.1768367737531662, + 0.35956063866615295 + ], + "sources": 8 + }, + { + "entropy_mean": 1.620237112045288, + "mean_weights": [ + 0.07192602753639221, + 0.0598294772207737, + 0.03551005944609642, + 0.03596649691462517, + 0.052555397152900696, + 0.08522239327430725, + 0.2067522257566452, + 0.45223796367645264 + ], + "sources": 8 + }, + { + "entropy_mean": 1.922456979751587, + "mean_weights": [ + 0.11432191729545593, + 0.10134892165660858, + 0.06334333121776581, + 0.05966266989707947, + 0.07981826364994049, + 0.11505833268165588, + 0.20316767692565918, + 0.26327887177467346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8568212985992432, + "mean_weights": [ + 0.07325166463851929, + 0.05971681326627731, + 0.03186998516321182, + 0.03226577863097191, + 0.04611574858427048, + 0.07693300396203995, + 0.17066776752471924, + 0.3028550446033478, + 0.20632418990135193 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9784047603607178, + "mean_weights": [ + 0.08589711785316467, + 0.07969395816326141, + 0.03702075034379959, + 0.037476278841495514, + 0.0731067955493927, + 0.12099474668502808, + 0.22441011667251587, + 0.20659996569156647, + 0.1348002701997757 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9687256813049316, + "mean_weights": [ + 0.07957327365875244, + 0.0893080085515976, + 0.03895219415426254, + 0.03689703717827797, + 0.0598301999270916, + 0.09126800298690796, + 0.16929352283477783, + 0.23949456214904785, + 0.1953832060098648 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05895887315273285, + 0.05461842566728592, + 0.07232892513275146, + 0.07006148993968964, + 0.12035266309976578, + 0.10343272238969803, + 0.14036840200424194, + 0.12118023633956909, + 0.16676878929138184, + 0.11889537423849106, + 0.13824602961540222, + 0.15437477827072144, + 0.1504182070493698, + 0.12633125483989716, + 0.11159109324216843, + 0.09292281419038773, + 0.11971895396709442, + 0.08979442715644836, + 0.08137664943933487, + 0.06381704658269882, + 0.09776479750871658, + 0.03986503183841705, + 0.06484787911176682, + 0.05485111102461815, + 0.07206991314888, + 0.046776797622442245, + 0.04656966030597687, + 0.04702001437544823, + 0.055061157792806625, + 0.04538688063621521, + 0.05328623950481415, + 0.053778134286403656 + ], + "loss_nats": 1.3475289344787598, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7764334678649902, + "mean_weights": [ + 0.08629432320594788, + 0.05034743621945381, + 0.046479545533657074, + 0.04277106374502182, + 0.04524604231119156, + 0.05579439178109169, + 0.07300648093223572, + 0.2586928606033325, + 0.3413679003715515 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06033274158835411, + 0.16565655171871185, + 0.19572967290878296, + 0.24343863129615784, + 0.20718541741371155, + 0.21182885766029358, + 0.280988872051239, + 0.28418517112731934, + 0.11816111207008362, + 0.12311498820781708, + 0.17678114771842957, + 0.18273214995861053, + 0.05406087636947632, + 0.06521740555763245, + 0.0933222770690918, + 0.10707039386034012, + 0.04759044572710991, + 0.06429450213909149, + 0.09436589479446411, + 0.1264011114835739, + 0.052182719111442566, + 0.0799982100725174, + 0.0912039652466774, + 0.12911832332611084, + 0.03087303601205349, + 0.06644809991121292, + 0.07178688794374466, + 0.1084718257188797, + 0.0293489471077919, + 0.10412788391113281, + 0.10697942227125168, + 0.23786598443984985 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.570489106001332e-05, + 7.69118414609693e-05, + 7.822692714398727e-05, + 6.853767990833148e-05, + 6.88084473949857e-05, + 7.10485182935372e-05, + 8.91982635948807e-05, + 8.665756467962638e-05, + 0.000171030915225856, + 0.00012665522808674723, + 0.00023480529489461333, + 0.00016227969899773598, + 0.00023323859204538167, + 0.000136202055728063, + 9.528713417239487e-05, + 6.303648115135729e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.484533489332534e-05, + "first_to_last_ratio": 0.5672633797074002, + "imbalance_abs_log_ratio": 0.5669315686327029, + "last_quartile_mean": 0.0001319410657742992, + "mean": 0.00011485184586490504, + "normalized": [ + 0.6591525847051819, + 0.6696613439842941, + 0.6811116230208613, + 0.5967486146365397, + 0.5991061517280438, + 0.6186101560536372, + 0.7766376145125305, + 0.7545160813658816, + 1.4891438090340474, + 1.1027705051926284, + 2.0444189914962623, + 1.4129481139434028, + 2.0307779147035174, + 1.1858934847966764, + 0.8296526142424977, + 0.5488503965839976 + ], + "population_cv": 0.4839100882418841 + }, + "activation_output_rms_by_block": [ + 0.16869333386421204, + 0.24958865344524384, + 0.23942448198795319, + 0.31786683201789856, + 0.1331564038991928, + 0.19950589537620544, + 0.06165311112999916, + 0.10097606480121613, + 0.05752932280302048, + 0.11994150280952454, + 0.07362622767686844, + 0.12106145173311234, + 0.06137929856777191, + 0.09901073575019836, + 0.0852273628115654, + 0.19495683908462524 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2438933253288269, + "first_to_last_ratio": 2.2143221757549307, + "imbalance_abs_log_ratio": 0.7949463412042855, + "last_quartile_mean": 0.11014355905354023, + "mean": 0.142724844859913, + "normalized": [ + 1.1819479224502754, + 1.7487400577679353, + 1.6775249062134392, + 2.2271303383085868, + 0.9329588273848762, + 1.3978357837559683, + 0.43197182095740094, + 0.7074876480007805, + 0.4030785450107621, + 0.8403687734062649, + 0.5158613256797294, + 0.8482156827841455, + 0.430053356358641, + 0.6937175923882012, + 0.5971445468742151, + 1.3659628726587785 + ], + "population_cv": 0.529368044298578 + }, + "bits_per_byte": 1.883020158101202, + "branch_output_rms_by_sublayer": [ + 0.06029115617275238, + 0.1536618322134018, + 0.06518179178237915, + 0.06745198369026184, + 0.23389145731925964, + 0.026561493054032326, + 0.09204838424921036, + 0.022710271179676056, + 0.12803514301776886, + 0.017566412687301636, + 0.07296492159366608, + 0.024595333263278008, + 0.05156426504254341, + 0.023977436125278473, + 0.0450289212167263, + 0.03369171544909477, + 0.04221063107252121, + 0.034665971994400024, + 0.05162743479013443, + 0.07154636830091476, + 0.04606551304459572, + 0.05490076169371605, + 0.03287900984287262, + 0.08022980391979218, + 0.027681373059749603, + 0.05210135132074356, + 0.019983114674687386, + 0.06804364919662476, + 0.02240605279803276, + 0.08063697069883347, + 0.016830744221806526, + 0.13339926302433014 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005343193187437567, + 0.0003255019836390766, + 0.00031277964721318437, + 0.00022314651478922115, + 0.0002396348782003974, + 0.00021519463019459167, + 0.00025456643186586784, + 0.0003008996254324453, + 0.00059437697364117, + 0.0008147654749400623, + 0.001560733102975404, + 0.001320101058102649, + 0.0015032932063515777, + 0.0009169853654825589, + 0.0006174439085696804, + 0.00045539974927406244 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003489368660963097, + "first_to_last_ratio": 0.3995701760811125, + "imbalance_abs_log_ratio": 0.9173658694246768, + "last_quartile_mean": 0.0008732805574194699, + "mean": 0.0006368213668384816, + "normalized": [ + 0.8390411292202721, + 0.5111354621391565, + 0.4911575890834005, + 0.3504067646113047, + 0.3762984263390404, + 0.33791992762889184, + 0.39974543117117817, + 0.4725024019314401, + 0.9333496088423855, + 1.2794254674352237, + 2.450817739869046, + 2.072953463631928, + 2.3606199236290095, + 1.4399412664731395, + 0.9695715953046595, + 0.7151138026899253 + ], + "population_cv": 0.7016607196207993 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2716493308544159, + "mean_weights": [ + 0.9192699193954468, + 0.0807301327586174 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5622239112854004, + "mean_weights": [ + 0.72159743309021, + 0.27840256690979004 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48637402057647705, + "mean_weights": [ + 0.7995253801345825, + 0.20047461986541748 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6475558280944824, + "mean_weights": [ + 0.5858595371246338, + 0.41414040327072144 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9933564066886902, + "mean_weights": [ + 0.4998217821121216, + 0.2229493409395218, + 0.2772288918495178 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489425659179688, + "mean_weights": [ + 0.2563193440437317, + 0.269240140914917, + 0.4744405150413513 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9815441966056824, + "mean_weights": [ + 0.5070900917053223, + 0.19802016019821167, + 0.29488980770111084 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071545958518982, + "mean_weights": [ + 0.2598530650138855, + 0.3191677927970886, + 0.4209791421890259 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3049530982971191, + "mean_weights": [ + 0.3487145006656647, + 0.13604819774627686, + 0.23624856770038605, + 0.27898871898651123 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3723504543304443, + "mean_weights": [ + 0.2135777771472931, + 0.23673927783966064, + 0.23828449845314026, + 0.311398446559906 + ], + "sources": 4 + }, + { + "entropy_mean": 1.265662670135498, + "mean_weights": [ + 0.24546673893928528, + 0.10275381803512573, + 0.2848597466945648, + 0.36691969633102417 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3751296997070312, + "mean_weights": [ + 0.23200613260269165, + 0.24026888608932495, + 0.22661332786083221, + 0.30111163854599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5548532009124756, + "mean_weights": [ + 0.2040066421031952, + 0.09975787997245789, + 0.22992464900016785, + 0.259371280670166, + 0.20693954825401306 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5716381072998047, + "mean_weights": [ + 0.1984836459159851, + 0.17718908190727234, + 0.15249167382717133, + 0.17045220732688904, + 0.301383376121521 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4117064476013184, + "mean_weights": [ + 0.25685301423072815, + 0.12795110046863556, + 0.08630626648664474, + 0.10788753628730774, + 0.4210020899772644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5975096225738525, + "mean_weights": [ + 0.2208021879196167, + 0.19316574931144714, + 0.17618897557258606, + 0.17661380767822266, + 0.23322927951812744 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7283259630203247, + "mean_weights": [ + 0.1650083363056183, + 0.10661210119724274, + 0.12329171597957611, + 0.13269363343715668, + 0.19600069522857666, + 0.27639350295066833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7028545141220093, + "mean_weights": [ + 0.17663681507110596, + 0.13835734128952026, + 0.11904168128967285, + 0.11434897780418396, + 0.12915222346782684, + 0.3224629759788513 + ], + "sources": 6 + }, + { + "entropy_mean": 1.27894127368927, + "mean_weights": [ + 0.17268770933151245, + 0.10967432707548141, + 0.03322022780776024, + 0.031795307993888855, + 0.09717383980751038, + 0.555448591709137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7646939754486084, + "mean_weights": [ + 0.1884503960609436, + 0.1533900499343872, + 0.15274858474731445, + 0.14330559968948364, + 0.1338358074426651, + 0.2282695174217224 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4344873428344727, + "mean_weights": [ + 0.09210026264190674, + 0.06219030171632767, + 0.029036182910203934, + 0.0315997451543808, + 0.07091090828180313, + 0.1949847936630249, + 0.5191777944564819 + ], + "sources": 7 + }, + { + "entropy_mean": 1.802895426750183, + "mean_weights": [ + 0.13523127138614655, + 0.10325458645820618, + 0.10260490328073502, + 0.09636874496936798, + 0.09404134750366211, + 0.12264730036258698, + 0.3458518385887146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4361302852630615, + "mean_weights": [ + 0.10137400776147842, + 0.06668317317962646, + 0.03099493309855461, + 0.033589236438274384, + 0.07286111265420914, + 0.16290563344955444, + 0.5315918922424316 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8628063201904297, + "mean_weights": [ + 0.14754432439804077, + 0.11970964819192886, + 0.10172373056411743, + 0.09891469776630402, + 0.10695713013410568, + 0.14022418856620789, + 0.28492626547813416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6038438081741333, + "mean_weights": [ + 0.07592754065990448, + 0.048247914761304855, + 0.024802275002002716, + 0.026826059445738792, + 0.049207836389541626, + 0.09689367562532425, + 0.29891306161880493, + 0.3791816532611847 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7748796939849854, + "mean_weights": [ + 0.09652430564165115, + 0.07308399677276611, + 0.04972345381975174, + 0.050231896340847015, + 0.0702696442604065, + 0.09669448435306549, + 0.16264159977436066, + 0.40083062648773193 + ], + "sources": 8 + }, + { + "entropy_mean": 1.526896357536316, + "mean_weights": [ + 0.06852421164512634, + 0.05069870874285698, + 0.02793668583035469, + 0.029434790834784508, + 0.046449482440948486, + 0.08027796447277069, + 0.19853916764259338, + 0.49813902378082275 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9032639265060425, + "mean_weights": [ + 0.11537295579910278, + 0.09104456007480621, + 0.05989263206720352, + 0.057612620294094086, + 0.07744243741035461, + 0.11209231615066528, + 0.20047283172607422, + 0.2860696315765381 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7657151222229004, + "mean_weights": [ + 0.06711487472057343, + 0.04881071671843529, + 0.02479729615151882, + 0.025201791897416115, + 0.040198031812906265, + 0.06934815645217896, + 0.15364184975624084, + 0.3216337561607361, + 0.24925348162651062 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9778118133544922, + "mean_weights": [ + 0.08567827194929123, + 0.08225203305482864, + 0.030744101852178574, + 0.03173041343688965, + 0.07107861340045929, + 0.128017395734787, + 0.2095082402229309, + 0.20655101537704468, + 0.15443992614746094 + ], + "sources": 9 + }, + { + "entropy_mean": 1.88002610206604, + "mean_weights": [ + 0.07342863082885742, + 0.07307932525873184, + 0.030178282409906387, + 0.029849819839000702, + 0.050495631992816925, + 0.08296258747577667, + 0.14574018120765686, + 0.2550249397754669, + 0.25924059748649597 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.061662707477808, + 0.0571652427315712, + 0.07283993810415268, + 0.07206818461418152, + 0.1187756210565567, + 0.10985641181468964, + 0.15539979934692383, + 0.12964090704917908, + 0.18306857347488403, + 0.129257932305336, + 0.14960098266601562, + 0.16845102608203888, + 0.16322582960128784, + 0.13869591057300568, + 0.11515536159276962, + 0.08842873573303223, + 0.1271047443151474, + 0.08747127652168274, + 0.08377104997634888, + 0.05908897519111633, + 0.10309968143701553, + 0.03653678670525551, + 0.06588983535766602, + 0.052161477506160736, + 0.07435949891805649, + 0.04259056970477104, + 0.04427788779139519, + 0.043995074927806854, + 0.053472407162189484, + 0.04156629368662834, + 0.04980214312672615, + 0.0496271476149559 + ], + "loss_nats": 1.3052101135253906, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.609398603439331, + "mean_weights": [ + 0.0809592753648758, + 0.04013277217745781, + 0.03442784771323204, + 0.03263507038354874, + 0.03443540260195732, + 0.047240644693374634, + 0.05792279541492462, + 0.2393622249364853, + 0.43288397789001465 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.06029115617275238, + 0.16869333386421204, + 0.20056062936782837, + 0.24958865344524384, + 0.23389145731925964, + 0.23942448198795319, + 0.31467917561531067, + 0.31786683201789856, + 0.12803514301776886, + 0.1331564038991928, + 0.19432871043682098, + 0.19950589537620544, + 0.05156426504254341, + 0.06165311112999916, + 0.08753261715173721, + 0.10097606480121613, + 0.04221063107252121, + 0.05752932280302048, + 0.08379971981048584, + 0.11994150280952454, + 0.04606551304459572, + 0.07362622767686844, + 0.08261217921972275, + 0.12106145173311234, + 0.027681373059749603, + 0.06137929856777191, + 0.06537999957799911, + 0.09901073575019836, + 0.02240605279803276, + 0.0852273628115654, + 0.08763451874256134, + 0.19495683908462524 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.015965601418904, + "cross_entropy_nats": 5.55624395608902, + "step": 0 + }, + { + "bits_per_byte": 5.1077262486126465, + "cross_entropy_nats": 3.540406048297882, + "step": 100 + }, + { + "bits_per_byte": 3.1449978072640112, + "cross_entropy_nats": 2.1799463629722595, + "step": 500 + }, + { + "bits_per_byte": 1.978178932745523, + "cross_entropy_nats": 1.3711691498756409, + "step": 2000 + }, + { + "bits_per_byte": 1.8301036761704788, + "cross_entropy_nats": 1.2685312032699585, + "step": 4000 + }, + { + "bits_per_byte": 1.72661337398776, + "cross_entropy_nats": 1.1967971920967102, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "ad1807a65c19f598a4bde3ce03f75bb64bba14b79befbf1740b5aaddab905279", + "final_model_state": "6df5cb12d848445089dff93c7b22def426897ab70173e1fcca78c3db41e914dc", + "final_optimizer_state": "a8e9a566e68720356f0b3f878d687f8c4cac616f59f729e02056853409f8d145", + "final_public_parameters": "16b3a7903fb4172275d1673bd0f390d3b3c03a9d0d151ac66d732eb978cff06f", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.54310071476275, + "measured_steps": 7980, + "median_ms": 54.49852900346741, + "p95_ms": 54.8562506082817, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.030921126746609, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.566610336303711, + "step": 1, + "unclipped_grad_norm": 17.857467651367188 + }, + { + "bits_per_byte": 7.460191884344154, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.171010971069336, + "step": 10, + "unclipped_grad_norm": 12.879206657409668 + }, + { + "bits_per_byte": 6.7765918040416455, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6971755027771, + "step": 20, + "unclipped_grad_norm": 4.5564117431640625 + }, + { + "bits_per_byte": 6.553465640395858, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.542516231536865, + "step": 30, + "unclipped_grad_norm": 3.0218148231506348 + }, + { + "bits_per_byte": 6.42131485940348, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.450916290283203, + "step": 40, + "unclipped_grad_norm": 2.865441083908081 + }, + { + "bits_per_byte": 6.256338093806218, + "learning_rate": 3.75e-05, + "loss_nats": 4.3365631103515625, + "step": 50, + "unclipped_grad_norm": 2.8344333171844482 + }, + { + "bits_per_byte": 5.946641367160921, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.1218976974487305, + "step": 60, + "unclipped_grad_norm": 2.6930737495422363 + }, + { + "bits_per_byte": 5.708490111205155, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9568238258361816, + "step": 70, + "unclipped_grad_norm": 2.6865241527557373 + }, + { + "bits_per_byte": 5.509431199721109, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8188467025756836, + "step": 80, + "unclipped_grad_norm": 2.448140859603882 + }, + { + "bits_per_byte": 5.311447867735559, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.681615114212036, + "step": 90, + "unclipped_grad_norm": 2.2759759426116943 + }, + { + "bits_per_byte": 5.0864742664542035, + "learning_rate": 7.5e-05, + "loss_nats": 3.5256752967834473, + "step": 100, + "unclipped_grad_norm": 2.1487066745758057 + }, + { + "bits_per_byte": 4.9338616775927715, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4198923110961914, + "step": 110, + "unclipped_grad_norm": 1.9959912300109863 + }, + { + "bits_per_byte": 4.731707078393195, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2797694206237793, + "step": 120, + "unclipped_grad_norm": 1.7048262357711792 + }, + { + "bits_per_byte": 4.582112785054405, + "learning_rate": 9.75e-05, + "loss_nats": 3.1760785579681396, + "step": 130, + "unclipped_grad_norm": 1.5641754865646362 + }, + { + "bits_per_byte": 4.436114304979855, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0748801231384277, + "step": 140, + "unclipped_grad_norm": 1.380750060081482 + }, + { + "bits_per_byte": 4.229324085408923, + "learning_rate": 0.0001125, + "loss_nats": 2.931544065475464, + "step": 150, + "unclipped_grad_norm": 1.1272099018096924 + }, + { + "bits_per_byte": 4.108468093029308, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8477730751037598, + "step": 160, + "unclipped_grad_norm": 0.8926234245300293 + }, + { + "bits_per_byte": 4.025982461953091, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7905983924865723, + "step": 170, + "unclipped_grad_norm": 0.915916919708252 + }, + { + "bits_per_byte": 3.9299425341970746, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7240285873413086, + "step": 180, + "unclipped_grad_norm": 0.7168256640434265 + }, + { + "bits_per_byte": 3.8518424607178594, + "learning_rate": 0.0001425, + "loss_nats": 2.669893741607666, + "step": 190, + "unclipped_grad_norm": 0.8937639594078064 + }, + { + "bits_per_byte": 3.7890044716415114, + "learning_rate": 0.00015, + "loss_nats": 2.626337766647339, + "step": 200, + "unclipped_grad_norm": 0.7755346894264221 + }, + { + "bits_per_byte": 3.7186869570791403, + "learning_rate": 0.0001575, + "loss_nats": 2.5775973796844482, + "step": 210, + "unclipped_grad_norm": 0.731957197189331 + }, + { + "bits_per_byte": 3.706538446587493, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.56917667388916, + "step": 220, + "unclipped_grad_norm": 0.8748186230659485 + }, + { + "bits_per_byte": 3.655573795761412, + "learning_rate": 0.0001725, + "loss_nats": 2.53385066986084, + "step": 230, + "unclipped_grad_norm": 0.8331225514411926 + }, + { + "bits_per_byte": 3.652975825837478, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5320498943328857, + "step": 240, + "unclipped_grad_norm": 1.103366494178772 + }, + { + "bits_per_byte": 3.571490789823981, + "learning_rate": 0.0001875, + "loss_nats": 2.4755687713623047, + "step": 250, + "unclipped_grad_norm": 0.6647337675094604 + }, + { + "bits_per_byte": 3.5855342051628347, + "learning_rate": 0.000195, + "loss_nats": 2.4853029251098633, + "step": 260, + "unclipped_grad_norm": 0.8029841780662537 + }, + { + "bits_per_byte": 3.576924065730039, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.479334831237793, + "step": 270, + "unclipped_grad_norm": 0.7433680891990662 + }, + { + "bits_per_byte": 3.5153632198189984, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.43666410446167, + "step": 280, + "unclipped_grad_norm": 0.824142336845398 + }, + { + "bits_per_byte": 3.511655617831698, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.434094190597534, + "step": 290, + "unclipped_grad_norm": 0.8560227751731873 + }, + { + "bits_per_byte": 3.4998641433232422, + "learning_rate": 0.000225, + "loss_nats": 2.4259209632873535, + "step": 300, + "unclipped_grad_norm": 0.8836220502853394 + }, + { + "bits_per_byte": 3.4156380478202935, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.367539882659912, + "step": 310, + "unclipped_grad_norm": 0.9033657908439636 + }, + { + "bits_per_byte": 3.481727196928552, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.4133493900299072, + "step": 320, + "unclipped_grad_norm": 1.119826316833496 + }, + { + "bits_per_byte": 3.4809216301919106, + "learning_rate": 0.0002475, + "loss_nats": 2.4127910137176514, + "step": 330, + "unclipped_grad_norm": 0.9842678308486938 + }, + { + "bits_per_byte": 3.422977579428515, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3726272583007812, + "step": 340, + "unclipped_grad_norm": 1.075486421585083 + }, + { + "bits_per_byte": 3.385370477129646, + "learning_rate": 0.0002625, + "loss_nats": 2.346560001373291, + "step": 350, + "unclipped_grad_norm": 0.7245537042617798 + }, + { + "bits_per_byte": 3.385430671057448, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.346601724624634, + "step": 360, + "unclipped_grad_norm": 1.202561616897583 + }, + { + "bits_per_byte": 3.3953991294667585, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.353511333465576, + "step": 370, + "unclipped_grad_norm": 0.7641725540161133 + }, + { + "bits_per_byte": 3.295281837024086, + "learning_rate": 0.000285, + "loss_nats": 2.2841153144836426, + "step": 380, + "unclipped_grad_norm": 1.061362862586975 + }, + { + "bits_per_byte": 3.380484450018634, + "learning_rate": 0.0002925, + "loss_nats": 2.3431732654571533, + "step": 390, + "unclipped_grad_norm": 1.0704491138458252 + }, + { + "bits_per_byte": 3.354825670405808, + "learning_rate": 0.0003, + "loss_nats": 2.325387954711914, + "step": 400, + "unclipped_grad_norm": 1.066178560256958 + }, + { + "bits_per_byte": 3.38879568360423, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.3489341735839844, + "step": 410, + "unclipped_grad_norm": 0.9359472990036011 + }, + { + "bits_per_byte": 3.3128674070401103, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.296304702758789, + "step": 420, + "unclipped_grad_norm": 0.9575046896934509 + }, + { + "bits_per_byte": 3.3331142365955855, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.3103387355804443, + "step": 430, + "unclipped_grad_norm": 0.9350728988647461 + }, + { + "bits_per_byte": 3.2769842588334903, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.271432399749756, + "step": 440, + "unclipped_grad_norm": 1.028943419456482 + }, + { + "bits_per_byte": 3.3396626480098375, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.31487774848938, + "step": 450, + "unclipped_grad_norm": 1.1271060705184937 + }, + { + "bits_per_byte": 3.2591895579140173, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2590980529785156, + "step": 460, + "unclipped_grad_norm": 1.1319539546966553 + }, + { + "bits_per_byte": 3.2327510089621794, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.240772247314453, + "step": 470, + "unclipped_grad_norm": 1.1909105777740479 + }, + { + "bits_per_byte": 3.2186394884935843, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2309908866882324, + "step": 480, + "unclipped_grad_norm": 1.3586266040802002 + }, + { + "bits_per_byte": 3.169725902761693, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1970865726470947, + "step": 490, + "unclipped_grad_norm": 1.5543664693832397 + }, + { + "bits_per_byte": 3.101864644418944, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1500487327575684, + "step": 500, + "unclipped_grad_norm": 1.9330527782440186 + }, + { + "bits_per_byte": 3.1523112835006266, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1850156784057617, + "step": 510, + "unclipped_grad_norm": 1.367343544960022 + }, + { + "bits_per_byte": 3.137257298105303, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1745810508728027, + "step": 520, + "unclipped_grad_norm": 1.4051828384399414 + }, + { + "bits_per_byte": 2.96818390576449, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.0573883056640625, + "step": 530, + "unclipped_grad_norm": 0.88100665807724 + }, + { + "bits_per_byte": 2.9700385666713944, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.058673858642578, + "step": 540, + "unclipped_grad_norm": 1.1694613695144653 + }, + { + "bits_per_byte": 2.981698990399893, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.066756248474121, + "step": 550, + "unclipped_grad_norm": 2.2637624740600586 + }, + { + "bits_per_byte": 2.9860140351100406, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.06974720954895, + "step": 560, + "unclipped_grad_norm": 2.0154106616973877 + }, + { + "bits_per_byte": 2.908624249978889, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0161046981811523, + "step": 570, + "unclipped_grad_norm": 1.2818057537078857 + }, + { + "bits_per_byte": 2.9258029090429716, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0280120372772217, + "step": 580, + "unclipped_grad_norm": 1.5401909351348877 + }, + { + "bits_per_byte": 2.865440266260492, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.986171841621399, + "step": 590, + "unclipped_grad_norm": 1.2538588047027588 + }, + { + "bits_per_byte": 2.8203701488100825, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.954931616783142, + "step": 600, + "unclipped_grad_norm": 1.5663292407989502 + }, + { + "bits_per_byte": 2.8417604910111445, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9697582721710205, + "step": 610, + "unclipped_grad_norm": 1.3150807619094849 + }, + { + "bits_per_byte": 2.806068931477586, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9450187683105469, + "step": 620, + "unclipped_grad_norm": 1.0626381635665894 + }, + { + "bits_per_byte": 2.766462358879786, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9175655841827393, + "step": 630, + "unclipped_grad_norm": 2.55611515045166 + }, + { + "bits_per_byte": 2.7411267346679313, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.900004267692566, + "step": 640, + "unclipped_grad_norm": 0.9799621105194092 + }, + { + "bits_per_byte": 2.7289859633955733, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8915889263153076, + "step": 650, + "unclipped_grad_norm": 1.4919346570968628 + }, + { + "bits_per_byte": 2.7446909031244124, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.9024747610092163, + "step": 660, + "unclipped_grad_norm": 1.184576153755188 + }, + { + "bits_per_byte": 2.706914225931793, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.876289963722229, + "step": 670, + "unclipped_grad_norm": 1.2165472507476807 + }, + { + "bits_per_byte": 2.7108362902847194, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8790085315704346, + "step": 680, + "unclipped_grad_norm": 2.3380022048950195 + }, + { + "bits_per_byte": 2.6699751042488753, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.850685715675354, + "step": 690, + "unclipped_grad_norm": 1.790702223777771 + }, + { + "bits_per_byte": 2.6284332108809165, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8218910694122314, + "step": 700, + "unclipped_grad_norm": 1.6820380687713623 + }, + { + "bits_per_byte": 2.6224609413470548, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.817751407623291, + "step": 710, + "unclipped_grad_norm": 1.3093626499176025 + }, + { + "bits_per_byte": 2.6378994799497075, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8284525871276855, + "step": 720, + "unclipped_grad_norm": 1.689491868019104 + }, + { + "bits_per_byte": 2.604935393276182, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8056036233901978, + "step": 730, + "unclipped_grad_norm": 1.4546315670013428 + }, + { + "bits_per_byte": 2.542083817570416, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.762038230895996, + "step": 740, + "unclipped_grad_norm": 1.6848912239074707 + }, + { + "bits_per_byte": 2.5572903515770546, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7725785970687866, + "step": 750, + "unclipped_grad_norm": 1.477650761604309 + }, + { + "bits_per_byte": 2.448532478772384, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.6971933841705322, + "step": 760, + "unclipped_grad_norm": 1.4033355712890625 + }, + { + "bits_per_byte": 2.5017822630804796, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7341033220291138, + "step": 770, + "unclipped_grad_norm": 1.3305320739746094 + }, + { + "bits_per_byte": 2.4567315796696176, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7028765678405762, + "step": 780, + "unclipped_grad_norm": 1.7414066791534424 + }, + { + "bits_per_byte": 2.463544844366199, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.70759916305542, + "step": 790, + "unclipped_grad_norm": 1.3770233392715454 + }, + { + "bits_per_byte": 2.390281954925151, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6568171977996826, + "step": 800, + "unclipped_grad_norm": 1.4013909101486206 + }, + { + "bits_per_byte": 2.4192710065892844, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6769108772277832, + "step": 810, + "unclipped_grad_norm": 1.360757827758789 + }, + { + "bits_per_byte": 2.3909581907083437, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6572859287261963, + "step": 820, + "unclipped_grad_norm": 1.4178366661071777 + }, + { + "bits_per_byte": 2.4097108349931227, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6702842712402344, + "step": 830, + "unclipped_grad_norm": 1.1894793510437012 + }, + { + "bits_per_byte": 2.395687713607071, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6605641841888428, + "step": 840, + "unclipped_grad_norm": 1.7937202453613281 + }, + { + "bits_per_byte": 2.326945216161301, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6129155158996582, + "step": 850, + "unclipped_grad_norm": 1.4669246673583984 + }, + { + "bits_per_byte": 2.3940583499727963, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6594347953796387, + "step": 860, + "unclipped_grad_norm": 1.3730710744857788 + }, + { + "bits_per_byte": 2.3742401011832714, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.645697832107544, + "step": 870, + "unclipped_grad_norm": 1.3468742370605469 + }, + { + "bits_per_byte": 2.3406023584663167, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6223819255828857, + "step": 880, + "unclipped_grad_norm": 1.3746341466903687 + }, + { + "bits_per_byte": 2.2996515694693733, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5939970016479492, + "step": 890, + "unclipped_grad_norm": 1.5287024974822998 + }, + { + "bits_per_byte": 2.338895602639154, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6211988925933838, + "step": 900, + "unclipped_grad_norm": 1.2028048038482666 + }, + { + "bits_per_byte": 2.322601450348459, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6099046468734741, + "step": 910, + "unclipped_grad_norm": 1.5355594158172607 + }, + { + "bits_per_byte": 2.307329906899794, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5993192195892334, + "step": 920, + "unclipped_grad_norm": 1.5230286121368408 + }, + { + "bits_per_byte": 2.2540851100885737, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5624127388000488, + "step": 930, + "unclipped_grad_norm": 1.3632712364196777 + }, + { + "bits_per_byte": 2.306159220995371, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5985077619552612, + "step": 940, + "unclipped_grad_norm": 1.4536393880844116 + }, + { + "bits_per_byte": 2.330239371855927, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6151988506317139, + "step": 950, + "unclipped_grad_norm": 1.4820353984832764 + }, + { + "bits_per_byte": 2.256347369878023, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5639808177947998, + "step": 960, + "unclipped_grad_norm": 1.5091183185577393 + }, + { + "bits_per_byte": 2.2135099312011146, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5342881679534912, + "step": 970, + "unclipped_grad_norm": 1.3972843885421753 + }, + { + "bits_per_byte": 2.3381242604500345, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6206642389297485, + "step": 980, + "unclipped_grad_norm": 1.3591992855072021 + }, + { + "bits_per_byte": 2.202303885636196, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5265207290649414, + "step": 990, + "unclipped_grad_norm": 1.2794140577316284 + }, + { + "bits_per_byte": 2.194666824041982, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5212271213531494, + "step": 1000, + "unclipped_grad_norm": 1.2117714881896973 + }, + { + "bits_per_byte": 2.205168084703665, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5285060405731201, + "step": 1010, + "unclipped_grad_norm": 1.4173126220703125 + }, + { + "bits_per_byte": 2.1627664500640433, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.4991154670715332, + "step": 1020, + "unclipped_grad_norm": 1.5233873128890991 + }, + { + "bits_per_byte": 2.171931921476475, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.505468487739563, + "step": 1030, + "unclipped_grad_norm": 1.5871013402938843 + }, + { + "bits_per_byte": 2.2533218510840447, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5618836879730225, + "step": 1040, + "unclipped_grad_norm": 1.408625602722168 + }, + { + "bits_per_byte": 2.203781560572409, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5275449752807617, + "step": 1050, + "unclipped_grad_norm": 1.9470032453536987 + }, + { + "bits_per_byte": 2.2074819392883733, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5301098823547363, + "step": 1060, + "unclipped_grad_norm": 1.358320713043213 + }, + { + "bits_per_byte": 2.2549842353872847, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5630359649658203, + "step": 1070, + "unclipped_grad_norm": 1.3524690866470337 + }, + { + "bits_per_byte": 2.232926944466177, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.547747015953064, + "step": 1080, + "unclipped_grad_norm": 1.3045321702957153 + }, + { + "bits_per_byte": 2.1562297334700493, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.494584560394287, + "step": 1090, + "unclipped_grad_norm": 1.3637882471084595 + }, + { + "bits_per_byte": 2.1766043461351154, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5087071657180786, + "step": 1100, + "unclipped_grad_norm": 1.3327077627182007 + }, + { + "bits_per_byte": 2.113153238908442, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.464726209640503, + "step": 1110, + "unclipped_grad_norm": 1.3556562662124634 + }, + { + "bits_per_byte": 2.158234707213808, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4959743022918701, + "step": 1120, + "unclipped_grad_norm": 1.5276546478271484 + }, + { + "bits_per_byte": 2.1778080527085044, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5095415115356445, + "step": 1130, + "unclipped_grad_norm": 1.3574931621551514 + }, + { + "bits_per_byte": 2.120840347454057, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4700545072555542, + "step": 1140, + "unclipped_grad_norm": 1.49063241481781 + }, + { + "bits_per_byte": 2.172341240185528, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5057522058486938, + "step": 1150, + "unclipped_grad_norm": 1.270938515663147 + }, + { + "bits_per_byte": 2.081790654679751, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.442987322807312, + "step": 1160, + "unclipped_grad_norm": 1.387803077697754 + }, + { + "bits_per_byte": 2.1352082940550883, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4800136089324951, + "step": 1170, + "unclipped_grad_norm": 1.2486525774002075 + }, + { + "bits_per_byte": 2.1410305947173978, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4840493202209473, + "step": 1180, + "unclipped_grad_norm": 1.3077996969223022 + }, + { + "bits_per_byte": 2.1241448221077346, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.472344994544983, + "step": 1190, + "unclipped_grad_norm": 1.846550464630127 + }, + { + "bits_per_byte": 2.128028362346868, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.475036859512329, + "step": 1200, + "unclipped_grad_norm": 1.1341090202331543 + }, + { + "bits_per_byte": 2.1484036629425374, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4891599416732788, + "step": 1210, + "unclipped_grad_norm": 1.2377034425735474 + }, + { + "bits_per_byte": 2.096246140450072, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4530071020126343, + "step": 1220, + "unclipped_grad_norm": 1.182817816734314 + }, + { + "bits_per_byte": 2.11190705262029, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.463862419128418, + "step": 1230, + "unclipped_grad_norm": 1.3176664113998413 + }, + { + "bits_per_byte": 2.0884407078406633, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.447596788406372, + "step": 1240, + "unclipped_grad_norm": 1.3174140453338623 + }, + { + "bits_per_byte": 2.01052448381522, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.3935893774032593, + "step": 1250, + "unclipped_grad_norm": 1.3046211004257202 + }, + { + "bits_per_byte": 2.0807407005962335, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4422595500946045, + "step": 1260, + "unclipped_grad_norm": 1.3089001178741455 + }, + { + "bits_per_byte": 2.0962698740558907, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4530235528945923, + "step": 1270, + "unclipped_grad_norm": 1.318946361541748 + }, + { + "bits_per_byte": 2.1189218809836823, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4687247276306152, + "step": 1280, + "unclipped_grad_norm": 1.1660429239273071 + }, + { + "bits_per_byte": 2.0943455601753866, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4516897201538086, + "step": 1290, + "unclipped_grad_norm": 1.5281683206558228 + }, + { + "bits_per_byte": 2.0472271293532014, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.419029712677002, + "step": 1300, + "unclipped_grad_norm": 1.416712999343872 + }, + { + "bits_per_byte": 2.0644360573638356, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4309580326080322, + "step": 1310, + "unclipped_grad_norm": 1.2950114011764526 + }, + { + "bits_per_byte": 2.044286569988749, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4169914722442627, + "step": 1320, + "unclipped_grad_norm": 1.1484317779541016 + }, + { + "bits_per_byte": 2.0443660259734475, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4170465469360352, + "step": 1330, + "unclipped_grad_norm": 1.206559181213379 + }, + { + "bits_per_byte": 2.040463051694767, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4143412113189697, + "step": 1340, + "unclipped_grad_norm": 1.1644346714019775 + }, + { + "bits_per_byte": 2.073596541279392, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.437307596206665, + "step": 1350, + "unclipped_grad_norm": 1.295727014541626 + }, + { + "bits_per_byte": 2.028423750186418, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4059962034225464, + "step": 1360, + "unclipped_grad_norm": 1.2224998474121094 + }, + { + "bits_per_byte": 2.049744783379182, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4207748174667358, + "step": 1370, + "unclipped_grad_norm": 1.2044527530670166 + }, + { + "bits_per_byte": 2.01407248590252, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.396048665046692, + "step": 1380, + "unclipped_grad_norm": 1.241443157196045 + }, + { + "bits_per_byte": 2.0615071928196422, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4289278984069824, + "step": 1390, + "unclipped_grad_norm": 1.1473987102508545 + }, + { + "bits_per_byte": 2.1662895146669676, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.501557469367981, + "step": 1400, + "unclipped_grad_norm": 1.1963618993759155 + }, + { + "bits_per_byte": 2.064517577140345, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4310145378112793, + "step": 1410, + "unclipped_grad_norm": 1.5827157497406006 + }, + { + "bits_per_byte": 2.0563607839752236, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4253606796264648, + "step": 1420, + "unclipped_grad_norm": 1.2223409414291382 + }, + { + "bits_per_byte": 1.996439276692207, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.3838262557983398, + "step": 1430, + "unclipped_grad_norm": 1.1598049402236938 + }, + { + "bits_per_byte": 2.0215600945731342, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4012386798858643, + "step": 1440, + "unclipped_grad_norm": 1.1466177701950073 + }, + { + "bits_per_byte": 2.0019069491284376, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3876161575317383, + "step": 1450, + "unclipped_grad_norm": 1.1313462257385254 + }, + { + "bits_per_byte": 2.1132414660083345, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4647873640060425, + "step": 1460, + "unclipped_grad_norm": 1.1495121717453003 + }, + { + "bits_per_byte": 2.102186421210873, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4571245908737183, + "step": 1470, + "unclipped_grad_norm": 1.337440013885498 + }, + { + "bits_per_byte": 2.0400563127254765, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4140592813491821, + "step": 1480, + "unclipped_grad_norm": 1.06326162815094 + }, + { + "bits_per_byte": 2.057447370363375, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4261138439178467, + "step": 1490, + "unclipped_grad_norm": 1.1691441535949707 + }, + { + "bits_per_byte": 1.9426222936015662, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3465231657028198, + "step": 1500, + "unclipped_grad_norm": 1.0966376066207886 + }, + { + "bits_per_byte": 1.953619208263036, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3541456460952759, + "step": 1510, + "unclipped_grad_norm": 1.2253862619400024 + }, + { + "bits_per_byte": 1.9856435817322469, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3763432502746582, + "step": 1520, + "unclipped_grad_norm": 1.1588174104690552 + }, + { + "bits_per_byte": 2.043245215037775, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4162696599960327, + "step": 1530, + "unclipped_grad_norm": 1.1396243572235107 + }, + { + "bits_per_byte": 2.030725050037613, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4075913429260254, + "step": 1540, + "unclipped_grad_norm": 1.1333942413330078 + }, + { + "bits_per_byte": 2.081739919797746, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4429521560668945, + "step": 1550, + "unclipped_grad_norm": 1.1568241119384766 + }, + { + "bits_per_byte": 1.9638389333252566, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.361229419708252, + "step": 1560, + "unclipped_grad_norm": 1.0923500061035156 + }, + { + "bits_per_byte": 1.9637097743544585, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3611398935317993, + "step": 1570, + "unclipped_grad_norm": 1.181723952293396 + }, + { + "bits_per_byte": 1.984943956308537, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3758583068847656, + "step": 1580, + "unclipped_grad_norm": 1.15809965133667 + }, + { + "bits_per_byte": 1.9967120411764756, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3840153217315674, + "step": 1590, + "unclipped_grad_norm": 1.2961064577102661 + }, + { + "bits_per_byte": 1.9713934432470563, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3664658069610596, + "step": 1600, + "unclipped_grad_norm": 1.1767901182174683 + }, + { + "bits_per_byte": 1.9459826626167747, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3488523960113525, + "step": 1610, + "unclipped_grad_norm": 1.1538722515106201 + }, + { + "bits_per_byte": 1.9305324291938633, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3381431102752686, + "step": 1620, + "unclipped_grad_norm": 1.0682140588760376 + }, + { + "bits_per_byte": 1.9835519287324535, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3748934268951416, + "step": 1630, + "unclipped_grad_norm": 1.1328599452972412 + }, + { + "bits_per_byte": 1.952673819631243, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3534903526306152, + "step": 1640, + "unclipped_grad_norm": 1.1611915826797485 + }, + { + "bits_per_byte": 1.9528083100642177, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.353583574295044, + "step": 1650, + "unclipped_grad_norm": 1.2886978387832642 + }, + { + "bits_per_byte": 1.9660846827802232, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.362786054611206, + "step": 1660, + "unclipped_grad_norm": 1.1841517686843872 + }, + { + "bits_per_byte": 1.9119278619714772, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3252474069595337, + "step": 1670, + "unclipped_grad_norm": 1.1872814893722534 + }, + { + "bits_per_byte": 1.93973676868539, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3445230722427368, + "step": 1680, + "unclipped_grad_norm": 1.1895164251327515 + }, + { + "bits_per_byte": 1.9710771671521197, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3662465810775757, + "step": 1690, + "unclipped_grad_norm": 1.0209845304489136 + }, + { + "bits_per_byte": 1.9189868898761413, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3301403522491455, + "step": 1700, + "unclipped_grad_norm": 1.2754039764404297 + }, + { + "bits_per_byte": 1.9587719804655366, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3577172756195068, + "step": 1710, + "unclipped_grad_norm": 0.9888415336608887 + }, + { + "bits_per_byte": 1.8892168649944399, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3095053434371948, + "step": 1720, + "unclipped_grad_norm": 1.1411211490631104 + }, + { + "bits_per_byte": 1.970308060737461, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3657134771347046, + "step": 1730, + "unclipped_grad_norm": 1.1278765201568604 + }, + { + "bits_per_byte": 1.9812284431192968, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3732829093933105, + "step": 1740, + "unclipped_grad_norm": 1.0859284400939941 + }, + { + "bits_per_byte": 1.9480999410315467, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3503199815750122, + "step": 1750, + "unclipped_grad_norm": 1.063671588897705 + }, + { + "bits_per_byte": 1.9687923776354073, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3646628856658936, + "step": 1760, + "unclipped_grad_norm": 1.02903151512146 + }, + { + "bits_per_byte": 1.9746323925107556, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3687108755111694, + "step": 1770, + "unclipped_grad_norm": 1.1287299394607544 + }, + { + "bits_per_byte": 2.0187137817013547, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3992657661437988, + "step": 1780, + "unclipped_grad_norm": 1.1014562845230103 + }, + { + "bits_per_byte": 1.8992608237874788, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.31646728515625, + "step": 1790, + "unclipped_grad_norm": 1.2110038995742798 + }, + { + "bits_per_byte": 2.0498698147663594, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.4208614826202393, + "step": 1800, + "unclipped_grad_norm": 1.5255318880081177 + }, + { + "bits_per_byte": 1.8942502812372417, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3129942417144775, + "step": 1810, + "unclipped_grad_norm": 1.0947346687316895 + }, + { + "bits_per_byte": 1.9627021280030534, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3604414463043213, + "step": 1820, + "unclipped_grad_norm": 1.1853052377700806 + }, + { + "bits_per_byte": 1.9780426364947141, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3710746765136719, + "step": 1830, + "unclipped_grad_norm": 1.0857312679290771 + }, + { + "bits_per_byte": 1.975353859731125, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.369210958480835, + "step": 1840, + "unclipped_grad_norm": 1.1014374494552612 + }, + { + "bits_per_byte": 1.8969483450639775, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3148643970489502, + "step": 1850, + "unclipped_grad_norm": 1.0865846872329712 + }, + { + "bits_per_byte": 1.943441103002324, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.347090721130371, + "step": 1860, + "unclipped_grad_norm": 0.9979931116104126 + }, + { + "bits_per_byte": 2.018713093770751, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3992652893066406, + "step": 1870, + "unclipped_grad_norm": 1.1247007846832275 + }, + { + "bits_per_byte": 1.895716605318498, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3140106201171875, + "step": 1880, + "unclipped_grad_norm": 1.2656805515289307 + }, + { + "bits_per_byte": 1.939341724536358, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3442492485046387, + "step": 1890, + "unclipped_grad_norm": 1.1298930644989014 + }, + { + "bits_per_byte": 1.9376002282137212, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3430421352386475, + "step": 1900, + "unclipped_grad_norm": 1.1912769079208374 + }, + { + "bits_per_byte": 1.9616757355427041, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3597300052642822, + "step": 1910, + "unclipped_grad_norm": 1.0634745359420776 + }, + { + "bits_per_byte": 1.8880833273426032, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3087196350097656, + "step": 1920, + "unclipped_grad_norm": 1.0458215475082397 + }, + { + "bits_per_byte": 1.9484389188363973, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3505549430847168, + "step": 1930, + "unclipped_grad_norm": 1.0484954118728638 + }, + { + "bits_per_byte": 1.9728711181832694, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3674900531768799, + "step": 1940, + "unclipped_grad_norm": 1.0497392416000366 + }, + { + "bits_per_byte": 1.9061248233660644, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3212250471115112, + "step": 1950, + "unclipped_grad_norm": 1.2195848226547241 + }, + { + "bits_per_byte": 1.8858399856447488, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3071646690368652, + "step": 1960, + "unclipped_grad_norm": 1.0152307748794556 + }, + { + "bits_per_byte": 1.8880905506139394, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3087246417999268, + "step": 1970, + "unclipped_grad_norm": 1.1332975625991821 + }, + { + "bits_per_byte": 1.9225495104887642, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.33260977268219, + "step": 1980, + "unclipped_grad_norm": 1.1374293565750122 + }, + { + "bits_per_byte": 1.8947043154355196, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3133089542388916, + "step": 1990, + "unclipped_grad_norm": 1.0456688404083252 + }, + { + "bits_per_byte": 1.977381019236845, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.37061607837677, + "step": 2000, + "unclipped_grad_norm": 1.1185020208358765 + }, + { + "bits_per_byte": 1.9636655748131868, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3611092567443848, + "step": 2010, + "unclipped_grad_norm": 1.1275893449783325 + }, + { + "bits_per_byte": 1.8762688071589317, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3005304336547852, + "step": 2020, + "unclipped_grad_norm": 1.0362930297851562 + }, + { + "bits_per_byte": 1.8977705941177525, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3154343366622925, + "step": 2030, + "unclipped_grad_norm": 1.0457504987716675 + }, + { + "bits_per_byte": 1.8942691993288368, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3130073547363281, + "step": 2040, + "unclipped_grad_norm": 0.9790334105491638 + }, + { + "bits_per_byte": 1.854481013051726, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2854282855987549, + "step": 2050, + "unclipped_grad_norm": 1.067064881324768 + }, + { + "bits_per_byte": 1.8548413167052835, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2856780290603638, + "step": 2060, + "unclipped_grad_norm": 1.0005096197128296 + }, + { + "bits_per_byte": 1.9705749778116002, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3658984899520874, + "step": 2070, + "unclipped_grad_norm": 1.1014602184295654 + }, + { + "bits_per_byte": 1.9435265783798028, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3471499681472778, + "step": 2080, + "unclipped_grad_norm": 1.1029977798461914 + }, + { + "bits_per_byte": 1.8586646630166146, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2883281707763672, + "step": 2090, + "unclipped_grad_norm": 1.0793615579605103 + }, + { + "bits_per_byte": 1.8191048696998495, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2609074115753174, + "step": 2100, + "unclipped_grad_norm": 1.0433158874511719 + }, + { + "bits_per_byte": 1.851254274556238, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2831916809082031, + "step": 2110, + "unclipped_grad_norm": 1.1101045608520508 + }, + { + "bits_per_byte": 1.8581610978148881, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2879791259765625, + "step": 2120, + "unclipped_grad_norm": 1.1086524724960327 + }, + { + "bits_per_byte": 1.9049420986760808, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3204052448272705, + "step": 2130, + "unclipped_grad_norm": 1.0729707479476929 + }, + { + "bits_per_byte": 1.941282376768694, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3455944061279297, + "step": 2140, + "unclipped_grad_norm": 1.0528757572174072 + }, + { + "bits_per_byte": 1.843757550805128, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2779953479766846, + "step": 2150, + "unclipped_grad_norm": 1.1246051788330078 + }, + { + "bits_per_byte": 1.9154739722496175, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3277053833007812, + "step": 2160, + "unclipped_grad_norm": 1.0607515573501587 + }, + { + "bits_per_byte": 1.850249895875199, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2824954986572266, + "step": 2170, + "unclipped_grad_norm": 1.0301073789596558 + }, + { + "bits_per_byte": 1.9307078514977434, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3382647037506104, + "step": 2180, + "unclipped_grad_norm": 1.0744229555130005 + }, + { + "bits_per_byte": 1.895962540509232, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3141810894012451, + "step": 2190, + "unclipped_grad_norm": 1.1302496194839478 + }, + { + "bits_per_byte": 1.8514286649642127, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2833125591278076, + "step": 2200, + "unclipped_grad_norm": 1.043583869934082 + }, + { + "bits_per_byte": 1.8687825743495246, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2953413724899292, + "step": 2210, + "unclipped_grad_norm": 1.1252996921539307 + }, + { + "bits_per_byte": 1.862911258631719, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.291271686553955, + "step": 2220, + "unclipped_grad_norm": 1.0556633472442627 + }, + { + "bits_per_byte": 1.8208848901362797, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.262141227722168, + "step": 2230, + "unclipped_grad_norm": 1.0693137645721436 + }, + { + "bits_per_byte": 1.90406292336487, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.319795846939087, + "step": 2240, + "unclipped_grad_norm": 1.0752713680267334 + }, + { + "bits_per_byte": 1.8964874315596651, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.314544916152954, + "step": 2250, + "unclipped_grad_norm": 1.0330802202224731 + }, + { + "bits_per_byte": 1.935825367256817, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3418118953704834, + "step": 2260, + "unclipped_grad_norm": 1.078423023223877 + }, + { + "bits_per_byte": 1.832309869615747, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.270060420036316, + "step": 2270, + "unclipped_grad_norm": 1.056177020072937 + }, + { + "bits_per_byte": 1.887026837918353, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.3079873323440552, + "step": 2280, + "unclipped_grad_norm": 1.0121575593948364 + }, + { + "bits_per_byte": 1.834445722156812, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.271540880203247, + "step": 2290, + "unclipped_grad_norm": 1.0369646549224854 + }, + { + "bits_per_byte": 1.8213642057842343, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.262473464012146, + "step": 2300, + "unclipped_grad_norm": 0.9950895309448242 + }, + { + "bits_per_byte": 1.8373876573824712, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2735800743103027, + "step": 2310, + "unclipped_grad_norm": 1.0178841352462769 + }, + { + "bits_per_byte": 1.8979250345382275, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.315541386604309, + "step": 2320, + "unclipped_grad_norm": 0.9894719123840332 + }, + { + "bits_per_byte": 1.8294843666447218, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2681019306182861, + "step": 2330, + "unclipped_grad_norm": 0.9816614389419556 + }, + { + "bits_per_byte": 1.848164090285535, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2810497283935547, + "step": 2340, + "unclipped_grad_norm": 1.0238531827926636 + }, + { + "bits_per_byte": 1.905264050198496, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3206284046173096, + "step": 2350, + "unclipped_grad_norm": 1.0559433698654175 + }, + { + "bits_per_byte": 1.8307938425483912, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2690095901489258, + "step": 2360, + "unclipped_grad_norm": 1.0712460279464722 + }, + { + "bits_per_byte": 1.7703326537069675, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2271010875701904, + "step": 2370, + "unclipped_grad_norm": 1.1017504930496216 + }, + { + "bits_per_byte": 1.886526368404342, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.3076404333114624, + "step": 2380, + "unclipped_grad_norm": 1.1038753986358643 + }, + { + "bits_per_byte": 1.8630619154338748, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2913761138916016, + "step": 2390, + "unclipped_grad_norm": 1.0329313278198242 + }, + { + "bits_per_byte": 1.9135943738583379, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3264025449752808, + "step": 2400, + "unclipped_grad_norm": 1.1860445737838745 + }, + { + "bits_per_byte": 1.835037686441082, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2719511985778809, + "step": 2410, + "unclipped_grad_norm": 1.153725266456604 + }, + { + "bits_per_byte": 1.871177432762789, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2970013618469238, + "step": 2420, + "unclipped_grad_norm": 1.0672882795333862 + }, + { + "bits_per_byte": 1.8586395535495885, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2883107662200928, + "step": 2430, + "unclipped_grad_norm": 1.0162642002105713 + }, + { + "bits_per_byte": 1.8218378460047104, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2628017663955688, + "step": 2440, + "unclipped_grad_norm": 1.065905213356018 + }, + { + "bits_per_byte": 1.9384147380482075, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.34360671043396, + "step": 2450, + "unclipped_grad_norm": 1.218255639076233 + }, + { + "bits_per_byte": 1.8158471743272062, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2586493492126465, + "step": 2460, + "unclipped_grad_norm": 1.0468804836273193 + }, + { + "bits_per_byte": 1.8206396428761493, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2619712352752686, + "step": 2470, + "unclipped_grad_norm": 1.1381754875183105 + }, + { + "bits_per_byte": 1.8851444878046595, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.3066825866699219, + "step": 2480, + "unclipped_grad_norm": 1.035730242729187 + }, + { + "bits_per_byte": 1.8941151028736636, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.3129005432128906, + "step": 2490, + "unclipped_grad_norm": 1.0220366716384888 + }, + { + "bits_per_byte": 1.8318142156159603, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2697168588638306, + "step": 2500, + "unclipped_grad_norm": 0.982593834400177 + }, + { + "bits_per_byte": 1.8129355080480989, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2566311359405518, + "step": 2510, + "unclipped_grad_norm": 1.091998815536499 + }, + { + "bits_per_byte": 1.841321932503609, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2763071060180664, + "step": 2520, + "unclipped_grad_norm": 1.000153660774231 + }, + { + "bits_per_byte": 1.848013777448681, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2809455394744873, + "step": 2530, + "unclipped_grad_norm": 1.0528815984725952 + }, + { + "bits_per_byte": 1.8343394368785788, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2714672088623047, + "step": 2540, + "unclipped_grad_norm": 0.9793087840080261 + }, + { + "bits_per_byte": 1.7887870800751502, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2398927211761475, + "step": 2550, + "unclipped_grad_norm": 1.0650514364242554 + }, + { + "bits_per_byte": 1.7584782335482958, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2188842296600342, + "step": 2560, + "unclipped_grad_norm": 0.9875280857086182 + }, + { + "bits_per_byte": 1.81654955147333, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2591361999511719, + "step": 2570, + "unclipped_grad_norm": 1.0060385465621948 + }, + { + "bits_per_byte": 1.7945154782100887, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2438633441925049, + "step": 2580, + "unclipped_grad_norm": 1.0824097394943237 + }, + { + "bits_per_byte": 1.7956424805211926, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2446445226669312, + "step": 2590, + "unclipped_grad_norm": 0.9988609552383423 + }, + { + "bits_per_byte": 1.815006351147138, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2580665349960327, + "step": 2600, + "unclipped_grad_norm": 0.9940101504325867 + }, + { + "bits_per_byte": 1.840550934279791, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2757726907730103, + "step": 2610, + "unclipped_grad_norm": 1.0804052352905273 + }, + { + "bits_per_byte": 1.7607050649116673, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2204277515411377, + "step": 2620, + "unclipped_grad_norm": 1.0194178819656372 + }, + { + "bits_per_byte": 1.7479795526263755, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2116070985794067, + "step": 2630, + "unclipped_grad_norm": 0.9564489126205444 + }, + { + "bits_per_byte": 1.7532458333784458, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2152574062347412, + "step": 2640, + "unclipped_grad_norm": 0.9848498106002808 + }, + { + "bits_per_byte": 1.813315245741204, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2568943500518799, + "step": 2650, + "unclipped_grad_norm": 0.9513642191886902 + }, + { + "bits_per_byte": 1.7231679595517, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1944090127944946, + "step": 2660, + "unclipped_grad_norm": 1.0000529289245605 + }, + { + "bits_per_byte": 1.802969457395901, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.249723196029663, + "step": 2670, + "unclipped_grad_norm": 0.9922699928283691 + }, + { + "bits_per_byte": 1.9163316497294702, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.328299880027771, + "step": 2680, + "unclipped_grad_norm": 1.192367672920227 + }, + { + "bits_per_byte": 1.8243539521868333, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2645457983016968, + "step": 2690, + "unclipped_grad_norm": 1.0074772834777832 + }, + { + "bits_per_byte": 1.751833683832211, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2142785787582397, + "step": 2700, + "unclipped_grad_norm": 0.9746342301368713 + }, + { + "bits_per_byte": 1.826312318632208, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2659032344818115, + "step": 2710, + "unclipped_grad_norm": 1.0221014022827148 + }, + { + "bits_per_byte": 1.8721807795479226, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.297696828842163, + "step": 2720, + "unclipped_grad_norm": 1.0561023950576782 + }, + { + "bits_per_byte": 1.742347808741222, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2077034711837769, + "step": 2730, + "unclipped_grad_norm": 0.946500301361084 + }, + { + "bits_per_byte": 1.826287381147833, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2658859491348267, + "step": 2740, + "unclipped_grad_norm": 1.0504826307296753 + }, + { + "bits_per_byte": 1.7872662374935704, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.23883855342865, + "step": 2750, + "unclipped_grad_norm": 1.0072195529937744 + }, + { + "bits_per_byte": 1.9127833036768689, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3258403539657593, + "step": 2760, + "unclipped_grad_norm": 1.1406866312026978 + }, + { + "bits_per_byte": 1.8309943743192973, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.269148588180542, + "step": 2770, + "unclipped_grad_norm": 1.0432147979736328 + }, + { + "bits_per_byte": 1.835324725485372, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2721501588821411, + "step": 2780, + "unclipped_grad_norm": 1.0946613550186157 + }, + { + "bits_per_byte": 1.8014186898330709, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2486482858657837, + "step": 2790, + "unclipped_grad_norm": 1.0307797193527222 + }, + { + "bits_per_byte": 1.8122052697125355, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2561249732971191, + "step": 2800, + "unclipped_grad_norm": 1.0519623756408691 + }, + { + "bits_per_byte": 1.7642310532196568, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2228717803955078, + "step": 2810, + "unclipped_grad_norm": 1.0939170122146606 + }, + { + "bits_per_byte": 1.8221561858914574, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2630224227905273, + "step": 2820, + "unclipped_grad_norm": 1.006143569946289 + }, + { + "bits_per_byte": 1.7646739085456284, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.223178744316101, + "step": 2830, + "unclipped_grad_norm": 0.9335127472877502 + }, + { + "bits_per_byte": 1.8147093371090979, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2578606605529785, + "step": 2840, + "unclipped_grad_norm": 0.9985573887825012 + }, + { + "bits_per_byte": 1.758499215431701, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2188987731933594, + "step": 2850, + "unclipped_grad_norm": 0.9980565905570984 + }, + { + "bits_per_byte": 1.783597331602714, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.236295461654663, + "step": 2860, + "unclipped_grad_norm": 1.0390338897705078 + }, + { + "bits_per_byte": 1.7758223399225082, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2309062480926514, + "step": 2870, + "unclipped_grad_norm": 1.0219858884811401 + }, + { + "bits_per_byte": 1.7873183482367818, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2388746738433838, + "step": 2880, + "unclipped_grad_norm": 1.002677083015442 + }, + { + "bits_per_byte": 1.7586128959639213, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2189775705337524, + "step": 2890, + "unclipped_grad_norm": 1.0140609741210938 + }, + { + "bits_per_byte": 1.7806794739481757, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2342729568481445, + "step": 2900, + "unclipped_grad_norm": 0.9781115651130676 + }, + { + "bits_per_byte": 1.7540770255300655, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2158335447311401, + "step": 2910, + "unclipped_grad_norm": 0.9601871967315674 + }, + { + "bits_per_byte": 1.798447689539416, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.246588945388794, + "step": 2920, + "unclipped_grad_norm": 1.0120164155960083 + }, + { + "bits_per_byte": 1.749404256906123, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2125946283340454, + "step": 2930, + "unclipped_grad_norm": 0.9387297630310059 + }, + { + "bits_per_byte": 1.7688945347804526, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2261042594909668, + "step": 2940, + "unclipped_grad_norm": 1.073227047920227 + }, + { + "bits_per_byte": 1.8185380148826058, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.260514497756958, + "step": 2950, + "unclipped_grad_norm": 1.033491849899292 + }, + { + "bits_per_byte": 1.8193900169349801, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2611050605773926, + "step": 2960, + "unclipped_grad_norm": 1.0577077865600586 + }, + { + "bits_per_byte": 1.799499191466791, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2473177909851074, + "step": 2970, + "unclipped_grad_norm": 1.0431829690933228 + }, + { + "bits_per_byte": 1.8318936716006589, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.269771933555603, + "step": 2980, + "unclipped_grad_norm": 1.3554320335388184 + }, + { + "bits_per_byte": 1.7557352822496848, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2169829607009888, + "step": 2990, + "unclipped_grad_norm": 1.0419063568115234 + }, + { + "bits_per_byte": 1.8299745171996809, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2684416770935059, + "step": 3000, + "unclipped_grad_norm": 1.0811892747879028 + }, + { + "bits_per_byte": 1.7247794369902845, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1955260038375854, + "step": 3010, + "unclipped_grad_norm": 1.0030642747879028 + }, + { + "bits_per_byte": 1.7923558920632043, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2423664331436157, + "step": 3020, + "unclipped_grad_norm": 1.002471685409546 + }, + { + "bits_per_byte": 1.7403096423458468, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2062907218933105, + "step": 3030, + "unclipped_grad_norm": 1.0230085849761963 + }, + { + "bits_per_byte": 1.7392983843587735, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.205589771270752, + "step": 3040, + "unclipped_grad_norm": 1.0195822715759277 + }, + { + "bits_per_byte": 1.8747281865725025, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2994625568389893, + "step": 3050, + "unclipped_grad_norm": 1.0621304512023926 + }, + { + "bits_per_byte": 1.80563828417199, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.251573085784912, + "step": 3060, + "unclipped_grad_norm": 1.0812959671020508 + }, + { + "bits_per_byte": 1.7309672007856773, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.199815034866333, + "step": 3070, + "unclipped_grad_norm": 1.048651099205017 + }, + { + "bits_per_byte": 1.80954315025983, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2542797327041626, + "step": 3080, + "unclipped_grad_norm": 1.0254682302474976 + }, + { + "bits_per_byte": 1.787755528135275, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2391777038574219, + "step": 3090, + "unclipped_grad_norm": 1.0376405715942383 + }, + { + "bits_per_byte": 1.748834134418513, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2121994495391846, + "step": 3100, + "unclipped_grad_norm": 1.0169665813446045 + }, + { + "bits_per_byte": 1.8895020122295705, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.30970299243927, + "step": 3110, + "unclipped_grad_norm": 1.009265661239624 + }, + { + "bits_per_byte": 1.772648744066137, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2287064790725708, + "step": 3120, + "unclipped_grad_norm": 1.0459308624267578 + }, + { + "bits_per_byte": 1.7184131552032957, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1911132335662842, + "step": 3130, + "unclipped_grad_norm": 0.9758033752441406 + }, + { + "bits_per_byte": 1.7503415623533254, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2132443189620972, + "step": 3140, + "unclipped_grad_norm": 1.0032713413238525 + }, + { + "bits_per_byte": 1.7368366246943234, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.203883409500122, + "step": 3150, + "unclipped_grad_norm": 0.9544663429260254 + }, + { + "bits_per_byte": 1.7448823170669872, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2094602584838867, + "step": 3160, + "unclipped_grad_norm": 0.9832491278648376 + }, + { + "bits_per_byte": 1.8183364512157945, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2603747844696045, + "step": 3170, + "unclipped_grad_norm": 1.080575942993164 + }, + { + "bits_per_byte": 1.796097030667423, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2449595928192139, + "step": 3180, + "unclipped_grad_norm": 1.0869723558425903 + }, + { + "bits_per_byte": 1.7798367589589479, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2336888313293457, + "step": 3190, + "unclipped_grad_norm": 1.0730317831039429 + }, + { + "bits_per_byte": 1.7660776309419706, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2241517305374146, + "step": 3200, + "unclipped_grad_norm": 1.0053741931915283 + }, + { + "bits_per_byte": 1.736418362887425, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2035934925079346, + "step": 3210, + "unclipped_grad_norm": 0.9884828925132751 + }, + { + "bits_per_byte": 1.7094793444215763, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1849207878112793, + "step": 3220, + "unclipped_grad_norm": 0.9791789054870605 + }, + { + "bits_per_byte": 1.7767474346014993, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.231547474861145, + "step": 3230, + "unclipped_grad_norm": 1.0538209676742554 + }, + { + "bits_per_byte": 1.783878351254224, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.236490249633789, + "step": 3240, + "unclipped_grad_norm": 1.0962731838226318 + }, + { + "bits_per_byte": 1.7619694813608107, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.221304178237915, + "step": 3250, + "unclipped_grad_norm": 0.9704980850219727 + }, + { + "bits_per_byte": 1.7647327266122235, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2232195138931274, + "step": 3260, + "unclipped_grad_norm": 0.9566147923469543 + }, + { + "bits_per_byte": 1.793783004100064, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.243355631828308, + "step": 3270, + "unclipped_grad_norm": 1.0367501974105835 + }, + { + "bits_per_byte": 1.7882781834112471, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2395399808883667, + "step": 3280, + "unclipped_grad_norm": 1.1419988870620728 + }, + { + "bits_per_byte": 1.7570841421804018, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2179179191589355, + "step": 3290, + "unclipped_grad_norm": 1.0371057987213135 + }, + { + "bits_per_byte": 1.7704131415875712, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2271568775177002, + "step": 3300, + "unclipped_grad_norm": 1.0890001058578491 + }, + { + "bits_per_byte": 1.73421681297373, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.202067494392395, + "step": 3310, + "unclipped_grad_norm": 1.0024704933166504 + }, + { + "bits_per_byte": 1.7556792159055035, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2169440984725952, + "step": 3320, + "unclipped_grad_norm": 1.0351438522338867 + }, + { + "bits_per_byte": 1.751791376100099, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2142492532730103, + "step": 3330, + "unclipped_grad_norm": 0.967126727104187 + }, + { + "bits_per_byte": 1.7023927992927743, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1800087690353394, + "step": 3340, + "unclipped_grad_norm": 0.9899824857711792 + }, + { + "bits_per_byte": 1.7427576433982281, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2079875469207764, + "step": 3350, + "unclipped_grad_norm": 1.044270396232605 + }, + { + "bits_per_byte": 1.7489617455454531, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2122879028320312, + "step": 3360, + "unclipped_grad_norm": 1.0146241188049316 + }, + { + "bits_per_byte": 1.6929069242017853, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1734336614608765, + "step": 3370, + "unclipped_grad_norm": 0.9934349656105042 + }, + { + "bits_per_byte": 1.7374774320514381, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2043275833129883, + "step": 3380, + "unclipped_grad_norm": 1.0039739608764648 + }, + { + "bits_per_byte": 1.7582206035373031, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.218705654144287, + "step": 3390, + "unclipped_grad_norm": 1.007129430770874 + }, + { + "bits_per_byte": 1.7465608677394082, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2106237411499023, + "step": 3400, + "unclipped_grad_norm": 0.9504732489585876 + }, + { + "bits_per_byte": 1.744059208099958, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2088897228240967, + "step": 3410, + "unclipped_grad_norm": 1.002007007598877 + }, + { + "bits_per_byte": 1.7186439559207538, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1912732124328613, + "step": 3420, + "unclipped_grad_norm": 0.9931132197380066 + }, + { + "bits_per_byte": 1.762099672227514, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.221394419670105, + "step": 3430, + "unclipped_grad_norm": 0.943431556224823 + }, + { + "bits_per_byte": 1.763225298677411, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2221746444702148, + "step": 3440, + "unclipped_grad_norm": 0.9422610402107239 + }, + { + "bits_per_byte": 1.8310394337738234, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2691798210144043, + "step": 3450, + "unclipped_grad_norm": 1.1188790798187256 + }, + { + "bits_per_byte": 1.822499291279929, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2632602453231812, + "step": 3460, + "unclipped_grad_norm": 1.14117431640625 + }, + { + "bits_per_byte": 1.7955301758501792, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2445666790008545, + "step": 3470, + "unclipped_grad_norm": 1.0871022939682007 + }, + { + "bits_per_byte": 1.7607453088519693, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2204556465148926, + "step": 3480, + "unclipped_grad_norm": 1.0999677181243896 + }, + { + "bits_per_byte": 1.6814413568167144, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1654863357543945, + "step": 3490, + "unclipped_grad_norm": 1.0303022861480713 + }, + { + "bits_per_byte": 1.6926489502254911, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1732548475265503, + "step": 3500, + "unclipped_grad_norm": 1.0132246017456055 + }, + { + "bits_per_byte": 1.7370531508517597, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2040334939956665, + "step": 3510, + "unclipped_grad_norm": 1.0184191465377808 + }, + { + "bits_per_byte": 1.7040419409318979, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1811518669128418, + "step": 3520, + "unclipped_grad_norm": 0.9375280141830444 + }, + { + "bits_per_byte": 1.7216300906876847, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1933430433273315, + "step": 3530, + "unclipped_grad_norm": 1.0306724309921265 + }, + { + "bits_per_byte": 1.7320362449434406, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2005560398101807, + "step": 3540, + "unclipped_grad_norm": 0.9943498373031616 + }, + { + "bits_per_byte": 1.7834315403272825, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2361805438995361, + "step": 3550, + "unclipped_grad_norm": 1.066209316253662 + }, + { + "bits_per_byte": 1.7059953198803977, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1825058460235596, + "step": 3560, + "unclipped_grad_norm": 1.0326995849609375 + }, + { + "bits_per_byte": 1.768657198722262, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2259397506713867, + "step": 3570, + "unclipped_grad_norm": 1.0416048765182495 + }, + { + "bits_per_byte": 1.8330426876910733, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2705683708190918, + "step": 3580, + "unclipped_grad_norm": 1.1256591081619263 + }, + { + "bits_per_byte": 1.759539710469421, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2196199893951416, + "step": 3590, + "unclipped_grad_norm": 1.0512219667434692 + }, + { + "bits_per_byte": 1.727328735824024, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1972930431365967, + "step": 3600, + "unclipped_grad_norm": 0.9710429906845093 + }, + { + "bits_per_byte": 1.7248793589104356, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1955952644348145, + "step": 3610, + "unclipped_grad_norm": 1.0369946956634521 + }, + { + "bits_per_byte": 1.7248757472747676, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1955927610397339, + "step": 3620, + "unclipped_grad_norm": 1.0838762521743774 + }, + { + "bits_per_byte": 1.7412607064051182, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2069499492645264, + "step": 3630, + "unclipped_grad_norm": 0.9884288311004639 + }, + { + "bits_per_byte": 1.6889442719432552, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.170686960220337, + "step": 3640, + "unclipped_grad_norm": 1.0365123748779297 + }, + { + "bits_per_byte": 1.7701531038194667, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2269766330718994, + "step": 3650, + "unclipped_grad_norm": 1.1143370866775513 + }, + { + "bits_per_byte": 1.6641216719789245, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1534812450408936, + "step": 3660, + "unclipped_grad_norm": 1.0202206373214722 + }, + { + "bits_per_byte": 1.6856676584790171, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1684157848358154, + "step": 3670, + "unclipped_grad_norm": 1.0151705741882324 + }, + { + "bits_per_byte": 1.7531821997976265, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.2152132987976074, + "step": 3680, + "unclipped_grad_norm": 1.013437271118164 + }, + { + "bits_per_byte": 1.7126154480600597, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1870945692062378, + "step": 3690, + "unclipped_grad_norm": 1.0570411682128906 + }, + { + "bits_per_byte": 1.6629353356532728, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1526589393615723, + "step": 3700, + "unclipped_grad_norm": 1.0999462604522705 + }, + { + "bits_per_byte": 1.7333302424085324, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.2014529705047607, + "step": 3710, + "unclipped_grad_norm": 1.0060557126998901 + }, + { + "bits_per_byte": 1.714764887230543, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1885844469070435, + "step": 3720, + "unclipped_grad_norm": 1.061936378479004 + }, + { + "bits_per_byte": 1.761044214699169, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2206628322601318, + "step": 3730, + "unclipped_grad_norm": 1.0892959833145142 + }, + { + "bits_per_byte": 1.704870553343755, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1817262172698975, + "step": 3740, + "unclipped_grad_norm": 1.0097936391830444 + }, + { + "bits_per_byte": 1.6862241943372092, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1688015460968018, + "step": 3750, + "unclipped_grad_norm": 1.0313822031021118 + }, + { + "bits_per_byte": 1.6824576023006628, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.16619074344635, + "step": 3760, + "unclipped_grad_norm": 1.0350043773651123 + }, + { + "bits_per_byte": 1.7407710717981117, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2066105604171753, + "step": 3770, + "unclipped_grad_norm": 1.0363857746124268 + }, + { + "bits_per_byte": 1.6889002443846344, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.170656442642212, + "step": 3780, + "unclipped_grad_norm": 0.9539183974266052 + }, + { + "bits_per_byte": 1.7337478162848272, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.20174241065979, + "step": 3790, + "unclipped_grad_norm": 0.9895081520080566 + }, + { + "bits_per_byte": 1.6773657119565675, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1626613140106201, + "step": 3800, + "unclipped_grad_norm": 0.9700848460197449 + }, + { + "bits_per_byte": 1.6665356204664348, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1551544666290283, + "step": 3810, + "unclipped_grad_norm": 1.059192180633545 + }, + { + "bits_per_byte": 1.7701960994821824, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.227006435394287, + "step": 3820, + "unclipped_grad_norm": 0.9839220643043518 + }, + { + "bits_per_byte": 1.6973205149708774, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1764929294586182, + "step": 3830, + "unclipped_grad_norm": 1.0503867864608765 + }, + { + "bits_per_byte": 1.6894750104038179, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1710548400878906, + "step": 3840, + "unclipped_grad_norm": 1.0203979015350342 + }, + { + "bits_per_byte": 1.7223587811793903, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1938481330871582, + "step": 3850, + "unclipped_grad_norm": 1.0148154497146606 + }, + { + "bits_per_byte": 1.6589467140144625, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1498942375183105, + "step": 3860, + "unclipped_grad_norm": 0.9799250364303589 + }, + { + "bits_per_byte": 1.719725382829379, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1920228004455566, + "step": 3870, + "unclipped_grad_norm": 0.9761002659797668 + }, + { + "bits_per_byte": 1.70201718918329, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1797484159469604, + "step": 3880, + "unclipped_grad_norm": 1.047680139541626 + }, + { + "bits_per_byte": 1.7301960305792086, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1992805004119873, + "step": 3890, + "unclipped_grad_norm": 1.0218281745910645 + }, + { + "bits_per_byte": 1.728929550338255, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1984026432037354, + "step": 3900, + "unclipped_grad_norm": 1.1127806901931763 + }, + { + "bits_per_byte": 1.7357486624449652, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.2031292915344238, + "step": 3910, + "unclipped_grad_norm": 0.9941173791885376 + }, + { + "bits_per_byte": 1.7061793413168207, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.182633399963379, + "step": 3920, + "unclipped_grad_norm": 0.9868062734603882 + }, + { + "bits_per_byte": 1.6839027725158628, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1671924591064453, + "step": 3930, + "unclipped_grad_norm": 1.0294983386993408 + }, + { + "bits_per_byte": 1.6929858642385314, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1734883785247803, + "step": 3940, + "unclipped_grad_norm": 1.0079408884048462 + }, + { + "bits_per_byte": 1.7648030675164263, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2232682704925537, + "step": 3950, + "unclipped_grad_norm": 1.091759204864502 + }, + { + "bits_per_byte": 1.7722222270919972, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2284108400344849, + "step": 3960, + "unclipped_grad_norm": 1.075370192527771 + }, + { + "bits_per_byte": 1.7441799399208635, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.2089734077453613, + "step": 3970, + "unclipped_grad_norm": 1.0237821340560913 + }, + { + "bits_per_byte": 1.7199797451700052, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1921991109848022, + "step": 3980, + "unclipped_grad_norm": 1.0290383100509644 + }, + { + "bits_per_byte": 1.7549034021674614, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2164063453674316, + "step": 3990, + "unclipped_grad_norm": 1.0765279531478882 + }, + { + "bits_per_byte": 1.7501554771250918, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.2131153345108032, + "step": 4000, + "unclipped_grad_norm": 1.037840485572815 + }, + { + "bits_per_byte": 1.783230320625773, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2360410690307617, + "step": 4010, + "unclipped_grad_norm": 1.1067007780075073 + }, + { + "bits_per_byte": 1.634854696350997, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.133194923400879, + "step": 4020, + "unclipped_grad_norm": 0.9555695652961731 + }, + { + "bits_per_byte": 1.6858748975733067, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1685594320297241, + "step": 4030, + "unclipped_grad_norm": 0.9827634692192078 + }, + { + "bits_per_byte": 1.6858767893824662, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1685607433319092, + "step": 4040, + "unclipped_grad_norm": 1.0647450685501099 + }, + { + "bits_per_byte": 1.6753074235910415, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1612346172332764, + "step": 4050, + "unclipped_grad_norm": 1.0498672723770142 + }, + { + "bits_per_byte": 1.7132395731000407, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1875271797180176, + "step": 4060, + "unclipped_grad_norm": 1.0244723558425903 + }, + { + "bits_per_byte": 1.659084988065756, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1499900817871094, + "step": 4070, + "unclipped_grad_norm": 0.954911470413208 + }, + { + "bits_per_byte": 1.7001991605810192, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1784882545471191, + "step": 4080, + "unclipped_grad_norm": 1.0178123712539673 + }, + { + "bits_per_byte": 1.6512532421107655, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.144561529159546, + "step": 4090, + "unclipped_grad_norm": 0.988084614276886 + }, + { + "bits_per_byte": 1.6312373852553994, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1306875944137573, + "step": 4100, + "unclipped_grad_norm": 0.9640962481498718 + }, + { + "bits_per_byte": 1.7389465078551083, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.205345869064331, + "step": 4110, + "unclipped_grad_norm": 1.0591799020767212 + }, + { + "bits_per_byte": 1.670625711869254, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.157989501953125, + "step": 4120, + "unclipped_grad_norm": 1.0143104791641235 + }, + { + "bits_per_byte": 1.6531302607622822, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1458625793457031, + "step": 4130, + "unclipped_grad_norm": 1.0070092678070068 + }, + { + "bits_per_byte": 1.6853878427060633, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1682218313217163, + "step": 4140, + "unclipped_grad_norm": 1.0397000312805176 + }, + { + "bits_per_byte": 1.716280570332597, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1896350383758545, + "step": 4150, + "unclipped_grad_norm": 1.0394762754440308 + }, + { + "bits_per_byte": 1.7032425655706875, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1805977821350098, + "step": 4160, + "unclipped_grad_norm": 1.0539857149124146 + }, + { + "bits_per_byte": 1.6362312454885026, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1341490745544434, + "step": 4170, + "unclipped_grad_norm": 1.0312221050262451 + }, + { + "bits_per_byte": 1.6176134356020002, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.121244192123413, + "step": 4180, + "unclipped_grad_norm": 0.9745424389839172 + }, + { + "bits_per_byte": 1.7779335989445002, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.2323696613311768, + "step": 4190, + "unclipped_grad_norm": 1.067311406135559 + }, + { + "bits_per_byte": 1.675829562919061, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1615965366363525, + "step": 4200, + "unclipped_grad_norm": 1.045929193496704 + }, + { + "bits_per_byte": 1.681460618873611, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1654996871948242, + "step": 4210, + "unclipped_grad_norm": 1.0254682302474976 + }, + { + "bits_per_byte": 1.7232076875440492, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1944365501403809, + "step": 4220, + "unclipped_grad_norm": 1.0498327016830444 + }, + { + "bits_per_byte": 1.6892706950145928, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1709132194519043, + "step": 4230, + "unclipped_grad_norm": 1.0495660305023193 + }, + { + "bits_per_byte": 1.7317139494557237, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.2003326416015625, + "step": 4240, + "unclipped_grad_norm": 1.1651132106781006 + }, + { + "bits_per_byte": 1.744415040204593, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.2091363668441772, + "step": 4250, + "unclipped_grad_norm": 1.0255383253097534 + }, + { + "bits_per_byte": 1.73589312787169, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.2032294273376465, + "step": 4260, + "unclipped_grad_norm": 1.1120972633361816 + }, + { + "bits_per_byte": 1.7234627378152787, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1946133375167847, + "step": 4270, + "unclipped_grad_norm": 1.016249656677246 + }, + { + "bits_per_byte": 1.7430716837187037, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.208205223083496, + "step": 4280, + "unclipped_grad_norm": 1.0722230672836304 + }, + { + "bits_per_byte": 1.7030208799337254, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1804441213607788, + "step": 4290, + "unclipped_grad_norm": 1.03997004032135 + }, + { + "bits_per_byte": 1.6244537015747667, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1259855031967163, + "step": 4300, + "unclipped_grad_norm": 1.037449836730957 + }, + { + "bits_per_byte": 1.7698973656176336, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2267993688583374, + "step": 4310, + "unclipped_grad_norm": 1.235188603401184 + }, + { + "bits_per_byte": 1.6511122163370582, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1444637775421143, + "step": 4320, + "unclipped_grad_norm": 1.1049147844314575 + }, + { + "bits_per_byte": 1.6449793150072904, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1402127742767334, + "step": 4330, + "unclipped_grad_norm": 1.0132696628570557 + }, + { + "bits_per_byte": 1.6583323919855806, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1494684219360352, + "step": 4340, + "unclipped_grad_norm": 1.0222140550613403 + }, + { + "bits_per_byte": 1.5761074866080718, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0924744606018066, + "step": 4350, + "unclipped_grad_norm": 1.0156723260879517 + }, + { + "bits_per_byte": 1.7219245249859618, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1935471296310425, + "step": 4360, + "unclipped_grad_norm": 1.0623725652694702 + }, + { + "bits_per_byte": 1.6842240361076746, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1674151420593262, + "step": 4370, + "unclipped_grad_norm": 1.070603847503662 + }, + { + "bits_per_byte": 1.6398215553079147, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1366376876831055, + "step": 4380, + "unclipped_grad_norm": 1.0265616178512573 + }, + { + "bits_per_byte": 1.6765849107216504, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1621201038360596, + "step": 4390, + "unclipped_grad_norm": 1.020019769668579 + }, + { + "bits_per_byte": 1.6365060737645816, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1343395709991455, + "step": 4400, + "unclipped_grad_norm": 1.0334678888320923 + }, + { + "bits_per_byte": 1.6461109608499676, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1409971714019775, + "step": 4410, + "unclipped_grad_norm": 1.064432144165039 + }, + { + "bits_per_byte": 1.6319834459948421, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1312047243118286, + "step": 4420, + "unclipped_grad_norm": 1.0356578826904297 + }, + { + "bits_per_byte": 1.6589714795161867, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1499114036560059, + "step": 4430, + "unclipped_grad_norm": 1.0542991161346436 + }, + { + "bits_per_byte": 1.6678407968038327, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1560591459274292, + "step": 4440, + "unclipped_grad_norm": 1.1220062971115112 + }, + { + "bits_per_byte": 1.6463864770566499, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.141188144683838, + "step": 4450, + "unclipped_grad_norm": 1.069725751876831 + }, + { + "bits_per_byte": 1.6193611233000682, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1224555969238281, + "step": 4460, + "unclipped_grad_norm": 1.0096163749694824 + }, + { + "bits_per_byte": 1.7261988098078551, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.196509838104248, + "step": 4470, + "unclipped_grad_norm": 1.0757471323013306 + }, + { + "bits_per_byte": 1.6253762165139949, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1266249418258667, + "step": 4480, + "unclipped_grad_norm": 1.0172395706176758 + }, + { + "bits_per_byte": 1.5785597872267245, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.0941742658615112, + "step": 4490, + "unclipped_grad_norm": 1.081101894378662 + }, + { + "bits_per_byte": 1.6683827141367014, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1564347743988037, + "step": 4500, + "unclipped_grad_norm": 1.027239441871643 + }, + { + "bits_per_byte": 1.6472935135573004, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1418168544769287, + "step": 4510, + "unclipped_grad_norm": 1.0604325532913208 + }, + { + "bits_per_byte": 1.6464067710094517, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1412022113800049, + "step": 4520, + "unclipped_grad_norm": 1.0223703384399414 + }, + { + "bits_per_byte": 1.6663331968863693, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.155014157295227, + "step": 4530, + "unclipped_grad_norm": 1.096794843673706 + }, + { + "bits_per_byte": 1.5983706407622622, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1079061031341553, + "step": 4540, + "unclipped_grad_norm": 0.991118311882019 + }, + { + "bits_per_byte": 1.6821887934173643, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1660044193267822, + "step": 4550, + "unclipped_grad_norm": 1.0131337642669678 + }, + { + "bits_per_byte": 1.6779839895864195, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1630898714065552, + "step": 4560, + "unclipped_grad_norm": 1.0416655540466309 + }, + { + "bits_per_byte": 1.6489940780090317, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1429955959320068, + "step": 4570, + "unclipped_grad_norm": 1.0762525796890259 + }, + { + "bits_per_byte": 1.5938887728807776, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.104799509048462, + "step": 4580, + "unclipped_grad_norm": 1.039039969444275 + }, + { + "bits_per_byte": 1.7129355077733153, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1873164176940918, + "step": 4590, + "unclipped_grad_norm": 1.1662163734436035 + }, + { + "bits_per_byte": 1.6753789683738005, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1612842082977295, + "step": 4600, + "unclipped_grad_norm": 1.0834790468215942 + }, + { + "bits_per_byte": 1.6489213293477167, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1429451704025269, + "step": 4610, + "unclipped_grad_norm": 1.0408029556274414 + }, + { + "bits_per_byte": 1.5772676815707922, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.0932786464691162, + "step": 4620, + "unclipped_grad_norm": 0.9947658777236938 + }, + { + "bits_per_byte": 1.6629637127906651, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1526786088943481, + "step": 4630, + "unclipped_grad_norm": 1.0645713806152344 + }, + { + "bits_per_byte": 1.6351708004632826, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1334140300750732, + "step": 4640, + "unclipped_grad_norm": 1.0213215351104736 + }, + { + "bits_per_byte": 1.6143973600308656, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1190149784088135, + "step": 4650, + "unclipped_grad_norm": 1.0158205032348633 + }, + { + "bits_per_byte": 1.6442494206370286, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1397068500518799, + "step": 4660, + "unclipped_grad_norm": 1.032319188117981 + }, + { + "bits_per_byte": 1.6788175894951514, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1636676788330078, + "step": 4670, + "unclipped_grad_norm": 1.1467437744140625 + }, + { + "bits_per_byte": 1.6228953667752988, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1249053478240967, + "step": 4680, + "unclipped_grad_norm": 1.0684930086135864 + }, + { + "bits_per_byte": 1.6116814100084402, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1171324253082275, + "step": 4690, + "unclipped_grad_norm": 1.0818438529968262 + }, + { + "bits_per_byte": 1.7118176205427071, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1865415573120117, + "step": 4700, + "unclipped_grad_norm": 1.0913734436035156 + }, + { + "bits_per_byte": 1.6314229545356804, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1308162212371826, + "step": 4710, + "unclipped_grad_norm": 1.0516728162765503 + }, + { + "bits_per_byte": 1.6067035441618671, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1136820316314697, + "step": 4720, + "unclipped_grad_norm": 1.027355670928955 + }, + { + "bits_per_byte": 1.723685455348146, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.194767713546753, + "step": 4730, + "unclipped_grad_norm": 1.0737906694412231 + }, + { + "bits_per_byte": 1.6626813172779484, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1524828672409058, + "step": 4740, + "unclipped_grad_norm": 1.093795657157898 + }, + { + "bits_per_byte": 1.6942468410346574, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1743624210357666, + "step": 4750, + "unclipped_grad_norm": 1.098819375038147 + }, + { + "bits_per_byte": 1.567997816689287, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.086853265762329, + "step": 4760, + "unclipped_grad_norm": 0.9886804819107056 + }, + { + "bits_per_byte": 1.674730077832095, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1608344316482544, + "step": 4770, + "unclipped_grad_norm": 1.0352309942245483 + }, + { + "bits_per_byte": 1.6338455021557339, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.132495403289795, + "step": 4780, + "unclipped_grad_norm": 1.037882685661316 + }, + { + "bits_per_byte": 1.6704422063807836, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1578623056411743, + "step": 4790, + "unclipped_grad_norm": 1.117195725440979 + }, + { + "bits_per_byte": 1.6306944360266256, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1303112506866455, + "step": 4800, + "unclipped_grad_norm": 1.0563780069351196 + }, + { + "bits_per_byte": 1.6131430905581232, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1181455850601196, + "step": 4810, + "unclipped_grad_norm": 1.0533764362335205 + }, + { + "bits_per_byte": 1.675093477173368, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1610863208770752, + "step": 4820, + "unclipped_grad_norm": 1.0993541479110718 + }, + { + "bits_per_byte": 1.6710443176414542, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1582796573638916, + "step": 4830, + "unclipped_grad_norm": 1.2536643743515015 + }, + { + "bits_per_byte": 1.6771435103716528, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1625072956085205, + "step": 4840, + "unclipped_grad_norm": 1.074759840965271 + }, + { + "bits_per_byte": 1.6143529885069432, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1189842224121094, + "step": 4850, + "unclipped_grad_norm": 1.0702130794525146 + }, + { + "bits_per_byte": 1.6332604171774985, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1320898532867432, + "step": 4860, + "unclipped_grad_norm": 1.0851078033447266 + }, + { + "bits_per_byte": 1.6331224870915066, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1319942474365234, + "step": 4870, + "unclipped_grad_norm": 1.08431875705719 + }, + { + "bits_per_byte": 1.7854224114936703, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.237560510635376, + "step": 4880, + "unclipped_grad_norm": 1.1104563474655151 + }, + { + "bits_per_byte": 1.5755340964500952, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0920770168304443, + "step": 4890, + "unclipped_grad_norm": 1.042919635772705 + }, + { + "bits_per_byte": 1.6730817961062259, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1596919298171997, + "step": 4900, + "unclipped_grad_norm": 1.0552027225494385 + }, + { + "bits_per_byte": 1.6497614645971816, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1435275077819824, + "step": 4910, + "unclipped_grad_norm": 1.0571706295013428 + }, + { + "bits_per_byte": 1.5965801293841295, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.106665015220642, + "step": 4920, + "unclipped_grad_norm": 1.0640807151794434 + }, + { + "bits_per_byte": 1.6007018655947076, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1095219850540161, + "step": 4930, + "unclipped_grad_norm": 1.0152149200439453 + }, + { + "bits_per_byte": 1.6331477685411835, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1320117712020874, + "step": 4940, + "unclipped_grad_norm": 1.100314736366272 + }, + { + "bits_per_byte": 1.6360496318091915, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1340231895446777, + "step": 4950, + "unclipped_grad_norm": 1.079222559928894 + }, + { + "bits_per_byte": 1.6569551549174713, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1485137939453125, + "step": 4960, + "unclipped_grad_norm": 1.0604397058486938 + }, + { + "bits_per_byte": 1.638398398872025, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1356512308120728, + "step": 4970, + "unclipped_grad_norm": 1.0775542259216309 + }, + { + "bits_per_byte": 1.680847672705936, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1650748252868652, + "step": 4980, + "unclipped_grad_norm": 1.1589142084121704 + }, + { + "bits_per_byte": 1.633999426628256, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1326020956039429, + "step": 4990, + "unclipped_grad_norm": 1.075969934463501 + }, + { + "bits_per_byte": 1.6444650868812105, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1398563385009766, + "step": 5000, + "unclipped_grad_norm": 1.0691334009170532 + }, + { + "bits_per_byte": 1.6738108305632333, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1601972579956055, + "step": 5010, + "unclipped_grad_norm": 1.0487086772918701 + }, + { + "bits_per_byte": 1.6477535671483583, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.142135739326477, + "step": 5020, + "unclipped_grad_norm": 1.1224150657653809 + }, + { + "bits_per_byte": 1.5805776596692978, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0955729484558105, + "step": 5030, + "unclipped_grad_norm": 0.9987351894378662 + }, + { + "bits_per_byte": 1.6501410303076358, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.143790602684021, + "step": 5040, + "unclipped_grad_norm": 1.069442868232727 + }, + { + "bits_per_byte": 1.6202205206064295, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1230512857437134, + "step": 5050, + "unclipped_grad_norm": 1.1084743738174438 + }, + { + "bits_per_byte": 1.6928149134835737, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1733698844909668, + "step": 5060, + "unclipped_grad_norm": 1.0974880456924438 + }, + { + "bits_per_byte": 1.6259845191500966, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1270465850830078, + "step": 5070, + "unclipped_grad_norm": 1.0709881782531738 + }, + { + "bits_per_byte": 1.6998562271751987, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1782505512237549, + "step": 5080, + "unclipped_grad_norm": 1.1272931098937988 + }, + { + "bits_per_byte": 1.6260173678364114, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.127069354057312, + "step": 5090, + "unclipped_grad_norm": 1.0618640184402466 + }, + { + "bits_per_byte": 1.6234274810970684, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1252741813659668, + "step": 5100, + "unclipped_grad_norm": 1.1292778253555298 + }, + { + "bits_per_byte": 1.6568031222541084, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1484084129333496, + "step": 5110, + "unclipped_grad_norm": 1.1496297121047974 + }, + { + "bits_per_byte": 1.5598939941806313, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0812361240386963, + "step": 5120, + "unclipped_grad_norm": 1.035429835319519 + }, + { + "bits_per_byte": 1.6914442117561972, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.172419786453247, + "step": 5130, + "unclipped_grad_norm": 1.0999172925949097 + }, + { + "bits_per_byte": 1.6623124144918477, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1522271633148193, + "step": 5140, + "unclipped_grad_norm": 1.0919309854507446 + }, + { + "bits_per_byte": 1.6100444791375281, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1159977912902832, + "step": 5150, + "unclipped_grad_norm": 1.0898085832595825 + }, + { + "bits_per_byte": 1.6321538808018472, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1313228607177734, + "step": 5160, + "unclipped_grad_norm": 1.0807921886444092 + }, + { + "bits_per_byte": 1.6844840738757791, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.167595386505127, + "step": 5170, + "unclipped_grad_norm": 1.176076889038086 + }, + { + "bits_per_byte": 1.65267932225172, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.145550012588501, + "step": 5180, + "unclipped_grad_norm": 1.110230803489685 + }, + { + "bits_per_byte": 1.5426296877564192, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0692694187164307, + "step": 5190, + "unclipped_grad_norm": 1.0403128862380981 + }, + { + "bits_per_byte": 1.6380544335702993, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1354128122329712, + "step": 5200, + "unclipped_grad_norm": 1.1156731843948364 + }, + { + "bits_per_byte": 1.6025732088187459, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1108191013336182, + "step": 5210, + "unclipped_grad_norm": 1.112536907196045 + }, + { + "bits_per_byte": 1.595609287320009, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.105992078781128, + "step": 5220, + "unclipped_grad_norm": 1.0583246946334839 + }, + { + "bits_per_byte": 1.6907153492818405, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1719145774841309, + "step": 5230, + "unclipped_grad_norm": 1.1788045167922974 + }, + { + "bits_per_byte": 1.6472453584150588, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1417834758758545, + "step": 5240, + "unclipped_grad_norm": 1.1392273902893066 + }, + { + "bits_per_byte": 1.6431225903085755, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1389257907867432, + "step": 5250, + "unclipped_grad_norm": 1.087467074394226 + }, + { + "bits_per_byte": 1.5745493237912547, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0913944244384766, + "step": 5260, + "unclipped_grad_norm": 1.0672780275344849 + }, + { + "bits_per_byte": 1.625834722261195, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.126942753791809, + "step": 5270, + "unclipped_grad_norm": 1.0955538749694824 + }, + { + "bits_per_byte": 1.5722745092682924, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0898176431655884, + "step": 5280, + "unclipped_grad_norm": 1.1191643476486206 + }, + { + "bits_per_byte": 1.581137635180507, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.095961093902588, + "step": 5290, + "unclipped_grad_norm": 1.0905182361602783 + }, + { + "bits_per_byte": 1.5625956696430352, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0831087827682495, + "step": 5300, + "unclipped_grad_norm": 1.0970019102096558 + }, + { + "bits_per_byte": 1.575699715742876, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0921918153762817, + "step": 5310, + "unclipped_grad_norm": 1.085243582725525 + }, + { + "bits_per_byte": 1.5934223559316376, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1044762134552002, + "step": 5320, + "unclipped_grad_norm": 1.1693148612976074 + }, + { + "bits_per_byte": 1.6214211314921028, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1238834857940674, + "step": 5330, + "unclipped_grad_norm": 1.1232852935791016 + }, + { + "bits_per_byte": 1.6839337293930183, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1672139167785645, + "step": 5340, + "unclipped_grad_norm": 1.1369119882583618 + }, + { + "bits_per_byte": 1.5416975417887426, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0686233043670654, + "step": 5350, + "unclipped_grad_norm": 1.0887452363967896 + }, + { + "bits_per_byte": 1.5866414239734188, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.099776029586792, + "step": 5360, + "unclipped_grad_norm": 1.0925503969192505 + }, + { + "bits_per_byte": 1.6646420914804352, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1538419723510742, + "step": 5370, + "unclipped_grad_norm": 1.1924686431884766 + }, + { + "bits_per_byte": 1.586491799067168, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0996723175048828, + "step": 5380, + "unclipped_grad_norm": 1.0599968433380127 + }, + { + "bits_per_byte": 1.5783984675002152, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0940624475479126, + "step": 5390, + "unclipped_grad_norm": 1.0973831415176392 + }, + { + "bits_per_byte": 1.642996527025493, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1388384103775024, + "step": 5400, + "unclipped_grad_norm": 1.1477714776992798 + }, + { + "bits_per_byte": 1.5308346016122951, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0610936880111694, + "step": 5410, + "unclipped_grad_norm": 1.0651572942733765 + }, + { + "bits_per_byte": 1.6877225071915258, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1698400974273682, + "step": 5420, + "unclipped_grad_norm": 1.230097770690918 + }, + { + "bits_per_byte": 1.606558218821888, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1135812997817993, + "step": 5430, + "unclipped_grad_norm": 1.1141976118087769 + }, + { + "bits_per_byte": 1.6079992614534675, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1145801544189453, + "step": 5440, + "unclipped_grad_norm": 1.0613634586334229 + }, + { + "bits_per_byte": 1.6121270170568258, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1174412965774536, + "step": 5450, + "unclipped_grad_norm": 1.1033883094787598 + }, + { + "bits_per_byte": 1.6549515570349194, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.147125005722046, + "step": 5460, + "unclipped_grad_norm": 1.0616251230239868 + }, + { + "bits_per_byte": 1.6138821000088808, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1186578273773193, + "step": 5470, + "unclipped_grad_norm": 1.1187942028045654 + }, + { + "bits_per_byte": 1.566549894751673, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.085849642753601, + "step": 5480, + "unclipped_grad_norm": 1.095421314239502 + }, + { + "bits_per_byte": 1.6100902265226575, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1160295009613037, + "step": 5490, + "unclipped_grad_norm": 1.1169681549072266 + }, + { + "bits_per_byte": 1.6345392801693144, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1329762935638428, + "step": 5500, + "unclipped_grad_norm": 1.0606133937835693 + }, + { + "bits_per_byte": 1.6112232482265416, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1168148517608643, + "step": 5510, + "unclipped_grad_norm": 1.082831859588623 + }, + { + "bits_per_byte": 1.6200284159854157, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1229181289672852, + "step": 5520, + "unclipped_grad_norm": 1.1288034915924072 + }, + { + "bits_per_byte": 1.6500037881522474, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.1436954736709595, + "step": 5530, + "unclipped_grad_norm": 1.1526131629943848 + }, + { + "bits_per_byte": 1.6647621353707376, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1539251804351807, + "step": 5540, + "unclipped_grad_norm": 1.1722888946533203 + }, + { + "bits_per_byte": 1.5974680758105342, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.1072804927825928, + "step": 5550, + "unclipped_grad_norm": 1.073315143585205 + }, + { + "bits_per_byte": 1.594942510582614, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1055299043655396, + "step": 5560, + "unclipped_grad_norm": 1.0879392623901367 + }, + { + "bits_per_byte": 1.6466642290377933, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1413806676864624, + "step": 5570, + "unclipped_grad_norm": 1.21271812915802 + }, + { + "bits_per_byte": 1.5387917229197647, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0666091442108154, + "step": 5580, + "unclipped_grad_norm": 1.0752229690551758 + }, + { + "bits_per_byte": 1.5950678859350929, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.105616807937622, + "step": 5590, + "unclipped_grad_norm": 1.115165114402771 + }, + { + "bits_per_byte": 1.60302053569364, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1111291646957397, + "step": 5600, + "unclipped_grad_norm": 1.0947556495666504 + }, + { + "bits_per_byte": 1.6150159816260192, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1194437742233276, + "step": 5610, + "unclipped_grad_norm": 1.0436956882476807 + }, + { + "bits_per_byte": 1.5445598490470525, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.070607304573059, + "step": 5620, + "unclipped_grad_norm": 1.1328603029251099 + }, + { + "bits_per_byte": 1.5627000631121089, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0831811428070068, + "step": 5630, + "unclipped_grad_norm": 1.0480417013168335 + }, + { + "bits_per_byte": 1.5462631652211978, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.07178795337677, + "step": 5640, + "unclipped_grad_norm": 1.1326069831848145 + }, + { + "bits_per_byte": 1.6566736193180087, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1483186483383179, + "step": 5650, + "unclipped_grad_norm": 1.174943447113037 + }, + { + "bits_per_byte": 1.6152889180929386, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1196329593658447, + "step": 5660, + "unclipped_grad_norm": 1.1866281032562256 + }, + { + "bits_per_byte": 1.6043573568387968, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.112055778503418, + "step": 5670, + "unclipped_grad_norm": 1.1592957973480225 + }, + { + "bits_per_byte": 1.6221047625292826, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1243573427200317, + "step": 5680, + "unclipped_grad_norm": 1.1609989404678345 + }, + { + "bits_per_byte": 1.6421648189259206, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1382619142532349, + "step": 5690, + "unclipped_grad_norm": 1.1130645275115967 + }, + { + "bits_per_byte": 1.598431178655366, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1079480648040771, + "step": 5700, + "unclipped_grad_norm": 1.1522045135498047 + }, + { + "bits_per_byte": 1.6208697551134366, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1235013008117676, + "step": 5710, + "unclipped_grad_norm": 1.1070868968963623 + }, + { + "bits_per_byte": 1.648464543427025, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.14262855052948, + "step": 5720, + "unclipped_grad_norm": 1.2136235237121582 + }, + { + "bits_per_byte": 1.5559942875723172, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0785330533981323, + "step": 5730, + "unclipped_grad_norm": 1.1237554550170898 + }, + { + "bits_per_byte": 1.6165538504900345, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1205097436904907, + "step": 5740, + "unclipped_grad_norm": 1.1293457746505737 + }, + { + "bits_per_byte": 1.5847578699811693, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0984704494476318, + "step": 5750, + "unclipped_grad_norm": 1.1838749647140503 + }, + { + "bits_per_byte": 1.5751051717188433, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0917797088623047, + "step": 5760, + "unclipped_grad_norm": 1.1379411220550537 + }, + { + "bits_per_byte": 1.5366526027083331, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0651264190673828, + "step": 5770, + "unclipped_grad_norm": 1.1725109815597534 + }, + { + "bits_per_byte": 1.6123841311198657, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.117619514465332, + "step": 5780, + "unclipped_grad_norm": 1.1070845127105713 + }, + { + "bits_per_byte": 1.5625800192218067, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0830979347229004, + "step": 5790, + "unclipped_grad_norm": 1.0731046199798584 + }, + { + "bits_per_byte": 1.581076581339451, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0959187746047974, + "step": 5800, + "unclipped_grad_norm": 1.1906232833862305 + }, + { + "bits_per_byte": 1.6024834338749956, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.1107568740844727, + "step": 5810, + "unclipped_grad_norm": 1.198943853378296 + }, + { + "bits_per_byte": 1.6068543729466738, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.1137865781784058, + "step": 5820, + "unclipped_grad_norm": 1.1112759113311768 + }, + { + "bits_per_byte": 1.567694095327863, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0866427421569824, + "step": 5830, + "unclipped_grad_norm": 1.2037192583084106 + }, + { + "bits_per_byte": 1.5486041930647423, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0734106302261353, + "step": 5840, + "unclipped_grad_norm": 1.1501859426498413 + }, + { + "bits_per_byte": 1.540087784176667, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0675075054168701, + "step": 5850, + "unclipped_grad_norm": 1.1106252670288086 + }, + { + "bits_per_byte": 1.591153904766757, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.1029038429260254, + "step": 5860, + "unclipped_grad_norm": 1.267377257347107 + }, + { + "bits_per_byte": 1.6187461133405827, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1220293045043945, + "step": 5870, + "unclipped_grad_norm": 1.1557857990264893 + }, + { + "bits_per_byte": 1.6194158137830426, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1224935054779053, + "step": 5880, + "unclipped_grad_norm": 1.1605706214904785 + }, + { + "bits_per_byte": 1.6297550667876128, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1296601295471191, + "step": 5890, + "unclipped_grad_norm": 1.1718353033065796 + }, + { + "bits_per_byte": 1.5804839291245776, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0955079793930054, + "step": 5900, + "unclipped_grad_norm": 1.1900224685668945 + }, + { + "bits_per_byte": 1.5770434161940672, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.093123197555542, + "step": 5910, + "unclipped_grad_norm": 1.1988046169281006 + }, + { + "bits_per_byte": 1.5761890063845807, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0925309658050537, + "step": 5920, + "unclipped_grad_norm": 1.1799134016036987 + }, + { + "bits_per_byte": 1.6240858306845711, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1257305145263672, + "step": 5930, + "unclipped_grad_norm": 1.1704758405685425 + }, + { + "bits_per_byte": 1.6230097352381225, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.124984622001648, + "step": 5940, + "unclipped_grad_norm": 1.1741182804107666 + }, + { + "bits_per_byte": 1.5942451209333652, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.1050465106964111, + "step": 5950, + "unclipped_grad_norm": 1.223347544670105 + }, + { + "bits_per_byte": 1.587844614598855, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1006100177764893, + "step": 5960, + "unclipped_grad_norm": 1.1676011085510254 + }, + { + "bits_per_byte": 1.6162547726601841, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.120302438735962, + "step": 5970, + "unclipped_grad_norm": 1.2063930034637451 + }, + { + "bits_per_byte": 1.5550433954956966, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.077873945236206, + "step": 5980, + "unclipped_grad_norm": 1.1823816299438477 + }, + { + "bits_per_byte": 1.5403349232459567, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0676788091659546, + "step": 5990, + "unclipped_grad_norm": 1.1445503234863281 + }, + { + "bits_per_byte": 1.6087774829486217, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1151195764541626, + "step": 6000, + "unclipped_grad_norm": 1.182093858718872 + }, + { + "bits_per_byte": 1.585380103211991, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0989017486572266, + "step": 6010, + "unclipped_grad_norm": 1.1548501253128052 + }, + { + "bits_per_byte": 1.57159036228316, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0893434286117554, + "step": 6020, + "unclipped_grad_norm": 1.132990837097168 + }, + { + "bits_per_byte": 1.6225496816470646, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.1246657371520996, + "step": 6030, + "unclipped_grad_norm": 1.1933883428573608 + }, + { + "bits_per_byte": 1.5568839538252304, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0791497230529785, + "step": 6040, + "unclipped_grad_norm": 1.1697132587432861 + }, + { + "bits_per_byte": 1.5810753774608948, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0959179401397705, + "step": 6050, + "unclipped_grad_norm": 1.1444475650787354 + }, + { + "bits_per_byte": 1.6146938581209531, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.119220495223999, + "step": 6060, + "unclipped_grad_norm": 1.1966716051101685 + }, + { + "bits_per_byte": 1.5827436091742642, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.097074270248413, + "step": 6070, + "unclipped_grad_norm": 1.1267356872558594 + }, + { + "bits_per_byte": 1.5673551175230125, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0864077806472778, + "step": 6080, + "unclipped_grad_norm": 1.097261905670166 + }, + { + "bits_per_byte": 1.5774123189801679, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0933789014816284, + "step": 6090, + "unclipped_grad_norm": 1.1282033920288086 + }, + { + "bits_per_byte": 1.5779040173789847, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.093719720840454, + "step": 6100, + "unclipped_grad_norm": 1.1281672716140747 + }, + { + "bits_per_byte": 1.5504769121499873, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0747087001800537, + "step": 6110, + "unclipped_grad_norm": 1.1312857866287231 + }, + { + "bits_per_byte": 1.5543487575688617, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0773924589157104, + "step": 6120, + "unclipped_grad_norm": 1.157880187034607 + }, + { + "bits_per_byte": 1.5931889754744168, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.1043144464492798, + "step": 6130, + "unclipped_grad_norm": 1.1723217964172363 + }, + { + "bits_per_byte": 1.6084328297162926, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.1148806810379028, + "step": 6140, + "unclipped_grad_norm": 1.1961045265197754 + }, + { + "bits_per_byte": 1.542341444833573, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0690696239471436, + "step": 6150, + "unclipped_grad_norm": 1.1969319581985474 + }, + { + "bits_per_byte": 1.6357159854665178, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1337919235229492, + "step": 6160, + "unclipped_grad_norm": 1.2412800788879395 + }, + { + "bits_per_byte": 1.4931134748159045, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.034947395324707, + "step": 6170, + "unclipped_grad_norm": 1.0997314453125 + }, + { + "bits_per_byte": 1.5659319610871227, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0854213237762451, + "step": 6180, + "unclipped_grad_norm": 1.128227710723877 + }, + { + "bits_per_byte": 1.581150533879322, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0959700345993042, + "step": 6190, + "unclipped_grad_norm": 1.1780462265014648 + }, + { + "bits_per_byte": 1.5351257407339731, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.064068078994751, + "step": 6200, + "unclipped_grad_norm": 1.13433837890625 + }, + { + "bits_per_byte": 1.5793001725386888, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0946874618530273, + "step": 6210, + "unclipped_grad_norm": 1.1643598079681396 + }, + { + "bits_per_byte": 1.5609047362197521, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0819367170333862, + "step": 6220, + "unclipped_grad_norm": 1.131072998046875 + }, + { + "bits_per_byte": 1.5253499028936293, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0572919845581055, + "step": 6230, + "unclipped_grad_norm": 1.1126173734664917 + }, + { + "bits_per_byte": 1.5400140036194467, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0674563646316528, + "step": 6240, + "unclipped_grad_norm": 1.1826318502426147 + }, + { + "bits_per_byte": 1.5344780540708238, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0636191368103027, + "step": 6250, + "unclipped_grad_norm": 1.1302508115768433 + }, + { + "bits_per_byte": 1.600364263651064, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.109287977218628, + "step": 6260, + "unclipped_grad_norm": 1.1724376678466797 + }, + { + "bits_per_byte": 1.5544997583363194, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.077497124671936, + "step": 6270, + "unclipped_grad_norm": 1.081700086593628 + }, + { + "bits_per_byte": 1.6754886933050508, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.161360263824463, + "step": 6280, + "unclipped_grad_norm": 1.2664587497711182 + }, + { + "bits_per_byte": 1.5413214157313058, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0683625936508179, + "step": 6290, + "unclipped_grad_norm": 1.2083820104599 + }, + { + "bits_per_byte": 1.5888871734283854, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.101332664489746, + "step": 6300, + "unclipped_grad_norm": 1.2475577592849731 + }, + { + "bits_per_byte": 1.6228484155116132, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.1248728036880493, + "step": 6310, + "unclipped_grad_norm": 1.2506906986236572 + }, + { + "bits_per_byte": 1.5728649257087044, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0902268886566162, + "step": 6320, + "unclipped_grad_norm": 1.1333374977111816 + }, + { + "bits_per_byte": 1.5253509347895344, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0572926998138428, + "step": 6330, + "unclipped_grad_norm": 1.180550217628479 + }, + { + "bits_per_byte": 1.5807446548232857, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0956887006759644, + "step": 6340, + "unclipped_grad_norm": 1.1644010543823242 + }, + { + "bits_per_byte": 1.5717518539923203, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0894553661346436, + "step": 6350, + "unclipped_grad_norm": 1.142453908920288 + }, + { + "bits_per_byte": 1.5231024336121541, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0557341575622559, + "step": 6360, + "unclipped_grad_norm": 1.1504549980163574 + }, + { + "bits_per_byte": 1.6077399116159663, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.1144003868103027, + "step": 6370, + "unclipped_grad_norm": 1.2130403518676758 + }, + { + "bits_per_byte": 1.5316929670227515, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0616886615753174, + "step": 6380, + "unclipped_grad_norm": 1.1643673181533813 + }, + { + "bits_per_byte": 1.5801346323606753, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0952658653259277, + "step": 6390, + "unclipped_grad_norm": 1.1411337852478027 + }, + { + "bits_per_byte": 1.5548263533903077, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.077723503112793, + "step": 6400, + "unclipped_grad_norm": 1.1154334545135498 + }, + { + "bits_per_byte": 1.5450890396637573, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0709741115570068, + "step": 6410, + "unclipped_grad_norm": 1.137816071510315 + }, + { + "bits_per_byte": 1.5323659351355776, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0621551275253296, + "step": 6420, + "unclipped_grad_norm": 1.1425901651382446 + }, + { + "bits_per_byte": 1.5643414655319436, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0843188762664795, + "step": 6430, + "unclipped_grad_norm": 1.2278902530670166 + }, + { + "bits_per_byte": 1.5636070996127593, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0838098526000977, + "step": 6440, + "unclipped_grad_norm": 1.204923152923584 + }, + { + "bits_per_byte": 1.5427707135301267, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0693671703338623, + "step": 6450, + "unclipped_grad_norm": 1.1532135009765625 + }, + { + "bits_per_byte": 1.5522318231193915, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0759251117706299, + "step": 6460, + "unclipped_grad_norm": 1.5724140405654907 + }, + { + "bits_per_byte": 1.5677630603708592, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0866905450820923, + "step": 6470, + "unclipped_grad_norm": 1.182875394821167 + }, + { + "bits_per_byte": 1.5839062118940967, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0978801250457764, + "step": 6480, + "unclipped_grad_norm": 1.211335301399231 + }, + { + "bits_per_byte": 1.561875234318571, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0826094150543213, + "step": 6490, + "unclipped_grad_norm": 1.2880244255065918 + }, + { + "bits_per_byte": 1.577221590220361, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0932466983795166, + "step": 6500, + "unclipped_grad_norm": 1.2181490659713745 + }, + { + "bits_per_byte": 1.5067461955844976, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0443968772888184, + "step": 6510, + "unclipped_grad_norm": 1.1337521076202393 + }, + { + "bits_per_byte": 1.4936307986296997, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0353059768676758, + "step": 6520, + "unclipped_grad_norm": 1.2117457389831543 + }, + { + "bits_per_byte": 1.5784743118492457, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0941150188446045, + "step": 6530, + "unclipped_grad_norm": 1.2337595224380493 + }, + { + "bits_per_byte": 1.5448394928373554, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0708011388778687, + "step": 6540, + "unclipped_grad_norm": 1.1313438415527344 + }, + { + "bits_per_byte": 1.5964084906985685, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.1065460443496704, + "step": 6550, + "unclipped_grad_norm": 1.2166260480880737 + }, + { + "bits_per_byte": 1.5631484218829081, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0834919214248657, + "step": 6560, + "unclipped_grad_norm": 1.1645575761795044 + }, + { + "bits_per_byte": 1.4810062401604644, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0265552997589111, + "step": 6570, + "unclipped_grad_norm": 1.2005696296691895 + }, + { + "bits_per_byte": 1.5898447728283895, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.1019964218139648, + "step": 6580, + "unclipped_grad_norm": 1.1795482635498047 + }, + { + "bits_per_byte": 1.5634753609021985, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0837185382843018, + "step": 6590, + "unclipped_grad_norm": 1.1887481212615967 + }, + { + "bits_per_byte": 1.47844799822888, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0247820615768433, + "step": 6600, + "unclipped_grad_norm": 1.1429756879806519 + }, + { + "bits_per_byte": 1.6001293353499855, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.1091251373291016, + "step": 6610, + "unclipped_grad_norm": 1.1761175394058228 + }, + { + "bits_per_byte": 1.6007649832275743, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1095657348632812, + "step": 6620, + "unclipped_grad_norm": 1.2725090980529785 + }, + { + "bits_per_byte": 1.5866947385951862, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0998129844665527, + "step": 6630, + "unclipped_grad_norm": 1.242309808731079 + }, + { + "bits_per_byte": 1.557243053600232, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0793986320495605, + "step": 6640, + "unclipped_grad_norm": 1.2104532718658447 + }, + { + "bits_per_byte": 1.5462134622350983, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0717535018920898, + "step": 6650, + "unclipped_grad_norm": 1.1630686521530151 + }, + { + "bits_per_byte": 1.588834890702523, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1012964248657227, + "step": 6660, + "unclipped_grad_norm": 1.204805612564087 + }, + { + "bits_per_byte": 1.587033544417386, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.1000478267669678, + "step": 6670, + "unclipped_grad_norm": 1.2499597072601318 + }, + { + "bits_per_byte": 1.5912992301067361, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.1030045747756958, + "step": 6680, + "unclipped_grad_norm": 1.2160732746124268 + }, + { + "bits_per_byte": 1.56477726956923, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0846209526062012, + "step": 6690, + "unclipped_grad_norm": 1.2467725276947021 + }, + { + "bits_per_byte": 1.6040180350686444, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.1118205785751343, + "step": 6700, + "unclipped_grad_norm": 1.267776370048523 + }, + { + "bits_per_byte": 1.537844786444114, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0659527778625488, + "step": 6710, + "unclipped_grad_norm": 1.2301596403121948 + }, + { + "bits_per_byte": 1.5327245189626266, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.062403678894043, + "step": 6720, + "unclipped_grad_norm": 1.180388331413269 + }, + { + "bits_per_byte": 1.5337890915714674, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0631415843963623, + "step": 6730, + "unclipped_grad_norm": 1.2269387245178223 + }, + { + "bits_per_byte": 1.5501341507268178, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.074471116065979, + "step": 6740, + "unclipped_grad_norm": 1.1682806015014648 + }, + { + "bits_per_byte": 1.5184903748639662, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0525373220443726, + "step": 6750, + "unclipped_grad_norm": 1.23702871799469 + }, + { + "bits_per_byte": 1.500455070215936, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0400362014770508, + "step": 6760, + "unclipped_grad_norm": 1.1704752445220947 + }, + { + "bits_per_byte": 1.5177950490065277, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0520553588867188, + "step": 6770, + "unclipped_grad_norm": 1.209053874015808 + }, + { + "bits_per_byte": 1.5265771710901863, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0581426620483398, + "step": 6780, + "unclipped_grad_norm": 1.2882649898529053 + }, + { + "bits_per_byte": 1.6213150181965206, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.1238099336624146, + "step": 6790, + "unclipped_grad_norm": 1.1679272651672363 + }, + { + "bits_per_byte": 1.5863846538756805, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0995980501174927, + "step": 6800, + "unclipped_grad_norm": 1.280809998512268 + }, + { + "bits_per_byte": 1.578056394007649, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.093825340270996, + "step": 6810, + "unclipped_grad_norm": 1.24360191822052 + }, + { + "bits_per_byte": 1.5169633409069554, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0514788627624512, + "step": 6820, + "unclipped_grad_norm": 1.2292150259017944 + }, + { + "bits_per_byte": 1.5347601056182387, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.063814640045166, + "step": 6830, + "unclipped_grad_norm": 1.242634892463684 + }, + { + "bits_per_byte": 1.5555476486280264, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.078223466873169, + "step": 6840, + "unclipped_grad_norm": 1.2094796895980835 + }, + { + "bits_per_byte": 1.5217322478327302, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0547844171524048, + "step": 6850, + "unclipped_grad_norm": 1.1779916286468506 + }, + { + "bits_per_byte": 1.549958384457636, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.074349284172058, + "step": 6860, + "unclipped_grad_norm": 1.2639429569244385 + }, + { + "bits_per_byte": 1.5571720247654257, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.079349398612976, + "step": 6870, + "unclipped_grad_norm": 1.2982949018478394 + }, + { + "bits_per_byte": 1.597770421310751, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.107490062713623, + "step": 6880, + "unclipped_grad_norm": 1.2857638597488403 + }, + { + "bits_per_byte": 1.4860104193526196, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0300239324569702, + "step": 6890, + "unclipped_grad_norm": 1.2555214166641235 + }, + { + "bits_per_byte": 1.5064882216082034, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0442180633544922, + "step": 6900, + "unclipped_grad_norm": 1.202001690864563 + }, + { + "bits_per_byte": 1.685866814388716, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1685538291931152, + "step": 6910, + "unclipped_grad_norm": 1.3379162549972534 + }, + { + "bits_per_byte": 1.5053493524941899, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.043428659439087, + "step": 6920, + "unclipped_grad_norm": 1.1770343780517578 + }, + { + "bits_per_byte": 1.516240325842728, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0509777069091797, + "step": 6930, + "unclipped_grad_norm": 1.2326538562774658 + }, + { + "bits_per_byte": 1.5463919802266939, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0718772411346436, + "step": 6940, + "unclipped_grad_norm": 1.2218632698059082 + }, + { + "bits_per_byte": 1.5410003241221448, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0681400299072266, + "step": 6950, + "unclipped_grad_norm": 1.2248505353927612 + }, + { + "bits_per_byte": 1.574234251574874, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0911760330200195, + "step": 6960, + "unclipped_grad_norm": 1.2728900909423828 + }, + { + "bits_per_byte": 1.556295085228676, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0787415504455566, + "step": 6970, + "unclipped_grad_norm": 1.2423036098480225 + }, + { + "bits_per_byte": 1.6109532354646872, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1166276931762695, + "step": 6980, + "unclipped_grad_norm": 1.2731002569198608 + }, + { + "bits_per_byte": 1.5507145921734797, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.074873447418213, + "step": 6990, + "unclipped_grad_norm": 1.2125478982925415 + }, + { + "bits_per_byte": 1.506560110356264, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0442678928375244, + "step": 7000, + "unclipped_grad_norm": 1.2366911172866821 + }, + { + "bits_per_byte": 1.5357397187975532, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0644936561584473, + "step": 7010, + "unclipped_grad_norm": 1.2639423608779907 + }, + { + "bits_per_byte": 1.5037485379799589, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0423190593719482, + "step": 7020, + "unclipped_grad_norm": 1.2011712789535522 + }, + { + "bits_per_byte": 1.528263632964547, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0593116283416748, + "step": 7030, + "unclipped_grad_norm": 1.1849545240402222 + }, + { + "bits_per_byte": 1.5983039114937276, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.1078598499298096, + "step": 7040, + "unclipped_grad_norm": 1.3057087659835815 + }, + { + "bits_per_byte": 1.5441888824691414, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.070350170135498, + "step": 7050, + "unclipped_grad_norm": 1.2155181169509888 + }, + { + "bits_per_byte": 1.5741694141154987, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0911310911178589, + "step": 7060, + "unclipped_grad_norm": 1.3050142526626587 + }, + { + "bits_per_byte": 1.5094996378248111, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0463054180145264, + "step": 7070, + "unclipped_grad_norm": 1.1835323572158813 + }, + { + "bits_per_byte": 1.4442604269771167, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0010850429534912, + "step": 7080, + "unclipped_grad_norm": 1.167786717414856 + }, + { + "bits_per_byte": 1.5378045425038123, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.065924882888794, + "step": 7090, + "unclipped_grad_norm": 1.240479588508606 + }, + { + "bits_per_byte": 1.5019334330827527, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0410609245300293, + "step": 7100, + "unclipped_grad_norm": 1.1977860927581787 + }, + { + "bits_per_byte": 1.5780336922977352, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0938096046447754, + "step": 7110, + "unclipped_grad_norm": 1.2797318696975708 + }, + { + "bits_per_byte": 1.56051364767169, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0816656351089478, + "step": 7120, + "unclipped_grad_norm": 1.297847032546997 + }, + { + "bits_per_byte": 1.4993894657111901, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0392975807189941, + "step": 7130, + "unclipped_grad_norm": 1.224601149559021 + }, + { + "bits_per_byte": 1.6077283887783587, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.1143923997879028, + "step": 7140, + "unclipped_grad_norm": 1.3082928657531738 + }, + { + "bits_per_byte": 1.624090990164097, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.1257340908050537, + "step": 7150, + "unclipped_grad_norm": 1.2189027070999146 + }, + { + "bits_per_byte": 1.5477993142587043, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0728527307510376, + "step": 7160, + "unclipped_grad_norm": 1.2660733461380005 + }, + { + "bits_per_byte": 1.5516317756505311, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0755091905593872, + "step": 7170, + "unclipped_grad_norm": 1.2526509761810303 + }, + { + "bits_per_byte": 1.4692106380683878, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0183792114257812, + "step": 7180, + "unclipped_grad_norm": 1.1470739841461182 + }, + { + "bits_per_byte": 1.4673153892558797, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0170655250549316, + "step": 7190, + "unclipped_grad_norm": 1.2286535501480103 + }, + { + "bits_per_byte": 1.553365876719181, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0767111778259277, + "step": 7200, + "unclipped_grad_norm": 1.2306180000305176 + }, + { + "bits_per_byte": 1.561453876823957, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0823173522949219, + "step": 7210, + "unclipped_grad_norm": 1.2458006143569946 + }, + { + "bits_per_byte": 1.5086355969868763, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0457065105438232, + "step": 7220, + "unclipped_grad_norm": 1.1648746728897095 + }, + { + "bits_per_byte": 1.5304512522835219, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0608279705047607, + "step": 7230, + "unclipped_grad_norm": 1.2855236530303955 + }, + { + "bits_per_byte": 1.493701999447157, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0353553295135498, + "step": 7240, + "unclipped_grad_norm": 1.2490638494491577 + }, + { + "bits_per_byte": 1.5430964486708607, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0695929527282715, + "step": 7250, + "unclipped_grad_norm": 1.286012887954712 + }, + { + "bits_per_byte": 1.5196966611771179, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0533734560012817, + "step": 7260, + "unclipped_grad_norm": 1.2369425296783447 + }, + { + "bits_per_byte": 1.5993731276341416, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.1086009740829468, + "step": 7270, + "unclipped_grad_norm": 1.2697807550430298 + }, + { + "bits_per_byte": 1.5305549578219921, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0608998537063599, + "step": 7280, + "unclipped_grad_norm": 1.2743209600448608 + }, + { + "bits_per_byte": 1.5476247518680786, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0727317333221436, + "step": 7290, + "unclipped_grad_norm": 1.2424356937408447 + }, + { + "bits_per_byte": 1.5365045256459402, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0650237798690796, + "step": 7300, + "unclipped_grad_norm": 1.298574686050415 + }, + { + "bits_per_byte": 1.5764911799021466, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0927404165267944, + "step": 7310, + "unclipped_grad_norm": 1.3189024925231934 + }, + { + "bits_per_byte": 1.5490052566065542, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0736886262893677, + "step": 7320, + "unclipped_grad_norm": 1.2497724294662476 + }, + { + "bits_per_byte": 1.5554974296939745, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0781886577606201, + "step": 7330, + "unclipped_grad_norm": 1.279703140258789 + }, + { + "bits_per_byte": 1.5241250424541843, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0564429759979248, + "step": 7340, + "unclipped_grad_norm": 1.2488007545471191 + }, + { + "bits_per_byte": 1.57090466745417, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0888681411743164, + "step": 7350, + "unclipped_grad_norm": 1.238014578819275 + }, + { + "bits_per_byte": 1.5500565865512788, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0744173526763916, + "step": 7360, + "unclipped_grad_norm": 1.2180218696594238 + }, + { + "bits_per_byte": 1.5597980278614498, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.081169605255127, + "step": 7370, + "unclipped_grad_norm": 1.2368392944335938 + }, + { + "bits_per_byte": 1.6215841710451209, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.1239964962005615, + "step": 7380, + "unclipped_grad_norm": 1.3172329664230347 + }, + { + "bits_per_byte": 1.5928735592927343, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.1040958166122437, + "step": 7390, + "unclipped_grad_norm": 1.3043875694274902 + }, + { + "bits_per_byte": 1.5057738056765193, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0437228679656982, + "step": 7400, + "unclipped_grad_norm": 1.2024799585342407 + }, + { + "bits_per_byte": 1.5152624324899222, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.050299882888794, + "step": 7410, + "unclipped_grad_norm": 1.2088910341262817 + }, + { + "bits_per_byte": 1.597047062281222, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.1069886684417725, + "step": 7420, + "unclipped_grad_norm": 1.3177212476730347 + }, + { + "bits_per_byte": 1.5299019396966662, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0604472160339355, + "step": 7430, + "unclipped_grad_norm": 1.290786862373352 + }, + { + "bits_per_byte": 1.5250764504787575, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0571024417877197, + "step": 7440, + "unclipped_grad_norm": 1.2367006540298462 + }, + { + "bits_per_byte": 1.4892672548120085, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0322813987731934, + "step": 7450, + "unclipped_grad_norm": 1.3495734930038452 + }, + { + "bits_per_byte": 1.5046927227331957, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.042973518371582, + "step": 7460, + "unclipped_grad_norm": 1.2188435792922974 + }, + { + "bits_per_byte": 1.5481360562890936, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.073086142539978, + "step": 7470, + "unclipped_grad_norm": 1.1978274583816528 + }, + { + "bits_per_byte": 1.4779339420854511, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.024425745010376, + "step": 7480, + "unclipped_grad_norm": 1.25648832321167 + }, + { + "bits_per_byte": 1.5058897219832008, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0438032150268555, + "step": 7490, + "unclipped_grad_norm": 1.2401517629623413 + }, + { + "bits_per_byte": 1.5728836718176484, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0902398824691772, + "step": 7500, + "unclipped_grad_norm": 1.2475262880325317 + }, + { + "bits_per_byte": 1.5378353273983165, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0659462213516235, + "step": 7510, + "unclipped_grad_norm": 1.2484221458435059 + }, + { + "bits_per_byte": 1.5312337733449477, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0613703727722168, + "step": 7520, + "unclipped_grad_norm": 1.1939175128936768 + }, + { + "bits_per_byte": 1.5599504044901142, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.081275224685669, + "step": 7530, + "unclipped_grad_norm": 1.2631168365478516 + }, + { + "bits_per_byte": 1.5976060058965262, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.1073760986328125, + "step": 7540, + "unclipped_grad_norm": 1.3131186962127686 + }, + { + "bits_per_byte": 1.5182279293387495, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.052355408668518, + "step": 7550, + "unclipped_grad_norm": 1.2190253734588623 + }, + { + "bits_per_byte": 1.5653272700866891, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0850021839141846, + "step": 7560, + "unclipped_grad_norm": 1.2437095642089844 + }, + { + "bits_per_byte": 1.566524613301996, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.085832118988037, + "step": 7570, + "unclipped_grad_norm": 1.2195208072662354 + }, + { + "bits_per_byte": 1.514368810636039, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.049680471420288, + "step": 7580, + "unclipped_grad_norm": 1.2027956247329712 + }, + { + "bits_per_byte": 1.5218865162705542, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0548913478851318, + "step": 7590, + "unclipped_grad_norm": 1.22147798538208 + }, + { + "bits_per_byte": 1.5124059726417418, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.048319935798645, + "step": 7600, + "unclipped_grad_norm": 1.2925957441329956 + }, + { + "bits_per_byte": 1.451400458710337, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0060341358184814, + "step": 7610, + "unclipped_grad_norm": 1.22130286693573 + }, + { + "bits_per_byte": 1.554240580481469, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.077317476272583, + "step": 7620, + "unclipped_grad_norm": 1.2743861675262451 + }, + { + "bits_per_byte": 1.5089792183233002, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0459446907043457, + "step": 7630, + "unclipped_grad_norm": 1.1950352191925049 + }, + { + "bits_per_byte": 1.523324463214418, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.055888056755066, + "step": 7640, + "unclipped_grad_norm": 1.2816646099090576 + }, + { + "bits_per_byte": 1.6003928127711071, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1093077659606934, + "step": 7650, + "unclipped_grad_norm": 1.3542457818984985 + }, + { + "bits_per_byte": 1.5396506042781735, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.067204475402832, + "step": 7660, + "unclipped_grad_norm": 1.277113437652588 + }, + { + "bits_per_byte": 1.515347907867401, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0503591299057007, + "step": 7670, + "unclipped_grad_norm": 1.3265182971954346 + }, + { + "bits_per_byte": 1.4941618810555641, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0356740951538086, + "step": 7680, + "unclipped_grad_norm": 1.2858244180679321 + }, + { + "bits_per_byte": 1.5394590156051124, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0670716762542725, + "step": 7690, + "unclipped_grad_norm": 1.2726463079452515 + }, + { + "bits_per_byte": 1.5472200766905984, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0724512338638306, + "step": 7700, + "unclipped_grad_norm": 1.3285934925079346 + }, + { + "bits_per_byte": 1.4615679010466955, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.013081669807434, + "step": 7710, + "unclipped_grad_norm": 1.2261079549789429 + }, + { + "bits_per_byte": 1.4667946257890672, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0167045593261719, + "step": 7720, + "unclipped_grad_norm": 1.24894380569458 + }, + { + "bits_per_byte": 1.5217016349208765, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0547631978988647, + "step": 7730, + "unclipped_grad_norm": 1.2570104598999023 + }, + { + "bits_per_byte": 1.5186754281962946, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0526655912399292, + "step": 7740, + "unclipped_grad_norm": 1.2608413696289062 + }, + { + "bits_per_byte": 1.530832365837834, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0610921382904053, + "step": 7750, + "unclipped_grad_norm": 1.268287181854248 + }, + { + "bits_per_byte": 1.5045843736631521, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.042898416519165, + "step": 7760, + "unclipped_grad_norm": 1.2538435459136963 + }, + { + "bits_per_byte": 1.5028014295216574, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.041662573814392, + "step": 7770, + "unclipped_grad_norm": 1.3365142345428467 + }, + { + "bits_per_byte": 1.6460007199707645, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.1409207582473755, + "step": 7780, + "unclipped_grad_norm": 1.3043659925460815 + }, + { + "bits_per_byte": 1.5196737874845532, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0533576011657715, + "step": 7790, + "unclipped_grad_norm": 1.2850605249404907 + }, + { + "bits_per_byte": 1.559599903847656, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0810322761535645, + "step": 7800, + "unclipped_grad_norm": 1.3824384212493896 + }, + { + "bits_per_byte": 1.4742115495901766, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0218455791473389, + "step": 7810, + "unclipped_grad_norm": 1.2828388214111328 + }, + { + "bits_per_byte": 1.5144011433744011, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0497028827667236, + "step": 7820, + "unclipped_grad_norm": 1.3298958539962769 + }, + { + "bits_per_byte": 1.549842984098907, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0742692947387695, + "step": 7830, + "unclipped_grad_norm": 1.2448348999023438 + }, + { + "bits_per_byte": 1.5177059619933808, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0519936084747314, + "step": 7840, + "unclipped_grad_norm": 1.234107255935669 + }, + { + "bits_per_byte": 1.486972662284197, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0306909084320068, + "step": 7850, + "unclipped_grad_norm": 1.2269891500473022 + }, + { + "bits_per_byte": 1.4944172752920954, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0358511209487915, + "step": 7860, + "unclipped_grad_norm": 1.266727328300476 + }, + { + "bits_per_byte": 1.549148518154723, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0737879276275635, + "step": 7870, + "unclipped_grad_norm": 1.3428150415420532 + }, + { + "bits_per_byte": 1.522268833703422, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0551563501358032, + "step": 7880, + "unclipped_grad_norm": 1.305385947227478 + }, + { + "bits_per_byte": 1.5751703531435204, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0918248891830444, + "step": 7890, + "unclipped_grad_norm": 1.2760140895843506 + }, + { + "bits_per_byte": 1.5609104116472305, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0819406509399414, + "step": 7900, + "unclipped_grad_norm": 1.2964600324630737 + }, + { + "bits_per_byte": 1.5527064952357728, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.07625412940979, + "step": 7910, + "unclipped_grad_norm": 1.3213040828704834 + }, + { + "bits_per_byte": 1.4860267577044515, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0300352573394775, + "step": 7920, + "unclipped_grad_norm": 1.2681679725646973 + }, + { + "bits_per_byte": 1.482107273091288, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0273184776306152, + "step": 7930, + "unclipped_grad_norm": 1.2925180196762085 + }, + { + "bits_per_byte": 1.5584990428994832, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.08026921749115, + "step": 7940, + "unclipped_grad_norm": 1.4040197134017944 + }, + { + "bits_per_byte": 1.488459280318255, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0317213535308838, + "step": 7950, + "unclipped_grad_norm": 1.3141639232635498 + }, + { + "bits_per_byte": 1.530331896323823, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0607452392578125, + "step": 7960, + "unclipped_grad_norm": 1.263742446899414 + }, + { + "bits_per_byte": 1.5034190192209056, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.042090654373169, + "step": 7970, + "unclipped_grad_norm": 1.2354859113693237 + }, + { + "bits_per_byte": 1.5362255697862408, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0648304224014282, + "step": 7980, + "unclipped_grad_norm": 1.2929013967514038 + }, + { + "bits_per_byte": 1.5535658925421343, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0768498182296753, + "step": 7990, + "unclipped_grad_norm": 1.2544541358947754 + }, + { + "bits_per_byte": 1.4716885641020192, + "learning_rate": 3e-05, + "loss_nats": 1.020096778869629, + "step": 8000, + "unclipped_grad_norm": 1.2711973190307617 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073001.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073001.json new file mode 100644 index 0000000..50ea641 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073001.json @@ -0,0 +1,8614 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "6574e479d5e2794b43a6244a723ba3a2b034f8ef3bfeae24909948fb4a990d99", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002768727717921138, + 0.0002554995589889586, + 0.00023783760843798518, + 0.00022115583124104887, + 0.00020655356638599187, + 0.00019376208365429193, + 0.00018488570640329272, + 0.00017590694187674671, + 0.0001684112212387845, + 0.0001647331955609843, + 0.00015843863366171718, + 0.0001538397918920964, + 0.00015132295084185898, + 0.00014730033581145108, + 0.00014253577683120966, + 0.0001394751889165491, + 0.0001364792260574177, + 0.0001337067224085331, + 0.00013106675760354847, + 0.00012915431580040604, + 0.00012709009752143174, + 0.00012574320135172457, + 0.0001242366706719622, + 0.0001228937617270276, + 0.00012132258416386321, + 0.00011988351616309956, + 0.00011879197700181976, + 0.00011764531518565491, + 0.00011639683361863717, + 0.00011549647024367005, + 0.00011481995898066089, + 0.00011415599146857858 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002190592585975537, + "first_to_last_ratio": 1.8672887091156776, + "imbalance_abs_log_ratio": 0.6244874905881238, + "last_quartile_mean": 0.00011731408085324802, + "mean": 0.0001546067051094724, + "normalized": [ + 1.7908199492128654, + 1.6525774791465027, + 1.538339545297078, + 1.4304413969915148, + 1.3359935860462033, + 1.2532579587482624, + 1.1958453307208163, + 1.1377704592578457, + 1.0892879524179597, + 1.065498391187767, + 1.0247850088360109, + 0.9950395863049215, + 0.978760596021445, + 0.9527422223192203, + 0.9219249367631522, + 0.9021289782858439, + 0.8827510162691898, + 0.8648183939620171, + 0.8477430361816715, + 0.835373315206192, + 0.8220219002238166, + 0.8133101424203405, + 0.80356586464988, + 0.7948798963150414, + 0.7847174809007043, + 0.775409553409819, + 0.7683494510649245, + 0.7609328140222236, + 0.7528576043077825, + 0.7470340316863389, + 0.7426583400723811, + 0.7383637817502684 + ], + "population_cv": 0.27888268733798954 + }, + "activation_output_rms_by_block": [ + 0.028270306065678596, + 0.0284538883715868, + 0.02867252007126808, + 0.02881420962512493, + 0.029061447829008102, + 0.02929086796939373, + 0.029533835127949715, + 0.029811719432473183, + 0.030115574598312378, + 0.030315518379211426, + 0.030812423676252365, + 0.03120259940624237, + 0.03179442882537842, + 0.03198441118001938, + 0.03240228444337845, + 0.0329107940196991, + 0.033366311341524124, + 0.033811621367931366, + 0.03420914337038994, + 0.03470747172832489, + 0.03500960022211075, + 0.03567802160978317, + 0.035978782922029495, + 0.036836784332990646, + 0.03769209608435631, + 0.038124579936265945, + 0.038542140275239944, + 0.03904740884900093, + 0.0400773324072361, + 0.04093807935714722, + 0.04163828864693642, + 0.042313966900110245 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.028988599311560392, + "first_to_last_ratio": 0.728416493900548, + "imbalance_abs_log_ratio": 0.31688228737002583, + "last_quartile_mean": 0.03979673655703664, + "mean": 0.03379432682413608, + "normalized": [ + 0.8365399971656723, + 0.8419723381282118, + 0.8484418174825138, + 0.8526345198433033, + 0.8599504875549783, + 0.8667392051281828, + 0.873928789339177, + 0.8821515986281313, + 0.8911429055838948, + 0.8970593951159853, + 0.9117632032322649, + 0.9233088017589188, + 0.9408214873116122, + 0.9464432106153376, + 0.9588084003566117, + 0.9738555879797565, + 0.9873346942272493, + 1.0005117587897308, + 1.0122747391422395, + 1.0270206567197142, + 1.0359608701276664, + 1.055739970659861, + 1.0646397281194921, + 1.0900286466627183, + 1.1153379761196018, + 1.1281355043603702, + 1.1404914344295491, + 1.1554427183059932, + 1.1859189447920195, + 1.2113891059344621, + 1.2321088348236646, + 1.2521026715611152 + ], + "population_cv": 0.12455242949918699 + }, + "bits_per_byte": 7.95915895791699, + "branch_output_rms_by_sublayer": [ + 0.0022265268489718437, + 0.002717922907322645, + 0.0021652730647474527, + 0.002733128145337105, + 0.0025650423485785723, + 0.002695153234526515, + 0.002537536434829235, + 0.002696225419640541, + 0.0027307753916829824, + 0.0027451792266219854, + 0.0029185698367655277, + 0.0026901187375187874, + 0.002936159959062934, + 0.0027115964330732822, + 0.0030124466866254807, + 0.0027617362793534994, + 0.0031113612931221724, + 0.002723218873143196, + 0.0034121612552553415, + 0.0027921765577048063, + 0.0034701526165008545, + 0.002744668396189809, + 0.0039346772246062756, + 0.0027058592531830072, + 0.004295050632208586, + 0.0026920263189822435, + 0.004176756367087364, + 0.0027182905469089746, + 0.004023758694529533, + 0.0027150253299623728, + 0.004447397775948048, + 0.0027387896552681923, + 0.004260339308530092, + 0.00275799329392612, + 0.005327676888555288, + 0.002721488941460848, + 0.00474865036085248, + 0.0027454239316284657, + 0.00498866755515337, + 0.002719792304560542, + 0.0052231852896511555, + 0.0027601351030170918, + 0.005184618756175041, + 0.002755985129624605, + 0.0052438946440815926, + 0.0027470553759485483, + 0.0059704589657485485, + 0.0027777766808867455, + 0.0060942270793020725, + 0.0026787330862134695, + 0.005698176100850105, + 0.002644060645252466, + 0.006003713235259056, + 0.002661966485902667, + 0.006405353546142578, + 0.002804658142849803, + 0.006667613051831722, + 0.0026854604948312044, + 0.006290186662226915, + 0.0027102255262434483, + 0.0072579155676066875, + 0.0027289679273962975, + 0.006407012697309256, + 0.0026884053368121386 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.005888019831616423, + 0.005779358340731243, + 0.0060348395662513914, + 0.005586875351280001, + 0.005559071516712478, + 0.0054082748429522395, + 0.005481135467826832, + 0.0053805923812947415, + 0.004971746645687392, + 0.005439263395635174, + 0.005139424262259116, + 0.005696496994849179, + 0.005782818048228185, + 0.0054294510739720785, + 0.005067071208887471, + 0.005218073376917496, + 0.005181036474299639, + 0.005605812513830226, + 0.005170579144406511, + 0.004834742615880465, + 0.005150211089068095, + 0.00476651480071292, + 0.004776402486441839, + 0.00519066249960807, + 0.0053994045182471796, + 0.004795508114414784, + 0.004895792813412353, + 0.00496559115560922, + 0.00502742614163479, + 0.004657478744424888, + 0.005287606581362069, + 0.004870555415551657 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005639770912333169, + "first_to_last_ratio": 1.130799174678946, + "imbalance_abs_log_ratio": 0.12292461698112994, + "last_quartile_mean": 0.0049874204355821175, + "mean": 0.005263682419187692, + "normalized": [ + 1.1186122875029152, + 1.0979686615711766, + 1.146505257280075, + 1.061400537941684, + 1.0561183357962487, + 1.0274698228824493, + 1.0413119621059315, + 1.0222106792919112, + 0.9445377303850805, + 1.0333570611721248, + 0.976393302818647, + 1.0822265747043835, + 1.0986259404914112, + 1.0314929058371969, + 0.962647592570647, + 0.9913351455049914, + 0.9842988352437859, + 1.0649982402804865, + 0.9823121405573801, + 0.9185095586801336, + 0.9784425956805525, + 0.9055475655859389, + 0.9074260386664719, + 0.9861275977985596, + 1.0257846291342996, + 0.9110557462459602, + 0.9301079403205877, + 0.9433683038148654, + 0.9551157804103686, + 0.884832779319472, + 1.0045451378463044, + 0.9253133125579595 + ], + "population_cv": 0.06750436074287791 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028027428314089775, + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346 + ], + "loss_nats": 5.516868591308594, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346, + 0.042313966900110245 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.172436016844586e-05, + 6.834617670392618e-05, + 6.473411485785618e-05, + 6.156626477604732e-05, + 5.8372206694912165e-05, + 5.494635479408316e-05, + 5.167145354789682e-05, + 4.809222446056083e-05, + 4.4674157834379e-05, + 4.1754625272005796e-05, + 3.8718964788131416e-05, + 3.608527549658902e-05, + 3.353153806529008e-05, + 3.1093299185158685e-05, + 2.904756365751382e-05, + 2.7216165108256973e-05, + 2.5600058506825007e-05, + 2.4208331524278037e-05, + 2.304886766069103e-05, + 2.1986410501995124e-05, + 2.1054642274975777e-05, + 2.025439971475862e-05, + 1.956623418664094e-05, + 1.9002633052878082e-05, + 1.8555338101577945e-05, + 1.8124846974387765e-05, + 1.7765629308996722e-05, + 1.744573273754213e-05, + 1.7174377717310563e-05, + 1.6944877643254586e-05, + 1.6726755347917788e-05, + 1.656309177633375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.9931644500466064e-05, + "first_to_last_ratio": 3.4418587232383575, + "imbalance_abs_log_ratio": 1.2360116519842717, + "last_quartile_mean": 1.7412581200915156e-05, + "mean": 3.3612405388794286e-05, + "normalized": [ + 2.1338657361414946, + 2.033361668508004, + 1.9258995037420696, + 1.831653047852752, + 1.7366268798594315, + 1.634704632367703, + 1.537273305799265, + 1.4307879458276982, + 1.3290973174229443, + 1.242238536309156, + 1.1519248426368067, + 1.0735701619444094, + 0.9975941226886675, + 0.9250542716447356, + 0.8641917566303572, + 0.8097059640167944, + 0.761625305023828, + 0.7202201462305526, + 0.6857250290208349, + 0.654115950574753, + 0.626394988143127, + 0.602587035366146, + 0.582113477459246, + 0.5653458249439413, + 0.5520383884148895, + 0.5392308811207612, + 0.5285438249212427, + 0.5190266074607738, + 0.510953545830914, + 0.5041257073765888, + 0.49763636831222313, + 0.4927672264078892 + ], + "population_cv": 0.513984504821457 + }, + "activation_output_rms_by_block": [ + 0.029439672827720642, + 0.030752120539546013, + 0.03267423436045647, + 0.034982286393642426, + 0.03797784447669983, + 0.04108294099569321, + 0.044510453939437866, + 0.04866119846701622, + 0.05275992304086685, + 0.05702463164925575, + 0.061875153332948685, + 0.06669966131448746, + 0.071797676384449, + 0.07734881341457367, + 0.08311311900615692, + 0.08836977183818817, + 0.09436330199241638, + 0.10090246796607971, + 0.10806354880332947, + 0.11583591252565384, + 0.1238454207777977, + 0.13231202960014343, + 0.1417466551065445, + 0.15185882151126862, + 0.16391249001026154, + 0.1746179163455963, + 0.18763093650341034, + 0.20137503743171692, + 0.21650369465351105, + 0.23270079493522644, + 0.2497490495443344, + 0.2673082947731018 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037510094000026584, + "first_to_last_ratio": 0.17716440452291277, + "imbalance_abs_log_ratio": 1.7306771383999247, + "last_quartile_mean": 0.21172477677464485, + "mean": 0.11005612107692286, + "normalized": [ + 0.2674969146617844, + 0.27942217332966024, + 0.29688702491721536, + 0.3178586165979067, + 0.3450770761778485, + 0.37329083192909013, + 0.4044341514483109, + 0.4421489508339555, + 0.4793910827003499, + 0.5181413908763769, + 0.5622145567869099, + 0.6060513550888119, + 0.6523733135594211, + 0.702812462021184, + 0.7551885183020911, + 0.8029519028425762, + 0.857410756158323, + 0.9168274056792781, + 0.9818949436515149, + 1.0525167650120182, + 1.1252933463940358, + 1.2022232685055743, + 1.2879488548162785, + 1.3798307629352853, + 1.4893536897933755, + 1.586626119809806, + 1.7048659780791944, + 1.8297486360705681, + 1.9672117510137168, + 2.114383031658748, + 2.269288133185925, + 2.428836235162865 + ], + "population_cv": 0.6268498746310547 + }, + "bits_per_byte": 5.1517691996807775, + "branch_output_rms_by_sublayer": [ + 0.0028861628379672766, + 0.004992613103240728, + 0.002704753540456295, + 0.005112498998641968, + 0.002895323559641838, + 0.005543495528399944, + 0.0029729304369539022, + 0.005716898012906313, + 0.00318311620503664, + 0.005760234780609608, + 0.003378538880497217, + 0.0057550352066755295, + 0.0039320578798651695, + 0.005740197841078043, + 0.0034711055923253298, + 0.006492512300610542, + 0.0037959336768835783, + 0.006124566774815321, + 0.004324590787291527, + 0.0062123509123921394, + 0.004316650331020355, + 0.006645897403359413, + 0.0046909768134355545, + 0.006303280591964722, + 0.0053819348104298115, + 0.006548910401761532, + 0.005383738782256842, + 0.006836008746176958, + 0.00522694643586874, + 0.006793942768126726, + 0.005513017997145653, + 0.006699956953525543, + 0.005531978793442249, + 0.0068945568054914474, + 0.006940840277820826, + 0.0070524453185498714, + 0.006634449120610952, + 0.007447477430105209, + 0.0070588430389761925, + 0.007686031982302666, + 0.007738462649285793, + 0.007927682250738144, + 0.008234621025621891, + 0.008245103061199188, + 0.008838997222483158, + 0.008133068680763245, + 0.009296931326389313, + 0.008390788920223713, + 0.013193055056035519, + 0.008053022436797619, + 0.00934307835996151, + 0.008179600350558758, + 0.012704792432487011, + 0.008694498799741268, + 0.012802425771951675, + 0.009316418319940567, + 0.014256707392632961, + 0.008604929782450199, + 0.015807906165719032, + 0.009046261198818684, + 0.0164310485124588, + 0.009124966338276863, + 0.01581309363245964, + 0.009515758603811264 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004018231655400003, + 0.0003405186139403606, + 0.0003221632759320079, + 0.0003173401597627506, + 0.0002956526561473367, + 0.0002913580859417252, + 0.00030034368266542127, + 0.00026684124508166604, + 0.0002497190139849492, + 0.00026578754174316435, + 0.0002404550275593826, + 0.00024493210822064133, + 0.0002532169075836586, + 0.00023865218150919594, + 0.0002121496513808785, + 0.0002091708005241373, + 0.0001980500112256327, + 0.00020258502349527826, + 0.00019483821933596074, + 0.0001851504169444552, + 0.00018439346409065922, + 0.00017525536186022104, + 0.000171573793780652, + 0.0001670148315601783, + 0.00018406810665917416, + 0.00014791927954418835, + 0.00017020274695916728, + 0.00015649953794173187, + 0.00016066786897129027, + 0.00016871534344057424, + 0.00016946426752177343, + 0.0001617385113992835 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031700511062640856, + "first_to_last_ratio": 1.9222979375866573, + "imbalance_abs_log_ratio": 0.653521312887717, + "last_quartile_mean": 0.0001649094578046479, + "mean": 0.00022650815319523427, + "normalized": [ + 1.773989853661721, + 1.5033393241560593, + 1.4223032212634112, + 1.401009876625613, + 1.305262755343313, + 1.286302862972861, + 1.3259729437047942, + 1.1780646361619815, + 1.1024725179305521, + 1.1734126917456873, + 1.061573387833537, + 1.0813390368757583, + 1.117915200895233, + 1.0536140891294732, + 0.9366093378458668, + 0.9234581518301755, + 0.8743615116358631, + 0.8943829201621014, + 0.8601819254074433, + 0.8174117104953322, + 0.814069875585118, + 0.7737265055936556, + 0.7574729269580301, + 0.7373457829406393, + 0.8126334706394442, + 0.6530417446682029, + 0.7514199579935843, + 0.6909223166321972, + 0.7093248817088087, + 0.7448532914184036, + 0.7481596804849096, + 0.7140516097002307 + ], + "population_cv": 0.27857383691000986 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0285640936344862, + 0.02861291915178299, + 0.029439672827720642, + 0.02950597181916237, + 0.030752120539546013, + 0.030888807028532028, + 0.03267423436045647, + 0.03270668536424637, + 0.034982286393642426, + 0.03526782616972923, + 0.03797784447669983, + 0.03822308033704758, + 0.04108294099569321, + 0.041396427899599075, + 0.044510453939437866, + 0.04485016688704491, + 0.04866119846701622, + 0.04912792518734932, + 0.05275992304086685, + 0.05317053198814392, + 0.05702463164925575, + 0.05767740681767464, + 0.061875153332948685, + 0.06268688291311264, + 0.06669966131448746, + 0.06753430515527725, + 0.071797676384449, + 0.07279384881258011, + 0.07734881341457367, + 0.078476682305336, + 0.08311311900615692, + 0.08398299664258957, + 0.08836977183818817, + 0.08972389250993729, + 0.09436330199241638, + 0.09631941467523575, + 0.10090246796607971, + 0.10291995853185654, + 0.10806354880332947, + 0.11053454130887985, + 0.11583591252565384, + 0.11846786737442017, + 0.1238454207777977, + 0.12666596472263336, + 0.13231202960014343, + 0.13596534729003906, + 0.1417466551065445, + 0.1458624303340912, + 0.15185882151126862, + 0.1581486165523529, + 0.16391249001026154, + 0.16865970194339752, + 0.1746179163455963, + 0.18122267723083496, + 0.18763093650341034, + 0.19437597692012787, + 0.20137503743171692, + 0.2100502997636795, + 0.21650369465351105, + 0.22599515318870544, + 0.23270079493522644, + 0.2427441030740738, + 0.2497490495443344, + 0.2599930763244629 + ], + "loss_nats": 3.570934295654297, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.02861291915178299, + 0.029439672827720642, + 0.02950597181916237, + 0.030752120539546013, + 0.030888807028532028, + 0.03267423436045647, + 0.03270668536424637, + 0.034982286393642426, + 0.03526782616972923, + 0.03797784447669983, + 0.03822308033704758, + 0.04108294099569321, + 0.041396427899599075, + 0.044510453939437866, + 0.04485016688704491, + 0.04866119846701622, + 0.04912792518734932, + 0.05275992304086685, + 0.05317053198814392, + 0.05702463164925575, + 0.05767740681767464, + 0.061875153332948685, + 0.06268688291311264, + 0.06669966131448746, + 0.06753430515527725, + 0.071797676384449, + 0.07279384881258011, + 0.07734881341457367, + 0.078476682305336, + 0.08311311900615692, + 0.08398299664258957, + 0.08836977183818817, + 0.08972389250993729, + 0.09436330199241638, + 0.09631941467523575, + 0.10090246796607971, + 0.10291995853185654, + 0.10806354880332947, + 0.11053454130887985, + 0.11583591252565384, + 0.11846786737442017, + 0.1238454207777977, + 0.12666596472263336, + 0.13231202960014343, + 0.13596534729003906, + 0.1417466551065445, + 0.1458624303340912, + 0.15185882151126862, + 0.1581486165523529, + 0.16391249001026154, + 0.16865970194339752, + 0.1746179163455963, + 0.18122267723083496, + 0.18763093650341034, + 0.19437597692012787, + 0.20137503743171692, + 0.2100502997636795, + 0.21650369465351105, + 0.22599515318870544, + 0.23270079493522644, + 0.2427441030740738, + 0.2497490495443344, + 0.2599930763244629, + 0.2673082947731018 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014451092283707112, + 0.00014379274216480553, + 0.00014628663484472781, + 0.0001445338421035558, + 0.00013843078340869397, + 0.0001292457600357011, + 0.00012031046207994223, + 0.00011329296830808744, + 0.0001077206979971379, + 0.0001003394863801077, + 9.358100214740261e-05, + 8.73711978783831e-05, + 8.221827738452703e-05, + 7.71504346630536e-05, + 7.201205153251067e-05, + 6.728569132974371e-05, + 6.256035703700036e-05, + 5.796365803689696e-05, + 5.370515646063723e-05, + 4.9705678975442424e-05, + 4.572505713440478e-05, + 4.197436646791175e-05, + 3.853226371575147e-05, + 3.558541357051581e-05, + 3.2570587791269645e-05, + 2.9896507840021513e-05, + 2.8078287868993357e-05, + 2.6684616386773996e-05, + 2.568897070887033e-05, + 2.5203555196640082e-05, + 2.4927356207626872e-05, + 2.5036461011040956e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013505051447282312, + "first_to_last_ratio": 4.954020049421058, + "imbalance_abs_log_ratio": 1.6001993781766728, + "last_quartile_mean": 2.7260792876404594e-05, + "mean": 7.412253904703903e-05, + "normalized": [ + 1.9496218652920512, + 1.9399327655728709, + 1.9735783032458267, + 1.9499310730820074, + 1.8675936521932173, + 1.743676912547211, + 1.6231292617160862, + 1.5284550389752622, + 1.4532785760182485, + 1.3536973728926245, + 1.2625174926619152, + 1.1787399487615544, + 1.1092210067487078, + 1.040849863684419, + 0.9715270477554875, + 0.90776290443914, + 0.8440126018524382, + 0.7819977402570161, + 0.7245455586263081, + 0.670587915828121, + 0.616884657787925, + 0.5662834410094119, + 0.5198454371793504, + 0.4800889719648283, + 0.4394154357097774, + 0.4033389603808477, + 0.3788090401379066, + 0.36000677701879086, + 0.34657434889767885, + 0.34002552422881266, + 0.3362992758762309, + 0.3377712276579264 + ], + "population_cv": 0.5690574657123064 + }, + "activation_output_rms_by_block": [ + 0.049516160041093826, + 0.057937320321798325, + 0.06586314737796783, + 0.0719481110572815, + 0.076947420835495, + 0.08064281940460205, + 0.08439230173826218, + 0.08712083846330643, + 0.0897950753569603, + 0.09305135160684586, + 0.09555979818105698, + 0.09745962917804718, + 0.10008277744054794, + 0.10209457576274872, + 0.1030229851603508, + 0.10483334213495255, + 0.10759729146957397, + 0.11058001965284348, + 0.11383581161499023, + 0.1180567741394043, + 0.1237289309501648, + 0.13110940158367157, + 0.14003770053386688, + 0.15068796277046204, + 0.16247524321079254, + 0.1767880618572235, + 0.1922861486673355, + 0.20968382060527802, + 0.22965215146541595, + 0.2505997121334076, + 0.27601101994514465, + 0.307341605424881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07179601490497589, + "first_to_last_ratio": 0.31823808816289667, + "imbalance_abs_log_ratio": 1.1449554714268964, + "last_quartile_mean": 0.22560472041368484, + "mean": 0.13002310344018042, + "normalized": [ + 0.3808258588741859, + 0.44559250463094413, + 0.506549571847971, + 0.5533486676879896, + 0.5917980635718029, + 0.6202191554495797, + 0.6490562023624398, + 0.6700412169702441, + 0.6906086147856962, + 0.7156524428726306, + 0.7349447571448028, + 0.7495562450013764, + 0.7697307231756155, + 0.785203345109504, + 0.7923436868875267, + 0.8062670353287107, + 0.8275244062227459, + 0.8504643923048483, + 0.8755044957634206, + 0.9079676689436854, + 0.9515918915678598, + 1.008354654786339, + 1.0770216740619014, + 1.1589322111496045, + 1.2495874880077935, + 1.359666529868349, + 1.4788613990882042, + 1.6126658651994643, + 1.7662411170724883, + 1.9273475674936549, + 2.1227844332459633, + 2.363746113522658 + ], + "population_cv": 0.4886061625493153 + }, + "bits_per_byte": 3.0864900833624125, + "branch_output_rms_by_sublayer": [ + 0.01111105177551508, + 0.01691170036792755, + 0.01229762565344572, + 0.013536608777940273, + 0.012528475373983383, + 0.011229105293750763, + 0.017575321719050407, + 0.009298570454120636, + 0.015520633198320866, + 0.008989781141281128, + 0.012922349385917187, + 0.008074384182691574, + 0.013718407601118088, + 0.0077475146390497684, + 0.012880085967481136, + 0.00855444185435772, + 0.019964123144745827, + 0.007989896461367607, + 0.02377571351826191, + 0.008533375337719917, + 0.02730517089366913, + 0.0086820675060153, + 0.02310827188193798, + 0.008458662778139114, + 0.019315505400300026, + 0.008968821726739407, + 0.031517889350652695, + 0.009528025984764099, + 0.018633879721164703, + 0.008953212760388851, + 0.016267014667391777, + 0.008700341917574406, + 0.016277894377708435, + 0.010046470910310745, + 0.022248439490795135, + 0.010587109252810478, + 0.016605904325842857, + 0.012151478789746761, + 0.01819506660103798, + 0.012493276968598366, + 0.017910068854689598, + 0.013204623945057392, + 0.017647475004196167, + 0.015671590343117714, + 0.02343275025486946, + 0.016670797020196915, + 0.03074510768055916, + 0.019418122246861458, + 0.017775433138012886, + 0.019835535436868668, + 0.017164910212159157, + 0.021999742835760117, + 0.01125776395201683, + 0.023901214823126793, + 0.007435314357280731, + 0.02528192289173603, + 0.006950905080884695, + 0.027673469856381416, + 0.007996064610779285, + 0.026783082634210587, + 0.010837847366929054, + 0.030296670272946358, + 0.021106943488121033, + 0.028989506885409355 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006755996472372422, + 0.0003902215048764158, + 0.00033650809592979206, + 0.00029675556364760884, + 0.00026186200490798707, + 0.0002455230107742763, + 0.00021966924842671773, + 0.00022418062520577193, + 0.00022548545903395049, + 0.00022091596273961857, + 0.00024014950731391624, + 0.00021478395214610848, + 0.0001956916368008953, + 0.00020741213919383669, + 0.0001699546796737774, + 0.00016808908215382032, + 0.00015850476160987511, + 0.00015061268581609982, + 0.00013300713992511992, + 0.0001284985461118983, + 0.0001312583398227583, + 0.00012527717278626568, + 0.0001234119340024874, + 0.00012326054887191204, + 0.00010914072106764207, + 0.00010077507233480893, + 9.59134879196041e-05, + 8.362219426315277e-05, + 8.733675859901561e-05, + 7.698995894839292e-05, + 8.92787325176121e-05, + 9.631544502550806e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003312899626257265, + "first_to_last_ratio": 3.584553340265607, + "imbalance_abs_log_ratio": 1.276633875010675, + "last_quartile_mean": 9.242154633446708e-05, + "mean": 0.00019081267561512152, + "normalized": [ + 3.540643435030279, + 2.045050223306504, + 1.763552105985586, + 1.5552193411206041, + 1.3723512028947988, + 1.2867227503769625, + 1.1512298526215383, + 1.17487281430902, + 1.1817111117332983, + 1.1577635606620644, + 1.2585616051960276, + 1.1256272753039647, + 1.0255693767201026, + 1.0869935056735807, + 0.8906886249872848, + 0.8809115097409158, + 0.8306825586869656, + 0.7893222257408777, + 0.697056102254977, + 0.6734277253733716, + 0.6878910921385158, + 0.6565453389425546, + 0.6467701037399387, + 0.6459767333305183, + 0.5719783589628198, + 0.5281361524329609, + 0.5026578428839092, + 0.43824234419217944, + 0.45770941745598764, + 0.4034845101364514, + 0.4678868017012227, + 0.5047643963642175 + ], + "population_cv": 0.6065148734399883 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.043906208127737045, + 0.04462560638785362, + 0.049516160041093826, + 0.052572574466466904, + 0.057937320321798325, + 0.060727521777153015, + 0.06586314737796783, + 0.06803189963102341, + 0.0719481110572815, + 0.07307156920433044, + 0.076947420835495, + 0.07774755358695984, + 0.08064281940460205, + 0.08214583247900009, + 0.08439230173826218, + 0.08468613773584366, + 0.08712083846330643, + 0.08735909312963486, + 0.0897950753569603, + 0.09098032861948013, + 0.09305135160684586, + 0.09323208779096603, + 0.09555979818105698, + 0.09555087983608246, + 0.09745962917804718, + 0.09809984266757965, + 0.10008277744054794, + 0.09976384043693542, + 0.10209457576274872, + 0.10044334083795547, + 0.1030229851603508, + 0.1023804098367691, + 0.10483334213495255, + 0.10451660305261612, + 0.10759729146957397, + 0.10710933804512024, + 0.11058001965284348, + 0.10916344821453094, + 0.11383581161499023, + 0.11319266259670258, + 0.1180567741394043, + 0.11788330227136612, + 0.1237289309501648, + 0.12335778772830963, + 0.13110940158367157, + 0.13146334886550903, + 0.14003770053386688, + 0.13964787125587463, + 0.15068796277046204, + 0.15031130611896515, + 0.16247524321079254, + 0.16256581246852875, + 0.1767880618572235, + 0.1762392520904541, + 0.1922861486673355, + 0.19244804978370667, + 0.20968382060527802, + 0.21054796874523163, + 0.22965215146541595, + 0.23175573348999023, + 0.2505997121334076, + 0.2547222673892975, + 0.27601101994514465, + 0.28623583912849426 + ], + "loss_nats": 2.1393918991088867, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04462560638785362, + 0.049516160041093826, + 0.052572574466466904, + 0.057937320321798325, + 0.060727521777153015, + 0.06586314737796783, + 0.06803189963102341, + 0.0719481110572815, + 0.07307156920433044, + 0.076947420835495, + 0.07774755358695984, + 0.08064281940460205, + 0.08214583247900009, + 0.08439230173826218, + 0.08468613773584366, + 0.08712083846330643, + 0.08735909312963486, + 0.0897950753569603, + 0.09098032861948013, + 0.09305135160684586, + 0.09323208779096603, + 0.09555979818105698, + 0.09555087983608246, + 0.09745962917804718, + 0.09809984266757965, + 0.10008277744054794, + 0.09976384043693542, + 0.10209457576274872, + 0.10044334083795547, + 0.1030229851603508, + 0.1023804098367691, + 0.10483334213495255, + 0.10451660305261612, + 0.10759729146957397, + 0.10710933804512024, + 0.11058001965284348, + 0.10916344821453094, + 0.11383581161499023, + 0.11319266259670258, + 0.1180567741394043, + 0.11788330227136612, + 0.1237289309501648, + 0.12335778772830963, + 0.13110940158367157, + 0.13146334886550903, + 0.14003770053386688, + 0.13964787125587463, + 0.15068796277046204, + 0.15031130611896515, + 0.16247524321079254, + 0.16256581246852875, + 0.1767880618572235, + 0.1762392520904541, + 0.1922861486673355, + 0.19244804978370667, + 0.20968382060527802, + 0.21054796874523163, + 0.22965215146541595, + 0.23175573348999023, + 0.2505997121334076, + 0.2547222673892975, + 0.27601101994514465, + 0.28623583912849426, + 0.307341605424881 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010485891834832728, + 0.00010495351307326928, + 0.00011622957390500233, + 0.00011138651461806148, + 0.00010787966311909258, + 0.000104747356090229, + 0.00010223284334642813, + 9.973083797376603e-05, + 9.735774074215442e-05, + 9.589978435542434e-05, + 9.408760524820536e-05, + 9.229916759068146e-05, + 9.143509669229388e-05, + 8.827183046378195e-05, + 8.612661622464657e-05, + 8.33196536405012e-05, + 8.087792957667261e-05, + 7.82960414653644e-05, + 8.221158350352198e-05, + 7.83153300289996e-05, + 7.789687515469268e-05, + 7.347232894971967e-05, + 6.715180643368512e-05, + 6.132270209491253e-05, + 5.5257889471249655e-05, + 5.0392085540806875e-05, + 4.541657472145744e-05, + 4.046381582156755e-05, + 3.4924181818496436e-05, + 2.9514063498936594e-05, + 2.4251618015114218e-05, + 2.1234091036603786e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010650240255927201, + "first_to_last_ratio": 2.8263626166920495, + "imbalance_abs_log_ratio": 1.038990590500678, + "last_quartile_mean": 3.768178999052907e-05, + "mean": 7.755673851761458e-05, + "normalized": [ + 1.3520284678359944, + 1.353248151989171, + 1.4986392688316106, + 1.4361938981325721, + 1.3909773048874547, + 1.3505900079390127, + 1.3181684183793927, + 1.2859080961883835, + 1.255309887999515, + 1.236511310150616, + 1.213145460298544, + 1.1900857276214447, + 1.1789445822496427, + 1.1381581040019337, + 1.1104981703825207, + 1.0743057951254327, + 1.0428227272386361, + 1.0095324140994133, + 1.060018575753417, + 1.009781116713833, + 1.0043856490561534, + 0.947336496531926, + 0.8658410309303259, + 0.7906818062106236, + 0.7124834092746121, + 0.6497447740064765, + 0.585591601574923, + 0.5217317875271078, + 0.4503049314092096, + 0.380548022816011, + 0.3126951761851902, + 0.2737878286589002 + ], + "population_cv": 0.34367475536074504 + }, + "activation_output_rms_by_block": [ + 0.09794864058494568, + 0.13700027763843536, + 0.20758503675460815, + 0.22207145392894745, + 0.22574232518672943, + 0.2265917956829071, + 0.22746813297271729, + 0.22716933488845825, + 0.22722382843494415, + 0.22613705694675446, + 0.22211141884326935, + 0.21741008758544922, + 0.21311214566230774, + 0.20640577375888824, + 0.19932369887828827, + 0.19335229694843292, + 0.19345322251319885, + 0.19645750522613525, + 0.20112650096416473, + 0.2038118690252304, + 0.20720544457435608, + 0.21031972765922546, + 0.2129739224910736, + 0.21647654473781586, + 0.22099705040454865, + 0.23361735045909882, + 0.2512694001197815, + 0.2773953676223755, + 0.3133004307746887, + 0.35808902978897095, + 0.4303363263607025, + 0.54518723487854 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1964471247047186, + "first_to_last_ratio": 0.5975141297159507, + "imbalance_abs_log_ratio": 0.5149773473959137, + "last_quartile_mean": 0.32877402380108833, + "mean": 0.2358959447592497, + "normalized": [ + 0.4152196880065486, + 0.5807657176059338, + 0.879985609614718, + 0.9413958097312304, + 0.9569572101678865, + 0.9605582491643161, + 0.964273180723248, + 0.9630065286637394, + 0.9632375353753703, + 0.958630540162719, + 0.9415652272867661, + 0.9216355448896473, + 0.9034158933075572, + 0.8749865283591098, + 0.8449644994182233, + 0.8196507877478102, + 0.8200786270854848, + 0.832814253872128, + 0.8526068609167068, + 0.8639905583507864, + 0.8783764586789548, + 0.8915783943376951, + 0.9028299435517223, + 0.9176781099766134, + 0.9368412442617167, + 0.9903406804959011, + 1.065170494457722, + 1.1759225785143497, + 1.32812978660755, + 1.5179956999872501, + 1.824263349672647, + 2.311134409007948 + ], + "population_cv": 0.335075401661643 + }, + "bits_per_byte": 2.08044695422856, + "branch_output_rms_by_sublayer": [ + 0.018880587071180344, + 0.05959044396877289, + 0.060257185250520706, + 0.016382697969675064, + 0.09490419179201126, + 0.0035255588591098785, + 0.0323294997215271, + 0.004731281660497189, + 0.02052305079996586, + 0.00493080448359251, + 0.01820369064807892, + 0.005357450805604458, + 0.019246362149715424, + 0.006479416508227587, + 0.024384424090385437, + 0.009427462704479694, + 0.017510300502181053, + 0.009214801713824272, + 0.01695968210697174, + 0.0109269879758358, + 0.019068937748670578, + 0.012604814022779465, + 0.019973911345005035, + 0.013778526335954666, + 0.022227400913834572, + 0.013526024296879768, + 0.027813225984573364, + 0.017681727185845375, + 0.027307361364364624, + 0.018625596538186073, + 0.03342962637543678, + 0.019818320870399475, + 0.030618051066994667, + 0.021072659641504288, + 0.025097575038671494, + 0.022579019889235497, + 0.025198329240083694, + 0.01933942921459675, + 0.03349718078970909, + 0.022255338728427887, + 0.03462665155529976, + 0.024717653170228004, + 0.03921883553266525, + 0.025107676163315773, + 0.043981269001960754, + 0.0301958154886961, + 0.038896940648555756, + 0.03620966896414757, + 0.04658674821257591, + 0.046204980462789536, + 0.033179834485054016, + 0.04984369874000549, + 0.02075173333287239, + 0.05490582063794136, + 0.011197025887668133, + 0.06379986554384232, + 0.012507086619734764, + 0.07197272032499313, + 0.011961712501943111, + 0.07922235131263733, + 0.010508585721254349, + 0.11548902839422226, + 0.021868467330932617, + 0.1684120148420334 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040070945427075937, + 0.00021125547826331077, + 0.00020379424936248455, + 0.00011927406183950472, + 9.446000613801463e-05, + 0.00010295610627477164, + 9.818733926828671e-05, + 0.00012673421109679544, + 0.00011573807851122185, + 0.00011146965874368272, + 0.0001232427596852727, + 0.00012517422740451753, + 0.000137325623762558, + 0.0001763842480315643, + 0.00017420922346804142, + 0.00019637547238758496, + 0.00020814888113781, + 0.0002138469295217413, + 0.0002110578549773679, + 0.0002602958562879386, + 0.0002719585176535882, + 0.00027346390406242154, + 0.00028937535779988523, + 0.00028561131716422746, + 0.00028061956919165, + 0.00023874030969169658, + 0.00021716224392712217, + 0.0001981302869037355, + 0.00018664421118786767, + 0.0001673158872138231, + 0.0001540361578610053, + 0.00014575183058398037 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016967136331424098, + "first_to_last_ratio": 0.8545520537501935, + "imbalance_abs_log_ratio": 0.1571778610785998, + "last_quartile_mean": 0.0001985500620701101, + "mean": 0.00019123279105231976, + "normalized": [ + 2.0954013799920355, + 1.104703210682696, + 1.0656867383519393, + 0.6237113477409442, + 0.4939529754192163, + 0.5383810261212141, + 0.5134440528111281, + 0.6627221743687356, + 0.605220882225982, + 0.582900339059453, + 0.6444645764311125, + 0.6545646628682571, + 0.71810709348998, + 0.9223535726323577, + 0.9109798717542076, + 1.0268922568507521, + 1.0884581038241612, + 1.1182545019867147, + 1.1036697933233854, + 1.3611465630741315, + 1.4221332866452938, + 1.4300052964640568, + 1.5132099270606496, + 1.493526897727945, + 1.4674239059496585, + 1.2484276800958218, + 1.1355910392360917, + 1.0360685832877303, + 0.976005266465014, + 0.8749330399515364, + 0.8054902980465429, + 0.7621696560612549 + ], + "population_cv": 0.3661316865844204 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0565960556268692, + 0.06250853836536407, + 0.09794864058494568, + 0.12857383489608765, + 0.13700027763843536, + 0.2065405696630478, + 0.20758503675460815, + 0.22102168202400208, + 0.22207145392894745, + 0.2248813658952713, + 0.22574232518672943, + 0.2264578640460968, + 0.2265917956829071, + 0.22759772837162018, + 0.22746813297271729, + 0.22716368734836578, + 0.22716933488845825, + 0.22757723927497864, + 0.22722382843494415, + 0.22673146426677704, + 0.22613705694675446, + 0.2244347184896469, + 0.22211141884326935, + 0.2183273881673813, + 0.21741008758544922, + 0.21451692283153534, + 0.21311214566230774, + 0.20762479305267334, + 0.20640577375888824, + 0.20071867108345032, + 0.19932369887828827, + 0.19245882332324982, + 0.19335229694843292, + 0.19078245759010315, + 0.19345322251319885, + 0.1916658580303192, + 0.19645750522613525, + 0.19844752550125122, + 0.20112650096416473, + 0.2010338455438614, + 0.2038118690252304, + 0.20468150079250336, + 0.20720544457435608, + 0.20727655291557312, + 0.21031972765922546, + 0.20860347151756287, + 0.2129739224910736, + 0.21107539534568787, + 0.21647654473781586, + 0.2111099362373352, + 0.22099705040454865, + 0.2196389138698578, + 0.23361735045909882, + 0.23286758363246918, + 0.2512694001197815, + 0.25084227323532104, + 0.2773953676223755, + 0.27732276916503906, + 0.3133004307746887, + 0.31342777609825134, + 0.35808902978897095, + 0.35860511660575867, + 0.4303363263607025, + 0.4337502121925354 + ], + "loss_nats": 1.4420559406280518, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06250853836536407, + 0.09794864058494568, + 0.12857383489608765, + 0.13700027763843536, + 0.2065405696630478, + 0.20758503675460815, + 0.22102168202400208, + 0.22207145392894745, + 0.2248813658952713, + 0.22574232518672943, + 0.2264578640460968, + 0.2265917956829071, + 0.22759772837162018, + 0.22746813297271729, + 0.22716368734836578, + 0.22716933488845825, + 0.22757723927497864, + 0.22722382843494415, + 0.22673146426677704, + 0.22613705694675446, + 0.2244347184896469, + 0.22211141884326935, + 0.2183273881673813, + 0.21741008758544922, + 0.21451692283153534, + 0.21311214566230774, + 0.20762479305267334, + 0.20640577375888824, + 0.20071867108345032, + 0.19932369887828827, + 0.19245882332324982, + 0.19335229694843292, + 0.19078245759010315, + 0.19345322251319885, + 0.1916658580303192, + 0.19645750522613525, + 0.19844752550125122, + 0.20112650096416473, + 0.2010338455438614, + 0.2038118690252304, + 0.20468150079250336, + 0.20720544457435608, + 0.20727655291557312, + 0.21031972765922546, + 0.20860347151756287, + 0.2129739224910736, + 0.21107539534568787, + 0.21647654473781586, + 0.2111099362373352, + 0.22099705040454865, + 0.2196389138698578, + 0.23361735045909882, + 0.23286758363246918, + 0.2512694001197815, + 0.25084227323532104, + 0.2773953676223755, + 0.27732276916503906, + 0.3133004307746887, + 0.31342777609825134, + 0.35808902978897095, + 0.35860511660575867, + 0.4303363263607025, + 0.4337502121925354, + 0.54518723487854 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010598741209832951, + 0.00010775360715342686, + 0.00011871542665176094, + 0.00011485091090435162, + 0.00011244894267292693, + 0.00011016445205314085, + 0.00010846564691746607, + 0.00010656306403689086, + 0.0001046601973939687, + 0.00010342278255848214, + 0.00010179441596847028, + 0.00010021102934842929, + 9.965883509721607e-05, + 9.537018195260316e-05, + 9.32025650399737e-05, + 9.038933058036491e-05, + 8.758632611716166e-05, + 8.443772094324231e-05, + 8.744140359340236e-05, + 8.20954010123387e-05, + 8.16298124846071e-05, + 7.812595140421763e-05, + 7.176945655373856e-05, + 6.528423546114936e-05, + 5.917211092310026e-05, + 5.3287756600184366e-05, + 4.791216633748263e-05, + 4.251671634847298e-05, + 3.64475381502416e-05, + 3.0126682759146206e-05, + 2.3703445549472235e-05, + 1.9818620785372332e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001106186828110367, + "first_to_last_ratio": 2.827449739094469, + "imbalance_abs_log_ratio": 1.0393751531099071, + "last_quartile_mean": 3.912312968168408e-05, + "mean": 8.203169204534788e-05, + "normalized": [ + 1.292030061256553, + 1.3135607040003474, + 1.4471897835062812, + 1.4000797501636437, + 1.3707987714158585, + 1.342949892978447, + 1.3222407610159408, + 1.2990474946924393, + 1.275850769189444, + 1.2607661743867122, + 1.2409157172108236, + 1.2216135843331344, + 1.2148821097353903, + 1.1626016674126585, + 1.1361775274420824, + 1.1018830445481589, + 1.067713269509827, + 1.0293304799389529, + 1.06594660445458, + 1.000776638460116, + 0.9951009231832177, + 0.9523874182801066, + 0.8748991367149143, + 0.7958416294163435, + 0.7213323222735595, + 0.6495996275528049, + 0.5840689755734447, + 0.5182962253779837, + 0.444310451747028, + 0.36725662982170226, + 0.28895473150022005, + 0.24159712290728336 + ], + "population_cv": 0.3450366052372569 + }, + "activation_output_rms_by_block": [ + 0.11079900711774826, + 0.18173734843730927, + 0.25566670298576355, + 0.2647266387939453, + 0.2669948637485504, + 0.2675777077674866, + 0.2675747275352478, + 0.26725223660469055, + 0.2656837999820709, + 0.265005499124527, + 0.25976043939590454, + 0.2535160183906555, + 0.24569463729858398, + 0.23344992101192474, + 0.22194796800613403, + 0.21299737691879272, + 0.21236176788806915, + 0.21684354543685913, + 0.22108736634254456, + 0.22472712397575378, + 0.22933396697044373, + 0.23535853624343872, + 0.2425893247127533, + 0.2497948557138443, + 0.26040923595428467, + 0.27719539403915405, + 0.2986113131046295, + 0.3278995454311371, + 0.36809566617012024, + 0.4197922646999359, + 0.5092319846153259, + 0.6714946031570435 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23529115412384272, + "first_to_last_ratio": 0.6008590681870453, + "imbalance_abs_log_ratio": 0.5093948674743478, + "last_quartile_mean": 0.39159125089645386, + "mean": 0.2751628558617085, + "normalized": [ + 0.40266701975732394, + 0.6604719516672225, + 0.9291468580855864, + 0.962072580490267, + 0.9703157895800326, + 0.9724339679842758, + 0.972423137190165, + 0.9712511369594387, + 0.9655511066348229, + 0.9630860178952118, + 0.9440243617999629, + 0.9213308155155495, + 0.8929062628353637, + 0.8484063747661208, + 0.806605845513105, + 0.7740775049443485, + 0.7717675673303741, + 0.7880552945919432, + 0.803478237097738, + 0.8167058859452209, + 0.8334481274816485, + 0.8553426860845104, + 0.881620900295765, + 0.9078073235269309, + 0.9463822256778773, + 1.007386673506787, + 1.0852166516788362, + 1.191656280802424, + 1.3377374828349577, + 1.5256138528774221, + 1.850656706627787, + 2.4403533720209807 + ], + "population_cv": 0.3576152457433685 + }, + "bits_per_byte": 1.9501283044158226, + "branch_output_rms_by_sublayer": [ + 0.02630079723894596, + 0.06842244416475296, + 0.10497251898050308, + 0.019157659262418747, + 0.09477359801530838, + 0.0031024799682199955, + 0.0338861346244812, + 0.0047894855961203575, + 0.022547677159309387, + 0.004550074227154255, + 0.021251287311315536, + 0.004262411035597324, + 0.01840241253376007, + 0.006118505261838436, + 0.02267448604106903, + 0.009554358199238777, + 0.022321712225675583, + 0.00963375624269247, + 0.01962294802069664, + 0.011137282475829124, + 0.019759519025683403, + 0.013955196365714073, + 0.021449215710163116, + 0.01607326976954937, + 0.026240134611725807, + 0.015075869858264923, + 0.039021123200654984, + 0.02189606800675392, + 0.031211141496896744, + 0.022710295394062996, + 0.03785466402769089, + 0.026124922558665276, + 0.0325845405459404, + 0.026591502130031586, + 0.03150996193289757, + 0.029410794377326965, + 0.02998937852680683, + 0.024189060553908348, + 0.04107306897640228, + 0.029402343556284904, + 0.040071673691272736, + 0.032556869089603424, + 0.04498095437884331, + 0.03419904410839081, + 0.05116673558950424, + 0.041954249143600464, + 0.04688612371683121, + 0.050006624311208725, + 0.052477724850177765, + 0.06379573792219162, + 0.04486159235239029, + 0.06914625316858292, + 0.032891806215047836, + 0.0747767761349678, + 0.015299716033041477, + 0.08387137204408646, + 0.017307085916399956, + 0.09213023632764816, + 0.017144370824098587, + 0.10062393546104431, + 0.017122838646173477, + 0.15051087737083435, + 0.027784937992691994, + 0.24076412618160248 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00032837002715831827, + 0.0002079177996867109, + 0.00018809406473136247, + 0.00010899326506499365, + 9.474430680287842e-05, + 8.318366756728252e-05, + 9.647881676005136e-05, + 0.00013404889675397192, + 0.00012865721885494213, + 0.0001321695767247016, + 0.00014056789256563647, + 0.00015176892928157556, + 0.00016698296163968515, + 0.00022933649843535653, + 0.000217708018553423, + 0.0002507549770035681, + 0.00025689149388127776, + 0.0002828493196393162, + 0.00027498157120012506, + 0.0003306027661240732, + 0.00032889440594708014, + 0.00033897343531885953, + 0.000372930739966371, + 0.0003793341532875068, + 0.0003649148263870391, + 0.0003190518951646816, + 0.00028800578855572547, + 0.0002568521570555004, + 0.00024004963937954607, + 0.00021095610212654392, + 0.00019368945551758278, + 0.00017448279084866967 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015522885556569618, + "first_to_last_ratio": 0.6063619309634984, + "imbalance_abs_log_ratio": 0.5002782253787689, + "last_quartile_mean": 0.00025600033187941115, + "mean": 0.00022728867056201116, + "normalized": [ + 1.4447267712304608, + 0.9147741467825811, + 0.8275558313851142, + 0.4795367265578555, + 0.41684570800914306, + 0.36598246345317736, + 0.42447701648080627, + 0.5897737728084399, + 0.56605205414249, + 0.581505344713791, + 0.6184553423540983, + 0.6677364468114501, + 0.7346734935216545, + 1.0090098106002459, + 0.9578480881387607, + 1.1032445056919573, + 1.1302432859766764, + 1.244449707677561, + 1.2098340428503753, + 1.4545501335662696, + 1.4470338760565185, + 1.491378494496396, + 1.6407801543483633, + 1.668953196609125, + 1.605512608634311, + 1.4037298664107178, + 1.2671365794149818, + 1.1300702160841907, + 1.05614432424626, + 0.928141739774877, + 0.8521738238806694, + 0.7676704272906798 + ], + "population_cv": 0.3864254083322711 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06033600866794586, + 0.06945198029279709, + 0.11079900711774826, + 0.17341101169586182, + 0.18173734843730927, + 0.25461602210998535, + 0.25566670298576355, + 0.26339152455329895, + 0.2647266387939453, + 0.2662918269634247, + 0.2669948637485504, + 0.2675604224205017, + 0.2675777077674866, + 0.2677975296974182, + 0.2675747275352478, + 0.2675320506095886, + 0.26725223660469055, + 0.26636025309562683, + 0.2656837999820709, + 0.2656107544898987, + 0.265005499124527, + 0.26352381706237793, + 0.25976043939590454, + 0.2553655505180359, + 0.2535160183906555, + 0.24811074137687683, + 0.24569463729858398, + 0.23600345849990845, + 0.23344992101192474, + 0.22498825192451477, + 0.22194796800613403, + 0.21307076513767242, + 0.21299737691879272, + 0.20943206548690796, + 0.21236176788806915, + 0.21104833483695984, + 0.21684354543685913, + 0.21818861365318298, + 0.22108736634254456, + 0.22058506309986115, + 0.22472712397575378, + 0.22517690062522888, + 0.22933396697044373, + 0.2308174967765808, + 0.23535853624343872, + 0.23589646816253662, + 0.2425893247127533, + 0.241608664393425, + 0.2497948557138443, + 0.24810902774333954, + 0.26040923595428467, + 0.26088401675224304, + 0.27719539403915405, + 0.2782979905605316, + 0.2986113131046295, + 0.2985531687736511, + 0.3278995454311371, + 0.3279455602169037, + 0.36809566617012024, + 0.3681766986846924, + 0.4197922646999359, + 0.4200114607810974, + 0.5092319846153259, + 0.5118712186813354 + ], + "loss_nats": 1.3517259359359741, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06945198029279709, + 0.11079900711774826, + 0.17341101169586182, + 0.18173734843730927, + 0.25461602210998535, + 0.25566670298576355, + 0.26339152455329895, + 0.2647266387939453, + 0.2662918269634247, + 0.2669948637485504, + 0.2675604224205017, + 0.2675777077674866, + 0.2677975296974182, + 0.2675747275352478, + 0.2675320506095886, + 0.26725223660469055, + 0.26636025309562683, + 0.2656837999820709, + 0.2656107544898987, + 0.265005499124527, + 0.26352381706237793, + 0.25976043939590454, + 0.2553655505180359, + 0.2535160183906555, + 0.24811074137687683, + 0.24569463729858398, + 0.23600345849990845, + 0.23344992101192474, + 0.22498825192451477, + 0.22194796800613403, + 0.21307076513767242, + 0.21299737691879272, + 0.20943206548690796, + 0.21236176788806915, + 0.21104833483695984, + 0.21684354543685913, + 0.21818861365318298, + 0.22108736634254456, + 0.22058506309986115, + 0.22472712397575378, + 0.22517690062522888, + 0.22933396697044373, + 0.2308174967765808, + 0.23535853624343872, + 0.23589646816253662, + 0.2425893247127533, + 0.241608664393425, + 0.2497948557138443, + 0.24810902774333954, + 0.26040923595428467, + 0.26088401675224304, + 0.27719539403915405, + 0.2782979905605316, + 0.2986113131046295, + 0.2985531687736511, + 0.3278995454311371, + 0.3279455602169037, + 0.36809566617012024, + 0.3681766986846924, + 0.4197922646999359, + 0.4200114607810974, + 0.5092319846153259, + 0.5118712186813354, + 0.6714946031570435 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018891454965341836, + 0.0001931450387928635, + 0.0002085021696984768, + 0.00020282011246308684, + 0.00019955210154876113, + 0.00019673256610985845, + 0.00019503913063090295, + 0.0001924382959259674, + 0.00019001582404598594, + 0.00018850783817470074, + 0.00018595026631373912, + 0.0001838041644077748, + 0.00018427071336191148, + 0.00017771514831110835, + 0.0001728432544041425, + 0.0001671368081588298, + 0.0001615467481315136, + 0.00015400326810777187, + 0.000160209892783314, + 0.00015067463391460478, + 0.00014840754738543183, + 0.00014097621897235513, + 0.00012854386295657605, + 0.00011642893514363095, + 0.00010425484651932493, + 9.334496280644089e-05, + 8.331386925419793e-05, + 7.313644891837612e-05, + 6.15216777077876e-05, + 4.904071465716697e-05, + 3.525121428538114e-05, + 2.4962779207271524e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019714299560291693, + "first_to_last_ratio": 3.0050767723956184, + "imbalance_abs_log_ratio": 1.1003031158789967, + "last_quartile_mean": 6.560331416949339e-05, + "mean": 0.00014728142508602104, + "normalized": [ + 1.282677360998382, + 1.311401208129643, + 1.4156718647765612, + 1.3770922731405397, + 1.3549033860326305, + 1.3357595229334251, + 1.324261566025645, + 1.3066026202121013, + 1.2901547084773621, + 1.2799159029361715, + 1.262550699826086, + 1.247979263511487, + 1.2511470014245618, + 1.206636534154339, + 1.1735577266664268, + 1.1348125404219307, + 1.0968575842959203, + 1.045639448544342, + 1.0877807075110928, + 1.023038946198422, + 1.0076460578701834, + 0.9571894004285788, + 0.8727771535446431, + 0.7905201560592559, + 0.707861473084158, + 0.6337863905914947, + 0.5656780493910737, + 0.4965761899415362, + 0.41771511697320496, + 0.33297284181304126, + 0.23934596141225786, + 0.1694903426735027 + ], + "population_cv": 0.361085575415688 + }, + "activation_output_rms_by_block": [ + 0.1016591340303421, + 0.18086770176887512, + 0.2325568050146103, + 0.2376081645488739, + 0.23870401084423065, + 0.23941391706466675, + 0.23885947465896606, + 0.2381371706724167, + 0.23675362765789032, + 0.23536528646945953, + 0.22870509326457977, + 0.22131286561489105, + 0.21254637837409973, + 0.19929367303848267, + 0.18746964633464813, + 0.17704539000988007, + 0.17596758902072906, + 0.1784123033285141, + 0.18117663264274597, + 0.18366514146327972, + 0.1872515082359314, + 0.19141052663326263, + 0.1969655156135559, + 0.2033752053976059, + 0.21314463019371033, + 0.22718842327594757, + 0.2457481175661087, + 0.269707053899765, + 0.30305343866348267, + 0.347743958234787, + 0.4299577474594116, + 0.6035616993904114 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2134757973253727, + "first_to_last_ratio": 0.6468706108937196, + "imbalance_abs_log_ratio": 0.4356089876432565, + "last_quartile_mean": 0.33001313358545303, + "mean": 0.23576961969956756, + "normalized": [ + 0.4311799550759871, + 0.7671374369579427, + 0.986373076017804, + 1.0077980566438083, + 1.0124460104249322, + 1.015457026947505, + 1.0131053990897376, + 1.010041798327817, + 1.0041735994636487, + 0.9982850494876174, + 0.9700363157730422, + 0.9386827102529232, + 0.9015002808459363, + 0.8452898778580343, + 0.7951391132306771, + 0.7509253746750002, + 0.7463539587711004, + 0.7567230398719659, + 0.7684477451913126, + 0.7790025775895867, + 0.7942138960674365, + 0.8118540755045961, + 0.8354151644496934, + 0.8626014057992686, + 0.9040377232033228, + 0.9636034683579899, + 1.0423230858973958, + 1.143943202875089, + 1.2853795117863462, + 1.4749311581276, + 1.8236350722679655, + 2.5599638331669174 + ], + "population_cv": 0.3675345504070441 + }, + "bits_per_byte": 1.8720084529317582, + "branch_output_rms_by_sublayer": [ + 0.03573180362582207, + 0.052011821419000626, + 0.10664675384759903, + 0.01823282428085804, + 0.06781794875860214, + 0.0026424091774970293, + 0.02450474537909031, + 0.003992235753685236, + 0.01725216954946518, + 0.0036646369844675064, + 0.018646538257598877, + 0.003319106763228774, + 0.015910238027572632, + 0.004927239380776882, + 0.018212314695119858, + 0.008043181151151657, + 0.016784843057394028, + 0.007957165129482746, + 0.01594853773713112, + 0.009037249721586704, + 0.0176989808678627, + 0.012198803015053272, + 0.019605962559580803, + 0.013432997278869152, + 0.022475004196166992, + 0.012972000986337662, + 0.03453320264816284, + 0.018862294033169746, + 0.026713354513049126, + 0.01990535296499729, + 0.0319337360560894, + 0.023401649668812752, + 0.02801610343158245, + 0.023707188665866852, + 0.028282491490244865, + 0.026515409350395203, + 0.02569115161895752, + 0.022723721340298653, + 0.03440641239285469, + 0.027487577870488167, + 0.034427426755428314, + 0.03072489984333515, + 0.039456624537706375, + 0.03320617973804474, + 0.043320003896951675, + 0.04054662957787514, + 0.04175209999084473, + 0.04858461394906044, + 0.043093886226415634, + 0.060716040432453156, + 0.03683279827237129, + 0.0668661892414093, + 0.03015786223113537, + 0.0722465068101883, + 0.015251493081450462, + 0.07915013283491135, + 0.017382971942424774, + 0.08651599287986755, + 0.015752796083688736, + 0.09494592994451523, + 0.018999194726347923, + 0.14218401908874512, + 0.03338775411248207, + 0.24767111241817474 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004570302696620844, + 0.00031493934518203267, + 0.0002143615651481701, + 0.00013786409436507824, + 0.0001291171736265367, + 0.00013682107280354788, + 0.00015837516984797755, + 0.00019563631023119442, + 0.00019674727625245423, + 0.000191343414336242, + 0.00021955065404037036, + 0.0002332172506517699, + 0.00027919484030541734, + 0.0003858941801364767, + 0.0003704235701666674, + 0.00042305543536477747, + 0.0004472346343569547, + 0.0004940425030518999, + 0.0004673224693273912, + 0.0005594278228367747, + 0.0005764247107952261, + 0.0005972083894683241, + 0.0006512089060709392, + 0.0006542639335938355, + 0.0006197665059356166, + 0.0005424206044372668, + 0.0004994517842756415, + 0.0004394000632885711, + 0.0004055610719669202, + 0.0003512581615295656, + 0.00030757820471853463, + 0.00025263519960351045 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021801812510832775, + "first_to_last_ratio": 0.5102716406035511, + "imbalance_abs_log_ratio": 0.6728120664256351, + "last_quartile_mean": 0.0004272589494694534, + "mean": 0.0003721492683555553, + "normalized": [ + 1.2280832142478726, + 0.8462715688618158, + 0.5760096374645209, + 0.3704537562958782, + 0.34694995961452973, + 0.36765105950128485, + 0.42556894051626687, + 0.5256931206546978, + 0.5286783905872947, + 0.5141577066152674, + 0.5899532061705127, + 0.6266766334979106, + 0.7502227306239702, + 1.0369338675356186, + 0.9953628870573538, + 1.1367896468913272, + 1.2017614231332092, + 1.3275385579419885, + 1.2557393203871798, + 1.5032350468100115, + 1.5489072794427852, + 1.6047549740115168, + 1.7498594285795221, + 1.7580685741634856, + 1.6653707494152192, + 1.4575350553128925, + 1.3420738041017997, + 1.1807091956143891, + 1.0897806510785424, + 0.9438636359052839, + 0.8264914937967075, + 0.6788544841693472 + ], + "population_cv": 0.43653500885793567 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06193641945719719, + 0.07578708976507187, + 0.1016591340303421, + 0.17375877499580383, + 0.18086770176887512, + 0.23162810504436493, + 0.2325568050146103, + 0.2364615947008133, + 0.2376081645488739, + 0.23820650577545166, + 0.23870401084423065, + 0.23957808315753937, + 0.23941391706466675, + 0.2393639236688614, + 0.23885947465896606, + 0.23879873752593994, + 0.2381371706724167, + 0.23766019940376282, + 0.23675362765789032, + 0.23612910509109497, + 0.23536528646945953, + 0.2328052520751953, + 0.22870509326457977, + 0.22353777289390564, + 0.22131286561489105, + 0.21547429263591766, + 0.21254637837409973, + 0.20247745513916016, + 0.19929367303848267, + 0.19119344651699066, + 0.18746964633464813, + 0.1784159243106842, + 0.17704539000988007, + 0.17436346411705017, + 0.17596758902072906, + 0.17439287900924683, + 0.1784123033285141, + 0.1789977103471756, + 0.18117663264274597, + 0.18044494092464447, + 0.18366514146327972, + 0.1840001940727234, + 0.1872515082359314, + 0.18770679831504822, + 0.19141052663326263, + 0.19118990004062653, + 0.1969655156135559, + 0.19665203988552094, + 0.2033752053976059, + 0.20245759189128876, + 0.21314463019371033, + 0.21310319006443024, + 0.22718842327594757, + 0.2287222445011139, + 0.2457481175661087, + 0.2456687092781067, + 0.269707053899765, + 0.26972076296806335, + 0.30305343866348267, + 0.30293965339660645, + 0.347743958234787, + 0.3480367064476013, + 0.4299577474594116, + 0.43200185894966125 + ], + "loss_nats": 1.2975773811340332, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07578708976507187, + 0.1016591340303421, + 0.17375877499580383, + 0.18086770176887512, + 0.23162810504436493, + 0.2325568050146103, + 0.2364615947008133, + 0.2376081645488739, + 0.23820650577545166, + 0.23870401084423065, + 0.23957808315753937, + 0.23941391706466675, + 0.2393639236688614, + 0.23885947465896606, + 0.23879873752593994, + 0.2381371706724167, + 0.23766019940376282, + 0.23675362765789032, + 0.23612910509109497, + 0.23536528646945953, + 0.2328052520751953, + 0.22870509326457977, + 0.22353777289390564, + 0.22131286561489105, + 0.21547429263591766, + 0.21254637837409973, + 0.20247745513916016, + 0.19929367303848267, + 0.19119344651699066, + 0.18746964633464813, + 0.1784159243106842, + 0.17704539000988007, + 0.17436346411705017, + 0.17596758902072906, + 0.17439287900924683, + 0.1784123033285141, + 0.1789977103471756, + 0.18117663264274597, + 0.18044494092464447, + 0.18366514146327972, + 0.1840001940727234, + 0.1872515082359314, + 0.18770679831504822, + 0.19141052663326263, + 0.19118990004062653, + 0.1969655156135559, + 0.19665203988552094, + 0.2033752053976059, + 0.20245759189128876, + 0.21314463019371033, + 0.21310319006443024, + 0.22718842327594757, + 0.2287222445011139, + 0.2457481175661087, + 0.2456687092781067, + 0.269707053899765, + 0.26972076296806335, + 0.30305343866348267, + 0.30293965339660645, + 0.347743958234787, + 0.3480367064476013, + 0.4299577474594116, + 0.43200185894966125, + 0.6035616993904114 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.95552135286859, + "cross_entropy_nats": 5.514347195625305, + "step": 0 + }, + { + "bits_per_byte": 5.166060485015187, + "cross_entropy_nats": 3.5808402597904205, + "step": 100 + }, + { + "bits_per_byte": 3.1277970484292927, + "cross_entropy_nats": 2.168023705482483, + "step": 500 + }, + { + "bits_per_byte": 1.9613801618593651, + "cross_entropy_nats": 1.359525129199028, + "step": 2000 + }, + { + "bits_per_byte": 1.7971937855298066, + "cross_entropy_nats": 1.2457198053598404, + "step": 4000 + }, + { + "bits_per_byte": 1.7178972072507075, + "cross_entropy_nats": 1.1907556056976318, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "final_optimizer_state": "06d1b38e173978179caca1f5bf426eaad5609e9e9a1b567ef88c8b18eeab7e4d", + "final_public_parameters": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 42.00224438344842, + "measured_steps": 7980, + "median_ms": 41.91726399585605, + "p95_ms": 42.462232409161516, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958522622108798, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516427516937256, + "step": 1, + "unclipped_grad_norm": 22.09486961364746 + }, + { + "bits_per_byte": 7.238571600789322, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.017395496368408, + "step": 10, + "unclipped_grad_norm": 11.172341346740723 + }, + { + "bits_per_byte": 6.729949421197048, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6648454666137695, + "step": 20, + "unclipped_grad_norm": 3.462282180786133 + }, + { + "bits_per_byte": 6.60904492970989, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.581040859222412, + "step": 30, + "unclipped_grad_norm": 3.06864070892334 + }, + { + "bits_per_byte": 6.370936325451538, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.415996551513672, + "step": 40, + "unclipped_grad_norm": 3.234333038330078 + }, + { + "bits_per_byte": 6.086515545038246, + "learning_rate": 3.75e-05, + "loss_nats": 4.218851089477539, + "step": 50, + "unclipped_grad_norm": 2.8305625915527344 + }, + { + "bits_per_byte": 5.8833778930063385, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.078046798706055, + "step": 60, + "unclipped_grad_norm": 2.670325517654419 + }, + { + "bits_per_byte": 5.715139992383416, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.961433172225952, + "step": 70, + "unclipped_grad_norm": 3.2375030517578125 + }, + { + "bits_per_byte": 5.566718964688812, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.858555555343628, + "step": 80, + "unclipped_grad_norm": 2.6123173236846924 + }, + { + "bits_per_byte": 5.355997221719156, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.712494373321533, + "step": 90, + "unclipped_grad_norm": 2.440842628479004 + }, + { + "bits_per_byte": 5.182009597112589, + "learning_rate": 7.5e-05, + "loss_nats": 3.591895341873169, + "step": 100, + "unclipped_grad_norm": 2.292849063873291 + }, + { + "bits_per_byte": 4.992510513259403, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4605445861816406, + "step": 110, + "unclipped_grad_norm": 2.057412624359131 + }, + { + "bits_per_byte": 4.778040580396842, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.311885356903076, + "step": 120, + "unclipped_grad_norm": 2.002248525619507 + }, + { + "bits_per_byte": 4.588237775182233, + "learning_rate": 9.75e-05, + "loss_nats": 3.180324077606201, + "step": 130, + "unclipped_grad_norm": 1.7531825304031372 + }, + { + "bits_per_byte": 4.430652823919056, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.071094512939453, + "step": 140, + "unclipped_grad_norm": 1.4980645179748535 + }, + { + "bits_per_byte": 4.274817968111058, + "learning_rate": 0.0001125, + "loss_nats": 2.963078022003174, + "step": 150, + "unclipped_grad_norm": 1.3038520812988281 + }, + { + "bits_per_byte": 4.169977000179788, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8904078006744385, + "step": 160, + "unclipped_grad_norm": 1.0998529195785522 + }, + { + "bits_per_byte": 4.028519206053318, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7923567295074463, + "step": 170, + "unclipped_grad_norm": 0.9818326234817505 + }, + { + "bits_per_byte": 3.950109907767851, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7380075454711914, + "step": 180, + "unclipped_grad_norm": 0.8779997229576111 + }, + { + "bits_per_byte": 3.846655463967837, + "learning_rate": 0.0001425, + "loss_nats": 2.6662983894348145, + "step": 190, + "unclipped_grad_norm": 1.2310962677001953 + }, + { + "bits_per_byte": 3.7855644746589534, + "learning_rate": 0.00015, + "loss_nats": 2.623953342437744, + "step": 200, + "unclipped_grad_norm": 1.0928086042404175 + }, + { + "bits_per_byte": 3.7365611139833117, + "learning_rate": 0.0001575, + "loss_nats": 2.589986801147461, + "step": 210, + "unclipped_grad_norm": 0.7771895527839661 + }, + { + "bits_per_byte": 3.6369384436445213, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5209336280822754, + "step": 220, + "unclipped_grad_norm": 0.7917730212211609 + }, + { + "bits_per_byte": 3.630690657903977, + "learning_rate": 0.0001725, + "loss_nats": 2.5166029930114746, + "step": 230, + "unclipped_grad_norm": 1.0061120986938477 + }, + { + "bits_per_byte": 3.503533565162051, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.428464412689209, + "step": 240, + "unclipped_grad_norm": 0.7126089930534363 + }, + { + "bits_per_byte": 3.5514149110234636, + "learning_rate": 0.0001875, + "loss_nats": 2.461653232574463, + "step": 250, + "unclipped_grad_norm": 1.2347668409347534 + }, + { + "bits_per_byte": 3.5407041754930297, + "learning_rate": 0.000195, + "loss_nats": 2.4542291164398193, + "step": 260, + "unclipped_grad_norm": 0.9977514147758484 + }, + { + "bits_per_byte": 3.5337584841552845, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4494147300720215, + "step": 270, + "unclipped_grad_norm": 0.7908692955970764 + }, + { + "bits_per_byte": 3.478037825102243, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.410792112350464, + "step": 280, + "unclipped_grad_norm": 1.0802956819534302 + }, + { + "bits_per_byte": 3.3705159916093237, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.336263656616211, + "step": 290, + "unclipped_grad_norm": 0.9241761565208435 + }, + { + "bits_per_byte": 3.410749956917471, + "learning_rate": 0.000225, + "loss_nats": 2.3641517162323, + "step": 300, + "unclipped_grad_norm": 0.9868955612182617 + }, + { + "bits_per_byte": 3.3596274260178975, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.328716278076172, + "step": 310, + "unclipped_grad_norm": 0.8471657037734985 + }, + { + "bits_per_byte": 3.3434163413475693, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174796104431152, + "step": 320, + "unclipped_grad_norm": 0.9364105463027954 + }, + { + "bits_per_byte": 3.3338296844231747, + "learning_rate": 0.0002475, + "loss_nats": 2.3108346462249756, + "step": 330, + "unclipped_grad_norm": 0.9398566484451294 + }, + { + "bits_per_byte": 3.4261592584694767, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3748326301574707, + "step": 340, + "unclipped_grad_norm": 0.949837863445282 + }, + { + "bits_per_byte": 3.364650007353695, + "learning_rate": 0.0002625, + "loss_nats": 2.332197666168213, + "step": 350, + "unclipped_grad_norm": 0.8382212519645691 + }, + { + "bits_per_byte": 3.2974426270495267, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2856130599975586, + "step": 360, + "unclipped_grad_norm": 1.0729857683181763 + }, + { + "bits_per_byte": 3.304221839181237, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2903120517730713, + "step": 370, + "unclipped_grad_norm": 0.9106518030166626 + }, + { + "bits_per_byte": 3.3613300542614395, + "learning_rate": 0.000285, + "loss_nats": 2.3298964500427246, + "step": 380, + "unclipped_grad_norm": 1.290928602218628 + }, + { + "bits_per_byte": 3.2658243046190027, + "learning_rate": 0.0002925, + "loss_nats": 2.2636969089508057, + "step": 390, + "unclipped_grad_norm": 1.1257888078689575 + }, + { + "bits_per_byte": 3.2284232375358677, + "learning_rate": 0.0003, + "loss_nats": 2.2377724647521973, + "step": 400, + "unclipped_grad_norm": 1.3445736169815063 + }, + { + "bits_per_byte": 3.1961283353568497, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2153873443603516, + "step": 410, + "unclipped_grad_norm": 0.7953492403030396 + }, + { + "bits_per_byte": 3.2387222466001364, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2449111938476562, + "step": 420, + "unclipped_grad_norm": 0.8595701456069946 + }, + { + "bits_per_byte": 3.2223075344711862, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2335333824157715, + "step": 430, + "unclipped_grad_norm": 1.2960833311080933 + }, + { + "bits_per_byte": 3.2629112624786885, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2616777420043945, + "step": 440, + "unclipped_grad_norm": 0.9954231977462769 + }, + { + "bits_per_byte": 3.169579029577856, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1969847679138184, + "step": 450, + "unclipped_grad_norm": 1.0229040384292603 + }, + { + "bits_per_byte": 3.210129442963591, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2250921726226807, + "step": 460, + "unclipped_grad_norm": 1.1146044731140137 + }, + { + "bits_per_byte": 3.160413042217472, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.19063138961792, + "step": 470, + "unclipped_grad_norm": 0.9123480319976807 + }, + { + "bits_per_byte": 3.1596232978847096, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1900839805603027, + "step": 480, + "unclipped_grad_norm": 0.7869184613227844 + }, + { + "bits_per_byte": 3.0856473683731847, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.138807773590088, + "step": 490, + "unclipped_grad_norm": 0.8606863021850586 + }, + { + "bits_per_byte": 3.05385947104891, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.116774082183838, + "step": 500, + "unclipped_grad_norm": 0.9675024151802063 + }, + { + "bits_per_byte": 3.0816491157059263, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1360363960266113, + "step": 510, + "unclipped_grad_norm": 0.8225864768028259 + }, + { + "bits_per_byte": 3.0308351216820006, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1008148193359375, + "step": 520, + "unclipped_grad_norm": 1.098574161529541 + }, + { + "bits_per_byte": 3.0891052515514326, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.141204595565796, + "step": 530, + "unclipped_grad_norm": 0.8098863363265991 + }, + { + "bits_per_byte": 3.0354875963531414, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.1040396690368652, + "step": 540, + "unclipped_grad_norm": 1.0088073015213013 + }, + { + "bits_per_byte": 3.0042414443790846, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0823814868927, + "step": 550, + "unclipped_grad_norm": 1.054882287979126 + }, + { + "bits_per_byte": 2.9880702596837563, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0711724758148193, + "step": 560, + "unclipped_grad_norm": 0.723493754863739 + }, + { + "bits_per_byte": 2.93916561704971, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0372743606567383, + "step": 570, + "unclipped_grad_norm": 1.0234330892562866 + }, + { + "bits_per_byte": 3.0140372322069284, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0891714096069336, + "step": 580, + "unclipped_grad_norm": 1.0687707662582397 + }, + { + "bits_per_byte": 2.915940047981291, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0211756229400635, + "step": 590, + "unclipped_grad_norm": 1.149570107460022 + }, + { + "bits_per_byte": 2.9613039117993742, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.052619457244873, + "step": 600, + "unclipped_grad_norm": 0.7275651693344116 + }, + { + "bits_per_byte": 2.880448160305384, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9965745210647583, + "step": 610, + "unclipped_grad_norm": 0.9942755699157715 + }, + { + "bits_per_byte": 2.9149380770573643, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0204811096191406, + "step": 620, + "unclipped_grad_norm": 0.8311108350753784 + }, + { + "bits_per_byte": 2.871522948656208, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.990388035774231, + "step": 630, + "unclipped_grad_norm": 1.2078016996383667 + }, + { + "bits_per_byte": 2.795491826466872, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9376872777938843, + "step": 640, + "unclipped_grad_norm": 0.7877771258354187 + }, + { + "bits_per_byte": 2.7606237198656447, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.9135185480117798, + "step": 650, + "unclipped_grad_norm": 1.1925899982452393 + }, + { + "bits_per_byte": 2.6852617821708162, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8612816333770752, + "step": 660, + "unclipped_grad_norm": 1.126668930053711 + }, + { + "bits_per_byte": 2.7363994475436653, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8967275619506836, + "step": 670, + "unclipped_grad_norm": 1.136749267578125 + }, + { + "bits_per_byte": 2.6585009377312616, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8427324295043945, + "step": 680, + "unclipped_grad_norm": 1.2957242727279663 + }, + { + "bits_per_byte": 2.734127384743117, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8951526880264282, + "step": 690, + "unclipped_grad_norm": 0.7534775137901306 + }, + { + "bits_per_byte": 2.6542058430086137, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8397552967071533, + "step": 700, + "unclipped_grad_norm": 0.8543418049812317 + }, + { + "bits_per_byte": 2.6774995172067744, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8559012413024902, + "step": 710, + "unclipped_grad_norm": 1.3297739028930664 + }, + { + "bits_per_byte": 2.6677150802338874, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8491191864013672, + "step": 720, + "unclipped_grad_norm": 0.7874022126197815 + }, + { + "bits_per_byte": 2.6473420154126788, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8349976539611816, + "step": 730, + "unclipped_grad_norm": 1.0262641906738281 + }, + { + "bits_per_byte": 2.582519862493327, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7900663614273071, + "step": 740, + "unclipped_grad_norm": 1.1010198593139648 + }, + { + "bits_per_byte": 2.5681288702170795, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7800912857055664, + "step": 750, + "unclipped_grad_norm": 0.9110153913497925 + }, + { + "bits_per_byte": 2.5014427693276766, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7338680028915405, + "step": 760, + "unclipped_grad_norm": 1.0652238130569458 + }, + { + "bits_per_byte": 2.634176399523829, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.8258719444274902, + "step": 770, + "unclipped_grad_norm": 0.9231922626495361 + }, + { + "bits_per_byte": 2.5180344516043642, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.745368480682373, + "step": 780, + "unclipped_grad_norm": 0.8394486308097839 + }, + { + "bits_per_byte": 2.482792454720161, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7209405899047852, + "step": 790, + "unclipped_grad_norm": 0.8228755593299866 + }, + { + "bits_per_byte": 2.430754632152696, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.684870719909668, + "step": 800, + "unclipped_grad_norm": 0.9281390905380249 + }, + { + "bits_per_byte": 2.489219790348206, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.725395679473877, + "step": 810, + "unclipped_grad_norm": 0.8680029511451721 + }, + { + "bits_per_byte": 2.4286650429447127, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.683422327041626, + "step": 820, + "unclipped_grad_norm": 1.0944814682006836 + }, + { + "bits_per_byte": 2.483867002322752, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7216854095458984, + "step": 830, + "unclipped_grad_norm": 0.8509150743484497 + }, + { + "bits_per_byte": 2.566762812021276, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7791444063186646, + "step": 840, + "unclipped_grad_norm": 1.2704709768295288 + }, + { + "bits_per_byte": 2.458091274507339, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.7038190364837646, + "step": 850, + "unclipped_grad_norm": 0.905615508556366 + }, + { + "bits_per_byte": 2.442831941826886, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.693242073059082, + "step": 860, + "unclipped_grad_norm": 0.7913721203804016 + }, + { + "bits_per_byte": 2.3506050414231483, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6293152570724487, + "step": 870, + "unclipped_grad_norm": 1.0574872493743896 + }, + { + "bits_per_byte": 2.3881724157296675, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6553549766540527, + "step": 880, + "unclipped_grad_norm": 0.923473060131073 + }, + { + "bits_per_byte": 2.321517099734769, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6091530323028564, + "step": 890, + "unclipped_grad_norm": 1.0251853466033936 + }, + { + "bits_per_byte": 2.330866592583624, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6156336069107056, + "step": 900, + "unclipped_grad_norm": 0.9670726656913757 + }, + { + "bits_per_byte": 2.356522276508734, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.633416771888733, + "step": 910, + "unclipped_grad_norm": 0.8682661652565002 + }, + { + "bits_per_byte": 2.329593920967239, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6147514581680298, + "step": 920, + "unclipped_grad_norm": 0.9293603301048279 + }, + { + "bits_per_byte": 2.356717820782765, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6335523128509521, + "step": 930, + "unclipped_grad_norm": 1.1330991983413696 + }, + { + "bits_per_byte": 2.3506738344834934, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.629362940788269, + "step": 940, + "unclipped_grad_norm": 0.8093944787979126 + }, + { + "bits_per_byte": 2.337553106066519, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6202683448791504, + "step": 950, + "unclipped_grad_norm": 1.0507299900054932 + }, + { + "bits_per_byte": 2.3753796582278888, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6464877128601074, + "step": 960, + "unclipped_grad_norm": 0.8050268888473511 + }, + { + "bits_per_byte": 2.2641018956227765, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.569355845451355, + "step": 970, + "unclipped_grad_norm": 0.8510206341743469 + }, + { + "bits_per_byte": 2.35890939570271, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6350713968276978, + "step": 980, + "unclipped_grad_norm": 0.8616457581520081 + }, + { + "bits_per_byte": 2.3101203254100433, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.6012533903121948, + "step": 990, + "unclipped_grad_norm": 0.8957601189613342 + }, + { + "bits_per_byte": 2.2887248237294555, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5864231586456299, + "step": 1000, + "unclipped_grad_norm": 1.0717068910598755 + }, + { + "bits_per_byte": 2.2687179099719343, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.572555422782898, + "step": 1010, + "unclipped_grad_norm": 0.9444129467010498 + }, + { + "bits_per_byte": 2.2310683279583023, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5464587211608887, + "step": 1020, + "unclipped_grad_norm": 0.9090887308120728 + }, + { + "bits_per_byte": 2.2341806979909666, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5486160516738892, + "step": 1030, + "unclipped_grad_norm": 0.8268627524375916 + }, + { + "bits_per_byte": 2.26161898209227, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5676348209381104, + "step": 1040, + "unclipped_grad_norm": 0.7536599636077881 + }, + { + "bits_per_byte": 2.2636605881406626, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5690499544143677, + "step": 1050, + "unclipped_grad_norm": 0.8793148398399353 + }, + { + "bits_per_byte": 2.1893296864377567, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.51752769947052, + "step": 1060, + "unclipped_grad_norm": 0.9039056301116943 + }, + { + "bits_per_byte": 2.2431745307178375, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5548501014709473, + "step": 1070, + "unclipped_grad_norm": 0.862832248210907 + }, + { + "bits_per_byte": 2.24520426996332, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5562570095062256, + "step": 1080, + "unclipped_grad_norm": 0.9674477577209473 + }, + { + "bits_per_byte": 2.1985577595351025, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5239241123199463, + "step": 1090, + "unclipped_grad_norm": 0.7859865427017212 + }, + { + "bits_per_byte": 2.2599985615558404, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.566511631011963, + "step": 1100, + "unclipped_grad_norm": 0.788394570350647 + }, + { + "bits_per_byte": 2.119760812354591, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.469306230545044, + "step": 1110, + "unclipped_grad_norm": 0.8257600665092468 + }, + { + "bits_per_byte": 2.1691607650231224, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5035476684570312, + "step": 1120, + "unclipped_grad_norm": 0.9345623254776001 + }, + { + "bits_per_byte": 2.1036112974732717, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4581122398376465, + "step": 1130, + "unclipped_grad_norm": 0.6939264535903931 + }, + { + "bits_per_byte": 2.2552164119659492, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5631968975067139, + "step": 1140, + "unclipped_grad_norm": 0.8951012492179871 + }, + { + "bits_per_byte": 2.1146625586524137, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4657723903656006, + "step": 1150, + "unclipped_grad_norm": 0.9783032536506653 + }, + { + "bits_per_byte": 2.1242843000375844, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4724416732788086, + "step": 1160, + "unclipped_grad_norm": 0.8440186977386475 + }, + { + "bits_per_byte": 2.174440976369912, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5072076320648193, + "step": 1170, + "unclipped_grad_norm": 0.9151178002357483 + }, + { + "bits_per_byte": 2.1585999983642408, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.496227502822876, + "step": 1180, + "unclipped_grad_norm": 0.7418354153633118 + }, + { + "bits_per_byte": 2.091859378974514, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4499664306640625, + "step": 1190, + "unclipped_grad_norm": 0.9675256609916687 + }, + { + "bits_per_byte": 2.0676216920057677, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4331661462783813, + "step": 1200, + "unclipped_grad_norm": 0.937224805355072 + }, + { + "bits_per_byte": 2.116782072841647, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4672415256500244, + "step": 1210, + "unclipped_grad_norm": 0.8679532408714294 + }, + { + "bits_per_byte": 2.1441637746508166, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4862210750579834, + "step": 1220, + "unclipped_grad_norm": 0.7687044143676758 + }, + { + "bits_per_byte": 2.083470065265426, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4441514015197754, + "step": 1230, + "unclipped_grad_norm": 0.7017022967338562 + }, + { + "bits_per_byte": 2.025742368676816, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4041376113891602, + "step": 1240, + "unclipped_grad_norm": 0.8267619013786316 + }, + { + "bits_per_byte": 2.1229982137744323, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4715502262115479, + "step": 1250, + "unclipped_grad_norm": 1.0167953968048096 + }, + { + "bits_per_byte": 2.098591123894586, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4546325206756592, + "step": 1260, + "unclipped_grad_norm": 0.839733898639679 + }, + { + "bits_per_byte": 2.0788139789586175, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.440924048423767, + "step": 1270, + "unclipped_grad_norm": 0.7740461230278015 + }, + { + "bits_per_byte": 1.9768289549275753, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.370233416557312, + "step": 1280, + "unclipped_grad_norm": 0.6849040389060974 + }, + { + "bits_per_byte": 2.098678835046526, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.45469331741333, + "step": 1290, + "unclipped_grad_norm": 0.7928483486175537 + }, + { + "bits_per_byte": 2.041355641652745, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4149599075317383, + "step": 1300, + "unclipped_grad_norm": 0.7687125205993652 + }, + { + "bits_per_byte": 2.0466036922438238, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4185975790023804, + "step": 1310, + "unclipped_grad_norm": 0.7732207179069519 + }, + { + "bits_per_byte": 2.038361079735922, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.41288423538208, + "step": 1320, + "unclipped_grad_norm": 0.8016889691352844 + }, + { + "bits_per_byte": 2.1092168999954937, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4619977474212646, + "step": 1330, + "unclipped_grad_norm": 0.7918544411659241 + }, + { + "bits_per_byte": 2.0672703314500547, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.432922601699829, + "step": 1340, + "unclipped_grad_norm": 0.9473646879196167 + }, + { + "bits_per_byte": 2.0426272813732247, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4158413410186768, + "step": 1350, + "unclipped_grad_norm": 0.7250363826751709 + }, + { + "bits_per_byte": 2.10636370781768, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4600200653076172, + "step": 1360, + "unclipped_grad_norm": 0.7928993105888367 + }, + { + "bits_per_byte": 2.104627027009267, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4588162899017334, + "step": 1370, + "unclipped_grad_norm": 0.794548749923706 + }, + { + "bits_per_byte": 2.0561698832327657, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4252283573150635, + "step": 1380, + "unclipped_grad_norm": 0.7989111542701721 + }, + { + "bits_per_byte": 2.0527231789268243, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4228392839431763, + "step": 1390, + "unclipped_grad_norm": 0.8791808485984802 + }, + { + "bits_per_byte": 1.9977298345042815, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.384720802307129, + "step": 1400, + "unclipped_grad_norm": 0.7230417132377625 + }, + { + "bits_per_byte": 2.02384763581226, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4028242826461792, + "step": 1410, + "unclipped_grad_norm": 0.7468129396438599 + }, + { + "bits_per_byte": 2.0636549121636167, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4304165840148926, + "step": 1420, + "unclipped_grad_norm": 0.7919746041297913 + }, + { + "bits_per_byte": 2.0795395737626077, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4414269924163818, + "step": 1430, + "unclipped_grad_norm": 0.6952440738677979 + }, + { + "bits_per_byte": 2.0428340045195617, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4159846305847168, + "step": 1440, + "unclipped_grad_norm": 0.9689956307411194 + }, + { + "bits_per_byte": 1.9802144334098097, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3725800514221191, + "step": 1450, + "unclipped_grad_norm": 0.7872179746627808 + }, + { + "bits_per_byte": 2.013908586436248, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.39593505859375, + "step": 1460, + "unclipped_grad_norm": 0.7631422281265259 + }, + { + "bits_per_byte": 1.9493597139491168, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3511931896209717, + "step": 1470, + "unclipped_grad_norm": 0.7200304865837097 + }, + { + "bits_per_byte": 2.0209578113298123, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4008212089538574, + "step": 1480, + "unclipped_grad_norm": 0.7176085114479065 + }, + { + "bits_per_byte": 2.0333618880406417, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.409419059753418, + "step": 1490, + "unclipped_grad_norm": 0.7242412567138672 + }, + { + "bits_per_byte": 2.033254742849154, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.4093447923660278, + "step": 1500, + "unclipped_grad_norm": 0.7850398421287537 + }, + { + "bits_per_byte": 2.0298035669942904, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4069526195526123, + "step": 1510, + "unclipped_grad_norm": 0.7650813460350037 + }, + { + "bits_per_byte": 1.981937699571455, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.373774528503418, + "step": 1520, + "unclipped_grad_norm": 0.7211347222328186 + }, + { + "bits_per_byte": 2.0555194448472025, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4247775077819824, + "step": 1530, + "unclipped_grad_norm": 0.7807993292808533 + }, + { + "bits_per_byte": 1.9955152139091212, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3831857442855835, + "step": 1540, + "unclipped_grad_norm": 0.9850755929946899 + }, + { + "bits_per_byte": 2.0666652964843193, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4325032234191895, + "step": 1550, + "unclipped_grad_norm": 0.8105318546295166 + }, + { + "bits_per_byte": 1.9693205363562067, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.365028977394104, + "step": 1560, + "unclipped_grad_norm": 0.6913962364196777 + }, + { + "bits_per_byte": 2.059840508950131, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4277726411819458, + "step": 1570, + "unclipped_grad_norm": 0.725780189037323 + }, + { + "bits_per_byte": 1.9988298355392002, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3854832649230957, + "step": 1580, + "unclipped_grad_norm": 0.7167323231697083 + }, + { + "bits_per_byte": 2.0030889858878176, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3884354829788208, + "step": 1590, + "unclipped_grad_norm": 0.7256693840026855 + }, + { + "bits_per_byte": 2.027964040560662, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4056775569915771, + "step": 1600, + "unclipped_grad_norm": 0.709171712398529 + }, + { + "bits_per_byte": 1.961726126459407, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3597649335861206, + "step": 1610, + "unclipped_grad_norm": 0.6563887000083923 + }, + { + "bits_per_byte": 1.9260072216843611, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3350064754486084, + "step": 1620, + "unclipped_grad_norm": 0.743120014667511 + }, + { + "bits_per_byte": 1.954758421342351, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3549352884292603, + "step": 1630, + "unclipped_grad_norm": 0.6699017286300659 + }, + { + "bits_per_byte": 1.9950130245686017, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3828376531600952, + "step": 1640, + "unclipped_grad_norm": 0.7793066501617432 + }, + { + "bits_per_byte": 2.0499475509245495, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4209153652191162, + "step": 1650, + "unclipped_grad_norm": 0.6984328031539917 + }, + { + "bits_per_byte": 1.996117325169792, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3836030960083008, + "step": 1660, + "unclipped_grad_norm": 0.7241947054862976 + }, + { + "bits_per_byte": 2.017874850330446, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3986842632293701, + "step": 1670, + "unclipped_grad_norm": 0.7746724486351013 + }, + { + "bits_per_byte": 2.0244201660569825, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4032211303710938, + "step": 1680, + "unclipped_grad_norm": 0.7699829936027527 + }, + { + "bits_per_byte": 2.06043436504356, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4281842708587646, + "step": 1690, + "unclipped_grad_norm": 0.7417407035827637 + }, + { + "bits_per_byte": 1.95363795437198, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.354158639907837, + "step": 1700, + "unclipped_grad_norm": 0.6731021404266357 + }, + { + "bits_per_byte": 1.8890682719840945, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.309402346611023, + "step": 1710, + "unclipped_grad_norm": 0.6633850932121277 + }, + { + "bits_per_byte": 1.8893066399381904, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3095675706863403, + "step": 1720, + "unclipped_grad_norm": 0.7114297151565552 + }, + { + "bits_per_byte": 1.8703618910323976, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.296436071395874, + "step": 1730, + "unclipped_grad_norm": 0.6913781762123108 + }, + { + "bits_per_byte": 1.988379825707474, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3782398700714111, + "step": 1740, + "unclipped_grad_norm": 0.7100496888160706 + }, + { + "bits_per_byte": 2.0086424776668284, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.392284870147705, + "step": 1750, + "unclipped_grad_norm": 0.7558912038803101 + }, + { + "bits_per_byte": 1.996569639541561, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3839166164398193, + "step": 1760, + "unclipped_grad_norm": 0.6800652742385864 + }, + { + "bits_per_byte": 2.0072186333003352, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3912979364395142, + "step": 1770, + "unclipped_grad_norm": 0.7516586780548096 + }, + { + "bits_per_byte": 1.8758278436421194, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.300224781036377, + "step": 1780, + "unclipped_grad_norm": 0.6571854948997498 + }, + { + "bits_per_byte": 1.963898439322455, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3612706661224365, + "step": 1790, + "unclipped_grad_norm": 0.6896912455558777 + }, + { + "bits_per_byte": 1.880059304783948, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3031578063964844, + "step": 1800, + "unclipped_grad_norm": 0.6719921827316284 + }, + { + "bits_per_byte": 1.9405041552735398, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3450549840927124, + "step": 1810, + "unclipped_grad_norm": 0.7259154915809631 + }, + { + "bits_per_byte": 1.9116848504858082, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3250789642333984, + "step": 1820, + "unclipped_grad_norm": 0.6911426782608032 + }, + { + "bits_per_byte": 1.887859061965878, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3085641860961914, + "step": 1830, + "unclipped_grad_norm": 0.6866450309753418 + }, + { + "bits_per_byte": 1.904207044826293, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3198957443237305, + "step": 1840, + "unclipped_grad_norm": 0.6746156215667725 + }, + { + "bits_per_byte": 1.913379911492712, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.326253890991211, + "step": 1850, + "unclipped_grad_norm": 0.6992620825767517 + }, + { + "bits_per_byte": 1.8924212457453158, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.311726450920105, + "step": 1860, + "unclipped_grad_norm": 0.7686212658882141 + }, + { + "bits_per_byte": 1.9034685513234884, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3193838596343994, + "step": 1870, + "unclipped_grad_norm": 0.6957508325576782 + }, + { + "bits_per_byte": 1.8975736739825146, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3152978420257568, + "step": 1880, + "unclipped_grad_norm": 0.686790406703949 + }, + { + "bits_per_byte": 1.9108474669587572, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3244985342025757, + "step": 1890, + "unclipped_grad_norm": 0.6941424608230591 + }, + { + "bits_per_byte": 1.9178445811091105, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3293485641479492, + "step": 1900, + "unclipped_grad_norm": 0.6622544527053833 + }, + { + "bits_per_byte": 1.857984643615103, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2878568172454834, + "step": 1910, + "unclipped_grad_norm": 0.6576761603355408 + }, + { + "bits_per_byte": 1.8615044405476613, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2902965545654297, + "step": 1920, + "unclipped_grad_norm": 0.7134246826171875 + }, + { + "bits_per_byte": 1.9495869030309065, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3513506650924683, + "step": 1930, + "unclipped_grad_norm": 0.68708336353302 + }, + { + "bits_per_byte": 1.8269496863363057, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2663450241088867, + "step": 1940, + "unclipped_grad_norm": 0.671497106552124 + }, + { + "bits_per_byte": 1.904890503880822, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3203694820404053, + "step": 1950, + "unclipped_grad_norm": 0.6914620995521545 + }, + { + "bits_per_byte": 1.81755221032786, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259831190109253, + "step": 1960, + "unclipped_grad_norm": 0.654005765914917 + }, + { + "bits_per_byte": 1.9009173606805894, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3176155090332031, + "step": 1970, + "unclipped_grad_norm": 0.6446248292922974 + }, + { + "bits_per_byte": 1.9534350148439619, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.354017972946167, + "step": 1980, + "unclipped_grad_norm": 0.6792307496070862 + }, + { + "bits_per_byte": 1.9578730271494766, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.357094168663025, + "step": 1990, + "unclipped_grad_norm": 0.8862876892089844 + }, + { + "bits_per_byte": 1.9120136813142579, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3253068923950195, + "step": 2000, + "unclipped_grad_norm": 0.6821991801261902 + }, + { + "bits_per_byte": 1.879799782963796, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3029779195785522, + "step": 2010, + "unclipped_grad_norm": 0.6441471576690674 + }, + { + "bits_per_byte": 1.8960643542385427, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3142516613006592, + "step": 2020, + "unclipped_grad_norm": 0.6215534806251526 + }, + { + "bits_per_byte": 1.9880719767624297, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3780264854431152, + "step": 2030, + "unclipped_grad_norm": 0.6645570993423462 + }, + { + "bits_per_byte": 1.9132447331291338, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.326160192489624, + "step": 2040, + "unclipped_grad_norm": 0.6700881719589233 + }, + { + "bits_per_byte": 1.943815681215903, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3473503589630127, + "step": 2050, + "unclipped_grad_norm": 0.6627214550971985 + }, + { + "bits_per_byte": 1.9209480080439298, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.331499695777893, + "step": 2060, + "unclipped_grad_norm": 0.6492555737495422 + }, + { + "bits_per_byte": 1.8667327131338907, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2939205169677734, + "step": 2070, + "unclipped_grad_norm": 0.733633279800415 + }, + { + "bits_per_byte": 1.8747223391623733, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2994585037231445, + "step": 2080, + "unclipped_grad_norm": 0.7179054021835327 + }, + { + "bits_per_byte": 1.870721334772701, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2966852188110352, + "step": 2090, + "unclipped_grad_norm": 0.6603640913963318 + }, + { + "bits_per_byte": 1.8546041526297437, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2855136394500732, + "step": 2100, + "unclipped_grad_norm": 0.6752005815505981 + }, + { + "bits_per_byte": 1.9160201891487578, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3280839920043945, + "step": 2110, + "unclipped_grad_norm": 0.6652648448944092 + }, + { + "bits_per_byte": 1.871099696604599, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969474792480469, + "step": 2120, + "unclipped_grad_norm": 0.7032200694084167 + }, + { + "bits_per_byte": 1.8333405616423677, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.2707748413085938, + "step": 2130, + "unclipped_grad_norm": 0.6664465069770813 + }, + { + "bits_per_byte": 1.966146252569232, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3628287315368652, + "step": 2140, + "unclipped_grad_norm": 0.7021564245223999 + }, + { + "bits_per_byte": 1.835432558607463, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2722249031066895, + "step": 2150, + "unclipped_grad_norm": 0.6662127375602722 + }, + { + "bits_per_byte": 1.8787831935145458, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3022732734680176, + "step": 2160, + "unclipped_grad_norm": 0.6593465209007263 + }, + { + "bits_per_byte": 1.9513302911627028, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3525590896606445, + "step": 2170, + "unclipped_grad_norm": 0.721863329410553 + }, + { + "bits_per_byte": 1.8313610413309367, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2694027423858643, + "step": 2180, + "unclipped_grad_norm": 0.661525547504425 + }, + { + "bits_per_byte": 1.8437549710653651, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.2779935598373413, + "step": 2190, + "unclipped_grad_norm": 0.6355888247489929 + }, + { + "bits_per_byte": 1.8767926663134598, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3008935451507568, + "step": 2200, + "unclipped_grad_norm": 0.650397777557373 + }, + { + "bits_per_byte": 1.8339084483555168, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2711684703826904, + "step": 2210, + "unclipped_grad_norm": 0.6309477686882019 + }, + { + "bits_per_byte": 1.8366491638796665, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2730681896209717, + "step": 2220, + "unclipped_grad_norm": 0.6397845149040222 + }, + { + "bits_per_byte": 1.9122785345965865, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3254904747009277, + "step": 2230, + "unclipped_grad_norm": 0.7579862475395203 + }, + { + "bits_per_byte": 1.8301518313127203, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2685645818710327, + "step": 2240, + "unclipped_grad_norm": 0.6486738324165344 + }, + { + "bits_per_byte": 1.854603636681791, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2855132818222046, + "step": 2250, + "unclipped_grad_norm": 0.6761392951011658 + }, + { + "bits_per_byte": 1.8292511581701518, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2679402828216553, + "step": 2260, + "unclipped_grad_norm": 0.6474469900131226 + }, + { + "bits_per_byte": 1.8217264012469514, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.26272451877594, + "step": 2270, + "unclipped_grad_norm": 0.6697631478309631 + }, + { + "bits_per_byte": 1.839495476751446, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.275041103363037, + "step": 2280, + "unclipped_grad_norm": 0.672606885433197 + }, + { + "bits_per_byte": 1.8894086256501519, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.309638261795044, + "step": 2290, + "unclipped_grad_norm": 0.7677630186080933 + }, + { + "bits_per_byte": 1.8035906587308173, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2501537799835205, + "step": 2300, + "unclipped_grad_norm": 0.6455734372138977 + }, + { + "bits_per_byte": 1.834394643309506, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2715054750442505, + "step": 2310, + "unclipped_grad_norm": 0.6606747508049011 + }, + { + "bits_per_byte": 1.8063805612931139, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2520875930786133, + "step": 2320, + "unclipped_grad_norm": 0.634936511516571 + }, + { + "bits_per_byte": 1.8747587994843562, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2994837760925293, + "step": 2330, + "unclipped_grad_norm": 0.67148357629776 + }, + { + "bits_per_byte": 1.7920827836136342, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.242177128791809, + "step": 2340, + "unclipped_grad_norm": 0.6197777390480042 + }, + { + "bits_per_byte": 1.8459198886744261, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2794941663742065, + "step": 2350, + "unclipped_grad_norm": 0.6368024349212646 + }, + { + "bits_per_byte": 1.8583048753110096, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.288078784942627, + "step": 2360, + "unclipped_grad_norm": 0.662507951259613 + }, + { + "bits_per_byte": 1.8678564477746282, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2946994304656982, + "step": 2370, + "unclipped_grad_norm": 0.6344826221466064 + }, + { + "bits_per_byte": 1.8307873072076584, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2690050601959229, + "step": 2380, + "unclipped_grad_norm": 0.680084764957428 + }, + { + "bits_per_byte": 1.888488346485385, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3090003728866577, + "step": 2390, + "unclipped_grad_norm": 0.6502870917320251 + }, + { + "bits_per_byte": 1.8376442554975587, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2737579345703125, + "step": 2400, + "unclipped_grad_norm": 0.6415471434593201 + }, + { + "bits_per_byte": 1.8272317378837208, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.26654052734375, + "step": 2410, + "unclipped_grad_norm": 0.6551245450973511 + }, + { + "bits_per_byte": 1.8084292186301916, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2535076141357422, + "step": 2420, + "unclipped_grad_norm": 0.6829442381858826 + }, + { + "bits_per_byte": 1.8282901191171306, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2672741413116455, + "step": 2430, + "unclipped_grad_norm": 0.6490575075149536 + }, + { + "bits_per_byte": 1.8846385148458211, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3063318729400635, + "step": 2440, + "unclipped_grad_norm": 0.6281237006187439 + }, + { + "bits_per_byte": 1.8146233457836665, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2578010559082031, + "step": 2450, + "unclipped_grad_norm": 0.634181559085846 + }, + { + "bits_per_byte": 1.8319019267679002, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2697776556015015, + "step": 2460, + "unclipped_grad_norm": 0.6390114426612854 + }, + { + "bits_per_byte": 1.8154657168075925, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2583849430084229, + "step": 2470, + "unclipped_grad_norm": 0.6522298455238342 + }, + { + "bits_per_byte": 1.8417133650169726, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.276578426361084, + "step": 2480, + "unclipped_grad_norm": 0.6301186680793762 + }, + { + "bits_per_byte": 1.7981788806561174, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.246402621269226, + "step": 2490, + "unclipped_grad_norm": 0.613478422164917 + }, + { + "bits_per_byte": 1.8422764362158974, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2769687175750732, + "step": 2500, + "unclipped_grad_norm": 0.6680112481117249 + }, + { + "bits_per_byte": 1.7934540012889635, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2431275844573975, + "step": 2510, + "unclipped_grad_norm": 0.6324169635772705 + }, + { + "bits_per_byte": 1.8373082013977726, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2735249996185303, + "step": 2520, + "unclipped_grad_norm": 0.6562070250511169 + }, + { + "bits_per_byte": 1.8465858054985669, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2799557447433472, + "step": 2530, + "unclipped_grad_norm": 0.67587810754776 + }, + { + "bits_per_byte": 1.8180633427662245, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2601854801177979, + "step": 2540, + "unclipped_grad_norm": 0.6210066080093384 + }, + { + "bits_per_byte": 1.8116361791208304, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2557305097579956, + "step": 2550, + "unclipped_grad_norm": 0.656908392906189 + }, + { + "bits_per_byte": 1.8229865181298233, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2635979652404785, + "step": 2560, + "unclipped_grad_norm": 0.6474040746688843 + }, + { + "bits_per_byte": 1.8231736352539618, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2637276649475098, + "step": 2570, + "unclipped_grad_norm": 0.6998929977416992 + }, + { + "bits_per_byte": 1.8511411099719701, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2831132411956787, + "step": 2580, + "unclipped_grad_norm": 0.6996825337409973 + }, + { + "bits_per_byte": 1.805761251767357, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.251658320426941, + "step": 2590, + "unclipped_grad_norm": 0.6730060577392578 + }, + { + "bits_per_byte": 1.7987187341971758, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.246776819229126, + "step": 2600, + "unclipped_grad_norm": 0.6812606453895569 + }, + { + "bits_per_byte": 1.7378126262379698, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2045599222183228, + "step": 2610, + "unclipped_grad_norm": 0.6152759194374084 + }, + { + "bits_per_byte": 1.8067871282797534, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2523694038391113, + "step": 2620, + "unclipped_grad_norm": 0.6381004452705383 + }, + { + "bits_per_byte": 1.815846142431301, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2586486339569092, + "step": 2630, + "unclipped_grad_norm": 0.6937149167060852 + }, + { + "bits_per_byte": 1.6830253170311609, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1665842533111572, + "step": 2640, + "unclipped_grad_norm": 0.5872177481651306 + }, + { + "bits_per_byte": 1.7705318096166665, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2272391319274902, + "step": 2650, + "unclipped_grad_norm": 0.6221135854721069 + }, + { + "bits_per_byte": 1.813369076310924, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2569316625595093, + "step": 2660, + "unclipped_grad_norm": 0.6537755727767944 + }, + { + "bits_per_byte": 1.8137383230623265, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2571876049041748, + "step": 2670, + "unclipped_grad_norm": 0.6585802435874939 + }, + { + "bits_per_byte": 1.7472491423081613, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2111008167266846, + "step": 2680, + "unclipped_grad_norm": 0.6200863122940063 + }, + { + "bits_per_byte": 1.9002486921340347, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3171520233154297, + "step": 2690, + "unclipped_grad_norm": 0.6396089792251587 + }, + { + "bits_per_byte": 1.8072413344606821, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.252684235572815, + "step": 2700, + "unclipped_grad_norm": 0.6738898158073425 + }, + { + "bits_per_byte": 1.7328827435509875, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2011427879333496, + "step": 2710, + "unclipped_grad_norm": 0.7000491619110107 + }, + { + "bits_per_byte": 1.7288494064229527, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1983470916748047, + "step": 2720, + "unclipped_grad_norm": 0.6326305270195007 + }, + { + "bits_per_byte": 1.776559801529408, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2314174175262451, + "step": 2730, + "unclipped_grad_norm": 0.647907555103302 + }, + { + "bits_per_byte": 1.783826928441616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2364546060562134, + "step": 2740, + "unclipped_grad_norm": 0.6690548658370972 + }, + { + "bits_per_byte": 1.7821005665922551, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2352579832077026, + "step": 2750, + "unclipped_grad_norm": 0.664342999458313 + }, + { + "bits_per_byte": 1.8057218677403093, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2516310214996338, + "step": 2760, + "unclipped_grad_norm": 0.6517228484153748 + }, + { + "bits_per_byte": 1.7048344369870736, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.1817011833190918, + "step": 2770, + "unclipped_grad_norm": 0.684334397315979 + }, + { + "bits_per_byte": 1.7173274287283988, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.19036066532135, + "step": 2780, + "unclipped_grad_norm": 0.6013396382331848 + }, + { + "bits_per_byte": 1.804812079517245, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2510004043579102, + "step": 2790, + "unclipped_grad_norm": 0.6293013095855713 + }, + { + "bits_per_byte": 1.8557677872454814, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2863202095031738, + "step": 2800, + "unclipped_grad_norm": 0.7250765562057495 + }, + { + "bits_per_byte": 1.7793906359626097, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.233379602432251, + "step": 2810, + "unclipped_grad_norm": 0.7088303565979004 + }, + { + "bits_per_byte": 1.7744306563117265, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2299416065216064, + "step": 2820, + "unclipped_grad_norm": 0.6359960436820984 + }, + { + "bits_per_byte": 1.8690950668261421, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.295557975769043, + "step": 2830, + "unclipped_grad_norm": 0.6622759103775024 + }, + { + "bits_per_byte": 1.7494551637707785, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2126299142837524, + "step": 2840, + "unclipped_grad_norm": 0.6139791011810303 + }, + { + "bits_per_byte": 1.7661483158114752, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.22420072555542, + "step": 2850, + "unclipped_grad_norm": 0.6314926743507385 + }, + { + "bits_per_byte": 1.7415515290677273, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2071515321731567, + "step": 2860, + "unclipped_grad_norm": 0.664522111415863 + }, + { + "bits_per_byte": 1.8281711071227333, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2671916484832764, + "step": 2870, + "unclipped_grad_norm": 0.6572005152702332 + }, + { + "bits_per_byte": 1.8333617155084239, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2707895040512085, + "step": 2880, + "unclipped_grad_norm": 0.6237366795539856 + }, + { + "bits_per_byte": 1.7312996432497951, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2000454664230347, + "step": 2890, + "unclipped_grad_norm": 0.6335425972938538 + }, + { + "bits_per_byte": 1.7034657990515074, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1807525157928467, + "step": 2900, + "unclipped_grad_norm": 0.6068143844604492 + }, + { + "bits_per_byte": 1.7597350827608012, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2197554111480713, + "step": 2910, + "unclipped_grad_norm": 0.6452642679214478 + }, + { + "bits_per_byte": 1.7579383800072375, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2185100317001343, + "step": 2920, + "unclipped_grad_norm": 0.6799261569976807 + }, + { + "bits_per_byte": 1.7601423376780443, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2200376987457275, + "step": 2930, + "unclipped_grad_norm": 0.6305575370788574 + }, + { + "bits_per_byte": 1.77748111259008, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2320560216903687, + "step": 2940, + "unclipped_grad_norm": 0.6869999766349792 + }, + { + "bits_per_byte": 1.7661192507434795, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2241805791854858, + "step": 2950, + "unclipped_grad_norm": 0.6630687713623047 + }, + { + "bits_per_byte": 1.7912226983766693, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2415809631347656, + "step": 2960, + "unclipped_grad_norm": 0.6753889918327332 + }, + { + "bits_per_byte": 1.7573228540997994, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218083381652832, + "step": 2970, + "unclipped_grad_norm": 0.6314569115638733 + }, + { + "bits_per_byte": 1.7556446473926801, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2169201374053955, + "step": 2980, + "unclipped_grad_norm": 0.685643196105957 + }, + { + "bits_per_byte": 1.778294762511312, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2326200008392334, + "step": 2990, + "unclipped_grad_norm": 0.7002031803131104 + }, + { + "bits_per_byte": 1.7854260231293384, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2375630140304565, + "step": 3000, + "unclipped_grad_norm": 0.6241077184677124 + }, + { + "bits_per_byte": 1.7589776711664014, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2192304134368896, + "step": 3010, + "unclipped_grad_norm": 0.5821192860603333 + }, + { + "bits_per_byte": 1.7213342805282006, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1931380033493042, + "step": 3020, + "unclipped_grad_norm": 0.6891434192657471 + }, + { + "bits_per_byte": 1.8083322204151049, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2534403800964355, + "step": 3030, + "unclipped_grad_norm": 0.6679258346557617 + }, + { + "bits_per_byte": 1.7077268412092843, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1837060451507568, + "step": 3040, + "unclipped_grad_norm": 0.6403138637542725 + }, + { + "bits_per_byte": 1.7462220619172086, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2103888988494873, + "step": 3050, + "unclipped_grad_norm": 0.6389932632446289 + }, + { + "bits_per_byte": 1.7619969985849488, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2213232517242432, + "step": 3060, + "unclipped_grad_norm": 0.676324188709259 + }, + { + "bits_per_byte": 1.6793940753408436, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.164067268371582, + "step": 3070, + "unclipped_grad_norm": 0.5887739062309265 + }, + { + "bits_per_byte": 1.723725699288448, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.1947956085205078, + "step": 3080, + "unclipped_grad_norm": 0.6651143431663513 + }, + { + "bits_per_byte": 1.792202311555984, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2422599792480469, + "step": 3090, + "unclipped_grad_norm": 0.7115829586982727 + }, + { + "bits_per_byte": 1.6949688242029795, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1748628616333008, + "step": 3100, + "unclipped_grad_norm": 0.648972749710083 + }, + { + "bits_per_byte": 1.7872602181007902, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2388343811035156, + "step": 3110, + "unclipped_grad_norm": 0.6126834154129028 + }, + { + "bits_per_byte": 1.7418012478767801, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2073246240615845, + "step": 3120, + "unclipped_grad_norm": 0.6380596160888672 + }, + { + "bits_per_byte": 1.7994572276999805, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.247288703918457, + "step": 3130, + "unclipped_grad_norm": 0.6736929416656494 + }, + { + "bits_per_byte": 1.7652089465724625, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2235496044158936, + "step": 3140, + "unclipped_grad_norm": 0.6578004956245422 + }, + { + "bits_per_byte": 1.7230821402089194, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1943495273590088, + "step": 3150, + "unclipped_grad_norm": 0.6959549784660339 + }, + { + "bits_per_byte": 1.745404112429705, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2098219394683838, + "step": 3160, + "unclipped_grad_norm": 0.6303315758705139 + }, + { + "bits_per_byte": 1.6651441088383039, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.154189944267273, + "step": 3170, + "unclipped_grad_norm": 0.6514579057693481 + }, + { + "bits_per_byte": 1.748495672561615, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2119648456573486, + "step": 3180, + "unclipped_grad_norm": 0.6430810689926147 + }, + { + "bits_per_byte": 1.7413002624148166, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.206977367401123, + "step": 3190, + "unclipped_grad_norm": 0.6930623650550842 + }, + { + "bits_per_byte": 1.7004863716079601, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178687334060669, + "step": 3200, + "unclipped_grad_norm": 0.6456758975982666 + }, + { + "bits_per_byte": 1.6959322710131128, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755306720733643, + "step": 3210, + "unclipped_grad_norm": 0.6182776689529419 + }, + { + "bits_per_byte": 1.7081593775762043, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.184005856513977, + "step": 3220, + "unclipped_grad_norm": 0.6259344816207886 + }, + { + "bits_per_byte": 1.8040880325571125, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2504985332489014, + "step": 3230, + "unclipped_grad_norm": 0.6837271451950073 + }, + { + "bits_per_byte": 1.680291136847744, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.164689064025879, + "step": 3240, + "unclipped_grad_norm": 0.6078165173530579 + }, + { + "bits_per_byte": 1.833327662943553, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2707659006118774, + "step": 3250, + "unclipped_grad_norm": 0.6908280849456787 + }, + { + "bits_per_byte": 1.6877314502893708, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1698462963104248, + "step": 3260, + "unclipped_grad_norm": 0.6267281174659729 + }, + { + "bits_per_byte": 1.7156755353668616, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1892156600952148, + "step": 3270, + "unclipped_grad_norm": 0.6226681470870972 + }, + { + "bits_per_byte": 1.77760270432424, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.232140302658081, + "step": 3280, + "unclipped_grad_norm": 0.62666916847229 + }, + { + "bits_per_byte": 1.7647146684338828, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2232069969177246, + "step": 3290, + "unclipped_grad_norm": 0.6992575526237488 + }, + { + "bits_per_byte": 1.6858417049216903, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1685364246368408, + "step": 3300, + "unclipped_grad_norm": 0.6361157894134521 + }, + { + "bits_per_byte": 1.7274259060217614, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.1973603963851929, + "step": 3310, + "unclipped_grad_norm": 0.6828696727752686 + }, + { + "bits_per_byte": 1.7324201102201664, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.200822114944458, + "step": 3320, + "unclipped_grad_norm": 0.6315762996673584 + }, + { + "bits_per_byte": 1.7279007501207935, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1976895332336426, + "step": 3330, + "unclipped_grad_norm": 0.6427719593048096 + }, + { + "bits_per_byte": 1.6987109947031034, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1774567365646362, + "step": 3340, + "unclipped_grad_norm": 0.6303005814552307 + }, + { + "bits_per_byte": 1.7007044456092542, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1788384914398193, + "step": 3350, + "unclipped_grad_norm": 0.6034587025642395 + }, + { + "bits_per_byte": 1.7301495952634756, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1992483139038086, + "step": 3360, + "unclipped_grad_norm": 0.6297745108604431 + }, + { + "bits_per_byte": 1.6889530430584494, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.170693039894104, + "step": 3370, + "unclipped_grad_norm": 0.6365876197814941 + }, + { + "bits_per_byte": 1.7185201284121325, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1911873817443848, + "step": 3380, + "unclipped_grad_norm": 0.612015962600708 + }, + { + "bits_per_byte": 1.7554757604295328, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2168030738830566, + "step": 3390, + "unclipped_grad_norm": 0.7148704528808594 + }, + { + "bits_per_byte": 1.805198180568432, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2512680292129517, + "step": 3400, + "unclipped_grad_norm": 0.7259010672569275 + }, + { + "bits_per_byte": 1.8248889902136676, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2649166584014893, + "step": 3410, + "unclipped_grad_norm": 0.6750312447547913 + }, + { + "bits_per_byte": 1.6908183668897072, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1719859838485718, + "step": 3420, + "unclipped_grad_norm": 0.6068294644355774 + }, + { + "bits_per_byte": 1.6181469257849768, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1216139793395996, + "step": 3430, + "unclipped_grad_norm": 0.6401124596595764 + }, + { + "bits_per_byte": 1.7073711810873, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1834595203399658, + "step": 3440, + "unclipped_grad_norm": 0.6253624558448792 + }, + { + "bits_per_byte": 1.7759955264519272, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.231026291847229, + "step": 3450, + "unclipped_grad_norm": 0.6825960874557495 + }, + { + "bits_per_byte": 1.7035373438342662, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.1808021068572998, + "step": 3460, + "unclipped_grad_norm": 0.6307380795478821 + }, + { + "bits_per_byte": 1.694909318205781, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1748216152191162, + "step": 3470, + "unclipped_grad_norm": 0.6624554395675659 + }, + { + "bits_per_byte": 1.640782594360936, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1373038291931152, + "step": 3480, + "unclipped_grad_norm": 0.6500871777534485 + }, + { + "bits_per_byte": 1.8527976468650806, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2842614650726318, + "step": 3490, + "unclipped_grad_norm": 0.6960679888725281 + }, + { + "bits_per_byte": 1.7002731131208901, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.178539514541626, + "step": 3500, + "unclipped_grad_norm": 0.7090469002723694 + }, + { + "bits_per_byte": 1.7481523951904927, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2117269039154053, + "step": 3510, + "unclipped_grad_norm": 0.6682069301605225 + }, + { + "bits_per_byte": 1.7343439081527179, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.202155590057373, + "step": 3520, + "unclipped_grad_norm": 0.6754779815673828 + }, + { + "bits_per_byte": 1.703965236669613, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1810986995697021, + "step": 3530, + "unclipped_grad_norm": 0.625368595123291 + }, + { + "bits_per_byte": 1.7204380789345546, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.192516803741455, + "step": 3540, + "unclipped_grad_norm": 0.6671978235244751 + }, + { + "bits_per_byte": 1.7171148581719324, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1902133226394653, + "step": 3550, + "unclipped_grad_norm": 0.6819398999214172 + }, + { + "bits_per_byte": 1.6516749435706812, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1448538303375244, + "step": 3560, + "unclipped_grad_norm": 0.6344290375709534 + }, + { + "bits_per_byte": 1.6805387918649863, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.164860725402832, + "step": 3570, + "unclipped_grad_norm": 0.6333562135696411 + }, + { + "bits_per_byte": 1.6952945593437136, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.17508864402771, + "step": 3580, + "unclipped_grad_norm": 0.655963659286499 + }, + { + "bits_per_byte": 1.7344106374212527, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2022018432617188, + "step": 3590, + "unclipped_grad_norm": 0.7093035578727722 + }, + { + "bits_per_byte": 1.7542852965202604, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2159779071807861, + "step": 3600, + "unclipped_grad_norm": 0.668968915939331 + }, + { + "bits_per_byte": 1.6751485116216442, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1611244678497314, + "step": 3610, + "unclipped_grad_norm": 0.6502929925918579 + }, + { + "bits_per_byte": 1.6413291552253781, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1376826763153076, + "step": 3620, + "unclipped_grad_norm": 0.6599135398864746 + }, + { + "bits_per_byte": 1.6914022479893867, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1723906993865967, + "step": 3630, + "unclipped_grad_norm": 0.6377425789833069 + }, + { + "bits_per_byte": 1.6712699588793862, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1584360599517822, + "step": 3640, + "unclipped_grad_norm": 0.6370565891265869 + }, + { + "bits_per_byte": 1.7179845744373456, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1908161640167236, + "step": 3650, + "unclipped_grad_norm": 0.6391654014587402 + }, + { + "bits_per_byte": 1.6386034021918534, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1357933282852173, + "step": 3660, + "unclipped_grad_norm": 0.651786208152771 + }, + { + "bits_per_byte": 1.7137386667528447, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.187873125076294, + "step": 3670, + "unclipped_grad_norm": 0.64645916223526 + }, + { + "bits_per_byte": 1.651732729741371, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1448938846588135, + "step": 3680, + "unclipped_grad_norm": 0.6200698018074036 + }, + { + "bits_per_byte": 1.6382305438047828, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1355348825454712, + "step": 3690, + "unclipped_grad_norm": 0.6166419386863708 + }, + { + "bits_per_byte": 1.6489116983192684, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.142938494682312, + "step": 3700, + "unclipped_grad_norm": 0.6672006845474243 + }, + { + "bits_per_byte": 1.7161945790071655, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.189575433731079, + "step": 3710, + "unclipped_grad_norm": 0.6522265672683716 + }, + { + "bits_per_byte": 1.6845286173823526, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1676262617111206, + "step": 3720, + "unclipped_grad_norm": 0.6373200416564941 + }, + { + "bits_per_byte": 1.6158851819434799, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1200462579727173, + "step": 3730, + "unclipped_grad_norm": 0.6535916328430176 + }, + { + "bits_per_byte": 1.6679503497524322, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.156135082244873, + "step": 3740, + "unclipped_grad_norm": 0.6719768643379211 + }, + { + "bits_per_byte": 1.6882193650698685, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1701844930648804, + "step": 3750, + "unclipped_grad_norm": 0.6530092358589172 + }, + { + "bits_per_byte": 1.7695160800806709, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2265350818634033, + "step": 3760, + "unclipped_grad_norm": 0.6655569672584534 + }, + { + "bits_per_byte": 1.7330591977507726, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2012650966644287, + "step": 3770, + "unclipped_grad_norm": 0.6482370495796204 + }, + { + "bits_per_byte": 1.555590988256044, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0782535076141357, + "step": 3780, + "unclipped_grad_norm": 0.6347804665565491 + }, + { + "bits_per_byte": 1.661505987841952, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1516681909561157, + "step": 3790, + "unclipped_grad_norm": 0.6471394896507263 + }, + { + "bits_per_byte": 1.671339095905033, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1584839820861816, + "step": 3800, + "unclipped_grad_norm": 0.6512831449508667 + }, + { + "bits_per_byte": 1.6910716973344282, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17216157913208, + "step": 3810, + "unclipped_grad_norm": 0.66470867395401 + }, + { + "bits_per_byte": 1.662058912064476, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1520514488220215, + "step": 3820, + "unclipped_grad_norm": 0.6363425254821777 + }, + { + "bits_per_byte": 1.7124487968713735, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.186979055404663, + "step": 3830, + "unclipped_grad_norm": 0.6397905945777893 + }, + { + "bits_per_byte": 1.6528970522877124, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1457009315490723, + "step": 3840, + "unclipped_grad_norm": 0.6409087777137756 + }, + { + "bits_per_byte": 1.5958772362900533, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106177806854248, + "step": 3850, + "unclipped_grad_norm": 0.6576514840126038 + }, + { + "bits_per_byte": 1.6597844415068153, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1504749059677124, + "step": 3860, + "unclipped_grad_norm": 0.6520916819572449 + }, + { + "bits_per_byte": 1.7387071080051073, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2051799297332764, + "step": 3870, + "unclipped_grad_norm": 0.6458169221878052 + }, + { + "bits_per_byte": 1.631997548572213, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1312144994735718, + "step": 3880, + "unclipped_grad_norm": 0.6073523163795471 + }, + { + "bits_per_byte": 1.6031751480967658, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.111236333847046, + "step": 3890, + "unclipped_grad_norm": 0.632803201675415 + }, + { + "bits_per_byte": 1.6778582702686387, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1630027294158936, + "step": 3900, + "unclipped_grad_norm": 0.7183825969696045 + }, + { + "bits_per_byte": 1.69528785202033, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1750839948654175, + "step": 3910, + "unclipped_grad_norm": 0.6837812662124634 + }, + { + "bits_per_byte": 1.6365686754494955, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.134382963180542, + "step": 3920, + "unclipped_grad_norm": 0.6382866501808167 + }, + { + "bits_per_byte": 1.6653448125918606, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1543290615081787, + "step": 3930, + "unclipped_grad_norm": 0.6285881400108337 + }, + { + "bits_per_byte": 1.6987961261152804, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1775157451629639, + "step": 3940, + "unclipped_grad_norm": 0.666064977645874 + }, + { + "bits_per_byte": 1.7229675997634448, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.194270133972168, + "step": 3950, + "unclipped_grad_norm": 0.6532164812088013 + }, + { + "bits_per_byte": 1.7031080751377128, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.180504560470581, + "step": 3960, + "unclipped_grad_norm": 0.651078999042511 + }, + { + "bits_per_byte": 1.631284336519085, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1307201385498047, + "step": 3970, + "unclipped_grad_norm": 0.6460844278335571 + }, + { + "bits_per_byte": 1.6744433827531067, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1606357097625732, + "step": 3980, + "unclipped_grad_norm": 0.6715673208236694 + }, + { + "bits_per_byte": 1.6984889651008395, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1773028373718262, + "step": 3990, + "unclipped_grad_norm": 0.7048564553260803 + }, + { + "bits_per_byte": 1.5840455178412955, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0979766845703125, + "step": 4000, + "unclipped_grad_norm": 0.6564667224884033 + }, + { + "bits_per_byte": 1.67799809216379, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1630996465682983, + "step": 4010, + "unclipped_grad_norm": 0.6846939921379089 + }, + { + "bits_per_byte": 1.645402220345762, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1405059099197388, + "step": 4020, + "unclipped_grad_norm": 0.6446395516395569 + }, + { + "bits_per_byte": 1.678588508604202, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1635088920593262, + "step": 4030, + "unclipped_grad_norm": 0.744084894657135 + }, + { + "bits_per_byte": 1.6238756678852169, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1255848407745361, + "step": 4040, + "unclipped_grad_norm": 0.6811469793319702 + }, + { + "bits_per_byte": 1.6502275375810198, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.143850564956665, + "step": 4050, + "unclipped_grad_norm": 0.6219719052314758 + }, + { + "bits_per_byte": 1.655425885185999, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.147453784942627, + "step": 4060, + "unclipped_grad_norm": 0.6728122234344482 + }, + { + "bits_per_byte": 1.6906052803852882, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1718382835388184, + "step": 4070, + "unclipped_grad_norm": 0.6714834570884705 + }, + { + "bits_per_byte": 1.7592499197027172, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2194191217422485, + "step": 4080, + "unclipped_grad_norm": 0.7062271237373352 + }, + { + "bits_per_byte": 1.656125338627058, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.14793860912323, + "step": 4090, + "unclipped_grad_norm": 0.7272074222564697 + }, + { + "bits_per_byte": 1.6359631245358075, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1339632272720337, + "step": 4100, + "unclipped_grad_norm": 0.6811342239379883 + }, + { + "bits_per_byte": 1.6284930580955814, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1287853717803955, + "step": 4110, + "unclipped_grad_norm": 0.6341036558151245 + }, + { + "bits_per_byte": 1.6773249520683131, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1626330614089966, + "step": 4120, + "unclipped_grad_norm": 0.6589056849479675 + }, + { + "bits_per_byte": 1.6258371300183072, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1269444227218628, + "step": 4130, + "unclipped_grad_norm": 0.6739165186882019 + }, + { + "bits_per_byte": 1.688376729195408, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1702935695648193, + "step": 4140, + "unclipped_grad_norm": 0.6829613447189331 + }, + { + "bits_per_byte": 1.6211284170203344, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.123680591583252, + "step": 4150, + "unclipped_grad_norm": 0.6710932850837708 + }, + { + "bits_per_byte": 1.6548134549662765, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1470292806625366, + "step": 4160, + "unclipped_grad_norm": 0.7125453948974609 + }, + { + "bits_per_byte": 1.6949535177470527, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1748522520065308, + "step": 4170, + "unclipped_grad_norm": 0.6789602041244507 + }, + { + "bits_per_byte": 1.7346840898361244, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.2023913860321045, + "step": 4180, + "unclipped_grad_norm": 0.7571533918380737 + }, + { + "bits_per_byte": 1.6432809863300202, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1390355825424194, + "step": 4190, + "unclipped_grad_norm": 0.6785401701927185 + }, + { + "bits_per_byte": 1.6844624040617706, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1675803661346436, + "step": 4200, + "unclipped_grad_norm": 0.7054091095924377 + }, + { + "bits_per_byte": 1.672555185229284, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1593269109725952, + "step": 4210, + "unclipped_grad_norm": 0.6699897050857544 + }, + { + "bits_per_byte": 1.6983758005165717, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1772243976593018, + "step": 4220, + "unclipped_grad_norm": 0.6806776523590088 + }, + { + "bits_per_byte": 1.6692897506373519, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1570634841918945, + "step": 4230, + "unclipped_grad_norm": 0.6995632648468018 + }, + { + "bits_per_byte": 1.611002250520183, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1166616678237915, + "step": 4240, + "unclipped_grad_norm": 0.6468433737754822 + }, + { + "bits_per_byte": 1.646616933808806, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.141347885131836, + "step": 4250, + "unclipped_grad_norm": 0.6655133962631226 + }, + { + "bits_per_byte": 1.61702336312689, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1208351850509644, + "step": 4260, + "unclipped_grad_norm": 0.6660183072090149 + }, + { + "bits_per_byte": 1.6213850151354217, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1238584518432617, + "step": 4270, + "unclipped_grad_norm": 0.6792552471160889 + }, + { + "bits_per_byte": 1.6175876382043708, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1212263107299805, + "step": 4280, + "unclipped_grad_norm": 0.6596150398254395 + }, + { + "bits_per_byte": 1.673857609844268, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1602296829223633, + "step": 4290, + "unclipped_grad_norm": 0.6736516952514648 + }, + { + "bits_per_byte": 1.5981075073064421, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1077237129211426, + "step": 4300, + "unclipped_grad_norm": 0.65042644739151 + }, + { + "bits_per_byte": 1.6804373221009774, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.164790391921997, + "step": 4310, + "unclipped_grad_norm": 0.7301046252250671 + }, + { + "bits_per_byte": 1.6671354959526443, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1555702686309814, + "step": 4320, + "unclipped_grad_norm": 0.6921650767326355 + }, + { + "bits_per_byte": 1.6208601240849883, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1234946250915527, + "step": 4330, + "unclipped_grad_norm": 0.6875792145729065 + }, + { + "bits_per_byte": 1.651699193124453, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.144870638847351, + "step": 4340, + "unclipped_grad_norm": 0.6566780209541321 + }, + { + "bits_per_byte": 1.638596522885819, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1357885599136353, + "step": 4350, + "unclipped_grad_norm": 0.6942516565322876 + }, + { + "bits_per_byte": 1.6271663839268258, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1278657913208008, + "step": 4360, + "unclipped_grad_norm": 0.6855849623680115 + }, + { + "bits_per_byte": 1.6394584999319433, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1363860368728638, + "step": 4370, + "unclipped_grad_norm": 0.7280991673469543 + }, + { + "bits_per_byte": 1.69825386481711, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1771398782730103, + "step": 4380, + "unclipped_grad_norm": 0.6875903010368347 + }, + { + "bits_per_byte": 1.6491371675745496, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.143094778060913, + "step": 4390, + "unclipped_grad_norm": 0.7626568078994751 + }, + { + "bits_per_byte": 1.5979716410122606, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1076295375823975, + "step": 4400, + "unclipped_grad_norm": 0.6716449856758118 + }, + { + "bits_per_byte": 1.6724951632841327, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.159285306930542, + "step": 4410, + "unclipped_grad_norm": 0.7023071646690369 + }, + { + "bits_per_byte": 1.6178187828871304, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1213865280151367, + "step": 4420, + "unclipped_grad_norm": 0.6506969332695007 + }, + { + "bits_per_byte": 1.6647528483075908, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.153918743133545, + "step": 4430, + "unclipped_grad_norm": 0.6848753094673157 + }, + { + "bits_per_byte": 1.604941065955825, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1124603748321533, + "step": 4440, + "unclipped_grad_norm": 0.6678289771080017 + }, + { + "bits_per_byte": 1.6073562183218915, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.114134430885315, + "step": 4450, + "unclipped_grad_norm": 0.6393097639083862 + }, + { + "bits_per_byte": 1.581368607880616, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0961211919784546, + "step": 4460, + "unclipped_grad_norm": 0.6358886957168579 + }, + { + "bits_per_byte": 1.6384745871863573, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1357040405273438, + "step": 4470, + "unclipped_grad_norm": 0.7018935680389404 + }, + { + "bits_per_byte": 1.6524677835911588, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1454033851623535, + "step": 4480, + "unclipped_grad_norm": 0.7296347618103027 + }, + { + "bits_per_byte": 1.6287001252072204, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1289288997650146, + "step": 4490, + "unclipped_grad_norm": 0.6741621494293213 + }, + { + "bits_per_byte": 1.644477125666771, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1398646831512451, + "step": 4500, + "unclipped_grad_norm": 0.656035304069519 + }, + { + "bits_per_byte": 1.6005383100937371, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1094086170196533, + "step": 4510, + "unclipped_grad_norm": 0.6776733994483948 + }, + { + "bits_per_byte": 1.6433757487706455, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.139101266860962, + "step": 4520, + "unclipped_grad_norm": 0.6692283153533936 + }, + { + "bits_per_byte": 1.6564529655769518, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1481657028198242, + "step": 4530, + "unclipped_grad_norm": 0.6900064945220947 + }, + { + "bits_per_byte": 1.7115691056122104, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1863692998886108, + "step": 4540, + "unclipped_grad_norm": 0.7291412353515625 + }, + { + "bits_per_byte": 1.568905197155239, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.087482213973999, + "step": 4550, + "unclipped_grad_norm": 0.6766284108161926 + }, + { + "bits_per_byte": 1.6280190739098037, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1284568309783936, + "step": 4560, + "unclipped_grad_norm": 0.7198469042778015 + }, + { + "bits_per_byte": 1.6109731854521872, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1166415214538574, + "step": 4570, + "unclipped_grad_norm": 0.6274889707565308 + }, + { + "bits_per_byte": 1.5632356170868957, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.083552360534668, + "step": 4580, + "unclipped_grad_norm": 0.7059792280197144 + }, + { + "bits_per_byte": 1.6166451732776426, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1205730438232422, + "step": 4590, + "unclipped_grad_norm": 0.7005748748779297 + }, + { + "bits_per_byte": 1.6071720249028174, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.114006757736206, + "step": 4600, + "unclipped_grad_norm": 0.6841927170753479 + }, + { + "bits_per_byte": 1.66910400937442, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1569347381591797, + "step": 4610, + "unclipped_grad_norm": 0.7007519006729126 + }, + { + "bits_per_byte": 1.6744798430750896, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.160660982131958, + "step": 4620, + "unclipped_grad_norm": 0.7218911051750183 + }, + { + "bits_per_byte": 1.648005693714523, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1423105001449585, + "step": 4630, + "unclipped_grad_norm": 0.7027076482772827 + }, + { + "bits_per_byte": 1.692054922149411, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.172843098640442, + "step": 4640, + "unclipped_grad_norm": 0.6847496628761292 + }, + { + "bits_per_byte": 1.6069941948418252, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1138834953308105, + "step": 4650, + "unclipped_grad_norm": 0.695062518119812 + }, + { + "bits_per_byte": 1.5888770264519845, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1013256311416626, + "step": 4660, + "unclipped_grad_norm": 0.7018181681632996 + }, + { + "bits_per_byte": 1.721095740591454, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1929726600646973, + "step": 4670, + "unclipped_grad_norm": 0.7332665920257568 + }, + { + "bits_per_byte": 1.7135371030860336, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1877334117889404, + "step": 4680, + "unclipped_grad_norm": 0.7111047506332397 + }, + { + "bits_per_byte": 1.6797499074454787, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1643139123916626, + "step": 4690, + "unclipped_grad_norm": 0.7494509816169739 + }, + { + "bits_per_byte": 1.6073170062774949, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1141072511672974, + "step": 4700, + "unclipped_grad_norm": 0.6868770718574524 + }, + { + "bits_per_byte": 1.6766973873753146, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1621980667114258, + "step": 4710, + "unclipped_grad_norm": 0.7650524377822876 + }, + { + "bits_per_byte": 1.6150646527162134, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1194775104522705, + "step": 4720, + "unclipped_grad_norm": 0.7059918642044067 + }, + { + "bits_per_byte": 1.5217877982289587, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0548229217529297, + "step": 4730, + "unclipped_grad_norm": 0.6677312850952148 + }, + { + "bits_per_byte": 1.5772625220912664, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.0932750701904297, + "step": 4740, + "unclipped_grad_norm": 0.6796184182167053 + }, + { + "bits_per_byte": 1.6299009080755444, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1297612190246582, + "step": 4750, + "unclipped_grad_norm": 0.7213169932365417 + }, + { + "bits_per_byte": 1.6096504668844014, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1157246828079224, + "step": 4760, + "unclipped_grad_norm": 0.7036519050598145 + }, + { + "bits_per_byte": 1.5962356481344513, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1064262390136719, + "step": 4770, + "unclipped_grad_norm": 0.7376033663749695 + }, + { + "bits_per_byte": 1.626534347684905, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1274276971817017, + "step": 4780, + "unclipped_grad_norm": 0.7321033477783203 + }, + { + "bits_per_byte": 1.5732460392630163, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0904910564422607, + "step": 4790, + "unclipped_grad_norm": 0.6640080809593201 + }, + { + "bits_per_byte": 1.5751923669228307, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.091840147972107, + "step": 4800, + "unclipped_grad_norm": 0.7126999497413635 + }, + { + "bits_per_byte": 1.637988564215019, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1353671550750732, + "step": 4810, + "unclipped_grad_norm": 0.7521406412124634 + }, + { + "bits_per_byte": 1.57020504203046, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0883831977844238, + "step": 4820, + "unclipped_grad_norm": 0.7331681847572327 + }, + { + "bits_per_byte": 1.6098945102659756, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.115893840789795, + "step": 4830, + "unclipped_grad_norm": 0.7234310507774353 + }, + { + "bits_per_byte": 1.6131886659606018, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1181771755218506, + "step": 4840, + "unclipped_grad_norm": 0.7273361682891846 + }, + { + "bits_per_byte": 1.6277132887565695, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1282448768615723, + "step": 4850, + "unclipped_grad_norm": 0.6802273988723755 + }, + { + "bits_per_byte": 1.5401256203598566, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0675337314605713, + "step": 4860, + "unclipped_grad_norm": 0.6796969771385193 + }, + { + "bits_per_byte": 1.619305056955887, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1224167346954346, + "step": 4870, + "unclipped_grad_norm": 0.7333647608757019 + }, + { + "bits_per_byte": 1.4835698135542255, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.028332233428955, + "step": 4880, + "unclipped_grad_norm": 0.7043541073799133 + }, + { + "bits_per_byte": 1.5411697270332447, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.068257451057434, + "step": 4890, + "unclipped_grad_norm": 0.741724967956543 + }, + { + "bits_per_byte": 1.588994318619873, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1014069318771362, + "step": 4900, + "unclipped_grad_norm": 0.6961487531661987 + }, + { + "bits_per_byte": 1.693890149016768, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1741151809692383, + "step": 4910, + "unclipped_grad_norm": 0.7309879064559937 + }, + { + "bits_per_byte": 1.6231887691776707, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1251087188720703, + "step": 4920, + "unclipped_grad_norm": 0.7229281067848206 + }, + { + "bits_per_byte": 1.5846615596966862, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0984036922454834, + "step": 4930, + "unclipped_grad_norm": 0.6784905791282654 + }, + { + "bits_per_byte": 1.5777433855830787, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0936083793640137, + "step": 4940, + "unclipped_grad_norm": 0.6933380365371704 + }, + { + "bits_per_byte": 1.5189858568811019, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0528807640075684, + "step": 4950, + "unclipped_grad_norm": 0.6876996159553528 + }, + { + "bits_per_byte": 1.5293708572708018, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0600790977478027, + "step": 4960, + "unclipped_grad_norm": 0.7548285126686096 + }, + { + "bits_per_byte": 1.613208615948102, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1181910037994385, + "step": 4970, + "unclipped_grad_norm": 0.7192531824111938 + }, + { + "bits_per_byte": 1.6464662770066503, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1412434577941895, + "step": 4980, + "unclipped_grad_norm": 0.7251392602920532 + }, + { + "bits_per_byte": 1.5932895853251714, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.104384183883667, + "step": 4990, + "unclipped_grad_norm": 0.770077109336853 + }, + { + "bits_per_byte": 1.5394951319617936, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670967102050781, + "step": 5000, + "unclipped_grad_norm": 0.689707338809967 + }, + { + "bits_per_byte": 1.5463551759394092, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0718517303466797, + "step": 5010, + "unclipped_grad_norm": 0.6723923087120056 + }, + { + "bits_per_byte": 1.5950135394174203, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.105579137802124, + "step": 5020, + "unclipped_grad_norm": 0.6902953386306763 + }, + { + "bits_per_byte": 1.575558174021216, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0920937061309814, + "step": 5030, + "unclipped_grad_norm": 0.7085601687431335 + }, + { + "bits_per_byte": 1.5476398863413545, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.072742223739624, + "step": 5040, + "unclipped_grad_norm": 0.7076513767242432 + }, + { + "bits_per_byte": 1.517178147237883, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0516277551651, + "step": 5050, + "unclipped_grad_norm": 0.6860785484313965 + }, + { + "bits_per_byte": 1.5405944450661087, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0678586959838867, + "step": 5060, + "unclipped_grad_norm": 0.6969667673110962 + }, + { + "bits_per_byte": 1.603204729112714, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1112568378448486, + "step": 5070, + "unclipped_grad_norm": 0.6925143003463745 + }, + { + "bits_per_byte": 1.568753508457178, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.0873770713806152, + "step": 5080, + "unclipped_grad_norm": 0.7227545380592346 + }, + { + "bits_per_byte": 1.5571398640097143, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.07932710647583, + "step": 5090, + "unclipped_grad_norm": 0.7346438765525818 + }, + { + "bits_per_byte": 1.5756090808858714, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0921289920806885, + "step": 5100, + "unclipped_grad_norm": 0.687656581401825 + }, + { + "bits_per_byte": 1.4874313400140482, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0310088396072388, + "step": 5110, + "unclipped_grad_norm": 0.7022747993469238 + }, + { + "bits_per_byte": 1.5848280389027214, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0985190868377686, + "step": 5120, + "unclipped_grad_norm": 0.6745842695236206 + }, + { + "bits_per_byte": 1.5742394110543998, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.091179609298706, + "step": 5130, + "unclipped_grad_norm": 0.6994429230690002 + }, + { + "bits_per_byte": 1.5480249554966363, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0730091333389282, + "step": 5140, + "unclipped_grad_norm": 0.7030988335609436 + }, + { + "bits_per_byte": 1.639304919424723, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.136279582977295, + "step": 5150, + "unclipped_grad_norm": 0.7419465184211731 + }, + { + "bits_per_byte": 1.5116879450743894, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0478222370147705, + "step": 5160, + "unclipped_grad_norm": 0.7094154953956604 + }, + { + "bits_per_byte": 1.6830882626813766, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1666278839111328, + "step": 5170, + "unclipped_grad_norm": 0.7866556644439697 + }, + { + "bits_per_byte": 1.5735232752962072, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0906832218170166, + "step": 5180, + "unclipped_grad_norm": 0.7095000743865967 + }, + { + "bits_per_byte": 1.51639820591622, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0510871410369873, + "step": 5190, + "unclipped_grad_norm": 0.701707124710083 + }, + { + "bits_per_byte": 1.549430569702138, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0739834308624268, + "step": 5200, + "unclipped_grad_norm": 0.7002435922622681 + }, + { + "bits_per_byte": 1.5581800150821326, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0800480842590332, + "step": 5210, + "unclipped_grad_norm": 0.6994377374649048 + }, + { + "bits_per_byte": 1.6434304392536199, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.139139175415039, + "step": 5220, + "unclipped_grad_norm": 0.764930009841919 + }, + { + "bits_per_byte": 1.5617875231666307, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.0825486183166504, + "step": 5230, + "unclipped_grad_norm": 0.7441672086715698 + }, + { + "bits_per_byte": 1.5466518460121477, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0720573663711548, + "step": 5240, + "unclipped_grad_norm": 0.688214123249054 + }, + { + "bits_per_byte": 1.5248055778036484, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0569146871566772, + "step": 5250, + "unclipped_grad_norm": 0.6841322779655457 + }, + { + "bits_per_byte": 1.5447307998020101, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0707257986068726, + "step": 5260, + "unclipped_grad_norm": 0.694692850112915 + }, + { + "bits_per_byte": 1.539800573149726, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0673084259033203, + "step": 5270, + "unclipped_grad_norm": 0.6789148449897766 + }, + { + "bits_per_byte": 1.5548948024853513, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0777709484100342, + "step": 5280, + "unclipped_grad_norm": 0.7349528074264526 + }, + { + "bits_per_byte": 1.5569278094012005, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.079180121421814, + "step": 5290, + "unclipped_grad_norm": 0.7407549023628235 + }, + { + "bits_per_byte": 1.5483455311578447, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0732313394546509, + "step": 5300, + "unclipped_grad_norm": 0.7291215658187866 + }, + { + "bits_per_byte": 1.5444855525418797, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0705558061599731, + "step": 5310, + "unclipped_grad_norm": 0.7253612279891968 + }, + { + "bits_per_byte": 1.5505493168460005, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0747588872909546, + "step": 5320, + "unclipped_grad_norm": 0.7087648510932922 + }, + { + "bits_per_byte": 1.5517671259967603, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0756030082702637, + "step": 5330, + "unclipped_grad_norm": 0.7454737424850464 + }, + { + "bits_per_byte": 1.5001372462771416, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.039815902709961, + "step": 5340, + "unclipped_grad_norm": 0.7009119391441345 + }, + { + "bits_per_byte": 1.5857225206698589, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0991390943527222, + "step": 5350, + "unclipped_grad_norm": 0.7474352121353149 + }, + { + "bits_per_byte": 1.4946740453898335, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0360291004180908, + "step": 5360, + "unclipped_grad_norm": 0.7168655395507812 + }, + { + "bits_per_byte": 1.4766953230339372, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0235671997070312, + "step": 5370, + "unclipped_grad_norm": 0.7242471575737 + }, + { + "bits_per_byte": 1.5813711876203789, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0961229801177979, + "step": 5380, + "unclipped_grad_norm": 0.7494375109672546 + }, + { + "bits_per_byte": 1.5460944502407012, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0716710090637207, + "step": 5390, + "unclipped_grad_norm": 0.7461474537849426 + }, + { + "bits_per_byte": 1.5453937929210861, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0711853504180908, + "step": 5400, + "unclipped_grad_norm": 0.7291334271430969 + }, + { + "bits_per_byte": 1.5672311180317404, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0863218307495117, + "step": 5410, + "unclipped_grad_norm": 0.720815896987915 + }, + { + "bits_per_byte": 1.541436472124733, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0684423446655273, + "step": 5420, + "unclipped_grad_norm": 0.743694543838501 + }, + { + "bits_per_byte": 1.5782952779096975, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0939909219741821, + "step": 5430, + "unclipped_grad_norm": 0.7555011510848999 + }, + { + "bits_per_byte": 1.508865365808429, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.045865774154663, + "step": 5440, + "unclipped_grad_norm": 0.7259441018104553 + }, + { + "bits_per_byte": 1.5083157092562716, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0454847812652588, + "step": 5450, + "unclipped_grad_norm": 0.7505731582641602 + }, + { + "bits_per_byte": 1.5045496331676778, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0428743362426758, + "step": 5460, + "unclipped_grad_norm": 0.7536665201187134 + }, + { + "bits_per_byte": 1.5449808625763646, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0708991289138794, + "step": 5470, + "unclipped_grad_norm": 0.7403784394264221 + }, + { + "bits_per_byte": 1.550094938682421, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0744439363479614, + "step": 5480, + "unclipped_grad_norm": 0.725517213344574 + }, + { + "bits_per_byte": 1.5697909078071826, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0880961418151855, + "step": 5490, + "unclipped_grad_norm": 0.776546061038971 + }, + { + "bits_per_byte": 1.546138821764624, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0717017650604248, + "step": 5500, + "unclipped_grad_norm": 0.7528209686279297 + }, + { + "bits_per_byte": 1.6319261757721049, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1311650276184082, + "step": 5510, + "unclipped_grad_norm": 0.8170714974403381 + }, + { + "bits_per_byte": 1.5554193495704827, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.078134536743164, + "step": 5520, + "unclipped_grad_norm": 0.7434027194976807 + }, + { + "bits_per_byte": 1.5251368163892103, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.057144284248352, + "step": 5530, + "unclipped_grad_norm": 0.7136013507843018 + }, + { + "bits_per_byte": 1.586380354309409, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.099595069885254, + "step": 5540, + "unclipped_grad_norm": 0.7905043959617615 + }, + { + "bits_per_byte": 1.5182255215816374, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0523537397384644, + "step": 5550, + "unclipped_grad_norm": 0.7156078815460205 + }, + { + "bits_per_byte": 1.5143791295950908, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0496876239776611, + "step": 5560, + "unclipped_grad_norm": 0.7718254923820496 + }, + { + "bits_per_byte": 1.4761380992451418, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0231809616088867, + "step": 5570, + "unclipped_grad_norm": 0.7351405620574951 + }, + { + "bits_per_byte": 1.5698151573609542, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0881129503250122, + "step": 5580, + "unclipped_grad_norm": 0.7265479564666748 + }, + { + "bits_per_byte": 1.5069491351125157, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0445375442504883, + "step": 5590, + "unclipped_grad_norm": 0.7370763421058655 + }, + { + "bits_per_byte": 1.5604274843636077, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0816059112548828, + "step": 5600, + "unclipped_grad_norm": 0.7772327661514282 + }, + { + "bits_per_byte": 1.5487797873512732, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0735323429107666, + "step": 5610, + "unclipped_grad_norm": 0.6945621371269226 + }, + { + "bits_per_byte": 1.5643851491252627, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0843491554260254, + "step": 5620, + "unclipped_grad_norm": 0.772422194480896 + }, + { + "bits_per_byte": 1.5390314667350675, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0667753219604492, + "step": 5630, + "unclipped_grad_norm": 0.7249912023544312 + }, + { + "bits_per_byte": 1.5236104703628028, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0560863018035889, + "step": 5640, + "unclipped_grad_norm": 0.7168114185333252 + }, + { + "bits_per_byte": 1.5290047062071148, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0598253011703491, + "step": 5650, + "unclipped_grad_norm": 0.7599905729293823 + }, + { + "bits_per_byte": 1.6052637054088437, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1126840114593506, + "step": 5660, + "unclipped_grad_norm": 0.8106476068496704 + }, + { + "bits_per_byte": 1.5110031101586539, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0473475456237793, + "step": 5670, + "unclipped_grad_norm": 0.7452697157859802 + }, + { + "bits_per_byte": 1.532641279359609, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0623459815979004, + "step": 5680, + "unclipped_grad_norm": 0.7387374043464661 + }, + { + "bits_per_byte": 1.6103865526300942, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1162348985671997, + "step": 5690, + "unclipped_grad_norm": 0.7787995338439941 + }, + { + "bits_per_byte": 1.5250291552497701, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0570696592330933, + "step": 5700, + "unclipped_grad_norm": 0.7521193027496338 + }, + { + "bits_per_byte": 1.5451155249919901, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0709924697875977, + "step": 5710, + "unclipped_grad_norm": 0.7714623808860779 + }, + { + "bits_per_byte": 1.5521905472831843, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0758965015411377, + "step": 5720, + "unclipped_grad_norm": 0.7254540324211121 + }, + { + "bits_per_byte": 1.554917504195265, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0777866840362549, + "step": 5730, + "unclipped_grad_norm": 0.7474860548973083 + }, + { + "bits_per_byte": 1.5468128217733552, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0721689462661743, + "step": 5740, + "unclipped_grad_norm": 0.7887082099914551 + }, + { + "bits_per_byte": 1.541234736475271, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0683025121688843, + "step": 5750, + "unclipped_grad_norm": 0.7614914774894714 + }, + { + "bits_per_byte": 1.5622339901282707, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0828580856323242, + "step": 5760, + "unclipped_grad_norm": 0.7596094608306885 + }, + { + "bits_per_byte": 1.5113494832174914, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0475876331329346, + "step": 5770, + "unclipped_grad_norm": 0.7298161387443542 + }, + { + "bits_per_byte": 1.4968841944360716, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.037561058998108, + "step": 5780, + "unclipped_grad_norm": 0.7402821779251099 + }, + { + "bits_per_byte": 1.524952966935438, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0570168495178223, + "step": 5790, + "unclipped_grad_norm": 0.7289884686470032 + }, + { + "bits_per_byte": 1.4886130328081264, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0318279266357422, + "step": 5800, + "unclipped_grad_norm": 0.7239735722541809 + }, + { + "bits_per_byte": 1.5780909625204724, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0938493013381958, + "step": 5810, + "unclipped_grad_norm": 0.7893304824829102 + }, + { + "bits_per_byte": 1.5071924905634866, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0447062253952026, + "step": 5820, + "unclipped_grad_norm": 0.7217836976051331 + }, + { + "bits_per_byte": 1.516913465938205, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0514442920684814, + "step": 5830, + "unclipped_grad_norm": 0.7600290775299072 + }, + { + "bits_per_byte": 1.5193552756151552, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0531368255615234, + "step": 5840, + "unclipped_grad_norm": 0.7521220445632935 + }, + { + "bits_per_byte": 1.4707125625583728, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0194202661514282, + "step": 5850, + "unclipped_grad_norm": 0.7758342027664185 + }, + { + "bits_per_byte": 1.5340661556220072, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0633336305618286, + "step": 5860, + "unclipped_grad_norm": 0.7845423817634583 + }, + { + "bits_per_byte": 1.5391652692374387, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0668680667877197, + "step": 5870, + "unclipped_grad_norm": 0.7568844556808472 + }, + { + "bits_per_byte": 1.550266749350633, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0745630264282227, + "step": 5880, + "unclipped_grad_norm": 0.7594192028045654 + }, + { + "bits_per_byte": 1.5444004211297027, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0704967975616455, + "step": 5890, + "unclipped_grad_norm": 0.722252607345581 + }, + { + "bits_per_byte": 1.5452520792167752, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071087121963501, + "step": 5900, + "unclipped_grad_norm": 0.7907101511955261 + }, + { + "bits_per_byte": 1.554658154357764, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0776069164276123, + "step": 5910, + "unclipped_grad_norm": 0.7964528799057007 + }, + { + "bits_per_byte": 1.4817756905404247, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0270886421203613, + "step": 5920, + "unclipped_grad_norm": 0.7341460585594177 + }, + { + "bits_per_byte": 1.5817024262059407, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.0963525772094727, + "step": 5930, + "unclipped_grad_norm": 0.8211336731910706 + }, + { + "bits_per_byte": 1.4638614616786019, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0146714448928833, + "step": 5940, + "unclipped_grad_norm": 0.7324587106704712 + }, + { + "bits_per_byte": 1.5159049596735457, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0507452487945557, + "step": 5950, + "unclipped_grad_norm": 0.7514658570289612 + }, + { + "bits_per_byte": 1.553970911684916, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0771305561065674, + "step": 5960, + "unclipped_grad_norm": 0.8067389130592346 + }, + { + "bits_per_byte": 1.5228774803048255, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0555782318115234, + "step": 5970, + "unclipped_grad_norm": 0.8027852177619934 + }, + { + "bits_per_byte": 1.564259601790133, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0842621326446533, + "step": 5980, + "unclipped_grad_norm": 0.7799409031867981 + }, + { + "bits_per_byte": 1.5324249251848234, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0621960163116455, + "step": 5990, + "unclipped_grad_norm": 0.7881770133972168 + }, + { + "bits_per_byte": 1.480203081180935, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.025998592376709, + "step": 6000, + "unclipped_grad_norm": 0.7544389367103577 + }, + { + "bits_per_byte": 1.5204526969103107, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.053897500038147, + "step": 6010, + "unclipped_grad_norm": 0.7601525783538818 + }, + { + "bits_per_byte": 1.4842316027947455, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0287909507751465, + "step": 6020, + "unclipped_grad_norm": 0.7522627711296082 + }, + { + "bits_per_byte": 1.524829999340071, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0569316148757935, + "step": 6030, + "unclipped_grad_norm": 0.7871755361557007 + }, + { + "bits_per_byte": 1.5740971814021363, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0910810232162476, + "step": 6040, + "unclipped_grad_norm": 0.8342390656471252 + }, + { + "bits_per_byte": 1.475741851217554, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0229063034057617, + "step": 6050, + "unclipped_grad_norm": 0.7723444104194641 + }, + { + "bits_per_byte": 1.4753325325085003, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0226225852966309, + "step": 6060, + "unclipped_grad_norm": 0.769443154335022 + }, + { + "bits_per_byte": 1.5574803896584226, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0795631408691406, + "step": 6070, + "unclipped_grad_norm": 0.8035310506820679 + }, + { + "bits_per_byte": 1.4788967009649812, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0250930786132812, + "step": 6080, + "unclipped_grad_norm": 0.7756995558738708 + }, + { + "bits_per_byte": 1.4841172343319218, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0287116765975952, + "step": 6090, + "unclipped_grad_norm": 0.7475231289863586 + }, + { + "bits_per_byte": 1.4853364193438883, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0295567512512207, + "step": 6100, + "unclipped_grad_norm": 0.7613393664360046 + }, + { + "bits_per_byte": 1.5560739155396666, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0785882472991943, + "step": 6110, + "unclipped_grad_norm": 0.8045461177825928 + }, + { + "bits_per_byte": 1.548306147130797, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0732040405273438, + "step": 6120, + "unclipped_grad_norm": 0.8037658333778381 + }, + { + "bits_per_byte": 1.482889794152714, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0278608798980713, + "step": 6130, + "unclipped_grad_norm": 0.7671685814857483 + }, + { + "bits_per_byte": 1.4915240111566304, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0338456630706787, + "step": 6140, + "unclipped_grad_norm": 0.7935377359390259 + }, + { + "bits_per_byte": 1.5420002312542613, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0688331127166748, + "step": 6150, + "unclipped_grad_norm": 0.7919352650642395 + }, + { + "bits_per_byte": 1.4144471484587233, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.9804200530052185, + "step": 6160, + "unclipped_grad_norm": 0.7595162987709045 + }, + { + "bits_per_byte": 1.5291687776560379, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0599390268325806, + "step": 6170, + "unclipped_grad_norm": 0.7883366942405701 + }, + { + "bits_per_byte": 1.4831051164315943, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0280101299285889, + "step": 6180, + "unclipped_grad_norm": 0.7718276977539062 + }, + { + "bits_per_byte": 1.4948415564917739, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0361452102661133, + "step": 6190, + "unclipped_grad_norm": 0.7770188450813293 + }, + { + "bits_per_byte": 1.5074203675758797, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0448641777038574, + "step": 6200, + "unclipped_grad_norm": 0.7592265009880066 + }, + { + "bits_per_byte": 1.5232448352470684, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.055832862854004, + "step": 6210, + "unclipped_grad_norm": 0.8102302551269531 + }, + { + "bits_per_byte": 1.5164056011702072, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.051092267036438, + "step": 6220, + "unclipped_grad_norm": 0.787604033946991 + }, + { + "bits_per_byte": 1.4586058438508853, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.011028528213501, + "step": 6230, + "unclipped_grad_norm": 0.7935925126075745 + }, + { + "bits_per_byte": 1.4934343944424144, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0351698398590088, + "step": 6240, + "unclipped_grad_norm": 0.8212590217590332 + }, + { + "bits_per_byte": 1.5503186881111937, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.074599027633667, + "step": 6250, + "unclipped_grad_norm": 0.8379690647125244 + }, + { + "bits_per_byte": 1.403218659157867, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9726370573043823, + "step": 6260, + "unclipped_grad_norm": 0.7395487427711487 + }, + { + "bits_per_byte": 1.5229197880369378, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.055607557296753, + "step": 6270, + "unclipped_grad_norm": 0.8249105215072632 + }, + { + "bits_per_byte": 1.509112332895068, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.046036958694458, + "step": 6280, + "unclipped_grad_norm": 0.7966778874397278 + }, + { + "bits_per_byte": 1.4545608119025921, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0082247257232666, + "step": 6290, + "unclipped_grad_norm": 0.7541455030441284 + }, + { + "bits_per_byte": 1.5209793077872527, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0542625188827515, + "step": 6300, + "unclipped_grad_norm": 0.8022943139076233 + }, + { + "bits_per_byte": 1.4739374092447013, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.021655559539795, + "step": 6310, + "unclipped_grad_norm": 0.7839627861976624 + }, + { + "bits_per_byte": 1.4204840834693098, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.98460453748703, + "step": 6320, + "unclipped_grad_norm": 0.7670372128486633 + }, + { + "bits_per_byte": 1.5195884840897251, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0532984733581543, + "step": 6330, + "unclipped_grad_norm": 0.7843371033668518 + }, + { + "bits_per_byte": 1.4664117924082465, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0164391994476318, + "step": 6340, + "unclipped_grad_norm": 0.7673349380493164 + }, + { + "bits_per_byte": 1.577301734135663, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0933022499084473, + "step": 6350, + "unclipped_grad_norm": 0.8766481280326843 + }, + { + "bits_per_byte": 1.4611757806027283, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0128098726272583, + "step": 6360, + "unclipped_grad_norm": 0.8388718962669373 + }, + { + "bits_per_byte": 1.4358727471032122, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9952711462974548, + "step": 6370, + "unclipped_grad_norm": 0.775425136089325 + }, + { + "bits_per_byte": 1.4795478272811478, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0255444049835205, + "step": 6380, + "unclipped_grad_norm": 0.7803093194961548 + }, + { + "bits_per_byte": 1.5296866174177859, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.060297966003418, + "step": 6390, + "unclipped_grad_norm": 0.8152122497558594 + }, + { + "bits_per_byte": 1.5162291469704219, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0509699583053589, + "step": 6400, + "unclipped_grad_norm": 0.792400598526001 + }, + { + "bits_per_byte": 1.5084371290077807, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0455689430236816, + "step": 6410, + "unclipped_grad_norm": 0.8240766525268555 + }, + { + "bits_per_byte": 1.4750114408993396, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0224000215530396, + "step": 6420, + "unclipped_grad_norm": 0.853355348110199 + }, + { + "bits_per_byte": 1.5300785658791023, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0605696439743042, + "step": 6430, + "unclipped_grad_norm": 0.8168259859085083 + }, + { + "bits_per_byte": 1.4651398087224652, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0155575275421143, + "step": 6440, + "unclipped_grad_norm": 0.8302746415138245 + }, + { + "bits_per_byte": 1.53347470728569, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0629236698150635, + "step": 6450, + "unclipped_grad_norm": 0.8165877461433411 + }, + { + "bits_per_byte": 1.585776179256928, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.099176287651062, + "step": 6460, + "unclipped_grad_norm": 0.8728512525558472 + }, + { + "bits_per_byte": 1.5696127337808887, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.087972640991211, + "step": 6470, + "unclipped_grad_norm": 0.8505071401596069 + }, + { + "bits_per_byte": 1.4561444281517368, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0093224048614502, + "step": 6480, + "unclipped_grad_norm": 0.7639529705047607 + }, + { + "bits_per_byte": 1.5541174409034513, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0772321224212646, + "step": 6490, + "unclipped_grad_norm": 0.8572817444801331 + }, + { + "bits_per_byte": 1.5173276001614828, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0517313480377197, + "step": 6500, + "unclipped_grad_norm": 0.825986921787262 + }, + { + "bits_per_byte": 1.4589312350263177, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.011254072189331, + "step": 6510, + "unclipped_grad_norm": 0.7764277458190918 + }, + { + "bits_per_byte": 1.4310413244838487, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9919222593307495, + "step": 6520, + "unclipped_grad_norm": 0.7579286098480225 + }, + { + "bits_per_byte": 1.4675702675444584, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.017242193222046, + "step": 6530, + "unclipped_grad_norm": 0.8082018494606018 + }, + { + "bits_per_byte": 1.5128018767040279, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.048594355583191, + "step": 6540, + "unclipped_grad_norm": 0.8025459051132202 + }, + { + "bits_per_byte": 1.4392037930764485, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9975800514221191, + "step": 6550, + "unclipped_grad_norm": 0.790289044380188 + }, + { + "bits_per_byte": 1.527554032547087, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0588197708129883, + "step": 6560, + "unclipped_grad_norm": 0.818975567817688 + }, + { + "bits_per_byte": 1.47142164702788, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.019911766052246, + "step": 6570, + "unclipped_grad_norm": 0.8489425778388977 + }, + { + "bits_per_byte": 1.514791200026558, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0499732494354248, + "step": 6580, + "unclipped_grad_norm": 0.8219936490058899 + }, + { + "bits_per_byte": 1.5010776474120593, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0404677391052246, + "step": 6590, + "unclipped_grad_norm": 0.7910152077674866 + }, + { + "bits_per_byte": 1.4169470882716653, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9821528792381287, + "step": 6600, + "unclipped_grad_norm": 0.7658081650733948 + }, + { + "bits_per_byte": 1.448498423459678, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0040225982666016, + "step": 6610, + "unclipped_grad_norm": 0.7563595771789551 + }, + { + "bits_per_byte": 1.4411885588587308, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9989557862281799, + "step": 6620, + "unclipped_grad_norm": 0.8031777143478394 + }, + { + "bits_per_byte": 1.471761484745985, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0201473236083984, + "step": 6630, + "unclipped_grad_norm": 0.802291989326477 + }, + { + "bits_per_byte": 1.511551906797557, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0477279424667358, + "step": 6640, + "unclipped_grad_norm": 0.8319684863090515 + }, + { + "bits_per_byte": 1.5027432993856658, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.041622281074524, + "step": 6650, + "unclipped_grad_norm": 0.7944393754005432 + }, + { + "bits_per_byte": 1.5208112807373597, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0541460514068604, + "step": 6660, + "unclipped_grad_norm": 0.7983800768852234 + }, + { + "bits_per_byte": 1.499563684136514, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.039418339729309, + "step": 6670, + "unclipped_grad_norm": 0.8299691677093506 + }, + { + "bits_per_byte": 1.4686320884308854, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0179781913757324, + "step": 6680, + "unclipped_grad_norm": 0.8146572113037109 + }, + { + "bits_per_byte": 1.4999098852127009, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0396583080291748, + "step": 6690, + "unclipped_grad_norm": 0.8203337788581848 + }, + { + "bits_per_byte": 1.5587066259590745, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0804131031036377, + "step": 6700, + "unclipped_grad_norm": 0.8495987057685852 + }, + { + "bits_per_byte": 1.5200210204566451, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0535982847213745, + "step": 6710, + "unclipped_grad_norm": 0.7851044535636902 + }, + { + "bits_per_byte": 1.5404384568017762, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0677505731582642, + "step": 6720, + "unclipped_grad_norm": 0.8347281217575073 + }, + { + "bits_per_byte": 1.4888245714686876, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0319745540618896, + "step": 6730, + "unclipped_grad_norm": 0.8212841153144836 + }, + { + "bits_per_byte": 1.4304995791336308, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9915467500686646, + "step": 6740, + "unclipped_grad_norm": 0.7957568168640137 + }, + { + "bits_per_byte": 1.5212219753076202, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0544307231903076, + "step": 6750, + "unclipped_grad_norm": 0.7967842817306519 + }, + { + "bits_per_byte": 1.4749558905031108, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0223615169525146, + "step": 6760, + "unclipped_grad_norm": 0.8139022588729858 + }, + { + "bits_per_byte": 1.3835263876600992, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9589874148368835, + "step": 6770, + "unclipped_grad_norm": 0.785581648349762 + }, + { + "bits_per_byte": 1.4903192726873364, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0330106019973755, + "step": 6780, + "unclipped_grad_norm": 0.8513718843460083 + }, + { + "bits_per_byte": 1.4442869123053497, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.001103401184082, + "step": 6790, + "unclipped_grad_norm": 0.8081458210945129 + }, + { + "bits_per_byte": 1.5181372944817448, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0522925853729248, + "step": 6800, + "unclipped_grad_norm": 0.8491948843002319 + }, + { + "bits_per_byte": 1.5056836867674672, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0436604022979736, + "step": 6810, + "unclipped_grad_norm": 0.8421627879142761 + }, + { + "bits_per_byte": 1.5049580919634769, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0431574583053589, + "step": 6820, + "unclipped_grad_norm": 0.8799064755439758 + }, + { + "bits_per_byte": 1.4726698971078425, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0207769870758057, + "step": 6830, + "unclipped_grad_norm": 0.8110744953155518 + }, + { + "bits_per_byte": 1.4402100635666468, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9982775449752808, + "step": 6840, + "unclipped_grad_norm": 0.8030053973197937 + }, + { + "bits_per_byte": 1.4826992373755579, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.027728796005249, + "step": 6850, + "unclipped_grad_norm": 0.8305994868278503 + }, + { + "bits_per_byte": 1.4689192994578262, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0181772708892822, + "step": 6860, + "unclipped_grad_norm": 0.8366835117340088 + }, + { + "bits_per_byte": 1.419515391188325, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9839330911636353, + "step": 6870, + "unclipped_grad_norm": 0.8165588974952698 + }, + { + "bits_per_byte": 1.4317985640955977, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9924471378326416, + "step": 6880, + "unclipped_grad_norm": 0.7981967926025391 + }, + { + "bits_per_byte": 1.426708565560662, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9889190196990967, + "step": 6890, + "unclipped_grad_norm": 0.8219130635261536 + }, + { + "bits_per_byte": 1.544462506866664, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0705398321151733, + "step": 6900, + "unclipped_grad_norm": 0.8717687129974365 + }, + { + "bits_per_byte": 1.4952656657088017, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0364391803741455, + "step": 6910, + "unclipped_grad_norm": 0.8866252899169922 + }, + { + "bits_per_byte": 1.4499126367977229, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0050028562545776, + "step": 6920, + "unclipped_grad_norm": 0.8052312135696411 + }, + { + "bits_per_byte": 1.437854245215128, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9966446161270142, + "step": 6930, + "unclipped_grad_norm": 0.8604745268821716 + }, + { + "bits_per_byte": 1.4202836376897292, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9844655990600586, + "step": 6940, + "unclipped_grad_norm": 0.7888393998146057 + }, + { + "bits_per_byte": 1.4300773617257627, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9912540912628174, + "step": 6950, + "unclipped_grad_norm": 0.8240157961845398 + }, + { + "bits_per_byte": 1.4910837355704216, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0335404872894287, + "step": 6960, + "unclipped_grad_norm": 0.833236575126648 + }, + { + "bits_per_byte": 1.496710663941351, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0374407768249512, + "step": 6970, + "unclipped_grad_norm": 0.828235387802124 + }, + { + "bits_per_byte": 1.4445393828368163, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0012784004211426, + "step": 6980, + "unclipped_grad_norm": 0.8125858902931213 + }, + { + "bits_per_byte": 1.4867339503647994, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0305254459381104, + "step": 6990, + "unclipped_grad_norm": 0.8594081401824951 + }, + { + "bits_per_byte": 1.533073471761227, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0626455545425415, + "step": 7000, + "unclipped_grad_norm": 0.8734340071678162 + }, + { + "bits_per_byte": 1.532040715942796, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.061929702758789, + "step": 7010, + "unclipped_grad_norm": 0.8863697648048401 + }, + { + "bits_per_byte": 1.459764662952399, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0118317604064941, + "step": 7020, + "unclipped_grad_norm": 0.8206458687782288 + }, + { + "bits_per_byte": 1.5177566968753855, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.052028775215149, + "step": 7030, + "unclipped_grad_norm": 0.8729474544525146 + }, + { + "bits_per_byte": 1.50918250181662, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0460855960845947, + "step": 7040, + "unclipped_grad_norm": 0.8456775546073914 + }, + { + "bits_per_byte": 1.4815705152379453, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0269464254379272, + "step": 7050, + "unclipped_grad_norm": 0.8360646367073059 + }, + { + "bits_per_byte": 1.4998292253494463, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0396023988723755, + "step": 7060, + "unclipped_grad_norm": 0.8408570289611816 + }, + { + "bits_per_byte": 1.453465282416596, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0074653625488281, + "step": 7070, + "unclipped_grad_norm": 0.8465698957443237 + }, + { + "bits_per_byte": 1.431578598285144, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9922946691513062, + "step": 7080, + "unclipped_grad_norm": 0.8291451930999756 + }, + { + "bits_per_byte": 1.4576069686146742, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.01033616065979, + "step": 7090, + "unclipped_grad_norm": 0.8671156764030457 + }, + { + "bits_per_byte": 1.4654204844086733, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0157520771026611, + "step": 7100, + "unclipped_grad_norm": 0.831190288066864 + }, + { + "bits_per_byte": 1.478559958934592, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0248596668243408, + "step": 7110, + "unclipped_grad_norm": 0.8307176828384399 + }, + { + "bits_per_byte": 1.5234764958777807, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0559934377670288, + "step": 7120, + "unclipped_grad_norm": 0.8790873289108276 + }, + { + "bits_per_byte": 1.486035012871693, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.030040979385376, + "step": 7130, + "unclipped_grad_norm": 0.8940773606300354 + }, + { + "bits_per_byte": 1.4639995637472447, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0147671699523926, + "step": 7140, + "unclipped_grad_norm": 0.8168482184410095 + }, + { + "bits_per_byte": 1.430851627619947, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.991790771484375, + "step": 7150, + "unclipped_grad_norm": 0.7878822088241577 + }, + { + "bits_per_byte": 1.5069802639723218, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.044559121131897, + "step": 7160, + "unclipped_grad_norm": 0.8418039679527283 + }, + { + "bits_per_byte": 1.4531985373251077, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0072804689407349, + "step": 7170, + "unclipped_grad_norm": 0.815416157245636 + }, + { + "bits_per_byte": 1.4725130489302556, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0206682682037354, + "step": 7180, + "unclipped_grad_norm": 0.8564274907112122 + }, + { + "bits_per_byte": 1.437259873173746, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9962326288223267, + "step": 7190, + "unclipped_grad_norm": 0.8627841472625732 + }, + { + "bits_per_byte": 1.4854186270510006, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.029613733291626, + "step": 7200, + "unclipped_grad_norm": 0.8707078695297241 + }, + { + "bits_per_byte": 1.5339277095880628, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0632376670837402, + "step": 7210, + "unclipped_grad_norm": 0.8451290726661682 + }, + { + "bits_per_byte": 1.5149174352922914, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.050060749053955, + "step": 7220, + "unclipped_grad_norm": 0.8569709062576294 + }, + { + "bits_per_byte": 1.4518221601702528, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.00632643699646, + "step": 7230, + "unclipped_grad_norm": 0.8588427901268005 + }, + { + "bits_per_byte": 1.4720953030713098, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0203787088394165, + "step": 7240, + "unclipped_grad_norm": 0.8185886144638062 + }, + { + "bits_per_byte": 1.4930615360553439, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0349113941192627, + "step": 7250, + "unclipped_grad_norm": 0.9033716917037964 + }, + { + "bits_per_byte": 1.392136183127593, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9649552702903748, + "step": 7260, + "unclipped_grad_norm": 0.8070151209831238 + }, + { + "bits_per_byte": 1.419755822934231, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9840997457504272, + "step": 7270, + "unclipped_grad_norm": 0.8482468724250793 + }, + { + "bits_per_byte": 1.499443640246212, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0393351316452026, + "step": 7280, + "unclipped_grad_norm": 0.9195974469184875 + }, + { + "bits_per_byte": 1.424472533125469, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.9873691201210022, + "step": 7290, + "unclipped_grad_norm": 0.84366774559021 + }, + { + "bits_per_byte": 1.449468749575846, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.004695177078247, + "step": 7300, + "unclipped_grad_norm": 0.8698141574859619 + }, + { + "bits_per_byte": 1.4526158601039845, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0068765878677368, + "step": 7310, + "unclipped_grad_norm": 0.8154390454292297 + }, + { + "bits_per_byte": 1.40692677709312, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9752073287963867, + "step": 7320, + "unclipped_grad_norm": 0.835978627204895 + }, + { + "bits_per_byte": 1.4261922736427717, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9885611534118652, + "step": 7330, + "unclipped_grad_norm": 0.8695172071456909 + }, + { + "bits_per_byte": 1.392467249730504, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.96518474817276, + "step": 7340, + "unclipped_grad_norm": 0.8504277467727661 + }, + { + "bits_per_byte": 1.4759263886019296, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0230342149734497, + "step": 7350, + "unclipped_grad_norm": 0.8514381051063538 + }, + { + "bits_per_byte": 1.4321242992363319, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9926729202270508, + "step": 7360, + "unclipped_grad_norm": 0.8621233105659485 + }, + { + "bits_per_byte": 1.4814723131443026, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0268783569335938, + "step": 7370, + "unclipped_grad_norm": 0.8794771432876587 + }, + { + "bits_per_byte": 1.401024074541532, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9711158871650696, + "step": 7380, + "unclipped_grad_norm": 0.8035550117492676 + }, + { + "bits_per_byte": 1.474961909895891, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.022365689277649, + "step": 7390, + "unclipped_grad_norm": 0.8444681763648987 + }, + { + "bits_per_byte": 1.4183169300857874, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9831023812294006, + "step": 7400, + "unclipped_grad_norm": 0.880998969078064 + }, + { + "bits_per_byte": 1.4093773578852642, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9769059419631958, + "step": 7410, + "unclipped_grad_norm": 0.8176257014274597 + }, + { + "bits_per_byte": 1.476101638923159, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.023155689239502, + "step": 7420, + "unclipped_grad_norm": 0.8866280913352966 + }, + { + "bits_per_byte": 1.489828262219123, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.032670259475708, + "step": 7430, + "unclipped_grad_norm": 0.8741231560707092 + }, + { + "bits_per_byte": 1.5070485410847143, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0446064472198486, + "step": 7440, + "unclipped_grad_norm": 0.8906005620956421 + }, + { + "bits_per_byte": 1.4470038942236803, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0029866695404053, + "step": 7450, + "unclipped_grad_norm": 0.833862841129303 + }, + { + "bits_per_byte": 1.4545604679372903, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0082244873046875, + "step": 7460, + "unclipped_grad_norm": 0.851222813129425 + }, + { + "bits_per_byte": 1.4397255884391662, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9979417324066162, + "step": 7470, + "unclipped_grad_norm": 0.8808716535568237 + }, + { + "bits_per_byte": 1.469680494670545, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.018704891204834, + "step": 7480, + "unclipped_grad_norm": 0.9037331938743591 + }, + { + "bits_per_byte": 1.5297206699826567, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.060321569442749, + "step": 7490, + "unclipped_grad_norm": 0.8992816209793091 + }, + { + "bits_per_byte": 1.48344512613235, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0282458066940308, + "step": 7500, + "unclipped_grad_norm": 0.8851720690727234 + }, + { + "bits_per_byte": 1.4271278592634655, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9892096519470215, + "step": 7510, + "unclipped_grad_norm": 0.8391884565353394 + }, + { + "bits_per_byte": 1.4636686691269847, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0145378112792969, + "step": 7520, + "unclipped_grad_norm": 0.8814444541931152 + }, + { + "bits_per_byte": 1.5040419403823309, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0425224304199219, + "step": 7530, + "unclipped_grad_norm": 0.8750982880592346 + }, + { + "bits_per_byte": 1.448819515068839, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0042451620101929, + "step": 7540, + "unclipped_grad_norm": 0.8588963150978088 + }, + { + "bits_per_byte": 1.4846842611318165, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0291047096252441, + "step": 7550, + "unclipped_grad_norm": 0.8553798198699951 + }, + { + "bits_per_byte": 1.4456509067093426, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0020488500595093, + "step": 7560, + "unclipped_grad_norm": 0.8477172255516052 + }, + { + "bits_per_byte": 1.4068580700241002, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9751597046852112, + "step": 7570, + "unclipped_grad_norm": 0.8558363914489746 + }, + { + "bits_per_byte": 1.5845748804406512, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0983436107635498, + "step": 7580, + "unclipped_grad_norm": 0.9179992079734802 + }, + { + "bits_per_byte": 1.435446746077025, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9949758648872375, + "step": 7590, + "unclipped_grad_norm": 0.8465545177459717 + }, + { + "bits_per_byte": 1.4142041369730542, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9802516102790833, + "step": 7600, + "unclipped_grad_norm": 0.8474127054214478 + }, + { + "bits_per_byte": 1.441303443269507, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9990354180335999, + "step": 7610, + "unclipped_grad_norm": 0.8852339386940002 + }, + { + "bits_per_byte": 1.4014744111128163, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9714280366897583, + "step": 7620, + "unclipped_grad_norm": 0.8255653977394104 + }, + { + "bits_per_byte": 1.3991135192730972, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.96979159116745, + "step": 7630, + "unclipped_grad_norm": 0.8985438942909241 + }, + { + "bits_per_byte": 1.4200022740729177, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9842705726623535, + "step": 7640, + "unclipped_grad_norm": 0.8505365252494812 + }, + { + "bits_per_byte": 1.5509151239443857, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.075012445449829, + "step": 7650, + "unclipped_grad_norm": 0.8840228915214539 + }, + { + "bits_per_byte": 1.4633312391659918, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0143039226531982, + "step": 7660, + "unclipped_grad_norm": 0.9004685878753662 + }, + { + "bits_per_byte": 1.5004719245157205, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0400478839874268, + "step": 7670, + "unclipped_grad_norm": 0.9166978597640991 + }, + { + "bits_per_byte": 1.4376227565670667, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9964841604232788, + "step": 7680, + "unclipped_grad_norm": 0.8822170495986938 + }, + { + "bits_per_byte": 1.4453855374790612, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0018649101257324, + "step": 7690, + "unclipped_grad_norm": 0.8656983375549316 + }, + { + "bits_per_byte": 1.396036663657836, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9676588773727417, + "step": 7700, + "unclipped_grad_norm": 0.840412974357605 + }, + { + "bits_per_byte": 1.4832948132954957, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0281416177749634, + "step": 7710, + "unclipped_grad_norm": 0.8699836730957031 + }, + { + "bits_per_byte": 1.4846416094344024, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0290751457214355, + "step": 7720, + "unclipped_grad_norm": 0.8882943391799927 + }, + { + "bits_per_byte": 1.4868813394965887, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0306276082992554, + "step": 7730, + "unclipped_grad_norm": 0.8993004560470581 + }, + { + "bits_per_byte": 1.4329027787054622, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9932125210762024, + "step": 7740, + "unclipped_grad_norm": 0.8985061049461365 + }, + { + "bits_per_byte": 1.457792709877606, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0104649066925049, + "step": 7750, + "unclipped_grad_norm": 0.8967045545578003 + }, + { + "bits_per_byte": 1.475737723633933, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0229034423828125, + "step": 7760, + "unclipped_grad_norm": 0.8843175768852234 + }, + { + "bits_per_byte": 1.4006856986759595, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9708813428878784, + "step": 7770, + "unclipped_grad_norm": 0.8820574879646301 + }, + { + "bits_per_byte": 1.4787585988963383, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.024997353553772, + "step": 7780, + "unclipped_grad_norm": 0.8838117718696594 + }, + { + "bits_per_byte": 1.4172842602586817, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.982386589050293, + "step": 7790, + "unclipped_grad_norm": 0.8568381071090698 + }, + { + "bits_per_byte": 1.4471380406913532, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0030796527862549, + "step": 7800, + "unclipped_grad_norm": 0.8466761708259583 + }, + { + "bits_per_byte": 1.4200249757828314, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9842863082885742, + "step": 7810, + "unclipped_grad_norm": 0.9039835929870605 + }, + { + "bits_per_byte": 1.410308987904988, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.9775516986846924, + "step": 7820, + "unclipped_grad_norm": 0.8540316224098206 + }, + { + "bits_per_byte": 1.4409317887609925, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 0.9987778067588806, + "step": 7830, + "unclipped_grad_norm": 0.9217174649238586 + }, + { + "bits_per_byte": 1.4393295123942291, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9976671934127808, + "step": 7840, + "unclipped_grad_norm": 0.894000232219696 + }, + { + "bits_per_byte": 1.3879004224194929, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9620192646980286, + "step": 7850, + "unclipped_grad_norm": 0.856924295425415 + }, + { + "bits_per_byte": 1.4481823193473922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0038034915924072, + "step": 7860, + "unclipped_grad_norm": 0.8965994119644165 + }, + { + "bits_per_byte": 1.4733294505739012, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.021234154701233, + "step": 7870, + "unclipped_grad_norm": 0.9309923648834229 + }, + { + "bits_per_byte": 1.3911279348369099, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9642564058303833, + "step": 7880, + "unclipped_grad_norm": 0.8823378086090088 + }, + { + "bits_per_byte": 1.46931073197119, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0184485912322998, + "step": 7890, + "unclipped_grad_norm": 0.9113782048225403 + }, + { + "bits_per_byte": 1.449501254296859, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0047177076339722, + "step": 7900, + "unclipped_grad_norm": 0.8579812049865723 + }, + { + "bits_per_byte": 1.457684704772864, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.010390043258667, + "step": 7910, + "unclipped_grad_norm": 0.8884916305541992 + }, + { + "bits_per_byte": 1.4556867823177908, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0090051889419556, + "step": 7920, + "unclipped_grad_norm": 0.8975357413291931 + }, + { + "bits_per_byte": 1.4187827450956494, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9834252595901489, + "step": 7930, + "unclipped_grad_norm": 0.844153106212616 + }, + { + "bits_per_byte": 1.4233246349222852, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9865734577178955, + "step": 7940, + "unclipped_grad_norm": 0.9031311273574829 + }, + { + "bits_per_byte": 1.4939792354803478, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0355474948883057, + "step": 7950, + "unclipped_grad_norm": 0.910707950592041 + }, + { + "bits_per_byte": 1.4252896226997183, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.987935483455658, + "step": 7960, + "unclipped_grad_norm": 0.8988999128341675 + }, + { + "bits_per_byte": 1.446669559950403, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0027549266815186, + "step": 7970, + "unclipped_grad_norm": 0.9068711400032043 + }, + { + "bits_per_byte": 1.484100552014788, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0287001132965088, + "step": 7980, + "unclipped_grad_norm": 0.9153758883476257 + }, + { + "bits_per_byte": 1.5191428770413398, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0529896020889282, + "step": 7990, + "unclipped_grad_norm": 0.9229548573493958 + }, + { + "bits_per_byte": 1.419257675186007, + "learning_rate": 3e-05, + "loss_nats": 0.9837544560432434, + "step": 8000, + "unclipped_grad_norm": 0.8808227777481079 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073002.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073002.json new file mode 100644 index 0000000..68b33f2 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073002.json @@ -0,0 +1,8614 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "890c06c98d3b195e1aa79852477608280a7bcd106a01b463c2d793e8ec58d94f", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00042041894630528986, + 0.000374794501112774, + 0.0003388339828234166, + 0.000304876797599718, + 0.0002806047268677503, + 0.0002645689237397164, + 0.00024906310136429965, + 0.00023202023294288665, + 0.00021741738601122051, + 0.00020566635066643357, + 0.0001967942516785115, + 0.0001872276043286547, + 0.00017897134239319712, + 0.00017287296941503882, + 0.00016580727242399007, + 0.00016062518989201635, + 0.0001559370430186391, + 0.00015203763905446976, + 0.00014800428471062332, + 0.00014486612053588033, + 0.00014247088984120637, + 0.00013994764594826847, + 0.00013763502647634596, + 0.00013544136891141534, + 0.00013363662583287805, + 0.0001319239236181602, + 0.00013088916603010148, + 0.00012932090612594038, + 0.00012785944272764027, + 0.00012667682312894613, + 0.0001251104404218495, + 0.00012353518104646355 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00030814765159448143, + "first_to_last_ratio": 2.395816319369912, + "imbalance_abs_log_ratio": 0.8737240159497589, + "last_quartile_mean": 0.00012861906361649744, + "mean": 0.00019174550334355445, + "normalized": [ + 2.1925882952885543, + 1.9546455826984732, + 1.7671026277801187, + 1.5900075479395406, + 1.4634227242606306, + 1.3797920635754504, + 1.2989253829752037, + 1.2100426288859099, + 1.1338851875012121, + 1.0726006455438846, + 1.0263304653664347, + 0.9764380445115058, + 0.9333796061570758, + 0.9015750899007978, + 0.8647257408008662, + 0.8376999047754503, + 0.8132500647967916, + 0.7929137132465669, + 0.7718787774931073, + 0.7555124788314888, + 0.7430207611488979, + 0.7298614231256383, + 0.7178005433052705, + 0.7063600791135226, + 0.6969478997034861, + 0.6880157360550423, + 0.6826192205174408, + 0.6744403590744623, + 0.6668184676985713, + 0.660650815377801, + 0.6524817439796045, + 0.644266378571199 + ], + "population_cv": 0.4034880502878489 + }, + "activation_output_rms_by_block": [ + 0.02817646972835064, + 0.028377581387758255, + 0.028678538277745247, + 0.028959807008504868, + 0.029224006459116936, + 0.029430421069264412, + 0.02957010455429554, + 0.029960129410028458, + 0.030406322330236435, + 0.03066711500287056, + 0.030792171135544777, + 0.031063809990882874, + 0.03129913657903671, + 0.03160789608955383, + 0.03180097043514252, + 0.032092027366161346, + 0.03248445317149162, + 0.032903194427490234, + 0.03316524624824524, + 0.033593542873859406, + 0.034045662730932236, + 0.03455357626080513, + 0.03491735830903053, + 0.03521261364221573, + 0.03560211881995201, + 0.0359802283346653, + 0.03664350509643555, + 0.03706779330968857, + 0.0375242717564106, + 0.03805854171514511, + 0.038408540189266205, + 0.039021097123622894 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029047132236883044, + "first_to_last_ratio": 0.7789886319526242, + "imbalance_abs_log_ratio": 0.24975882634672159, + "last_quartile_mean": 0.03728826204314828, + "mean": 0.03285275783855468, + "normalized": [ + 0.8576591915605899, + 0.863780798166523, + 0.8729415782588962, + 0.8815030735264118, + 0.8895449996231614, + 0.8958280219241153, + 0.9000798258584322, + 0.9119517319446604, + 0.9255333290331198, + 0.93347155674343, + 0.9372781209682287, + 0.945546493951495, + 0.9527095634663979, + 0.96210784631481, + 0.9679848063720901, + 0.9768442431490312, + 0.9887892313676379, + 1.0015352315073034, + 1.0095117862319563, + 1.0225486407851998, + 1.0363106469855512, + 1.051770948138011, + 1.0628440534770849, + 1.0718312847663467, + 1.0836873724546434, + 1.095196589323784, + 1.1153859677933096, + 1.128300810904444, + 1.1421954875390588, + 1.1584580479414466, + 1.1691115972063533, + 1.187757122716477 + ], + "population_cv": 0.09694557255182189 + }, + "bits_per_byte": 8.04236897991864, + "branch_output_rms_by_sublayer": [ + 0.0023512106854468584, + 0.0027520405128598213, + 0.0023248849902302027, + 0.0026936428621411324, + 0.0027838374953716993, + 0.002722234698012471, + 0.002502240240573883, + 0.002735150745138526, + 0.0026338943280279636, + 0.002727407729253173, + 0.0029348398093134165, + 0.002726956969127059, + 0.0029466808773577213, + 0.002693725284188986, + 0.0032394269946962595, + 0.0027635409496724606, + 0.003579996759071946, + 0.0027838009409606457, + 0.0033850702457129955, + 0.0026989516336470842, + 0.0034471398685127497, + 0.002707457635551691, + 0.0036693045403808355, + 0.0027280766516923904, + 0.0035536750219762325, + 0.002806172939017415, + 0.0038618813268840313, + 0.0027198586612939835, + 0.003969449549913406, + 0.002747237915173173, + 0.003774659475311637, + 0.0027643220964819193, + 0.004222253803163767, + 0.002741035772487521, + 0.004245356190949678, + 0.00275977305136621, + 0.004526515491306782, + 0.0027434700168669224, + 0.004249728750437498, + 0.0026767924427986145, + 0.004827029537409544, + 0.002736816881224513, + 0.004658437799662352, + 0.0027651384007185698, + 0.0048249042592942715, + 0.0027371037285774946, + 0.005304279271513224, + 0.0027228749822825193, + 0.005711541976779699, + 0.002716122195124626, + 0.005045780446380377, + 0.002763758646324277, + 0.005378549452871084, + 0.0027021593414247036, + 0.005505796521902084, + 0.0027199271135032177, + 0.005381250753998756, + 0.0027382206171751022, + 0.005084864795207977, + 0.002702005673199892, + 0.005992465652525425, + 0.002712319139391184, + 0.00573702622205019, + 0.002734891138970852 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.010411069784541356, + 0.009271735997530202, + 0.009345779773643629, + 0.008099773113238128, + 0.008225801589217078, + 0.008647498843153682, + 0.007992995665779355, + 0.0076659621960069, + 0.007406648616870928, + 0.006601475371863238, + 0.006568536350729326, + 0.006644454140273569, + 0.005847021988988907, + 0.006236102346072616, + 0.006448411665711761, + 0.005769055287848533, + 0.005701484582581849, + 0.005836584711958408, + 0.005498613632501928, + 0.004978853123018873, + 0.005187902378445178, + 0.005078997897134883, + 0.005158829428828408, + 0.005043411661395455, + 0.00572819320620205, + 0.004439465860302503, + 0.004811331873512697, + 0.004686191415610403, + 0.004548827932369003, + 0.00413026790671143, + 0.004686908144290253, + 0.004087931241888514 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00870757712038879, + "first_to_last_ratio": 1.8766776125890325, + "imbalance_abs_log_ratio": 0.6295029861071718, + "last_quartile_mean": 0.004639889697610857, + "mean": 0.006274566179006908, + "normalized": [ + 1.6592493389222875, + 1.4776696480708191, + 1.489470268862924, + 1.2908897415629936, + 1.3109753494573861, + 1.3781827454599171, + 1.2738722387727572, + 1.2217517478188766, + 1.1804240174646141, + 1.0521006844983298, + 1.0468510751716935, + 1.0589503641708666, + 0.9318607569319367, + 0.9938698817038568, + 1.027706375507919, + 0.9194349255810409, + 0.9086659411861104, + 0.930197330850717, + 0.8763336740154054, + 0.7934975870804967, + 0.8268145128188418, + 0.8094580170542961, + 0.8221810531042816, + 0.8037865116905484, + 0.9129225898305317, + 0.707533514453289, + 0.766799127820212, + 0.7468550465352011, + 0.7249629380893642, + 0.6582555333514927, + 0.7469692741422425, + 0.6515081880187487 + ], + "population_cv": 0.26225105130829346 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.027978211641311646, + 0.028061864897608757, + 0.02817646972835064, + 0.028305010870099068, + 0.028377581387758255, + 0.02853298746049404, + 0.028678538277745247, + 0.028806988149881363, + 0.028959807008504868, + 0.029115403071045876, + 0.029224006459116936, + 0.02931177243590355, + 0.029430421069264412, + 0.02947254478931427, + 0.02957010455429554, + 0.02984699048101902, + 0.029960129410028458, + 0.0302706528455019, + 0.030406322330236435, + 0.030570803210139275, + 0.03066711500287056, + 0.030674004927277565, + 0.030792171135544777, + 0.03094458393752575, + 0.031063809990882874, + 0.031205367296934128, + 0.03129913657903671, + 0.03147619590163231, + 0.03160789608955383, + 0.03167128562927246, + 0.03180097043514252, + 0.03195846453309059, + 0.032092027366161346, + 0.032371606677770615, + 0.03248445317149162, + 0.032751601189374924, + 0.032903194427490234, + 0.03311120718717575, + 0.03316524624824524, + 0.03350966051220894, + 0.033593542873859406, + 0.03394605591893196, + 0.034045662730932236, + 0.03444153815507889, + 0.03455357626080513, + 0.03483813256025314, + 0.03491735830903053, + 0.03510763868689537, + 0.03521261364221573, + 0.03553535044193268, + 0.03560211881995201, + 0.03590916842222214, + 0.0359802283346653, + 0.036534491926431656, + 0.03664350509643555, + 0.036974843591451645, + 0.03706779330968857, + 0.037423327565193176, + 0.0375242717564106, + 0.037971291691064835, + 0.03805854171514511, + 0.03832196444272995, + 0.038408540189266205, + 0.03889527916908264 + ], + "loss_nats": 5.574545383453369, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028061864897608757, + 0.02817646972835064, + 0.028305010870099068, + 0.028377581387758255, + 0.02853298746049404, + 0.028678538277745247, + 0.028806988149881363, + 0.028959807008504868, + 0.029115403071045876, + 0.029224006459116936, + 0.02931177243590355, + 0.029430421069264412, + 0.02947254478931427, + 0.02957010455429554, + 0.02984699048101902, + 0.029960129410028458, + 0.0302706528455019, + 0.030406322330236435, + 0.030570803210139275, + 0.03066711500287056, + 0.030674004927277565, + 0.030792171135544777, + 0.03094458393752575, + 0.031063809990882874, + 0.031205367296934128, + 0.03129913657903671, + 0.03147619590163231, + 0.03160789608955383, + 0.03167128562927246, + 0.03180097043514252, + 0.03195846453309059, + 0.032092027366161346, + 0.032371606677770615, + 0.03248445317149162, + 0.032751601189374924, + 0.032903194427490234, + 0.03311120718717575, + 0.03316524624824524, + 0.03350966051220894, + 0.033593542873859406, + 0.03394605591893196, + 0.034045662730932236, + 0.03444153815507889, + 0.03455357626080513, + 0.03483813256025314, + 0.03491735830903053, + 0.03510763868689537, + 0.03521261364221573, + 0.03553535044193268, + 0.03560211881995201, + 0.03590916842222214, + 0.0359802283346653, + 0.036534491926431656, + 0.03664350509643555, + 0.036974843591451645, + 0.03706779330968857, + 0.037423327565193176, + 0.0375242717564106, + 0.037971291691064835, + 0.03805854171514511, + 0.03832196444272995, + 0.038408540189266205, + 0.03889527916908264, + 0.039021097123622894 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.223192369565368e-05, + 6.862939335405827e-05, + 6.46814878564328e-05, + 6.090753959142603e-05, + 5.703999340767041e-05, + 5.302598947309889e-05, + 4.9269601731793955e-05, + 4.545194315141998e-05, + 4.1738676372915506e-05, + 3.8814047002233565e-05, + 3.584829028113745e-05, + 3.317802838864736e-05, + 3.07806913042441e-05, + 2.884516288759187e-05, + 2.6981158953276463e-05, + 2.530650635890197e-05, + 2.3913053155411035e-05, + 2.265583498228807e-05, + 2.164491161238402e-05, + 2.069839865725953e-05, + 1.9972700101789087e-05, + 1.928487472468987e-05, + 1.8700015061767772e-05, + 1.8183285646955483e-05, + 1.7775813830667175e-05, + 1.7374133676639758e-05, + 1.7024083717842586e-05, + 1.673021324677393e-05, + 1.646951204747893e-05, + 1.624442847969476e-05, + 1.604618228157051e-05, + 1.58860220835777e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.890473403269425e-05, + "first_to_last_ratio": 3.528539860533838, + "imbalance_abs_log_ratio": 1.2608841481311845, + "last_quartile_mean": 1.669379867053067e-05, + "mean": 3.222918428491539e-05, + "normalized": [ + 2.2411961487173366, + 2.1294176342582674, + 2.0069228958644927, + 1.8898256640002307, + 1.7698242966194933, + 1.64527867054697, + 1.528726302727252, + 1.4102728368677142, + 1.2950584167422121, + 1.2043136636380887, + 1.1122928202038285, + 1.029440524939878, + 0.9550564802426835, + 0.8950013327235407, + 0.8371654310191393, + 0.7852046808006393, + 0.7419689230733446, + 0.7029602357293278, + 0.6715935290523237, + 0.6422253344757239, + 0.6197085202413003, + 0.5983668266067783, + 0.5802199303728628, + 0.5641869644049918, + 0.5515440190333021, + 0.5390807760769664, + 0.5282195033961989, + 0.5191013554322059, + 0.5110123762948401, + 0.5040285331483811, + 0.49787739397055714, + 0.49290797877912856 + ], + "population_cv": 0.5370990514498413 + }, + "activation_output_rms_by_block": [ + 0.029305610805749893, + 0.03049159236252308, + 0.0324031226336956, + 0.03488599509000778, + 0.037802305072546005, + 0.04105908423662186, + 0.04460485652089119, + 0.04878050833940506, + 0.053197599947452545, + 0.057943083345890045, + 0.06266378611326218, + 0.06765592098236084, + 0.07297883927822113, + 0.07823853194713593, + 0.08444638550281525, + 0.09052036702632904, + 0.09696295112371445, + 0.10418187081813812, + 0.11156722903251648, + 0.1197449341416359, + 0.12838369607925415, + 0.13761533796787262, + 0.14720149338245392, + 0.15857642889022827, + 0.16927987337112427, + 0.1816006898880005, + 0.19378916919231415, + 0.20805081725120544, + 0.22281049191951752, + 0.2388531118631363, + 0.25510889291763306, + 0.2763713300228119 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03741663438268006, + "first_to_last_ratio": 0.17145265067740043, + "imbalance_abs_log_ratio": 1.763448139840426, + "last_quartile_mean": 0.2182330470532179, + "mean": 0.11303362209582701, + "normalized": [ + 0.2592645467992289, + 0.2697568369230271, + 0.286668002253569, + 0.30863378916082446, + 0.3344341654423688, + 0.36324664710658405, + 0.39461582928906175, + 0.4315575085973109, + 0.47063518766436574, + 0.5126181243379725, + 0.554381828622084, + 0.5985468724076087, + 0.6456383324277757, + 0.6921704400555022, + 0.747090856127956, + 0.8008269163451932, + 0.8578239759627604, + 0.9216892185390237, + 0.9870269302520681, + 1.0593744756769734, + 1.135800956377508, + 1.2174726016583441, + 1.3022806015864932, + 1.4029138052020595, + 1.497606377928977, + 1.6066077200821192, + 1.714438284814271, + 1.8406100184494245, + 1.9711877562467606, + 2.1131156149330748, + 2.256929294023315, + 2.445036484706394 + ], + "population_cv": 0.6334590447760624 + }, + "bits_per_byte": 5.11848367743279, + "branch_output_rms_by_sublayer": [ + 0.003032666863873601, + 0.004902590066194534, + 0.002639081096276641, + 0.004850045777857304, + 0.0032534960191696882, + 0.005442064255475998, + 0.0027798404917120934, + 0.005710980389267206, + 0.0031097980681806803, + 0.006003146059811115, + 0.0030368315055966377, + 0.0059938253834843636, + 0.0034559990745037794, + 0.0061976369470357895, + 0.0037613045424222946, + 0.0062723769806325436, + 0.004607585724443197, + 0.0063410610891878605, + 0.003909118939191103, + 0.006816320586949587, + 0.004674337804317474, + 0.006481682416051626, + 0.004910735413432121, + 0.0065277558751404285, + 0.00493645528331399, + 0.006758649833500385, + 0.004751654341816902, + 0.006488988175988197, + 0.006120393518358469, + 0.006713794078677893, + 0.005455858074128628, + 0.00696563022211194, + 0.005903614219278097, + 0.007100075017660856, + 0.006780208554118872, + 0.007398899644613266, + 0.007146476302295923, + 0.007193166296929121, + 0.007645847275853157, + 0.00761415995657444, + 0.00879710540175438, + 0.007752604782581329, + 0.00870877131819725, + 0.007873632945120335, + 0.008980454877018929, + 0.008168715983629227, + 0.011330032721161842, + 0.008200661279261112, + 0.010545967146754265, + 0.007979212328791618, + 0.011436101980507374, + 0.008451862260699272, + 0.01130830217152834, + 0.008476903662085533, + 0.013203900307416916, + 0.008634857833385468, + 0.01393965259194374, + 0.009264753200113773, + 0.014520598575472832, + 0.009397287853062153, + 0.015960348770022392, + 0.00835501030087471, + 0.02132205106317997, + 0.009183173067867756 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00028524855645157025, + 0.00022469127596424853, + 0.00022670239393809648, + 0.0001993403227560908, + 0.0002100203582478305, + 0.00019183315773228503, + 0.00020704106188439305, + 0.00020121077118127333, + 0.00020796371696885172, + 0.00017825031863398375, + 0.00018469615204762195, + 0.0001848551977445252, + 0.00018214824531137669, + 0.0001649843875278976, + 0.0001966104199015976, + 0.00016919176207710993, + 0.0001681445657707544, + 0.00017333427923610245, + 0.0001624042419733422, + 0.00015992743704390023, + 0.00016485453322609178, + 0.00015449118786750607, + 0.00015016403044324942, + 0.00015560032515501767, + 0.00015156764536984935, + 0.00014581309890539414, + 0.000137484036700603, + 0.0001500871171184727, + 0.00014447415143049674, + 0.00014218802742214665, + 0.00014221421491139387, + 0.00016697872359292516 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002182609872694735, + "first_to_last_ratio": 1.4787241905811908, + "imbalance_abs_log_ratio": 0.3911796826204884, + "last_quartile_mean": 0.0001476008769314102, + "mean": 0.00017764111607924995, + "normalized": [ + 1.6057575112527105, + 1.2648607536557492, + 1.2761819951466593, + 1.122151938446279, + 1.1822733547459552, + 1.0798916487706802, + 1.1655019201299495, + 1.1326813049942133, + 1.170695847666675, + 1.0034294006262718, + 1.0397151072008834, + 1.0406104274984227, + 1.0253721060281789, + 0.9287511313219521, + 1.1067844215405909, + 0.9524358201038854, + 0.9465408092557868, + 0.9757554054027329, + 0.9142266472863736, + 0.9002839014620511, + 0.9280201389443322, + 0.869681475084765, + 0.8453224892837178, + 0.8759251719945322, + 0.8532238972323919, + 0.8208296712138615, + 0.7739426532271284, + 0.8448895189980412, + 0.8132922975221759, + 0.8004229570293466, + 0.8005703749797919, + 0.9399779019539145 + ], + "population_cv": 0.17622384112301237 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028563687577843666, + 0.028535494580864906, + 0.029305610805749893, + 0.029406530782580376, + 0.03049159236252308, + 0.030669530853629112, + 0.0324031226336956, + 0.03261565789580345, + 0.03488599509000778, + 0.035065844655036926, + 0.037802305072546005, + 0.03803478553891182, + 0.04105908423662186, + 0.041278183460235596, + 0.04460485652089119, + 0.04512108862400055, + 0.04878050833940506, + 0.04953810200095177, + 0.053197599947452545, + 0.05369982123374939, + 0.057943083345890045, + 0.0585285909473896, + 0.06266378611326218, + 0.06350461393594742, + 0.06765592098236084, + 0.06854932755231857, + 0.07297883927822113, + 0.07389408349990845, + 0.07823853194713593, + 0.07984820008277893, + 0.08444638550281525, + 0.08570227771997452, + 0.09052036702632904, + 0.09210614860057831, + 0.09696295112371445, + 0.09899988770484924, + 0.10418187081813812, + 0.10663887858390808, + 0.11156722903251648, + 0.11457803845405579, + 0.1197449341416359, + 0.12300471216440201, + 0.12838369607925415, + 0.13199855387210846, + 0.13761533796787262, + 0.14150823652744293, + 0.14720149338245392, + 0.15258266031742096, + 0.15857642889022827, + 0.16335271298885345, + 0.16927987337112427, + 0.17534348368644714, + 0.1816006898880005, + 0.1876170039176941, + 0.19378916919231415, + 0.2015211284160614, + 0.20805081725120544, + 0.21581925451755524, + 0.22281049191951752, + 0.23164939880371094, + 0.2388531118631363, + 0.24887791275978088, + 0.25510889291763306, + 0.26933422684669495 + ], + "loss_nats": 3.5478625297546387, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028535494580864906, + 0.029305610805749893, + 0.029406530782580376, + 0.03049159236252308, + 0.030669530853629112, + 0.0324031226336956, + 0.03261565789580345, + 0.03488599509000778, + 0.035065844655036926, + 0.037802305072546005, + 0.03803478553891182, + 0.04105908423662186, + 0.041278183460235596, + 0.04460485652089119, + 0.04512108862400055, + 0.04878050833940506, + 0.04953810200095177, + 0.053197599947452545, + 0.05369982123374939, + 0.057943083345890045, + 0.0585285909473896, + 0.06266378611326218, + 0.06350461393594742, + 0.06765592098236084, + 0.06854932755231857, + 0.07297883927822113, + 0.07389408349990845, + 0.07823853194713593, + 0.07984820008277893, + 0.08444638550281525, + 0.08570227771997452, + 0.09052036702632904, + 0.09210614860057831, + 0.09696295112371445, + 0.09899988770484924, + 0.10418187081813812, + 0.10663887858390808, + 0.11156722903251648, + 0.11457803845405579, + 0.1197449341416359, + 0.12300471216440201, + 0.12838369607925415, + 0.13199855387210846, + 0.13761533796787262, + 0.14150823652744293, + 0.14720149338245392, + 0.15258266031742096, + 0.15857642889022827, + 0.16335271298885345, + 0.16927987337112427, + 0.17534348368644714, + 0.1816006898880005, + 0.1876170039176941, + 0.19378916919231415, + 0.2015211284160614, + 0.20805081725120544, + 0.21581925451755524, + 0.22281049191951752, + 0.23164939880371094, + 0.2388531118631363, + 0.24887791275978088, + 0.25510889291763306, + 0.26933422684669495, + 0.2763713300228119 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014093925710767508, + 0.00014282441406976432, + 0.00014458753867074847, + 0.0001413008722011, + 0.00013349838263820857, + 0.00012304198753554374, + 0.00011454652121756226, + 0.00010784466576296836, + 0.0001000517513602972, + 9.189492993755266e-05, + 8.563220035284758e-05, + 8.04013543529436e-05, + 7.555109914392233e-05, + 7.068048580549657e-05, + 6.581390334758908e-05, + 6.151734851300716e-05, + 5.7112174545181915e-05, + 5.2947088988730684e-05, + 4.8879119276534766e-05, + 4.527245255303569e-05, + 4.1345181671204045e-05, + 3.832591755781323e-05, + 3.484322223812342e-05, + 3.232451854273677e-05, + 3.010137334058527e-05, + 2.8228545488673262e-05, + 2.6716850697994232e-05, + 2.5478970201220363e-05, + 2.4533494070055895e-05, + 2.4085495169856586e-05, + 2.3973674615263008e-05, + 2.415898779872805e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013107295490044635, + "first_to_last_ratio": 5.058842318548806, + "imbalance_abs_log_ratio": 1.621137666322135, + "last_quartile_mean": 2.5909673922797083e-05, + "mean": 6.995168058665513e-05, + "normalized": [ + 2.01480873548251, + 2.041758151797876, + 2.06696304446375, + 2.019978233776078, + 1.9084371028489115, + 1.7589568471213655, + 1.6375092100276802, + 1.5417022844700914, + 1.430298036032952, + 1.3136915248764403, + 1.2241621592889058, + 1.14938417031087, + 1.0800469482692516, + 1.010418694915251, + 0.9408480653450457, + 0.8794263125215463, + 0.8164517859500485, + 0.756909462999114, + 0.6987554675828556, + 0.6471960669615769, + 0.591053443240527, + 0.5478913049177658, + 0.49810414769034944, + 0.46209781197027316, + 0.4303166569857621, + 0.4035434924784137, + 0.38193293533380923, + 0.3642367129358298, + 0.3507205811826659, + 0.34431617607842635, + 0.34271763614835193, + 0.34536679599570513 + ], + "population_cv": 0.5909500348424317 + }, + "activation_output_rms_by_block": [ + 0.04959565028548241, + 0.05870900675654411, + 0.06652694195508957, + 0.07286322116851807, + 0.07801168411970139, + 0.08261121064424515, + 0.08867742121219635, + 0.09177010506391525, + 0.09494514763355255, + 0.0981190949678421, + 0.10163456201553345, + 0.10214314609766006, + 0.10418473184108734, + 0.10623370856046677, + 0.10849270969629288, + 0.11100933700799942, + 0.11405495554208755, + 0.11736668646335602, + 0.12177617102861404, + 0.12738163769245148, + 0.13297238945960999, + 0.14011575281620026, + 0.15017743408679962, + 0.16121913492679596, + 0.17427310347557068, + 0.189283087849617, + 0.2044389545917511, + 0.22237628698349, + 0.24115440249443054, + 0.26191622018814087, + 0.2883519232273102, + 0.3199674189090729 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07359565515071154, + "first_to_last_ratio": 0.309589437408787, + "imbalance_abs_log_ratio": 1.1725082547452423, + "last_quartile_mean": 0.2377201747149229, + "mean": 0.13694853871129453, + "normalized": [ + 0.3621480795062482, + 0.4286939262660582, + 0.48578059014807806, + 0.5320481828734462, + 0.5696423258970309, + 0.6032281280372068, + 0.6475236760221296, + 0.6701064934864231, + 0.6932906953736058, + 0.7164668998381211, + 0.7421368856647125, + 0.745850573206854, + 0.7607582586968847, + 0.775719913188861, + 0.7922151686847111, + 0.8105916138471667, + 0.8328307597537082, + 0.8570130617514684, + 0.8892111750482725, + 0.9301423650893323, + 0.9709661067646236, + 1.0231270383365143, + 1.0965975650415178, + 1.177224207310945, + 1.2725444544024032, + 1.3821475543353592, + 1.4928158892059098, + 1.6237945222058072, + 1.7609125644108963, + 1.9125156255978355, + 2.105549470923482, + 2.336406229084387 + ], + "population_cv": 0.48939845088217687 + }, + "bits_per_byte": 3.103317553853433, + "branch_output_rms_by_sublayer": [ + 0.011270907707512379, + 0.01699911244213581, + 0.012035840190947056, + 0.01337367296218872, + 0.010826056823134422, + 0.010907532647252083, + 0.01394603680819273, + 0.008780849166214466, + 0.015218054875731468, + 0.009724137373268604, + 0.015542514622211456, + 0.009259181097149849, + 0.025958625599741936, + 0.009840037673711777, + 0.03120059333741665, + 0.009063811041414738, + 0.014079542830586433, + 0.008434661664068699, + 0.01716585084795952, + 0.009029604494571686, + 0.021518073976039886, + 0.01014146488159895, + 0.030795104801654816, + 0.00910214427858591, + 0.018123170360922813, + 0.009148034267127514, + 0.016974516212940216, + 0.00891866721212864, + 0.01868187077343464, + 0.00946173444390297, + 0.017949005588889122, + 0.00928061455488205, + 0.021634839475154877, + 0.01075725443661213, + 0.02261096052825451, + 0.010938199236989021, + 0.017853887751698494, + 0.012016859836876392, + 0.016204316169023514, + 0.012549139559268951, + 0.019116709008812904, + 0.014130656607449055, + 0.021943919360637665, + 0.015834644436836243, + 0.015276721678674221, + 0.01847725547850132, + 0.01698230765759945, + 0.019000399857759476, + 0.017433995380997658, + 0.019877459853887558, + 0.017341874539852142, + 0.022254064679145813, + 0.018439622595906258, + 0.023488705977797508, + 0.013915229588747025, + 0.02547476813197136, + 0.008195439353585243, + 0.027127297595143318, + 0.007334497291594744, + 0.02754027582705021, + 0.014406879432499409, + 0.029805582016706467, + 0.02094510942697525, + 0.03139834478497505 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005239687790313407, + 0.0004396407496139251, + 0.0004479555677531567, + 0.000484081397596248, + 0.0004298002646761996, + 0.0003778969963663625, + 0.0003311985651517251, + 0.0003381159607123189, + 0.00027120323619484496, + 0.00025249768611013433, + 0.0002492173720055997, + 0.0002512649978189958, + 0.00021872457711399188, + 0.0001974354513181695, + 0.0001883229322305711, + 0.00018548264077727127, + 0.00019625685908145198, + 0.00017137558371316787, + 0.00015305597823747372, + 0.00014260825992090635, + 0.00013921721401447933, + 0.00013266258711253505, + 0.00012717821667718618, + 0.00011256530678442586, + 0.0001087245205863228, + 0.00010436458537579927, + 0.00010507936295752309, + 9.703474953173527e-05, + 9.385369360633854e-05, + 9.122967598255955e-05, + 0.00010232602359400015, + 0.0001188300325189452 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004215822851126596, + "first_to_last_ratio": 4.105774523524949, + "imbalance_abs_log_ratio": 1.4123944031885163, + "last_quartile_mean": 0.00010268033051915298, + "mean": 0.00022447405700517828, + "normalized": [ + 2.3342063934775927, + 1.9585370152764836, + 1.995578347580821, + 2.156513781835738, + 1.9146990543601428, + 1.6834773755509973, + 1.475442505786247, + 1.506258519239571, + 1.2081718476206305, + 1.1248412822347198, + 1.1102279494144423, + 1.11934983120655, + 0.9743868847567816, + 0.8795468570054622, + 0.8389518804225415, + 0.8262987859349487, + 0.874296396206376, + 0.7634538529733728, + 0.6818426159328645, + 0.6352995166725061, + 0.6201928894227087, + 0.5909929587519093, + 0.5665608684314569, + 0.5014624330589309, + 0.48435227677029374, + 0.4649293854629791, + 0.4681136179362547, + 0.4322760092026885, + 0.41810485745430026, + 0.4064152321194739, + 0.45584788264257925, + 0.5293708952576376 + ], + "population_cv": 0.5698198347315865 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.044032808393239975, + 0.044310975819826126, + 0.04959565028548241, + 0.052685707807540894, + 0.05870900675654411, + 0.061390094459056854, + 0.06652694195508957, + 0.0693279355764389, + 0.07286322116851807, + 0.07403621077537537, + 0.07801168411970139, + 0.07902701944112778, + 0.08261121064424515, + 0.08570802956819534, + 0.08867742121219635, + 0.08839071542024612, + 0.09177010506391525, + 0.0920921340584755, + 0.09494514763355255, + 0.09543459117412567, + 0.0981190949678421, + 0.09975367039442062, + 0.10163456201553345, + 0.09967570006847382, + 0.10214314609766006, + 0.10169660300016403, + 0.10418473184108734, + 0.1038530021905899, + 0.10623370856046677, + 0.10576826333999634, + 0.10849270969629288, + 0.10825389623641968, + 0.11100933700799942, + 0.11050844192504883, + 0.11405495554208755, + 0.11365467309951782, + 0.11736668646335602, + 0.117034412920475, + 0.12177617102861404, + 0.12205220758914948, + 0.12738163769245148, + 0.12672542035579681, + 0.13297238945960999, + 0.13195016980171204, + 0.14011575281620026, + 0.13969174027442932, + 0.15017743408679962, + 0.15003392100334167, + 0.16121913492679596, + 0.16189023852348328, + 0.17427310347557068, + 0.1747252345085144, + 0.189283087849617, + 0.18889831006526947, + 0.2044389545917511, + 0.20492415130138397, + 0.22237628698349, + 0.22264817357063293, + 0.24115440249443054, + 0.2428991198539734, + 0.26191622018814087, + 0.267524778842926, + 0.2883519232273102, + 0.2990366220474243 + ], + "loss_nats": 2.1510558128356934, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.044310975819826126, + 0.04959565028548241, + 0.052685707807540894, + 0.05870900675654411, + 0.061390094459056854, + 0.06652694195508957, + 0.0693279355764389, + 0.07286322116851807, + 0.07403621077537537, + 0.07801168411970139, + 0.07902701944112778, + 0.08261121064424515, + 0.08570802956819534, + 0.08867742121219635, + 0.08839071542024612, + 0.09177010506391525, + 0.0920921340584755, + 0.09494514763355255, + 0.09543459117412567, + 0.0981190949678421, + 0.09975367039442062, + 0.10163456201553345, + 0.09967570006847382, + 0.10214314609766006, + 0.10169660300016403, + 0.10418473184108734, + 0.1038530021905899, + 0.10623370856046677, + 0.10576826333999634, + 0.10849270969629288, + 0.10825389623641968, + 0.11100933700799942, + 0.11050844192504883, + 0.11405495554208755, + 0.11365467309951782, + 0.11736668646335602, + 0.117034412920475, + 0.12177617102861404, + 0.12205220758914948, + 0.12738163769245148, + 0.12672542035579681, + 0.13297238945960999, + 0.13195016980171204, + 0.14011575281620026, + 0.13969174027442932, + 0.15017743408679962, + 0.15003392100334167, + 0.16121913492679596, + 0.16189023852348328, + 0.17427310347557068, + 0.1747252345085144, + 0.189283087849617, + 0.18889831006526947, + 0.2044389545917511, + 0.20492415130138397, + 0.22237628698349, + 0.22264817357063293, + 0.24115440249443054, + 0.2428991198539734, + 0.26191622018814087, + 0.267524778842926, + 0.2883519232273102, + 0.2990366220474243, + 0.3199674189090729 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010639085667207837, + 0.00010674773511709645, + 0.000114928770926781, + 0.00011333057773299515, + 0.00011114918743260205, + 0.00010892991122091189, + 0.00010597817890811712, + 0.00010435423610033467, + 0.00010266973549732938, + 0.0001015385496430099, + 0.00010008511890191585, + 9.485893679084256e-05, + 9.186805982608348e-05, + 8.913136116461828e-05, + 8.655320561956614e-05, + 8.490345499012619e-05, + 8.653065015096217e-05, + 8.836571214487776e-05, + 8.356331090908498e-05, + 8.248459926107898e-05, + 7.508730777772143e-05, + 7.080488285282627e-05, + 6.500465678982437e-05, + 6.0319798649288714e-05, + 5.538603363675065e-05, + 5.071960185887292e-05, + 4.5680528273805976e-05, + 4.0155850001610816e-05, + 3.449051291681826e-05, + 2.892977681767661e-05, + 2.4221473722718656e-05, + 2.1527655917452648e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010897618176386459, + "first_to_last_ratio": 2.8953050536910427, + "imbalance_abs_log_ratio": 1.0630904780785062, + "last_quartile_mean": 3.763892914321332e-05, + "mean": 7.927156963205562e-05, + "normalized": [ + 1.3421060938480058, + 1.3466080665814153, + 1.4498107134781197, + 1.429649725103549, + 1.4021317850587363, + 1.3741359194288378, + 1.3369002203440912, + 1.3164144041136308, + 1.29516465958575, + 1.2808949048733107, + 1.262560075023968, + 1.1966325030668217, + 1.1589029995557718, + 1.1243799205481557, + 1.0918568412522767, + 1.0710454628842516, + 1.0915723071032915, + 1.1147213629682524, + 1.054139728744489, + 1.0405319289618822, + 0.9472161094607345, + 0.8931939052231708, + 0.8200248473891447, + 0.760926003222431, + 0.6986872329364575, + 0.6398208348124227, + 0.5762536112989216, + 0.5065605511281904, + 0.4350930992903045, + 0.3649451745683369, + 0.305550575510795, + 0.2715684326344833 + ], + "population_cv": 0.35248901334980626 + }, + "activation_output_rms_by_block": [ + 0.09771591424942017, + 0.13586144149303436, + 0.1991395801305771, + 0.2241763025522232, + 0.22696681320667267, + 0.22763192653656006, + 0.22912801802158356, + 0.2290782928466797, + 0.22818906605243683, + 0.22590669989585876, + 0.22214065492153168, + 0.21565376222133636, + 0.20698806643486023, + 0.20169880986213684, + 0.1979832947254181, + 0.1976672261953354, + 0.2011667788028717, + 0.20515663921833038, + 0.205980584025383, + 0.20982402563095093, + 0.2095191776752472, + 0.2096652388572693, + 0.2105090171098709, + 0.2148621529340744, + 0.2245095670223236, + 0.2386746108531952, + 0.25724881887435913, + 0.2833212912082672, + 0.3200184106826782, + 0.3660464584827423, + 0.43435168266296387, + 0.5434449911117554 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19621228612959385, + "first_to_last_ratio": 0.5884274155428802, + "imbalance_abs_log_ratio": 0.5303016979635413, + "last_quartile_mean": 0.3334519788622856, + "mean": 0.23750704107806087, + "normalized": [ + 0.4114232311003534, + 0.5720312159014314, + 0.8384575852011324, + 0.9438722386278381, + 0.9556214088494162, + 0.9584218030055994, + 0.9647209488255828, + 0.9645115858750019, + 0.960767584054228, + 0.951157906184392, + 0.9353013448073788, + 0.9079889221071891, + 0.8715028636428086, + 0.8492329698800171, + 0.8335891594066359, + 0.8322583839961553, + 0.8469929055145556, + 0.8637918197587332, + 0.8672609582032722, + 0.8834433904719, + 0.8821598581845203, + 0.8827748344031583, + 0.8863274796164187, + 0.9046559291834054, + 0.9452754158498172, + 1.0049159375226715, + 1.0831208106786603, + 1.1928963870807885, + 1.3474059936500937, + 1.5412025547589332, + 1.8287949725254948, + 2.288121601132417 + ], + "population_cv": 0.33570692978684213 + }, + "bits_per_byte": 2.0753701983577404, + "branch_output_rms_by_sublayer": [ + 0.017155639827251434, + 0.059614092111587524, + 0.05783598870038986, + 0.017162656411528587, + 0.08590122312307358, + 0.003484122920781374, + 0.037979111075401306, + 0.0025395466946065426, + 0.02590934932231903, + 0.004171412438154221, + 0.027761045843362808, + 0.005880473647266626, + 0.021641533821821213, + 0.0071118478663265705, + 0.01924360916018486, + 0.00746425986289978, + 0.022704657167196274, + 0.00924856960773468, + 0.018269814550876617, + 0.010196910239756107, + 0.022024810314178467, + 0.012587273493409157, + 0.03360125049948692, + 0.015558412298560143, + 0.026457328349351883, + 0.018742306157946587, + 0.024426763877272606, + 0.01840486004948616, + 0.02595876343548298, + 0.019045589491724968, + 0.023261789232492447, + 0.01923760399222374, + 0.0244763121008873, + 0.01806296780705452, + 0.0326010137796402, + 0.017410961911082268, + 0.037907715886831284, + 0.019881773740053177, + 0.034735601395368576, + 0.020177578553557396, + 0.04502493515610695, + 0.02574281580746174, + 0.034673795104026794, + 0.027684781700372696, + 0.0429018959403038, + 0.03708433732390404, + 0.03208737075328827, + 0.0411839634180069, + 0.027529660612344742, + 0.04565756395459175, + 0.02247321978211403, + 0.048749130219221115, + 0.01957971602678299, + 0.05322333425283432, + 0.02326148748397827, + 0.06035253033041954, + 0.014175337739288807, + 0.07119441032409668, + 0.011726614087820053, + 0.07984230667352676, + 0.013617267832159996, + 0.10812961310148239, + 0.01758233644068241, + 0.16684016585350037 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00037331716215226093, + 0.0002052857320094383, + 0.00022573696371192266, + 0.00015002318346068796, + 0.00011232796492243865, + 0.00011469567077281986, + 0.0001328622706326584, + 0.00011987742098506258, + 0.00012268315232766492, + 0.00013855849890850403, + 0.00016581239426236876, + 0.00020970093409220108, + 0.00019415593885306704, + 0.0001873084275669645, + 0.00019997003245353254, + 0.00019081336552079618, + 0.00019544152058884818, + 0.0002331889851483562, + 0.00028258169221688257, + 0.00026717547757069314, + 0.00031847213451059053, + 0.00028331189819324093, + 0.0003090653888381074, + 0.000275741819401602, + 0.000255221776165695, + 0.00023469865916502512, + 0.00021558263955965886, + 0.0002009260839676739, + 0.00018496297858040776, + 0.0001671377375581853, + 0.00015254274522576858, + 0.00014770686125523138 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017926579608091117, + "first_to_last_ratio": 0.9200315924660546, + "imbalance_abs_log_ratio": 0.08334726989162704, + "last_quartile_mean": 0.00019484743518470574, + "mean": 0.0002052152347055736, + "normalized": [ + 1.8191493564689118, + 1.0003435286077362, + 1.1000010015620527, + 0.7310528561679606, + 0.5473665982138111, + 0.5589042691561181, + 0.6474288852057137, + 0.584154588508271, + 0.5978267281358565, + 0.6751862214678978, + 0.8079926156567426, + 1.021858510617222, + 0.946108797096017, + 0.9127413363617942, + 0.9744404831368078, + 0.9298206626547972, + 0.9523733502010805, + 1.1363141994936055, + 1.377001530233894, + 1.301928085183418, + 1.5518932352537513, + 1.3805597747151384, + 1.5060547979370522, + 1.3436712912528852, + 1.2436785043365166, + 1.1436707391717382, + 1.0505196647264496, + 0.9790992576937408, + 0.9013121216160088, + 0.8144509241625318, + 0.7433305107421713, + 0.7197655742623075 + ], + "population_cv": 0.31225216605899336 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05658925697207451, + 0.0616849884390831, + 0.09771591424942017, + 0.1262788027524948, + 0.13586144149303436, + 0.19808362424373627, + 0.1991395801305771, + 0.22383953630924225, + 0.2241763025522232, + 0.22702015936374664, + 0.22696681320667267, + 0.227730393409729, + 0.22763192653656006, + 0.22866052389144897, + 0.22912801802158356, + 0.22953395545482635, + 0.2290782928466797, + 0.22848571836948395, + 0.22818906605243683, + 0.22744755446910858, + 0.22590669989585876, + 0.22389201819896698, + 0.22214065492153168, + 0.21693368256092072, + 0.21565376222133636, + 0.209661602973938, + 0.20698806643486023, + 0.20248256623744965, + 0.20169880986213684, + 0.19645103812217712, + 0.1979832947254181, + 0.1954326629638672, + 0.1976672261953354, + 0.19818583130836487, + 0.2011667788028717, + 0.20395968854427338, + 0.20515663921833038, + 0.2042165845632553, + 0.205980584025383, + 0.20741206407546997, + 0.20982402563095093, + 0.20696893334388733, + 0.2095191776752472, + 0.20577766001224518, + 0.2096652388572693, + 0.20420841872692108, + 0.2105090171098709, + 0.20596569776535034, + 0.2148621529340744, + 0.21317537128925323, + 0.2245095670223236, + 0.22439339756965637, + 0.2386746108531952, + 0.23865793645381927, + 0.25724881887435913, + 0.25777244567871094, + 0.2833212912082672, + 0.28324905037879944, + 0.3200184106826782, + 0.3202260434627533, + 0.3660464584827423, + 0.366729199886322, + 0.43435168266296387, + 0.4362008571624756 + ], + "loss_nats": 1.4385370016098022, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.0616849884390831, + 0.09771591424942017, + 0.1262788027524948, + 0.13586144149303436, + 0.19808362424373627, + 0.1991395801305771, + 0.22383953630924225, + 0.2241763025522232, + 0.22702015936374664, + 0.22696681320667267, + 0.227730393409729, + 0.22763192653656006, + 0.22866052389144897, + 0.22912801802158356, + 0.22953395545482635, + 0.2290782928466797, + 0.22848571836948395, + 0.22818906605243683, + 0.22744755446910858, + 0.22590669989585876, + 0.22389201819896698, + 0.22214065492153168, + 0.21693368256092072, + 0.21565376222133636, + 0.209661602973938, + 0.20698806643486023, + 0.20248256623744965, + 0.20169880986213684, + 0.19645103812217712, + 0.1979832947254181, + 0.1954326629638672, + 0.1976672261953354, + 0.19818583130836487, + 0.2011667788028717, + 0.20395968854427338, + 0.20515663921833038, + 0.2042165845632553, + 0.205980584025383, + 0.20741206407546997, + 0.20982402563095093, + 0.20696893334388733, + 0.2095191776752472, + 0.20577766001224518, + 0.2096652388572693, + 0.20420841872692108, + 0.2105090171098709, + 0.20596569776535034, + 0.2148621529340744, + 0.21317537128925323, + 0.2245095670223236, + 0.22439339756965637, + 0.2386746108531952, + 0.23865793645381927, + 0.25724881887435913, + 0.25777244567871094, + 0.2833212912082672, + 0.28324905037879944, + 0.3200184106826782, + 0.3202260434627533, + 0.3660464584827423, + 0.366729199886322, + 0.43435168266296387, + 0.4362008571624756, + 0.5434449911117554 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001095466359402053, + 0.00011157916014781222, + 0.0001209566617035307, + 0.00011857408389914781, + 0.00011687575170071796, + 0.00011450610327301547, + 0.00011203528265468776, + 0.00011082263517891988, + 0.00010936566832242534, + 0.00010819115414051339, + 0.00010705165914259851, + 0.00010176753858104348, + 9.876617696136236e-05, + 9.541869076201692e-05, + 9.238059283234179e-05, + 9.015990508487448e-05, + 9.102027252083644e-05, + 9.192643483402207e-05, + 8.731557318242267e-05, + 8.749608969083056e-05, + 7.95004380051978e-05, + 7.466241368092597e-05, + 6.735928764101118e-05, + 6.187916733324528e-05, + 5.64747242606245e-05, + 5.17304906679783e-05, + 4.670304406317882e-05, + 4.1035571484826505e-05, + 3.507353176246397e-05, + 2.8758000553352758e-05, + 2.3154751033871435e-05, + 1.9675369912874885e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011436203931225464, + "first_to_last_ratio": 3.02339634825199, + "imbalance_abs_log_ratio": 1.1063808181034647, + "last_quartile_mean": 3.78256854673964e-05, + "mean": 8.318008940477739e-05, + "normalized": [ + 1.3169814642434559, + 1.341416689333394, + 1.45415402374626, + 1.4255104165869963, + 1.4050928838507029, + 1.376604715051423, + 1.3469002432720767, + 1.3323216646188754, + 1.3148058520377592, + 1.300685715953312, + 1.286986584273256, + 1.223460317358093, + 1.1873776244786027, + 1.1471337845969738, + 1.1106094438392846, + 1.0839120963928204, + 1.0942555264386233, + 1.1051495074341955, + 1.0497172316985723, + 1.0518874206187772, + 0.955762834280288, + 0.8975995844101339, + 0.8098006160250947, + 0.743918019036071, + 0.6789452219244779, + 0.6219095343387212, + 0.5614690293960716, + 0.49333406321717244, + 0.4216577790844446, + 0.34573178219860223, + 0.2783689125554313, + 0.23653941771003728 + ], + "population_cv": 0.36398106608216985 + }, + "activation_output_rms_by_block": [ + 0.11107808351516724, + 0.1828140765428543, + 0.2578570544719696, + 0.27254435420036316, + 0.2737041115760803, + 0.2742544412612915, + 0.27536308765411377, + 0.2744285464286804, + 0.2742423117160797, + 0.27115195989608765, + 0.26583269238471985, + 0.255132257938385, + 0.23856717348098755, + 0.22743643820285797, + 0.22118839621543884, + 0.22011704742908478, + 0.22253327071666718, + 0.2256278693675995, + 0.22584785521030426, + 0.22925184667110443, + 0.23080435395240784, + 0.23398378491401672, + 0.24050307273864746, + 0.24922524392604828, + 0.2630935311317444, + 0.27895617485046387, + 0.2995731830596924, + 0.3278428912162781, + 0.36820048093795776, + 0.42407870292663574, + 0.5123529434204102, + 0.6704596281051636 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24025546945631504, + "first_to_last_ratio": 0.6112286812568156, + "imbalance_abs_log_ratio": 0.4922841160907453, + "last_quartile_mean": 0.39306969195604324, + "mean": 0.2780639645643532, + "normalized": [ + 0.39946953820209985, + 0.6574533195240588, + 0.9273299935716515, + 0.9801498537480837, + 0.9843206832100536, + 0.9862998310153919, + 0.9902868503134846, + 0.9869259645298935, + 0.9862562095945767, + 0.975142393301143, + 0.9560127390156568, + 0.9175308218672072, + 0.8579578941656613, + 0.817928488357655, + 0.7954586872196038, + 0.7916058011110692, + 0.8002952524442108, + 0.8114243415938267, + 0.8122154755440653, + 0.824457232458282, + 0.8300405063777765, + 0.8414746775282531, + 0.8649199586701105, + 0.896287457875072, + 0.9461619075450383, + 1.0032086512450777, + 1.077353491413524, + 1.179019696888499, + 1.3241574884211362, + 1.5251120496359385, + 1.842572244926166, + 2.411170498685733 + ], + "population_cv": 0.35267348500337414 + }, + "bits_per_byte": 1.9134731260894795, + "branch_output_rms_by_sublayer": [ + 0.024391189217567444, + 0.06930837035179138, + 0.10445628315210342, + 0.019052622839808464, + 0.09584712982177734, + 0.0026203703600913286, + 0.031347230076789856, + 0.0018832145724445581, + 0.020275620743632317, + 0.002480348339304328, + 0.017658807337284088, + 0.005030750762671232, + 0.020567260682582855, + 0.0064065041951835155, + 0.016946211457252502, + 0.006859246641397476, + 0.017460251227021217, + 0.009737920947372913, + 0.022223282605409622, + 0.009925507940351963, + 0.02365109510719776, + 0.012353206984698772, + 0.04038562253117561, + 0.01845649816095829, + 0.03250120207667351, + 0.023668669164180756, + 0.029116366058588028, + 0.023308098316192627, + 0.029388509690761566, + 0.023618459701538086, + 0.026225870475172997, + 0.024062534794211388, + 0.02648484893143177, + 0.02197168581187725, + 0.03580358996987343, + 0.021145176142454147, + 0.04329143092036247, + 0.025955181568861008, + 0.04141230136156082, + 0.02680986560881138, + 0.05275771766901016, + 0.035374872386455536, + 0.044423554092645645, + 0.038993529975414276, + 0.05167620629072189, + 0.05052104964852333, + 0.03969942033290863, + 0.05720135569572449, + 0.0389031358063221, + 0.06300201267004013, + 0.026857485994696617, + 0.06565802544355392, + 0.030672859400510788, + 0.0703272894024849, + 0.02773989364504814, + 0.07888873666524887, + 0.019402364268898964, + 0.09176591783761978, + 0.018159963190555573, + 0.10585808008909225, + 0.02488313429057598, + 0.1433597356081009, + 0.024982891976833344, + 0.2408120483160019 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00037295757895409754, + 0.00022454763424927058, + 0.00021936409959100382, + 0.00010566073392567048, + 9.515815728211845e-05, + 0.00010551160558417538, + 0.00011265453132986426, + 0.00010566938037237213, + 0.00012588745811984293, + 0.0001281493774985002, + 0.0001672472035341268, + 0.0002455532725992925, + 0.00022493633083964273, + 0.0002220131528127045, + 0.0002449305149794571, + 0.0002379224969510024, + 0.00024418235837095776, + 0.00027770634611442576, + 0.000335912973129177, + 0.00033241348484404164, + 0.00039774019686300875, + 0.00036753844348542204, + 0.0003869475308616486, + 0.00034605021914329437, + 0.00032032390763334626, + 0.00028824702486555996, + 0.00027123288540817104, + 0.0002526057589974623, + 0.00022790421762734515, + 0.00020876185354851858, + 0.00018495058171630387, + 0.00016871013519521616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016769046516107157, + "first_to_last_ratio": 0.6977158937201501, + "imbalance_abs_log_ratio": 0.359943288132801, + "last_quartile_mean": 0.00024034204562399042, + "mean": 0.00023591848270084503, + "normalized": [ + 1.5808747779504164, + 0.9518017905108639, + 0.9298300713012261, + 0.4478696739485771, + 0.40335185354164543, + 0.44723755585512437, + 0.47751464845047825, + 0.44790632409401143, + 0.5336057466912151, + 0.5431934625529073, + 0.7089194607367985, + 1.040839487386263, + 0.9534493790589178, + 0.9410587516122122, + 1.0381997721223044, + 1.0084945199172823, + 1.035028521612661, + 1.1771284001795213, + 1.4238518715599293, + 1.4090184077080408, + 1.6859221554394308, + 1.5579044052749225, + 1.640174718113608, + 1.466821145938367, + 1.357773685072106, + 1.2218077259807991, + 1.1496890040281678, + 1.0707332299935883, + 0.9660295158660277, + 0.8848897770050418, + 0.7839596948867686, + 0.7151204656107762 + ], + "population_cv": 0.3768493091252366 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060260046273469925, + 0.06848448514938354, + 0.11107808351516724, + 0.17387914657592773, + 0.1828140765428543, + 0.25734034180641174, + 0.2578570544719696, + 0.2722952663898468, + 0.27254435420036316, + 0.27370211482048035, + 0.2737041115760803, + 0.27443060278892517, + 0.2742544412612915, + 0.2751515209674835, + 0.27536308765411377, + 0.2749122083187103, + 0.2744285464286804, + 0.2747279405593872, + 0.2742423117160797, + 0.2733822464942932, + 0.27115195989608765, + 0.26904159784317017, + 0.26583269238471985, + 0.2588832378387451, + 0.255132257938385, + 0.24577246606349945, + 0.23856717348098755, + 0.2310730367898941, + 0.22743643820285797, + 0.22001685202121735, + 0.22118839621543884, + 0.21798942983150482, + 0.22011704742908478, + 0.21957813203334808, + 0.22253327071666718, + 0.22450163960456848, + 0.2256278693675995, + 0.22364993393421173, + 0.22584785521030426, + 0.22638610005378723, + 0.22925184667110443, + 0.22674965858459473, + 0.23080435395240784, + 0.2277289777994156, + 0.23398378491401672, + 0.2319381833076477, + 0.24050307273864746, + 0.23799650371074677, + 0.24922524392604828, + 0.24935781955718994, + 0.2630935311317444, + 0.2632073760032654, + 0.27895617485046387, + 0.2799976170063019, + 0.2995731830596924, + 0.30053073167800903, + 0.3278428912162781, + 0.32797256112098694, + 0.36820048093795776, + 0.3684364855289459, + 0.42407870292663574, + 0.4252319037914276, + 0.5123529434204102, + 0.5140777826309204 + ], + "loss_nats": 1.3263185024261475, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06848448514938354, + 0.11107808351516724, + 0.17387914657592773, + 0.1828140765428543, + 0.25734034180641174, + 0.2578570544719696, + 0.2722952663898468, + 0.27254435420036316, + 0.27370211482048035, + 0.2737041115760803, + 0.27443060278892517, + 0.2742544412612915, + 0.2751515209674835, + 0.27536308765411377, + 0.2749122083187103, + 0.2744285464286804, + 0.2747279405593872, + 0.2742423117160797, + 0.2733822464942932, + 0.27115195989608765, + 0.26904159784317017, + 0.26583269238471985, + 0.2588832378387451, + 0.255132257938385, + 0.24577246606349945, + 0.23856717348098755, + 0.2310730367898941, + 0.22743643820285797, + 0.22001685202121735, + 0.22118839621543884, + 0.21798942983150482, + 0.22011704742908478, + 0.21957813203334808, + 0.22253327071666718, + 0.22450163960456848, + 0.2256278693675995, + 0.22364993393421173, + 0.22584785521030426, + 0.22638610005378723, + 0.22925184667110443, + 0.22674965858459473, + 0.23080435395240784, + 0.2277289777994156, + 0.23398378491401672, + 0.2319381833076477, + 0.24050307273864746, + 0.23799650371074677, + 0.24922524392604828, + 0.24935781955718994, + 0.2630935311317444, + 0.2632073760032654, + 0.27895617485046387, + 0.2799976170063019, + 0.2995731830596924, + 0.30053073167800903, + 0.3278428912162781, + 0.32797256112098694, + 0.36820048093795776, + 0.3684364855289459, + 0.42407870292663574, + 0.4252319037914276, + 0.5123529434204102, + 0.5140777826309204, + 0.6704596281051636 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018927091150544584, + 0.0001941375230671838, + 0.00020724961359519511, + 0.00020481550018303096, + 0.00020258830045349896, + 0.00020023793331347406, + 0.00019735096429940313, + 0.0001957487838808447, + 0.00019380047160666436, + 0.00019302338478155434, + 0.0001922047813422978, + 0.00018312338215764612, + 0.00017868881695903838, + 0.00017335887241642922, + 0.00016814567788969725, + 0.0001643431605771184, + 0.00016659151879139245, + 0.0001678116386756301, + 0.0001584039127919823, + 0.0001587578735779971, + 0.00014523210120387375, + 0.00013619904348161072, + 0.00012270425213500857, + 0.00011189868382643908, + 0.00010116446355823427, + 9.155859152087942e-05, + 8.147361222654581e-05, + 7.049509440548718e-05, + 5.894774221815169e-05, + 4.627152156899683e-05, + 3.385082527529448e-05, + 2.4755347112659365e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019892494128725957, + "first_to_last_ratio": 3.129490087873211, + "imbalance_abs_log_ratio": 1.140870080057971, + "last_quartile_mean": 6.356464973578113e-05, + "mean": 0.00014731888438745955, + "normalized": [ + 1.2847701928535473, + 1.3178047327360134, + 1.4068095509745602, + 1.3902867988353147, + 1.3751685759490058, + 1.3592142931712237, + 1.3396175590113453, + 1.3287419647165597, + 1.3155168287655152, + 1.310241966494184, + 1.304685289611514, + 1.2430407881451104, + 1.212938977253409, + 1.1767593349436627, + 1.141372191276318, + 1.1155607189155992, + 1.1308225655119981, + 1.1391047344227307, + 1.0752451286242999, + 1.077647813028859, + 0.9858349240678649, + 0.9245185642554635, + 0.8329159743857911, + 0.759567819778877, + 0.6867039753855606, + 0.6214993542855888, + 0.553042555034996, + 0.4785204198266929, + 0.40013703920751104, + 0.314090903969238, + 0.2297792670372409, + 0.16803919752440544 + ], + "population_cv": 0.3716245052237534 + }, + "activation_output_rms_by_block": [ + 0.10094190388917923, + 0.17939583957195282, + 0.23316815495491028, + 0.2387540638446808, + 0.23957562446594238, + 0.2399372160434723, + 0.24013783037662506, + 0.23953619599342346, + 0.2386971116065979, + 0.235844686627388, + 0.23058047890663147, + 0.2186841070652008, + 0.2011982500553131, + 0.1896955370903015, + 0.18346232175827026, + 0.1819538027048111, + 0.18335868418216705, + 0.18572981655597687, + 0.1859624832868576, + 0.18881060183048248, + 0.18956796824932098, + 0.19203881919384003, + 0.19704653322696686, + 0.20452122390270233, + 0.21571911871433258, + 0.22943545877933502, + 0.2464856505393982, + 0.26912161707878113, + 0.30166757106781006, + 0.3485357463359833, + 0.4315659701824188, + 0.6013737916946411 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2139308536425233, + "first_to_last_ratio": 0.6473178416327895, + "imbalance_abs_log_ratio": 0.4349178506515776, + "last_quartile_mean": 0.3304881155490875, + "mean": 0.2363282556179911, + "normalized": [ + 0.42712583658360814, + 0.7590960255803447, + 0.9866283417748029, + 1.0102645712860119, + 1.0137409250513, + 1.015270964599827, + 1.0161198446147375, + 1.0135740873093813, + 1.0100235834365736, + 0.9979538249062154, + 0.9756788425644265, + 0.9253405035862031, + 0.8513508024217668, + 0.8026781926446058, + 0.7763029489576774, + 0.7699197974825631, + 0.7758644166466246, + 0.7858976323855102, + 0.7868821390001439, + 0.7989336752676848, + 0.8021383975166516, + 0.8125935626768981, + 0.8337832169568393, + 0.8654116426657729, + 0.9127944441100949, + 0.9708338013978413, + 1.0429800274827308, + 1.138761915603517, + 1.2764769505828182, + 1.4747950702067896, + 1.8261293769290816, + 2.5446546377709565 + ], + "population_cv": 0.36483989965797836 + }, + "bits_per_byte": 1.8402207275901343, + "branch_output_rms_by_sublayer": [ + 0.033478349447250366, + 0.052373696118593216, + 0.10464497655630112, + 0.018782487139105797, + 0.07005563378334045, + 0.002347440691664815, + 0.020065777003765106, + 0.0017286264337599277, + 0.017071161419153214, + 0.002082002116367221, + 0.015527697280049324, + 0.004463158082216978, + 0.01603785529732704, + 0.005855648312717676, + 0.013071421533823013, + 0.006003717426210642, + 0.014386727474629879, + 0.008227710612118244, + 0.014956407248973846, + 0.008421593345701694, + 0.015932755544781685, + 0.010948682203888893, + 0.037222135812044144, + 0.016641829162836075, + 0.029781410470604897, + 0.021082323044538498, + 0.025465231388807297, + 0.02131747640669346, + 0.024801449850201607, + 0.021334858611226082, + 0.024093298241496086, + 0.021644918248057365, + 0.023488737642765045, + 0.020262837409973145, + 0.031102469190955162, + 0.020060395821928978, + 0.03728024289011955, + 0.024732086807489395, + 0.03451274335384369, + 0.025743624195456505, + 0.0439009852707386, + 0.03343674913048744, + 0.0384296216070652, + 0.037262238562107086, + 0.044400904327631, + 0.047452766448259354, + 0.03565425053238869, + 0.054450251162052155, + 0.032797131687402725, + 0.05981304869055748, + 0.022211335599422455, + 0.0634065493941307, + 0.02717488259077072, + 0.068015918135643, + 0.023526398465037346, + 0.07457657158374786, + 0.01840120367705822, + 0.08500036597251892, + 0.017178243026137352, + 0.09797899425029755, + 0.022575926035642624, + 0.13843585550785065, + 0.029057970270514488, + 0.24571193754673004 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004888726102119165, + 0.00034371112095624357, + 0.00027329239576199496, + 0.00013109360072049493, + 0.0001309003833444521, + 0.00015153657998531148, + 0.00016751171475037753, + 0.00015920301567702626, + 0.00019282855273444342, + 0.00018189099839941241, + 0.00022228035276414997, + 0.00039117902483509027, + 0.00035836801505216564, + 0.00037533451853890784, + 0.0004202645380768536, + 0.0004297743688669142, + 0.0004147505595850994, + 0.0004900318658719625, + 0.0005824864404877667, + 0.0005699482810090311, + 0.0006776225188364896, + 0.0006405760434101032, + 0.0006565350519436726, + 0.0006182198906110644, + 0.0005596621809681081, + 0.0005013865499612654, + 0.000471541721969693, + 0.0004314515507572245, + 0.0003811454676239625, + 0.0003426357383233983, + 0.00029149764497300414, + 0.0002412795793653285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023076517767597717, + "first_to_last_ratio": 0.5732227450358324, + "imbalance_abs_log_ratio": 0.5564809030131627, + "last_quartile_mean": 0.00040257505424274806, + "mean": 0.000384025402386654, + "normalized": [ + 1.2730215427772602, + 0.8950218366288691, + 0.7116518700677824, + 0.3413670030830513, + 0.34086386611648084, + 0.3946004067531388, + 0.4361995683340739, + 0.4145637624167724, + 0.5021244728500928, + 0.4736431425342962, + 0.5788167954065396, + 1.0186279928462485, + 0.9331883068801387, + 0.9773689870676945, + 1.0943665066554955, + 1.1191300528453012, + 1.0800081375020862, + 1.2760402380324216, + 1.5167914332429844, + 1.4841421360850018, + 1.7645252491766907, + 1.6680564343635333, + 1.7096136033278435, + 1.6098411375105153, + 1.4573571891075452, + 1.305607771895378, + 1.227892006724398, + 1.1234974251073624, + 0.9925006659851322, + 0.8922215462674452, + 0.7590582371931511, + 0.6282906752152749 + ], + "population_cv": 0.4280332727708177 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.061826109886169434, + 0.07441755384206772, + 0.10094190388917923, + 0.17219661176204681, + 0.17939583957195282, + 0.2327817976474762, + 0.23316815495491028, + 0.2385595589876175, + 0.2387540638446808, + 0.23968438804149628, + 0.23957562446594238, + 0.24012914299964905, + 0.2399372160434723, + 0.2401275336742401, + 0.24013783037662506, + 0.24013836681842804, + 0.23953619599342346, + 0.239403635263443, + 0.2386971116065979, + 0.23807503283023834, + 0.235844686627388, + 0.23385408520698547, + 0.23058047890663147, + 0.2232486605644226, + 0.2186841070652008, + 0.20881249010562897, + 0.2011982500553131, + 0.19461779296398163, + 0.1896955370903015, + 0.18348202109336853, + 0.18346232175827026, + 0.18089424073696136, + 0.1819538027048111, + 0.1813284158706665, + 0.18335868418216705, + 0.18508672714233398, + 0.18572981655597687, + 0.1841643899679184, + 0.1859624832868576, + 0.18672171235084534, + 0.18881060183048248, + 0.18648506700992584, + 0.18956796824932098, + 0.1869000494480133, + 0.19203881919384003, + 0.18984925746917725, + 0.19704653322696686, + 0.19547933340072632, + 0.20452122390270233, + 0.20447643101215363, + 0.21571911871433258, + 0.21594053506851196, + 0.22943545877933502, + 0.23029562830924988, + 0.2464856505393982, + 0.24702563881874084, + 0.26912161707878113, + 0.2691715657711029, + 0.30166757106781006, + 0.30165502429008484, + 0.3485357463359833, + 0.3492375612258911, + 0.4315659701824188, + 0.4323083460330963 + ], + "loss_nats": 1.2755438089370728, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07441755384206772, + 0.10094190388917923, + 0.17219661176204681, + 0.17939583957195282, + 0.2327817976474762, + 0.23316815495491028, + 0.2385595589876175, + 0.2387540638446808, + 0.23968438804149628, + 0.23957562446594238, + 0.24012914299964905, + 0.2399372160434723, + 0.2401275336742401, + 0.24013783037662506, + 0.24013836681842804, + 0.23953619599342346, + 0.239403635263443, + 0.2386971116065979, + 0.23807503283023834, + 0.235844686627388, + 0.23385408520698547, + 0.23058047890663147, + 0.2232486605644226, + 0.2186841070652008, + 0.20881249010562897, + 0.2011982500553131, + 0.19461779296398163, + 0.1896955370903015, + 0.18348202109336853, + 0.18346232175827026, + 0.18089424073696136, + 0.1819538027048111, + 0.1813284158706665, + 0.18335868418216705, + 0.18508672714233398, + 0.18572981655597687, + 0.1841643899679184, + 0.1859624832868576, + 0.18672171235084534, + 0.18881060183048248, + 0.18648506700992584, + 0.18956796824932098, + 0.1869000494480133, + 0.19203881919384003, + 0.18984925746917725, + 0.19704653322696686, + 0.19547933340072632, + 0.20452122390270233, + 0.20447643101215363, + 0.21571911871433258, + 0.21594053506851196, + 0.22943545877933502, + 0.23029562830924988, + 0.2464856505393982, + 0.24702563881874084, + 0.26912161707878113, + 0.2691715657711029, + 0.30166757106781006, + 0.30165502429008484, + 0.3485357463359833, + 0.3492375612258911, + 0.4315659701824188, + 0.4323083460330963, + 0.6013737916946411 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.037726566232575, + "cross_entropy_nats": 5.57132750749588, + "step": 0 + }, + { + "bits_per_byte": 5.132850850111892, + "cross_entropy_nats": 3.5578210949897766, + "step": 100 + }, + { + "bits_per_byte": 3.1258965111502706, + "cross_entropy_nats": 2.1667063534259796, + "step": 500 + }, + { + "bits_per_byte": 1.9712387018569424, + "cross_entropy_nats": 1.3663585484027863, + "step": 2000 + }, + { + "bits_per_byte": 1.7961544728728112, + "cross_entropy_nats": 1.2449994087219238, + "step": 4000 + }, + { + "bits_per_byte": 1.7269767303333703, + "cross_entropy_nats": 1.1970490515232086, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "final_optimizer_state": "4b960a87623e39ce5348146d7169b13bd4e5894a99361177367c7518bf6f603e", + "final_public_parameters": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 41.92772524377444, + "measured_steps": 7980, + "median_ms": 41.88774450449273, + "p95_ms": 42.070325661916286, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.02169460149312, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.560214996337891, + "step": 1, + "unclipped_grad_norm": 27.98912239074707 + }, + { + "bits_per_byte": 7.204295458472364, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 4.9936370849609375, + "step": 10, + "unclipped_grad_norm": 12.742182731628418 + }, + { + "bits_per_byte": 6.681083646642096, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.630974292755127, + "step": 20, + "unclipped_grad_norm": 3.4196884632110596 + }, + { + "bits_per_byte": 6.539581824957005, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.53289270401001, + "step": 30, + "unclipped_grad_norm": 3.0530965328216553 + }, + { + "bits_per_byte": 6.404133792582285, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43900728225708, + "step": 40, + "unclipped_grad_norm": 3.002453088760376 + }, + { + "bits_per_byte": 6.077271133589068, + "learning_rate": 3.75e-05, + "loss_nats": 4.2124433517456055, + "step": 50, + "unclipped_grad_norm": 2.940347909927368 + }, + { + "bits_per_byte": 5.8763623767123425, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.073184013366699, + "step": 60, + "unclipped_grad_norm": 2.7971668243408203 + }, + { + "bits_per_byte": 5.7093882046079605, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.957446336746216, + "step": 70, + "unclipped_grad_norm": 2.5766468048095703 + }, + { + "bits_per_byte": 5.512541333979312, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.82100248336792, + "step": 80, + "unclipped_grad_norm": 2.5652287006378174 + }, + { + "bits_per_byte": 5.341233887003791, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.702261209487915, + "step": 90, + "unclipped_grad_norm": 2.3708243370056152 + }, + { + "bits_per_byte": 5.171287854692499, + "learning_rate": 7.5e-05, + "loss_nats": 3.584463596343994, + "step": 100, + "unclipped_grad_norm": 2.273510217666626 + }, + { + "bits_per_byte": 5.001223154352113, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.466583728790283, + "step": 110, + "unclipped_grad_norm": 2.179899215698242 + }, + { + "bits_per_byte": 4.73609126012899, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.282808303833008, + "step": 120, + "unclipped_grad_norm": 1.9098186492919922 + }, + { + "bits_per_byte": 4.5869558165027025, + "learning_rate": 9.75e-05, + "loss_nats": 3.1794354915618896, + "step": 130, + "unclipped_grad_norm": 1.6915351152420044 + }, + { + "bits_per_byte": 4.392154507523416, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0444095134735107, + "step": 140, + "unclipped_grad_norm": 1.478805422782898 + }, + { + "bits_per_byte": 4.261904134823071, + "learning_rate": 0.0001125, + "loss_nats": 2.9541268348693848, + "step": 150, + "unclipped_grad_norm": 1.2380529642105103 + }, + { + "bits_per_byte": 4.155084334510974, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.880084991455078, + "step": 160, + "unclipped_grad_norm": 1.0523717403411865 + }, + { + "bits_per_byte": 4.00081108117281, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.77315092086792, + "step": 170, + "unclipped_grad_norm": 0.9717193841934204 + }, + { + "bits_per_byte": 3.86890313968345, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.6817193031311035, + "step": 180, + "unclipped_grad_norm": 1.2074095010757446 + }, + { + "bits_per_byte": 3.870022058809963, + "learning_rate": 0.0001425, + "loss_nats": 2.682494878768921, + "step": 190, + "unclipped_grad_norm": 0.712932288646698 + }, + { + "bits_per_byte": 3.7502069054333704, + "learning_rate": 0.00015, + "loss_nats": 2.599445343017578, + "step": 200, + "unclipped_grad_norm": 0.6916731595993042 + }, + { + "bits_per_byte": 3.7020534830182927, + "learning_rate": 0.0001575, + "loss_nats": 2.566067934036255, + "step": 210, + "unclipped_grad_norm": 0.8496056795120239 + }, + { + "bits_per_byte": 3.664806168425029, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.540250062942505, + "step": 220, + "unclipped_grad_norm": 0.7601449489593506 + }, + { + "bits_per_byte": 3.680203087226173, + "learning_rate": 0.0001725, + "loss_nats": 2.550922393798828, + "step": 230, + "unclipped_grad_norm": 0.7172863483428955 + }, + { + "bits_per_byte": 3.6667052008558563, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5415663719177246, + "step": 240, + "unclipped_grad_norm": 0.8026229739189148 + }, + { + "bits_per_byte": 3.532286312663899, + "learning_rate": 0.0001875, + "loss_nats": 2.448394298553467, + "step": 250, + "unclipped_grad_norm": 0.9514944553375244 + }, + { + "bits_per_byte": 3.530420644867339, + "learning_rate": 0.000195, + "loss_nats": 2.44710111618042, + "step": 260, + "unclipped_grad_norm": 0.7874091267585754 + }, + { + "bits_per_byte": 3.4939940314839926, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4218521118164062, + "step": 270, + "unclipped_grad_norm": 0.9165833592414856 + }, + { + "bits_per_byte": 3.5060328170443893, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.430196762084961, + "step": 280, + "unclipped_grad_norm": 0.971961259841919 + }, + { + "bits_per_byte": 3.3964946589527543, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3542706966400146, + "step": 290, + "unclipped_grad_norm": 0.7937634587287903 + }, + { + "bits_per_byte": 3.488020042123622, + "learning_rate": 0.000225, + "loss_nats": 2.4177112579345703, + "step": 300, + "unclipped_grad_norm": 1.0019733905792236 + }, + { + "bits_per_byte": 3.3886120061331084, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.348806858062744, + "step": 310, + "unclipped_grad_norm": 0.8179588913917542 + }, + { + "bits_per_byte": 3.3666969448642643, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3336164951324463, + "step": 320, + "unclipped_grad_norm": 0.8202242255210876 + }, + { + "bits_per_byte": 3.351965942887261, + "learning_rate": 0.0002475, + "loss_nats": 2.3234057426452637, + "step": 330, + "unclipped_grad_norm": 0.8768095374107361 + }, + { + "bits_per_byte": 3.357094121570688, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.326960325241089, + "step": 340, + "unclipped_grad_norm": 0.9328333139419556 + }, + { + "bits_per_byte": 3.3293237389705688, + "learning_rate": 0.0002625, + "loss_nats": 2.307711362838745, + "step": 350, + "unclipped_grad_norm": 0.7516238689422607 + }, + { + "bits_per_byte": 3.3205963073698848, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.301661968231201, + "step": 360, + "unclipped_grad_norm": 0.9922879338264465 + }, + { + "bits_per_byte": 3.3203087523776422, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3014626502990723, + "step": 370, + "unclipped_grad_norm": 0.7593456506729126 + }, + { + "bits_per_byte": 3.3612334000116544, + "learning_rate": 0.000285, + "loss_nats": 2.329829454421997, + "step": 380, + "unclipped_grad_norm": 0.8890619874000549 + }, + { + "bits_per_byte": 3.310503677486652, + "learning_rate": 0.0002925, + "loss_nats": 2.294666290283203, + "step": 390, + "unclipped_grad_norm": 0.8439806699752808 + }, + { + "bits_per_byte": 3.2443041155165395, + "learning_rate": 0.0003, + "loss_nats": 2.2487802505493164, + "step": 400, + "unclipped_grad_norm": 1.1809823513031006 + }, + { + "bits_per_byte": 3.208035726171987, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2236409187316895, + "step": 410, + "unclipped_grad_norm": 0.9897300601005554 + }, + { + "bits_per_byte": 3.232116049015194, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2403321266174316, + "step": 420, + "unclipped_grad_norm": 0.898377537727356 + }, + { + "bits_per_byte": 3.2113598068478635, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.225944995880127, + "step": 430, + "unclipped_grad_norm": 1.0354490280151367 + }, + { + "bits_per_byte": 3.2315632967753207, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2399489879608154, + "step": 440, + "unclipped_grad_norm": 0.9574596285820007 + }, + { + "bits_per_byte": 3.1852710706078806, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.2078616619110107, + "step": 450, + "unclipped_grad_norm": 0.916765570640564 + }, + { + "bits_per_byte": 3.155730642565081, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.1873857975006104, + "step": 460, + "unclipped_grad_norm": 0.9241403937339783 + }, + { + "bits_per_byte": 3.18185446326584, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.205493450164795, + "step": 470, + "unclipped_grad_norm": 0.9457137584686279 + }, + { + "bits_per_byte": 3.13667255709237, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.17417573928833, + "step": 480, + "unclipped_grad_norm": 0.7014725208282471 + }, + { + "bits_per_byte": 3.105025341576501, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1522395610809326, + "step": 490, + "unclipped_grad_norm": 0.840235710144043 + }, + { + "bits_per_byte": 3.1575567543519423, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1886515617370605, + "step": 500, + "unclipped_grad_norm": 1.1171972751617432 + }, + { + "bits_per_byte": 3.1160109053830136, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1598541736602783, + "step": 510, + "unclipped_grad_norm": 0.8018220067024231 + }, + { + "bits_per_byte": 3.0148496782496044, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0897345542907715, + "step": 520, + "unclipped_grad_norm": 1.302595853805542 + }, + { + "bits_per_byte": 3.0793170309602265, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1344199180603027, + "step": 530, + "unclipped_grad_norm": 0.6869481205940247 + }, + { + "bits_per_byte": 3.040265962324714, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.107351779937744, + "step": 540, + "unclipped_grad_norm": 1.3514107465744019 + }, + { + "bits_per_byte": 2.996858229177544, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.077263832092285, + "step": 550, + "unclipped_grad_norm": 0.9677014350891113 + }, + { + "bits_per_byte": 2.9987245849047075, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0785574913024902, + "step": 560, + "unclipped_grad_norm": 1.1580114364624023 + }, + { + "bits_per_byte": 2.9750642436949075, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.062157392501831, + "step": 570, + "unclipped_grad_norm": 1.1581705808639526 + }, + { + "bits_per_byte": 2.870633970333898, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.989771842956543, + "step": 580, + "unclipped_grad_norm": 0.9525179862976074 + }, + { + "bits_per_byte": 2.9300787417087224, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.030975818634033, + "step": 590, + "unclipped_grad_norm": 0.8967422842979431 + }, + { + "bits_per_byte": 2.847721925638002, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9738904237747192, + "step": 600, + "unclipped_grad_norm": 0.7550538182258606 + }, + { + "bits_per_byte": 2.905717055248704, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.014089584350586, + "step": 610, + "unclipped_grad_norm": 0.6918516755104065 + }, + { + "bits_per_byte": 2.79248952533076, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9356062412261963, + "step": 620, + "unclipped_grad_norm": 0.7874937653541565 + }, + { + "bits_per_byte": 2.8091388217954867, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9471466541290283, + "step": 630, + "unclipped_grad_norm": 0.9318027496337891 + }, + { + "bits_per_byte": 2.8177121569409977, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9530892372131348, + "step": 640, + "unclipped_grad_norm": 0.8821531534194946 + }, + { + "bits_per_byte": 2.7350290897815905, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.895777702331543, + "step": 650, + "unclipped_grad_norm": 1.1635478734970093 + }, + { + "bits_per_byte": 2.7242683072997553, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8883188962936401, + "step": 660, + "unclipped_grad_norm": 0.8677663207054138 + }, + { + "bits_per_byte": 2.7265658235326318, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.889911413192749, + "step": 670, + "unclipped_grad_norm": 0.8130120635032654 + }, + { + "bits_per_byte": 2.7610239234942022, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.9137959480285645, + "step": 680, + "unclipped_grad_norm": 0.9226085543632507 + }, + { + "bits_per_byte": 2.640049951016096, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8299431800842285, + "step": 690, + "unclipped_grad_norm": 0.8291181921958923 + }, + { + "bits_per_byte": 2.583675413924474, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7908673286437988, + "step": 700, + "unclipped_grad_norm": 0.9929485321044922 + }, + { + "bits_per_byte": 2.6777724536736933, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8560904264450073, + "step": 710, + "unclipped_grad_norm": 1.0362995862960815 + }, + { + "bits_per_byte": 2.7123457820113424, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8800548315048218, + "step": 720, + "unclipped_grad_norm": 0.9176086187362671 + }, + { + "bits_per_byte": 2.545722970462673, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.7645606994628906, + "step": 730, + "unclipped_grad_norm": 1.4210379123687744 + }, + { + "bits_per_byte": 2.557563288043974, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7727677822113037, + "step": 740, + "unclipped_grad_norm": 1.1384987831115723 + }, + { + "bits_per_byte": 2.56630310239552, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7788257598876953, + "step": 750, + "unclipped_grad_norm": 0.9141139984130859 + }, + { + "bits_per_byte": 2.5326694872621855, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7555127143859863, + "step": 760, + "unclipped_grad_norm": 0.9497804045677185 + }, + { + "bits_per_byte": 2.5270525338850063, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7516193389892578, + "step": 770, + "unclipped_grad_norm": 1.0209811925888062 + }, + { + "bits_per_byte": 2.4536180057583974, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7007184028625488, + "step": 780, + "unclipped_grad_norm": 0.8395339250564575 + }, + { + "bits_per_byte": 2.5403350979764427, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7608261108398438, + "step": 790, + "unclipped_grad_norm": 1.0305956602096558 + }, + { + "bits_per_byte": 2.476898265309791, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.7168550491333008, + "step": 800, + "unclipped_grad_norm": 0.7494933605194092 + }, + { + "bits_per_byte": 2.5328731147208075, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7556538581848145, + "step": 810, + "unclipped_grad_norm": 2.0184309482574463 + }, + { + "bits_per_byte": 2.4645244575455134, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7082781791687012, + "step": 820, + "unclipped_grad_norm": 1.0523004531860352 + }, + { + "bits_per_byte": 2.4679443325579205, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7106486558914185, + "step": 830, + "unclipped_grad_norm": 0.9168965816497803 + }, + { + "bits_per_byte": 2.457752984633092, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7035845518112183, + "step": 840, + "unclipped_grad_norm": 1.0069817304611206 + }, + { + "bits_per_byte": 2.4003085434704525, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6637670993804932, + "step": 850, + "unclipped_grad_norm": 0.9418690204620361 + }, + { + "bits_per_byte": 2.440524794565561, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6916428804397583, + "step": 860, + "unclipped_grad_norm": 1.019945502281189 + }, + { + "bits_per_byte": 2.404373525406246, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6665847301483154, + "step": 870, + "unclipped_grad_norm": 1.4541757106781006 + }, + { + "bits_per_byte": 2.4247352393724975, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6806983947753906, + "step": 880, + "unclipped_grad_norm": 0.9109940528869629 + }, + { + "bits_per_byte": 2.4066381929528076, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6681544780731201, + "step": 890, + "unclipped_grad_norm": 0.9647485613822937 + }, + { + "bits_per_byte": 2.3372799976169487, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6200790405273438, + "step": 900, + "unclipped_grad_norm": 0.8496110439300537 + }, + { + "bits_per_byte": 2.3871222896634996, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6546270847320557, + "step": 910, + "unclipped_grad_norm": 1.1169326305389404 + }, + { + "bits_per_byte": 2.3333503660273847, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.617355227470398, + "step": 920, + "unclipped_grad_norm": 0.8540288805961609 + }, + { + "bits_per_byte": 2.3443775496354062, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.624998688697815, + "step": 930, + "unclipped_grad_norm": 1.0937522649765015 + }, + { + "bits_per_byte": 2.285418285283967, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5841312408447266, + "step": 940, + "unclipped_grad_norm": 0.9093461632728577 + }, + { + "bits_per_byte": 2.261280348252721, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5674000978469849, + "step": 950, + "unclipped_grad_norm": 0.8109434843063354 + }, + { + "bits_per_byte": 2.2762792991997682, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5777965784072876, + "step": 960, + "unclipped_grad_norm": 0.8615142107009888 + }, + { + "bits_per_byte": 2.320710157136921, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.6085937023162842, + "step": 970, + "unclipped_grad_norm": 1.2632875442504883 + }, + { + "bits_per_byte": 2.270013111315582, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5734531879425049, + "step": 980, + "unclipped_grad_norm": 0.9684879183769226 + }, + { + "bits_per_byte": 2.2007698003905, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5254573822021484, + "step": 990, + "unclipped_grad_norm": 0.8230604529380798 + }, + { + "bits_per_byte": 2.245051377386703, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.556151032447815, + "step": 1000, + "unclipped_grad_norm": 0.7837699055671692 + }, + { + "bits_per_byte": 2.2494322914521314, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.559187650680542, + "step": 1010, + "unclipped_grad_norm": 0.8822826743125916 + }, + { + "bits_per_byte": 2.262538057378481, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5682718753814697, + "step": 1020, + "unclipped_grad_norm": 0.9820390939712524 + }, + { + "bits_per_byte": 2.1631948588473424, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4994124174118042, + "step": 1030, + "unclipped_grad_norm": 0.7792394161224365 + }, + { + "bits_per_byte": 2.258897528625017, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5657484531402588, + "step": 1040, + "unclipped_grad_norm": 0.8651536703109741 + }, + { + "bits_per_byte": 2.2449380408197843, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.556072473526001, + "step": 1050, + "unclipped_grad_norm": 0.8636346459388733 + }, + { + "bits_per_byte": 2.190915194496061, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5186266899108887, + "step": 1060, + "unclipped_grad_norm": 0.9602811932563782 + }, + { + "bits_per_byte": 2.2937991718431627, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5899404287338257, + "step": 1070, + "unclipped_grad_norm": 0.82843017578125 + }, + { + "bits_per_byte": 2.212529630091197, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5336086750030518, + "step": 1080, + "unclipped_grad_norm": 0.9238869547843933 + }, + { + "bits_per_byte": 2.140786207370522, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.4838799238204956, + "step": 1090, + "unclipped_grad_norm": 0.8506197333335876 + }, + { + "bits_per_byte": 2.2365177702335415, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5502359867095947, + "step": 1100, + "unclipped_grad_norm": 0.905985951423645 + }, + { + "bits_per_byte": 2.2504714106286445, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5599079132080078, + "step": 1110, + "unclipped_grad_norm": 0.8354071378707886 + }, + { + "bits_per_byte": 2.1399247462723503, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4832828044891357, + "step": 1120, + "unclipped_grad_norm": 0.8565837144851685 + }, + { + "bits_per_byte": 2.1407069233684743, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4838249683380127, + "step": 1130, + "unclipped_grad_norm": 0.7768547534942627 + }, + { + "bits_per_byte": 2.1434902905900377, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4857542514801025, + "step": 1140, + "unclipped_grad_norm": 0.8102613687515259 + }, + { + "bits_per_byte": 2.1476347285105297, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4886269569396973, + "step": 1150, + "unclipped_grad_norm": 0.7917956113815308 + }, + { + "bits_per_byte": 2.0881920209275155, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4474244117736816, + "step": 1160, + "unclipped_grad_norm": 0.8217911720275879 + }, + { + "bits_per_byte": 2.1987029128924305, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5240247249603271, + "step": 1170, + "unclipped_grad_norm": 0.7990338802337646 + }, + { + "bits_per_byte": 2.091903062567833, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4499967098236084, + "step": 1180, + "unclipped_grad_norm": 0.8429117202758789 + }, + { + "bits_per_byte": 2.129434492500322, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4760115146636963, + "step": 1190, + "unclipped_grad_norm": 0.878530740737915 + }, + { + "bits_per_byte": 2.113836010032367, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4651994705200195, + "step": 1200, + "unclipped_grad_norm": 0.7968210577964783 + }, + { + "bits_per_byte": 2.2311123555169234, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5464892387390137, + "step": 1210, + "unclipped_grad_norm": 0.8802653551101685 + }, + { + "bits_per_byte": 2.085374257175779, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4454712867736816, + "step": 1220, + "unclipped_grad_norm": 0.8048909306526184 + }, + { + "bits_per_byte": 2.0275156817898625, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4053667783737183, + "step": 1230, + "unclipped_grad_norm": 0.7767927646636963 + }, + { + "bits_per_byte": 2.034634215691725, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4103009700775146, + "step": 1240, + "unclipped_grad_norm": 0.769820511341095 + }, + { + "bits_per_byte": 2.2082001388383765, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5306077003479004, + "step": 1250, + "unclipped_grad_norm": 0.8053375482559204 + }, + { + "bits_per_byte": 2.1275602255712194, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4747123718261719, + "step": 1260, + "unclipped_grad_norm": 0.7535898685455322 + }, + { + "bits_per_byte": 1.9931237951488738, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3815281391143799, + "step": 1270, + "unclipped_grad_norm": 0.7098326683044434 + }, + { + "bits_per_byte": 2.1454299109264685, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4870986938476562, + "step": 1280, + "unclipped_grad_norm": 0.7225908637046814 + }, + { + "bits_per_byte": 2.0451564582368134, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4175944328308105, + "step": 1290, + "unclipped_grad_norm": 0.7847483158111572 + }, + { + "bits_per_byte": 2.0739233083160316, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4375340938568115, + "step": 1300, + "unclipped_grad_norm": 0.7665325403213501 + }, + { + "bits_per_byte": 2.153918458625104, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4929825067520142, + "step": 1310, + "unclipped_grad_norm": 0.8358681797981262 + }, + { + "bits_per_byte": 2.0810513012636918, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4424748420715332, + "step": 1320, + "unclipped_grad_norm": 0.7432150840759277 + }, + { + "bits_per_byte": 1.9944121171864873, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3824211359024048, + "step": 1330, + "unclipped_grad_norm": 0.9147225022315979 + }, + { + "bits_per_byte": 2.0106892431947467, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.393703579902649, + "step": 1340, + "unclipped_grad_norm": 0.7953330278396606 + }, + { + "bits_per_byte": 2.178856458948164, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.510268211364746, + "step": 1350, + "unclipped_grad_norm": 0.9230644702911377 + }, + { + "bits_per_byte": 2.000966204028218, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3869640827178955, + "step": 1360, + "unclipped_grad_norm": 0.766094446182251 + }, + { + "bits_per_byte": 2.099968876910648, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4555875062942505, + "step": 1370, + "unclipped_grad_norm": 0.802431583404541 + }, + { + "bits_per_byte": 2.0735674762113967, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.437287449836731, + "step": 1380, + "unclipped_grad_norm": 0.7674750685691833 + }, + { + "bits_per_byte": 2.0708314042188203, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4353909492492676, + "step": 1390, + "unclipped_grad_norm": 0.7390276789665222 + }, + { + "bits_per_byte": 2.082916281129648, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4437675476074219, + "step": 1400, + "unclipped_grad_norm": 0.8315935730934143 + }, + { + "bits_per_byte": 2.026164414102033, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4044301509857178, + "step": 1410, + "unclipped_grad_norm": 0.8864192366600037 + }, + { + "bits_per_byte": 2.0547052789780182, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.424213171005249, + "step": 1420, + "unclipped_grad_norm": 0.6849789023399353 + }, + { + "bits_per_byte": 2.0500001776157135, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4209518432617188, + "step": 1430, + "unclipped_grad_norm": 0.6987867951393127 + }, + { + "bits_per_byte": 2.022067959341132, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4015907049179077, + "step": 1440, + "unclipped_grad_norm": 0.7956437468528748 + }, + { + "bits_per_byte": 2.068799257216225, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4339823722839355, + "step": 1450, + "unclipped_grad_norm": 0.9165704250335693 + }, + { + "bits_per_byte": 2.0134160281241766, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3955936431884766, + "step": 1460, + "unclipped_grad_norm": 0.7257574200630188 + }, + { + "bits_per_byte": 2.0111035494006755, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3939907550811768, + "step": 1470, + "unclipped_grad_norm": 0.77181476354599 + }, + { + "bits_per_byte": 1.915849582359102, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3279657363891602, + "step": 1480, + "unclipped_grad_norm": 0.6826913356781006 + }, + { + "bits_per_byte": 2.0189380470780796, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.399421215057373, + "step": 1490, + "unclipped_grad_norm": 0.850294291973114 + }, + { + "bits_per_byte": 1.925917102775309, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3349440097808838, + "step": 1500, + "unclipped_grad_norm": 0.6999683380126953 + }, + { + "bits_per_byte": 1.9985412486510523, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3852832317352295, + "step": 1510, + "unclipped_grad_norm": 0.7689449787139893 + }, + { + "bits_per_byte": 1.9951115706275462, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3829059600830078, + "step": 1520, + "unclipped_grad_norm": 0.785367488861084 + }, + { + "bits_per_byte": 1.9521630311581806, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3531363010406494, + "step": 1530, + "unclipped_grad_norm": 0.6442116498947144 + }, + { + "bits_per_byte": 2.0327179849958115, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4089727401733398, + "step": 1540, + "unclipped_grad_norm": 0.7405364513397217 + }, + { + "bits_per_byte": 1.996995812550399, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3842120170593262, + "step": 1550, + "unclipped_grad_norm": 0.7386215329170227 + }, + { + "bits_per_byte": 2.0870359535484155, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4466230869293213, + "step": 1560, + "unclipped_grad_norm": 0.7484163641929626 + }, + { + "bits_per_byte": 2.0235489019477115, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4026172161102295, + "step": 1570, + "unclipped_grad_norm": 0.7196536064147949 + }, + { + "bits_per_byte": 2.00750653225788, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3914974927902222, + "step": 1580, + "unclipped_grad_norm": 0.7144435048103333 + }, + { + "bits_per_byte": 2.0128109931584413, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.395174264907837, + "step": 1590, + "unclipped_grad_norm": 0.8628523349761963 + }, + { + "bits_per_byte": 2.0153315708894866, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3969213962554932, + "step": 1600, + "unclipped_grad_norm": 0.715620756149292 + }, + { + "bits_per_byte": 2.048095125792106, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4196313619613647, + "step": 1610, + "unclipped_grad_norm": 0.7313905358314514 + }, + { + "bits_per_byte": 2.0567931483594926, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4256603717803955, + "step": 1620, + "unclipped_grad_norm": 0.761407732963562 + }, + { + "bits_per_byte": 1.907960738164025, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3224976062774658, + "step": 1630, + "unclipped_grad_norm": 0.7100295424461365 + }, + { + "bits_per_byte": 2.0356341228238413, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.410994052886963, + "step": 1640, + "unclipped_grad_norm": 0.7265889048576355 + }, + { + "bits_per_byte": 1.9881094689803178, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3780524730682373, + "step": 1650, + "unclipped_grad_norm": 0.7565467953681946 + }, + { + "bits_per_byte": 1.9576370669524927, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3569306135177612, + "step": 1660, + "unclipped_grad_norm": 0.6934499144554138 + }, + { + "bits_per_byte": 2.105078653450433, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4591293334960938, + "step": 1670, + "unclipped_grad_norm": 0.8928083181381226 + }, + { + "bits_per_byte": 2.0211131116635417, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4009288549423218, + "step": 1680, + "unclipped_grad_norm": 0.6971679925918579 + }, + { + "bits_per_byte": 1.9058530907777012, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.321036696434021, + "step": 1690, + "unclipped_grad_norm": 0.6719903349876404 + }, + { + "bits_per_byte": 1.9394077658742894, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3442950248718262, + "step": 1700, + "unclipped_grad_norm": 0.6790670156478882 + }, + { + "bits_per_byte": 2.021125322431753, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.4009373188018799, + "step": 1710, + "unclipped_grad_norm": 0.6948477029800415 + }, + { + "bits_per_byte": 2.0066268409987162, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.39088773727417, + "step": 1720, + "unclipped_grad_norm": 0.6923770904541016 + }, + { + "bits_per_byte": 1.9082544845316984, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3227012157440186, + "step": 1730, + "unclipped_grad_norm": 0.875411868095398 + }, + { + "bits_per_byte": 1.950284636645457, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3518342971801758, + "step": 1740, + "unclipped_grad_norm": 0.7188331484794617 + }, + { + "bits_per_byte": 2.0503372636114046, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4211854934692383, + "step": 1750, + "unclipped_grad_norm": 0.6843723058700562 + }, + { + "bits_per_byte": 1.9245816574913592, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.334018349647522, + "step": 1760, + "unclipped_grad_norm": 0.7317860722541809 + }, + { + "bits_per_byte": 1.9854330749675908, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.376197338104248, + "step": 1770, + "unclipped_grad_norm": 0.7310082912445068 + }, + { + "bits_per_byte": 1.9060752923626159, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3211907148361206, + "step": 1780, + "unclipped_grad_norm": 0.6923527717590332 + }, + { + "bits_per_byte": 1.8947768921141837, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.313359260559082, + "step": 1790, + "unclipped_grad_norm": 0.7431268692016602 + }, + { + "bits_per_byte": 1.9836379200578849, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.374953031539917, + "step": 1800, + "unclipped_grad_norm": 0.800605058670044 + }, + { + "bits_per_byte": 1.9736849400871523, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3680541515350342, + "step": 1810, + "unclipped_grad_norm": 0.6639943718910217 + }, + { + "bits_per_byte": 2.0327128255162856, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4089691638946533, + "step": 1820, + "unclipped_grad_norm": 0.7219430804252625 + }, + { + "bits_per_byte": 1.8641528013882975, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.2921322584152222, + "step": 1830, + "unclipped_grad_norm": 0.6714999079704285 + }, + { + "bits_per_byte": 1.9760018903595762, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3696601390838623, + "step": 1840, + "unclipped_grad_norm": 0.6850439310073853 + }, + { + "bits_per_byte": 1.8495459708852175, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2820075750350952, + "step": 1850, + "unclipped_grad_norm": 0.7066381573677063 + }, + { + "bits_per_byte": 2.002332434206672, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.387911081314087, + "step": 1860, + "unclipped_grad_norm": 0.7545463442802429 + }, + { + "bits_per_byte": 2.025226076758926, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4037797451019287, + "step": 1870, + "unclipped_grad_norm": 0.7131425142288208 + }, + { + "bits_per_byte": 1.8761874593650736, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3004740476608276, + "step": 1880, + "unclipped_grad_norm": 0.627357006072998 + }, + { + "bits_per_byte": 1.9627021280030534, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3604414463043213, + "step": 1890, + "unclipped_grad_norm": 0.6769031882286072 + }, + { + "bits_per_byte": 1.87336023656754, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2985143661499023, + "step": 1900, + "unclipped_grad_norm": 0.6752462983131409 + }, + { + "bits_per_byte": 1.8594791728511009, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2888927459716797, + "step": 1910, + "unclipped_grad_norm": 0.6314391493797302 + }, + { + "bits_per_byte": 1.9380384400081196, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3433458805084229, + "step": 1920, + "unclipped_grad_norm": 0.6901462078094482 + }, + { + "bits_per_byte": 1.8913296718602894, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3109698295593262, + "step": 1930, + "unclipped_grad_norm": 0.6903179287910461 + }, + { + "bits_per_byte": 1.8624152606666307, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2909278869628906, + "step": 1940, + "unclipped_grad_norm": 0.6572144627571106 + }, + { + "bits_per_byte": 1.930714558821127, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3382693529129028, + "step": 1950, + "unclipped_grad_norm": 0.7911640405654907 + }, + { + "bits_per_byte": 1.901616298173696, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3180999755859375, + "step": 1960, + "unclipped_grad_norm": 0.7476807236671448 + }, + { + "bits_per_byte": 1.8545080143279113, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2854470014572144, + "step": 1970, + "unclipped_grad_norm": 0.6583976745605469 + }, + { + "bits_per_byte": 1.8296589290353475, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2682229280471802, + "step": 1980, + "unclipped_grad_norm": 0.6357024908065796 + }, + { + "bits_per_byte": 1.9398576724889465, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.344606876373291, + "step": 1990, + "unclipped_grad_norm": 0.7643471956253052 + }, + { + "bits_per_byte": 1.866766077768158, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.2939436435699463, + "step": 2000, + "unclipped_grad_norm": 0.682447612285614 + }, + { + "bits_per_byte": 1.9452981716663407, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3483779430389404, + "step": 2010, + "unclipped_grad_norm": 0.6716283559799194 + }, + { + "bits_per_byte": 1.922642553102881, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.332674264907837, + "step": 2020, + "unclipped_grad_norm": 0.7301247119903564 + }, + { + "bits_per_byte": 1.9056240098867518, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3208779096603394, + "step": 2030, + "unclipped_grad_norm": 0.7755060791969299 + }, + { + "bits_per_byte": 1.9283054258478407, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3365994691848755, + "step": 2040, + "unclipped_grad_norm": 0.648145854473114 + }, + { + "bits_per_byte": 1.8707481640662356, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.296703815460205, + "step": 2050, + "unclipped_grad_norm": 0.6962859630584717 + }, + { + "bits_per_byte": 1.774047478965604, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2296760082244873, + "step": 2060, + "unclipped_grad_norm": 0.605278730392456 + }, + { + "bits_per_byte": 1.8770535639948187, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3010743856430054, + "step": 2070, + "unclipped_grad_norm": 0.6994346976280212 + }, + { + "bits_per_byte": 1.9817110264376179, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.37361741065979, + "step": 2080, + "unclipped_grad_norm": 0.7077776789665222 + }, + { + "bits_per_byte": 1.8423689628820616, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2770328521728516, + "step": 2090, + "unclipped_grad_norm": 0.6188594698905945 + }, + { + "bits_per_byte": 1.9159211271418608, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3280153274536133, + "step": 2100, + "unclipped_grad_norm": 0.6360442638397217 + }, + { + "bits_per_byte": 1.8300450300865345, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2684905529022217, + "step": 2110, + "unclipped_grad_norm": 0.6518478393554688 + }, + { + "bits_per_byte": 1.8775524856649717, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3014202117919922, + "step": 2120, + "unclipped_grad_norm": 0.677630603313446 + }, + { + "bits_per_byte": 1.88719968048247, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3081071376800537, + "step": 2130, + "unclipped_grad_norm": 0.6904116868972778 + }, + { + "bits_per_byte": 1.7861939256654409, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2380952835083008, + "step": 2140, + "unclipped_grad_norm": 0.5910601615905762 + }, + { + "bits_per_byte": 1.8339614190119824, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.271205186843872, + "step": 2150, + "unclipped_grad_norm": 0.7349988222122192 + }, + { + "bits_per_byte": 1.8980882460738961, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3156545162200928, + "step": 2160, + "unclipped_grad_norm": 0.6920630931854248 + }, + { + "bits_per_byte": 1.8727663804741106, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2981027364730835, + "step": 2170, + "unclipped_grad_norm": 0.7049499750137329 + }, + { + "bits_per_byte": 1.7856067768953952, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2376883029937744, + "step": 2180, + "unclipped_grad_norm": 0.652675211429596 + }, + { + "bits_per_byte": 1.9184699100276477, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3297820091247559, + "step": 2190, + "unclipped_grad_norm": 0.6759907007217407 + }, + { + "bits_per_byte": 1.9474407315307896, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.349863052368164, + "step": 2200, + "unclipped_grad_norm": 0.6637232899665833 + }, + { + "bits_per_byte": 1.9672947527116937, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3636248111724854, + "step": 2210, + "unclipped_grad_norm": 0.7057329416275024 + }, + { + "bits_per_byte": 1.8580620358079913, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2879104614257812, + "step": 2220, + "unclipped_grad_norm": 0.6423102021217346 + }, + { + "bits_per_byte": 1.8639622446111417, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2920001745224, + "step": 2230, + "unclipped_grad_norm": 0.664786159992218 + }, + { + "bits_per_byte": 1.8281711071227333, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2671916484832764, + "step": 2240, + "unclipped_grad_norm": 0.6536247730255127 + }, + { + "bits_per_byte": 1.8379201156695426, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.273949146270752, + "step": 2250, + "unclipped_grad_norm": 0.6435070633888245 + }, + { + "bits_per_byte": 1.8619839281782669, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2906289100646973, + "step": 2260, + "unclipped_grad_norm": 0.6457217335700989 + }, + { + "bits_per_byte": 1.9082623957336382, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.322706699371338, + "step": 2270, + "unclipped_grad_norm": 0.6787397861480713 + }, + { + "bits_per_byte": 1.7849188462919439, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2372114658355713, + "step": 2280, + "unclipped_grad_norm": 0.6086795330047607 + }, + { + "bits_per_byte": 1.8290736720744614, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2678172588348389, + "step": 2290, + "unclipped_grad_norm": 0.630083441734314 + }, + { + "bits_per_byte": 1.8820873242029221, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3045635223388672, + "step": 2300, + "unclipped_grad_norm": 0.6462850570678711 + }, + { + "bits_per_byte": 1.834706619838171, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2717217206954956, + "step": 2310, + "unclipped_grad_norm": 0.6394227743148804 + }, + { + "bits_per_byte": 1.8546760413778043, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2855634689331055, + "step": 2320, + "unclipped_grad_norm": 0.6594879031181335 + }, + { + "bits_per_byte": 1.7978681080060082, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2461872100830078, + "step": 2330, + "unclipped_grad_norm": 0.6738982796669006 + }, + { + "bits_per_byte": 1.8462244699491042, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.279705286026001, + "step": 2340, + "unclipped_grad_norm": 0.687351405620575 + }, + { + "bits_per_byte": 1.7549572327371814, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.216443657875061, + "step": 2350, + "unclipped_grad_norm": 0.6065738201141357 + }, + { + "bits_per_byte": 1.8467686230564342, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2800824642181396, + "step": 2360, + "unclipped_grad_norm": 0.676578938961029 + }, + { + "bits_per_byte": 1.8782005162934228, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3018693923950195, + "step": 2370, + "unclipped_grad_norm": 0.6810930967330933 + }, + { + "bits_per_byte": 1.7780921669485956, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2324795722961426, + "step": 2380, + "unclipped_grad_norm": 0.5974810719490051 + }, + { + "bits_per_byte": 1.7797094917973095, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2336006164550781, + "step": 2390, + "unclipped_grad_norm": 0.6242655515670776 + }, + { + "bits_per_byte": 1.8773909939558115, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.301308274269104, + "step": 2400, + "unclipped_grad_norm": 0.7230958342552185 + }, + { + "bits_per_byte": 1.843124310684651, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2775564193725586, + "step": 2410, + "unclipped_grad_norm": 0.6579370498657227 + }, + { + "bits_per_byte": 1.9371601246101633, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.342737078666687, + "step": 2420, + "unclipped_grad_norm": 0.6557210087776184 + }, + { + "bits_per_byte": 1.8373321069862427, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2735415697097778, + "step": 2430, + "unclipped_grad_norm": 0.6670473217964172 + }, + { + "bits_per_byte": 1.7749123797167932, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2302755117416382, + "step": 2440, + "unclipped_grad_norm": 0.6538338661193848 + }, + { + "bits_per_byte": 1.8761522029216466, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3004496097564697, + "step": 2450, + "unclipped_grad_norm": 0.6531840562820435 + }, + { + "bits_per_byte": 1.8790119304401935, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3024318218231201, + "step": 2460, + "unclipped_grad_norm": 0.6944200992584229 + }, + { + "bits_per_byte": 1.8171404838616947, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2595458030700684, + "step": 2470, + "unclipped_grad_norm": 0.6366217136383057 + }, + { + "bits_per_byte": 1.8241991678010567, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.264438509941101, + "step": 2480, + "unclipped_grad_norm": 0.6316180229187012 + }, + { + "bits_per_byte": 1.7491533342185142, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2124207019805908, + "step": 2490, + "unclipped_grad_norm": 0.6618465185165405 + }, + { + "bits_per_byte": 1.827854315079844, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2669720649719238, + "step": 2500, + "unclipped_grad_norm": 0.6852855682373047 + }, + { + "bits_per_byte": 1.8052024801347035, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2512710094451904, + "step": 2510, + "unclipped_grad_norm": 0.663596510887146 + }, + { + "bits_per_byte": 1.7913451500240836, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2416658401489258, + "step": 2520, + "unclipped_grad_norm": 0.6331848502159119 + }, + { + "bits_per_byte": 1.7932974970766782, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2430191040039062, + "step": 2530, + "unclipped_grad_norm": 0.6864507794380188 + }, + { + "bits_per_byte": 1.8207522915124645, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2620493173599243, + "step": 2540, + "unclipped_grad_norm": 0.6478230953216553 + }, + { + "bits_per_byte": 1.8469901367107453, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.280236005783081, + "step": 2550, + "unclipped_grad_norm": 0.6545907258987427 + }, + { + "bits_per_byte": 1.7922375679994107, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2422844171524048, + "step": 2560, + "unclipped_grad_norm": 0.6228907704353333 + }, + { + "bits_per_byte": 1.783868376260474, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2364833354949951, + "step": 2570, + "unclipped_grad_norm": 0.6730257868766785 + }, + { + "bits_per_byte": 1.8186578867902572, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.260597586631775, + "step": 2580, + "unclipped_grad_norm": 0.6890478730201721 + }, + { + "bits_per_byte": 1.7885676302126492, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2397406101226807, + "step": 2590, + "unclipped_grad_norm": 0.6227992177009583 + }, + { + "bits_per_byte": 1.8280013602463319, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2670739889144897, + "step": 2600, + "unclipped_grad_norm": 0.6276137828826904 + }, + { + "bits_per_byte": 1.7336515060003441, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2016756534576416, + "step": 2610, + "unclipped_grad_norm": 0.6445866823196411 + }, + { + "bits_per_byte": 1.7935037042750628, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2431620359420776, + "step": 2620, + "unclipped_grad_norm": 0.6606776714324951 + }, + { + "bits_per_byte": 1.8032969123631437, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2499501705169678, + "step": 2630, + "unclipped_grad_norm": 0.6334376931190491 + }, + { + "bits_per_byte": 1.810602563389145, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2550140619277954, + "step": 2640, + "unclipped_grad_norm": 0.6691069006919861 + }, + { + "bits_per_byte": 1.7587798911179091, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2190933227539062, + "step": 2650, + "unclipped_grad_norm": 0.6500447988510132 + }, + { + "bits_per_byte": 1.7476075541525593, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2113492488861084, + "step": 2660, + "unclipped_grad_norm": 0.5952810049057007 + }, + { + "bits_per_byte": 1.7865554331975544, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2383458614349365, + "step": 2670, + "unclipped_grad_norm": 0.6339060664176941 + }, + { + "bits_per_byte": 1.7817725956770598, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2350306510925293, + "step": 2680, + "unclipped_grad_norm": 0.6211492419242859 + }, + { + "bits_per_byte": 1.8492028654967463, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2817697525024414, + "step": 2690, + "unclipped_grad_norm": 0.6305774450302124 + }, + { + "bits_per_byte": 1.8342178451444189, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2713829278945923, + "step": 2700, + "unclipped_grad_norm": 0.6566506624221802 + }, + { + "bits_per_byte": 1.7120219359319322, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.186683177947998, + "step": 2710, + "unclipped_grad_norm": 0.6094632744789124 + }, + { + "bits_per_byte": 1.8346758349436665, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.271700382232666, + "step": 2720, + "unclipped_grad_norm": 0.6520996689796448 + }, + { + "bits_per_byte": 1.7481449999365055, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2117217779159546, + "step": 2730, + "unclipped_grad_norm": 0.598540186882019 + }, + { + "bits_per_byte": 1.7678683143027543, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2253929376602173, + "step": 2740, + "unclipped_grad_norm": 0.6463729739189148 + }, + { + "bits_per_byte": 1.7756850977671197, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2308111190795898, + "step": 2750, + "unclipped_grad_norm": 0.6158115863800049 + }, + { + "bits_per_byte": 1.8688916113501715, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2954169511795044, + "step": 2760, + "unclipped_grad_norm": 0.6701047420501709 + }, + { + "bits_per_byte": 1.7889463360098492, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2400031089782715, + "step": 2770, + "unclipped_grad_norm": 0.6446090936660767 + }, + { + "bits_per_byte": 1.7765037351852266, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2313785552978516, + "step": 2780, + "unclipped_grad_norm": 0.637730598449707 + }, + { + "bits_per_byte": 1.7761525466121648, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2311351299285889, + "step": 2790, + "unclipped_grad_norm": 0.7148120403289795 + }, + { + "bits_per_byte": 1.7209247898364963, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.1928541660308838, + "step": 2800, + "unclipped_grad_norm": 0.6417195200920105 + }, + { + "bits_per_byte": 1.792574482012451, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2425179481506348, + "step": 2810, + "unclipped_grad_norm": 0.6152215003967285 + }, + { + "bits_per_byte": 1.7960922151531988, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2449562549591064, + "step": 2820, + "unclipped_grad_norm": 0.6487079858779907 + }, + { + "bits_per_byte": 1.72477857707703, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.1955254077911377, + "step": 2830, + "unclipped_grad_norm": 0.6001560091972351 + }, + { + "bits_per_byte": 1.807270399528678, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.252704381942749, + "step": 2840, + "unclipped_grad_norm": 0.6889218688011169 + }, + { + "bits_per_byte": 1.7224509638802528, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.1939120292663574, + "step": 2850, + "unclipped_grad_norm": 0.6565119028091431 + }, + { + "bits_per_byte": 1.7068036383394527, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1830661296844482, + "step": 2860, + "unclipped_grad_norm": 0.637535035610199 + }, + { + "bits_per_byte": 1.8267986855688483, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2662403583526611, + "step": 2870, + "unclipped_grad_norm": 0.635680615901947 + }, + { + "bits_per_byte": 1.7536737262137922, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2155539989471436, + "step": 2880, + "unclipped_grad_norm": 0.6707039475440979 + }, + { + "bits_per_byte": 1.7805597740231751, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2341899871826172, + "step": 2890, + "unclipped_grad_norm": 0.6349734663963318 + }, + { + "bits_per_byte": 1.713862494261466, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1879589557647705, + "step": 2900, + "unclipped_grad_norm": 0.6683149337768555 + }, + { + "bits_per_byte": 1.7497669683167927, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.212846040725708, + "step": 2910, + "unclipped_grad_norm": 0.6716704368591309 + }, + { + "bits_per_byte": 1.8262012178397509, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2658262252807617, + "step": 2920, + "unclipped_grad_norm": 0.6724333763122559 + }, + { + "bits_per_byte": 1.761767401746047, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2211641073226929, + "step": 2930, + "unclipped_grad_norm": 0.6612268686294556 + }, + { + "bits_per_byte": 1.8132794733498245, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2568695545196533, + "step": 2940, + "unclipped_grad_norm": 0.6505576968193054 + }, + { + "bits_per_byte": 1.7731756989083804, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2290717363357544, + "step": 2950, + "unclipped_grad_norm": 0.6778621077537537 + }, + { + "bits_per_byte": 1.7513619354208945, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.213951587677002, + "step": 2960, + "unclipped_grad_norm": 0.6551554203033447 + }, + { + "bits_per_byte": 1.8079858473562673, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2532002925872803, + "step": 2970, + "unclipped_grad_norm": 0.6640533208847046 + }, + { + "bits_per_byte": 1.7386847502604952, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2051644325256348, + "step": 2980, + "unclipped_grad_norm": 0.643166184425354 + }, + { + "bits_per_byte": 1.8672367942835695, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2942699193954468, + "step": 2990, + "unclipped_grad_norm": 0.6637417078018188 + }, + { + "bits_per_byte": 1.7982774267150616, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2464709281921387, + "step": 3000, + "unclipped_grad_norm": 0.6690325736999512 + }, + { + "bits_per_byte": 1.762204753627191, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2214672565460205, + "step": 3010, + "unclipped_grad_norm": 0.6516304612159729 + }, + { + "bits_per_byte": 1.7294503138050674, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1987636089324951, + "step": 3020, + "unclipped_grad_norm": 0.6919088363647461 + }, + { + "bits_per_byte": 1.7609028449601596, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.220564842224121, + "step": 3030, + "unclipped_grad_norm": 0.6205207109451294 + }, + { + "bits_per_byte": 1.7350170482481948, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2026221752166748, + "step": 3040, + "unclipped_grad_norm": 0.7004191279411316 + }, + { + "bits_per_byte": 1.7472788953067604, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2111214399337769, + "step": 3050, + "unclipped_grad_norm": 0.6576176881790161 + }, + { + "bits_per_byte": 1.768669753455775, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.225948452949524, + "step": 3060, + "unclipped_grad_norm": 0.7110167145729065 + }, + { + "bits_per_byte": 1.7354397816040157, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.2029151916503906, + "step": 3070, + "unclipped_grad_norm": 0.6717135906219482 + }, + { + "bits_per_byte": 1.748681069859245, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2120933532714844, + "step": 3080, + "unclipped_grad_norm": 0.6585274934768677 + }, + { + "bits_per_byte": 1.7073046238014162, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.1834133863449097, + "step": 3090, + "unclipped_grad_norm": 0.6292386651039124 + }, + { + "bits_per_byte": 1.8085337840819162, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.253580093383789, + "step": 3100, + "unclipped_grad_norm": 0.6514226198196411 + }, + { + "bits_per_byte": 1.7739243393875863, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.229590654373169, + "step": 3110, + "unclipped_grad_norm": 0.6451593041419983 + }, + { + "bits_per_byte": 1.6671654209338944, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1555910110473633, + "step": 3120, + "unclipped_grad_norm": 0.6327124834060669 + }, + { + "bits_per_byte": 1.7555804978639082, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.216875672340393, + "step": 3130, + "unclipped_grad_norm": 0.6219494342803955 + }, + { + "bits_per_byte": 1.7189500850392896, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1914854049682617, + "step": 3140, + "unclipped_grad_norm": 0.6722466349601746 + }, + { + "bits_per_byte": 1.6998178750440565, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.178223967552185, + "step": 3150, + "unclipped_grad_norm": 0.6060081720352173 + }, + { + "bits_per_byte": 1.7570095017099274, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2178661823272705, + "step": 3160, + "unclipped_grad_norm": 0.6486148834228516 + }, + { + "bits_per_byte": 1.8187225522669817, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.260642409324646, + "step": 3170, + "unclipped_grad_norm": 0.6782575845718384 + }, + { + "bits_per_byte": 1.680190870962291, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1646195650100708, + "step": 3180, + "unclipped_grad_norm": 0.6483905911445618 + }, + { + "bits_per_byte": 1.7650935462137336, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.223469614982605, + "step": 3190, + "unclipped_grad_norm": 0.7115707993507385 + }, + { + "bits_per_byte": 1.6804184040093824, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1647772789001465, + "step": 3200, + "unclipped_grad_norm": 0.6299659609794617 + }, + { + "bits_per_byte": 1.7133916057634035, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1876325607299805, + "step": 3210, + "unclipped_grad_norm": 0.6491534113883972 + }, + { + "bits_per_byte": 1.710321027514899, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1855041980743408, + "step": 3220, + "unclipped_grad_norm": 0.6104377508163452 + }, + { + "bits_per_byte": 1.6818119794293238, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1657432317733765, + "step": 3230, + "unclipped_grad_norm": 0.6708391308784485 + }, + { + "bits_per_byte": 1.7427318460005987, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2079696655273438, + "step": 3240, + "unclipped_grad_norm": 0.6418254375457764 + }, + { + "bits_per_byte": 1.733884714474914, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2018373012542725, + "step": 3250, + "unclipped_grad_norm": 0.6041762828826904 + }, + { + "bits_per_byte": 1.83323737205185, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2707033157348633, + "step": 3260, + "unclipped_grad_norm": 0.7252490520477295 + }, + { + "bits_per_byte": 1.6948699341787334, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.174794316291809, + "step": 3270, + "unclipped_grad_norm": 0.6451477408409119 + }, + { + "bits_per_byte": 1.6338448142251303, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1324949264526367, + "step": 3280, + "unclipped_grad_norm": 0.6251513361930847 + }, + { + "bits_per_byte": 1.6883934115125416, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1703051328659058, + "step": 3290, + "unclipped_grad_norm": 0.6296114921569824 + }, + { + "bits_per_byte": 1.744128001160303, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.208937406539917, + "step": 3300, + "unclipped_grad_norm": 0.6343309283256531 + }, + { + "bits_per_byte": 1.735473146238283, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2029383182525635, + "step": 3310, + "unclipped_grad_norm": 0.6353676915168762 + }, + { + "bits_per_byte": 1.7632710460625405, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2222063541412354, + "step": 3320, + "unclipped_grad_norm": 0.6778877973556519 + }, + { + "bits_per_byte": 1.6989030993241172, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1775898933410645, + "step": 3330, + "unclipped_grad_norm": 0.629153847694397 + }, + { + "bits_per_byte": 1.7071973066272779, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.18333899974823, + "step": 3340, + "unclipped_grad_norm": 0.6583864688873291 + }, + { + "bits_per_byte": 1.70429750715108, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1813290119171143, + "step": 3350, + "unclipped_grad_norm": 0.6497669816017151 + }, + { + "bits_per_byte": 1.7138317093669615, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.187937617301941, + "step": 3360, + "unclipped_grad_norm": 0.6299119591712952 + }, + { + "bits_per_byte": 1.6462306607749682, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1410801410675049, + "step": 3370, + "unclipped_grad_norm": 0.6380488872528076 + }, + { + "bits_per_byte": 1.7443316286189245, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2090785503387451, + "step": 3380, + "unclipped_grad_norm": 0.702843189239502 + }, + { + "bits_per_byte": 1.7228774808543925, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1942076683044434, + "step": 3390, + "unclipped_grad_norm": 0.6957465410232544 + }, + { + "bits_per_byte": 1.7111405248462603, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1860722303390503, + "step": 3400, + "unclipped_grad_norm": 0.6491814255714417 + }, + { + "bits_per_byte": 1.7448699343161251, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.209451675415039, + "step": 3410, + "unclipped_grad_norm": 0.6583186388015747 + }, + { + "bits_per_byte": 1.7129637129280568, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1873359680175781, + "step": 3420, + "unclipped_grad_norm": 0.6641048192977905 + }, + { + "bits_per_byte": 1.670370317632723, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.157812476158142, + "step": 3430, + "unclipped_grad_norm": 0.629267692565918 + }, + { + "bits_per_byte": 1.6804600238108913, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1648061275482178, + "step": 3440, + "unclipped_grad_norm": 0.6077277064323425 + }, + { + "bits_per_byte": 1.6713492428814338, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1584910154342651, + "step": 3450, + "unclipped_grad_norm": 0.6522519588470459 + }, + { + "bits_per_byte": 1.669551508231965, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.1572449207305908, + "step": 3460, + "unclipped_grad_norm": 0.627102255821228 + }, + { + "bits_per_byte": 1.7251744811393164, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1957998275756836, + "step": 3470, + "unclipped_grad_norm": 0.6381968855857849 + }, + { + "bits_per_byte": 1.7823232841251224, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.235412359237671, + "step": 3480, + "unclipped_grad_norm": 0.6999886631965637 + }, + { + "bits_per_byte": 1.718624865846508, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1912599802017212, + "step": 3490, + "unclipped_grad_norm": 0.6294557452201843 + }, + { + "bits_per_byte": 1.6614911973339777, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1516579389572144, + "step": 3500, + "unclipped_grad_norm": 0.6351874470710754 + }, + { + "bits_per_byte": 1.6797325371977416, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.164301872253418, + "step": 3510, + "unclipped_grad_norm": 0.6315440535545349 + }, + { + "bits_per_byte": 1.717181759423118, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1902596950531006, + "step": 3520, + "unclipped_grad_norm": 0.6176374554634094 + }, + { + "bits_per_byte": 1.6772098956748858, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.162553310394287, + "step": 3530, + "unclipped_grad_norm": 0.6610027551651001 + }, + { + "bits_per_byte": 1.6855345439072493, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1683235168457031, + "step": 3540, + "unclipped_grad_norm": 0.658635139465332 + }, + { + "bits_per_byte": 1.747215777673894, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2110776901245117, + "step": 3550, + "unclipped_grad_norm": 0.6718185544013977 + }, + { + "bits_per_byte": 1.666718266041651, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1552810668945312, + "step": 3560, + "unclipped_grad_norm": 0.6717336773872375 + }, + { + "bits_per_byte": 1.7121595220526225, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1867785453796387, + "step": 3570, + "unclipped_grad_norm": 0.6859359741210938 + }, + { + "bits_per_byte": 1.739672790589702, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.205849289894104, + "step": 3580, + "unclipped_grad_norm": 0.6660984754562378 + }, + { + "bits_per_byte": 1.6956691375572928, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1753482818603516, + "step": 3590, + "unclipped_grad_norm": 0.6079815030097961 + }, + { + "bits_per_byte": 1.7521809168043032, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2145192623138428, + "step": 3600, + "unclipped_grad_norm": 0.7036373615264893 + }, + { + "bits_per_byte": 1.7346027420422663, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.202335000038147, + "step": 3610, + "unclipped_grad_norm": 0.6451415419578552 + }, + { + "bits_per_byte": 1.6701776970637565, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1576789617538452, + "step": 3620, + "unclipped_grad_norm": 0.620152473449707 + }, + { + "bits_per_byte": 1.6714409096343439, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1585545539855957, + "step": 3630, + "unclipped_grad_norm": 0.6908113956451416 + }, + { + "bits_per_byte": 1.6125346159393705, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.117723822593689, + "step": 3640, + "unclipped_grad_norm": 0.604874312877655 + }, + { + "bits_per_byte": 1.6950919637809974, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1749482154846191, + "step": 3650, + "unclipped_grad_norm": 0.6512246131896973 + }, + { + "bits_per_byte": 1.6897813115050047, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1712671518325806, + "step": 3660, + "unclipped_grad_norm": 0.647068440914154 + }, + { + "bits_per_byte": 1.7204222565306753, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1925058364868164, + "step": 3670, + "unclipped_grad_norm": 0.7147674560546875 + }, + { + "bits_per_byte": 1.6652036148355023, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1542311906814575, + "step": 3680, + "unclipped_grad_norm": 0.6392054557800293 + }, + { + "bits_per_byte": 1.7297106955384738, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.198944091796875, + "step": 3690, + "unclipped_grad_norm": 0.7120746374130249 + }, + { + "bits_per_byte": 1.6313792709423611, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1307859420776367, + "step": 3700, + "unclipped_grad_norm": 0.6207358241081238 + }, + { + "bits_per_byte": 1.6932649920808815, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1736818552017212, + "step": 3710, + "unclipped_grad_norm": 0.6575222611427307 + }, + { + "bits_per_byte": 1.7191832935138596, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1916470527648926, + "step": 3720, + "unclipped_grad_norm": 0.6928348541259766 + }, + { + "bits_per_byte": 1.6966805675270171, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1760493516921997, + "step": 3730, + "unclipped_grad_norm": 0.6457412242889404 + }, + { + "bits_per_byte": 1.6651184834233252, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1541721820831299, + "step": 3740, + "unclipped_grad_norm": 0.6149929761886597 + }, + { + "bits_per_byte": 1.690470445987012, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1717448234558105, + "step": 3750, + "unclipped_grad_norm": 0.6539504528045654 + }, + { + "bits_per_byte": 1.7257394441474005, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.196191430091858, + "step": 3760, + "unclipped_grad_norm": 0.6810935735702515 + }, + { + "bits_per_byte": 1.709021182639678, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.184603214263916, + "step": 3770, + "unclipped_grad_norm": 0.6034973859786987 + }, + { + "bits_per_byte": 1.7700676284419878, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2269173860549927, + "step": 3780, + "unclipped_grad_norm": 0.6868588328361511 + }, + { + "bits_per_byte": 1.6607432447853754, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.151139497756958, + "step": 3790, + "unclipped_grad_norm": 0.6561638712882996 + }, + { + "bits_per_byte": 1.6881032767805362, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1701040267944336, + "step": 3800, + "unclipped_grad_norm": 0.6706013679504395 + }, + { + "bits_per_byte": 1.6338372469884925, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1324896812438965, + "step": 3810, + "unclipped_grad_norm": 0.6373327374458313 + }, + { + "bits_per_byte": 1.6310115720348164, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1305310726165771, + "step": 3820, + "unclipped_grad_norm": 0.6431099772453308 + }, + { + "bits_per_byte": 1.699418359346102, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1779470443725586, + "step": 3830, + "unclipped_grad_norm": 0.6568827629089355 + }, + { + "bits_per_byte": 1.6534394855685337, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1460769176483154, + "step": 3840, + "unclipped_grad_norm": 0.6673737168312073 + }, + { + "bits_per_byte": 1.6227488375567638, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1248037815093994, + "step": 3850, + "unclipped_grad_norm": 0.6300650835037231 + }, + { + "bits_per_byte": 1.6095687751252414, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1156680583953857, + "step": 3860, + "unclipped_grad_norm": 0.6291298270225525 + }, + { + "bits_per_byte": 1.5641092889532788, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.084157943725586, + "step": 3870, + "unclipped_grad_norm": 0.619691789150238 + }, + { + "bits_per_byte": 1.670964173726152, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.158224105834961, + "step": 3880, + "unclipped_grad_norm": 0.6558656692504883 + }, + { + "bits_per_byte": 1.6390438497607132, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1360986232757568, + "step": 3890, + "unclipped_grad_norm": 0.6659069061279297 + }, + { + "bits_per_byte": 1.6436039697483404, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1392594575881958, + "step": 3900, + "unclipped_grad_norm": 0.6627041101455688 + }, + { + "bits_per_byte": 1.6594528589559516, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1502450704574585, + "step": 3910, + "unclipped_grad_norm": 0.6734365224838257 + }, + { + "bits_per_byte": 1.6941402117911224, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1742885112762451, + "step": 3920, + "unclipped_grad_norm": 0.6930717825889587 + }, + { + "bits_per_byte": 1.6772669939149722, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.162592887878418, + "step": 3930, + "unclipped_grad_norm": 0.7069801688194275 + }, + { + "bits_per_byte": 1.6121323485190024, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1174449920654297, + "step": 3940, + "unclipped_grad_norm": 0.6367635726928711 + }, + { + "bits_per_byte": 1.7182599186613772, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1910070180892944, + "step": 3950, + "unclipped_grad_norm": 0.690847635269165 + }, + { + "bits_per_byte": 1.6993077745015972, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1778703927993774, + "step": 3960, + "unclipped_grad_norm": 0.6660118699073792 + }, + { + "bits_per_byte": 1.6431734971732308, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1389610767364502, + "step": 3970, + "unclipped_grad_norm": 0.669096827507019 + }, + { + "bits_per_byte": 1.6275112091418058, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.12810480594635, + "step": 3980, + "unclipped_grad_norm": 0.6583772301673889 + }, + { + "bits_per_byte": 1.669720739160414, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1573622226715088, + "step": 3990, + "unclipped_grad_norm": 0.6523779630661011 + }, + { + "bits_per_byte": 1.5952216384249642, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1057233810424805, + "step": 4000, + "unclipped_grad_norm": 0.6264373064041138 + }, + { + "bits_per_byte": 1.6774286576067834, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1627049446105957, + "step": 4010, + "unclipped_grad_norm": 0.6995967030525208 + }, + { + "bits_per_byte": 1.6794896976947233, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1641335487365723, + "step": 4020, + "unclipped_grad_norm": 0.6873654127120972 + }, + { + "bits_per_byte": 1.6416081110850775, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.137876033782959, + "step": 4030, + "unclipped_grad_norm": 0.7148170471191406 + }, + { + "bits_per_byte": 1.62684030482079, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1276397705078125, + "step": 4040, + "unclipped_grad_norm": 0.6498348712921143 + }, + { + "bits_per_byte": 1.644293964143602, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1397377252578735, + "step": 4050, + "unclipped_grad_norm": 0.6837151646614075 + }, + { + "bits_per_byte": 1.6349320885438852, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1332485675811768, + "step": 4060, + "unclipped_grad_norm": 0.6708939671516418 + }, + { + "bits_per_byte": 1.769896849669681, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2267990112304688, + "step": 4070, + "unclipped_grad_norm": 0.7203540802001953 + }, + { + "bits_per_byte": 1.5942710903136454, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1050645112991333, + "step": 4080, + "unclipped_grad_norm": 0.6567085385322571 + }, + { + "bits_per_byte": 1.6276448396615262, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.128197431564331, + "step": 4090, + "unclipped_grad_norm": 0.6684355139732361 + }, + { + "bits_per_byte": 1.7285615074654084, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1981475353240967, + "step": 4100, + "unclipped_grad_norm": 0.7356466054916382 + }, + { + "bits_per_byte": 1.6342331510507786, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1327641010284424, + "step": 4110, + "unclipped_grad_norm": 0.655012309551239 + }, + { + "bits_per_byte": 1.6424126459258137, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1384336948394775, + "step": 4120, + "unclipped_grad_norm": 0.7107924222946167 + }, + { + "bits_per_byte": 1.6944599275390764, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.17451012134552, + "step": 4130, + "unclipped_grad_norm": 0.6998885869979858 + }, + { + "bits_per_byte": 1.7017107160994525, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.179535984992981, + "step": 4140, + "unclipped_grad_norm": 0.6857012510299683 + }, + { + "bits_per_byte": 1.6170882005862652, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.120880126953125, + "step": 4150, + "unclipped_grad_norm": 0.6785045266151428 + }, + { + "bits_per_byte": 1.6430269679546958, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138859510421753, + "step": 4160, + "unclipped_grad_norm": 0.6962723135948181 + }, + { + "bits_per_byte": 1.6682418603456448, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1563371419906616, + "step": 4170, + "unclipped_grad_norm": 0.7071301937103271 + }, + { + "bits_per_byte": 1.6402630347726797, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1369436979293823, + "step": 4180, + "unclipped_grad_norm": 0.6712477207183838 + }, + { + "bits_per_byte": 1.67100166594404, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.158250093460083, + "step": 4190, + "unclipped_grad_norm": 0.6872346997261047 + }, + { + "bits_per_byte": 1.6579426792987255, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.149198293685913, + "step": 4200, + "unclipped_grad_norm": 0.7030718326568604 + }, + { + "bits_per_byte": 1.6808303024581988, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1650627851486206, + "step": 4210, + "unclipped_grad_norm": 0.658928394317627 + }, + { + "bits_per_byte": 1.659455094730413, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1502466201782227, + "step": 4220, + "unclipped_grad_norm": 0.6767720580101013 + }, + { + "bits_per_byte": 1.6554740403282406, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1474871635437012, + "step": 4230, + "unclipped_grad_norm": 0.7309125065803528 + }, + { + "bits_per_byte": 1.6771517655388943, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.162513017654419, + "step": 4240, + "unclipped_grad_norm": 0.7190505862236023 + }, + { + "bits_per_byte": 1.6138194983239667, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1186144351959229, + "step": 4250, + "unclipped_grad_norm": 0.6701101660728455 + }, + { + "bits_per_byte": 1.6526860295751036, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1455546617507935, + "step": 4260, + "unclipped_grad_norm": 0.6757012009620667 + }, + { + "bits_per_byte": 1.5671267245626668, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.0862494707107544, + "step": 4270, + "unclipped_grad_norm": 0.647323727607727 + }, + { + "bits_per_byte": 1.640895586962553, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.13738214969635, + "step": 4280, + "unclipped_grad_norm": 0.666140079498291 + }, + { + "bits_per_byte": 1.6259669769197085, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1270344257354736, + "step": 4290, + "unclipped_grad_norm": 0.6681356430053711 + }, + { + "bits_per_byte": 1.5856241465935652, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0990709066390991, + "step": 4300, + "unclipped_grad_norm": 0.6779436469078064 + }, + { + "bits_per_byte": 1.5644216094472456, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.0843744277954102, + "step": 4310, + "unclipped_grad_norm": 0.6528535485267639 + }, + { + "bits_per_byte": 1.6990217673532124, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1776721477508545, + "step": 4320, + "unclipped_grad_norm": 0.6877329349517822 + }, + { + "bits_per_byte": 1.6415279671697756, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1378204822540283, + "step": 4330, + "unclipped_grad_norm": 0.7055521011352539 + }, + { + "bits_per_byte": 1.604049851859054, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1118426322937012, + "step": 4340, + "unclipped_grad_norm": 0.6865817308425903 + }, + { + "bits_per_byte": 1.5633534252027368, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0836340188980103, + "step": 4350, + "unclipped_grad_norm": 0.6733583211898804 + }, + { + "bits_per_byte": 1.6505902489916895, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1441019773483276, + "step": 4360, + "unclipped_grad_norm": 0.6889402270317078 + }, + { + "bits_per_byte": 1.6184719729951074, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1218392848968506, + "step": 4370, + "unclipped_grad_norm": 0.6762850284576416 + }, + { + "bits_per_byte": 1.556092145700658, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.0786008834838867, + "step": 4380, + "unclipped_grad_norm": 0.6198620200157166 + }, + { + "bits_per_byte": 1.7130244228038114, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1873780488967896, + "step": 4390, + "unclipped_grad_norm": 0.6756254434585571 + }, + { + "bits_per_byte": 1.6522954569749941, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1452839374542236, + "step": 4400, + "unclipped_grad_norm": 0.6876093745231628 + }, + { + "bits_per_byte": 1.661213789318136, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.151465654373169, + "step": 4410, + "unclipped_grad_norm": 0.6821660399436951 + }, + { + "bits_per_byte": 1.6753433679650718, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1612595319747925, + "step": 4420, + "unclipped_grad_norm": 0.7177299857139587 + }, + { + "bits_per_byte": 1.6206171125993192, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1233261823654175, + "step": 4430, + "unclipped_grad_norm": 0.6967712640762329 + }, + { + "bits_per_byte": 1.6227820302083802, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1248267889022827, + "step": 4440, + "unclipped_grad_norm": 0.6610331535339355 + }, + { + "bits_per_byte": 1.6554300127696198, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1474566459655762, + "step": 4450, + "unclipped_grad_norm": 0.6704992055892944 + }, + { + "bits_per_byte": 1.6458450756717338, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.140812873840332, + "step": 4460, + "unclipped_grad_norm": 0.7582810521125793 + }, + { + "bits_per_byte": 1.6144303806998315, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1190378665924072, + "step": 4470, + "unclipped_grad_norm": 0.7301307916641235 + }, + { + "bits_per_byte": 1.6560006512051826, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1478521823883057, + "step": 4480, + "unclipped_grad_norm": 0.7387185096740723 + }, + { + "bits_per_byte": 1.6943782357799166, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1744534969329834, + "step": 4490, + "unclipped_grad_norm": 0.7154380083084106 + }, + { + "bits_per_byte": 1.6097710267226562, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1158082485198975, + "step": 4500, + "unclipped_grad_norm": 0.6710496544837952 + }, + { + "bits_per_byte": 1.6061318738303991, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.113285779953003, + "step": 4510, + "unclipped_grad_norm": 0.7097434401512146 + }, + { + "bits_per_byte": 1.676021495557424, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1617295742034912, + "step": 4520, + "unclipped_grad_norm": 0.7576807141304016 + }, + { + "bits_per_byte": 1.635137779794317, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1333911418914795, + "step": 4530, + "unclipped_grad_norm": 0.6911670565605164 + }, + { + "bits_per_byte": 1.64362185594403, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.139271855354309, + "step": 4540, + "unclipped_grad_norm": 0.6786331534385681 + }, + { + "bits_per_byte": 1.650524379636409, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1440563201904297, + "step": 4550, + "unclipped_grad_norm": 0.6853471994400024 + }, + { + "bits_per_byte": 1.6363368428361325, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1342222690582275, + "step": 4560, + "unclipped_grad_norm": 0.7256327867507935 + }, + { + "bits_per_byte": 1.6365012582503573, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.134336233139038, + "step": 4570, + "unclipped_grad_norm": 0.697358250617981 + }, + { + "bits_per_byte": 1.6112284077060677, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1168184280395508, + "step": 4580, + "unclipped_grad_norm": 0.6777223348617554 + }, + { + "bits_per_byte": 1.5962301446896239, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1064224243164062, + "step": 4590, + "unclipped_grad_norm": 0.7330402731895447 + }, + { + "bits_per_byte": 1.5998985346325274, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1089651584625244, + "step": 4600, + "unclipped_grad_norm": 0.6789212822914124 + }, + { + "bits_per_byte": 1.596745060746307, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1067793369293213, + "step": 4610, + "unclipped_grad_norm": 0.6819480061531067 + }, + { + "bits_per_byte": 1.6577524664868712, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.14906644821167, + "step": 4620, + "unclipped_grad_norm": 0.7283186912536621 + }, + { + "bits_per_byte": 1.646706020821953, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1414096355438232, + "step": 4630, + "unclipped_grad_norm": 0.7197591662406921 + }, + { + "bits_per_byte": 1.6542598428131492, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1466455459594727, + "step": 4640, + "unclipped_grad_norm": 0.7083833813667297 + }, + { + "bits_per_byte": 1.6790158854915962, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1638051271438599, + "step": 4650, + "unclipped_grad_norm": 0.684671938419342 + }, + { + "bits_per_byte": 1.6168221434253804, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.12069571018219, + "step": 4660, + "unclipped_grad_norm": 0.7103760838508606 + }, + { + "bits_per_byte": 1.6034386255178876, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1114189624786377, + "step": 4670, + "unclipped_grad_norm": 0.6704027652740479 + }, + { + "bits_per_byte": 1.591610346722147, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1032202243804932, + "step": 4680, + "unclipped_grad_norm": 0.7572834491729736 + }, + { + "bits_per_byte": 1.6094411639983013, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.115579605102539, + "step": 4690, + "unclipped_grad_norm": 0.6773369312286377 + }, + { + "bits_per_byte": 1.6221410508686145, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.124382495880127, + "step": 4700, + "unclipped_grad_norm": 0.7124287486076355 + }, + { + "bits_per_byte": 1.6214163159778787, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.12388014793396, + "step": 4710, + "unclipped_grad_norm": 0.7075444459915161 + }, + { + "bits_per_byte": 1.6415025137374477, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1378028392791748, + "step": 4720, + "unclipped_grad_norm": 0.7044992446899414 + }, + { + "bits_per_byte": 1.654741394235565, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1469793319702148, + "step": 4730, + "unclipped_grad_norm": 0.7124809622764587 + }, + { + "bits_per_byte": 1.6107767812649019, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1165053844451904, + "step": 4740, + "unclipped_grad_norm": 0.7306708693504333 + }, + { + "bits_per_byte": 1.5756537963750956, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.0921599864959717, + "step": 4750, + "unclipped_grad_norm": 0.6872034668922424 + }, + { + "bits_per_byte": 1.550611402582962, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0748019218444824, + "step": 4760, + "unclipped_grad_norm": 0.6966266632080078 + }, + { + "bits_per_byte": 1.5558193812163896, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.0784118175506592, + "step": 4770, + "unclipped_grad_norm": 0.6661347150802612 + }, + { + "bits_per_byte": 1.5972586009417833, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.10713529586792, + "step": 4780, + "unclipped_grad_norm": 0.7209707498550415 + }, + { + "bits_per_byte": 1.6013201432245594, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1099505424499512, + "step": 4790, + "unclipped_grad_norm": 0.7224242091178894 + }, + { + "bits_per_byte": 1.6214809814546032, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.123924970626831, + "step": 4800, + "unclipped_grad_norm": 0.7595614790916443 + }, + { + "bits_per_byte": 1.598943858937588, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.108303427696228, + "step": 4810, + "unclipped_grad_norm": 0.6932185292243958 + }, + { + "bits_per_byte": 1.5882840302718095, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1009145975112915, + "step": 4820, + "unclipped_grad_norm": 0.7010871171951294 + }, + { + "bits_per_byte": 1.5957891811728115, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.106116771697998, + "step": 4830, + "unclipped_grad_norm": 0.6711499691009521 + }, + { + "bits_per_byte": 1.5337598545208206, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.0631213188171387, + "step": 4840, + "unclipped_grad_norm": 0.7005295157432556 + }, + { + "bits_per_byte": 1.6063714456630511, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1134518384933472, + "step": 4850, + "unclipped_grad_norm": 0.6924180388450623 + }, + { + "bits_per_byte": 1.58919708616524, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1015474796295166, + "step": 4860, + "unclipped_grad_norm": 0.6728304028511047 + }, + { + "bits_per_byte": 1.5949746713383253, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1055521965026855, + "step": 4870, + "unclipped_grad_norm": 0.6908511519432068 + }, + { + "bits_per_byte": 1.562132004416309, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0827873945236206, + "step": 4880, + "unclipped_grad_norm": 0.6754211187362671 + }, + { + "bits_per_byte": 1.5204766024987808, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0539140701293945, + "step": 4890, + "unclipped_grad_norm": 0.686815619468689 + }, + { + "bits_per_byte": 1.522231341485534, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0551303625106812, + "step": 4900, + "unclipped_grad_norm": 0.6894630789756775 + }, + { + "bits_per_byte": 1.5875051208460518, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.100374698638916, + "step": 4910, + "unclipped_grad_norm": 0.7035151124000549 + }, + { + "bits_per_byte": 1.5873929881576891, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.100296974182129, + "step": 4920, + "unclipped_grad_norm": 0.707631528377533 + }, + { + "bits_per_byte": 1.5818637459324498, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0964643955230713, + "step": 4930, + "unclipped_grad_norm": 0.7146247029304504 + }, + { + "bits_per_byte": 1.6602286726939939, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.150782823562622, + "step": 4940, + "unclipped_grad_norm": 0.7488285899162292 + }, + { + "bits_per_byte": 1.5109480757103777, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.047309398651123, + "step": 4950, + "unclipped_grad_norm": 0.6982936263084412 + }, + { + "bits_per_byte": 1.6013034609074257, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1099389791488647, + "step": 4960, + "unclipped_grad_norm": 0.7697318196296692 + }, + { + "bits_per_byte": 1.673659485830474, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1600923538208008, + "step": 4970, + "unclipped_grad_norm": 0.7759159207344055 + }, + { + "bits_per_byte": 1.5500858236019253, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.0744376182556152, + "step": 4980, + "unclipped_grad_norm": 0.7765741944313049 + }, + { + "bits_per_byte": 1.6026017579387892, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1108388900756836, + "step": 4990, + "unclipped_grad_norm": 0.7224560976028442 + }, + { + "bits_per_byte": 1.6136506113608193, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.118497371673584, + "step": 5000, + "unclipped_grad_norm": 0.7415750622749329 + }, + { + "bits_per_byte": 1.5910434919049032, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1028273105621338, + "step": 5010, + "unclipped_grad_norm": 0.6908667087554932 + }, + { + "bits_per_byte": 1.598975331762696, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1083252429962158, + "step": 5020, + "unclipped_grad_norm": 0.730786919593811 + }, + { + "bits_per_byte": 1.5824023955949522, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0968377590179443, + "step": 5030, + "unclipped_grad_norm": 0.7392095923423767 + }, + { + "bits_per_byte": 1.5712477728426413, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0891059637069702, + "step": 5040, + "unclipped_grad_norm": 0.7138375639915466 + }, + { + "bits_per_byte": 1.600523519585763, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.109398365020752, + "step": 5050, + "unclipped_grad_norm": 0.7417458295822144 + }, + { + "bits_per_byte": 1.556087158203783, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0785974264144897, + "step": 5060, + "unclipped_grad_norm": 0.766182541847229 + }, + { + "bits_per_byte": 1.6239489324944845, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1256356239318848, + "step": 5070, + "unclipped_grad_norm": 0.7354997992515564 + }, + { + "bits_per_byte": 1.6499730032577429, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1436741352081299, + "step": 5080, + "unclipped_grad_norm": 0.8061900734901428 + }, + { + "bits_per_byte": 1.621231606610852, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1237521171569824, + "step": 5090, + "unclipped_grad_norm": 0.789067804813385 + }, + { + "bits_per_byte": 1.5443199332490989, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0704410076141357, + "step": 5100, + "unclipped_grad_norm": 0.7130411267280579 + }, + { + "bits_per_byte": 1.5718306220464153, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0895099639892578, + "step": 5110, + "unclipped_grad_norm": 0.6817747950553894 + }, + { + "bits_per_byte": 1.5553593276253317, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0780929327011108, + "step": 5120, + "unclipped_grad_norm": 0.7432519793510437 + }, + { + "bits_per_byte": 1.611734380664906, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1171691417694092, + "step": 5130, + "unclipped_grad_norm": 0.7314459681510925 + }, + { + "bits_per_byte": 1.5509013653323167, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.075002908706665, + "step": 5140, + "unclipped_grad_norm": 0.7262561321258545 + }, + { + "bits_per_byte": 1.5519597465657264, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0757365226745605, + "step": 5150, + "unclipped_grad_norm": 0.7442956566810608 + }, + { + "bits_per_byte": 1.582139434121783, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0966554880142212, + "step": 5160, + "unclipped_grad_norm": 0.7529576420783997 + }, + { + "bits_per_byte": 1.5941560339202183, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1049847602844238, + "step": 5170, + "unclipped_grad_norm": 0.7270960211753845 + }, + { + "bits_per_byte": 1.6180967068509247, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1215791702270508, + "step": 5180, + "unclipped_grad_norm": 0.759490966796875 + }, + { + "bits_per_byte": 1.5759898504748815, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.092392921447754, + "step": 5190, + "unclipped_grad_norm": 0.7248204946517944 + }, + { + "bits_per_byte": 1.5225916451390915, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.05538010597229, + "step": 5200, + "unclipped_grad_norm": 0.7128092646598816 + }, + { + "bits_per_byte": 1.601221769148266, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1098823547363281, + "step": 5210, + "unclipped_grad_norm": 0.7404454946517944 + }, + { + "bits_per_byte": 1.5843179383602621, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.098165512084961, + "step": 5220, + "unclipped_grad_norm": 0.7370045781135559 + }, + { + "bits_per_byte": 1.614152456736037, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1188452243804932, + "step": 5230, + "unclipped_grad_norm": 0.8054904937744141 + }, + { + "bits_per_byte": 1.5701584347320763, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0883508920669556, + "step": 5240, + "unclipped_grad_norm": 0.7422505617141724 + }, + { + "bits_per_byte": 1.5364362485335477, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.064976453781128, + "step": 5250, + "unclipped_grad_norm": 0.7392460107803345 + }, + { + "bits_per_byte": 1.510300904995181, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0468608140945435, + "step": 5260, + "unclipped_grad_norm": 0.7152841091156006 + }, + { + "bits_per_byte": 1.6197508359869233, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1227257251739502, + "step": 5270, + "unclipped_grad_norm": 0.762697160243988 + }, + { + "bits_per_byte": 1.6105141637570344, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1163233518600464, + "step": 5280, + "unclipped_grad_norm": 0.7536966800689697 + }, + { + "bits_per_byte": 1.5885342650288148, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.101088047027588, + "step": 5290, + "unclipped_grad_norm": 0.7395375967025757 + }, + { + "bits_per_byte": 1.5242677880544004, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.056541919708252, + "step": 5300, + "unclipped_grad_norm": 0.6971617937088013 + }, + { + "bits_per_byte": 1.589467786857698, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1017351150512695, + "step": 5310, + "unclipped_grad_norm": 0.7348577380180359 + }, + { + "bits_per_byte": 1.5472704676073012, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.072486162185669, + "step": 5320, + "unclipped_grad_norm": 0.7314221858978271 + }, + { + "bits_per_byte": 1.6553672391020549, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1474131345748901, + "step": 5330, + "unclipped_grad_norm": 0.7674097418785095 + }, + { + "bits_per_byte": 1.5389176142201964, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0666964054107666, + "step": 5340, + "unclipped_grad_norm": 0.7431036829948425 + }, + { + "bits_per_byte": 1.593487021408362, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1045210361480713, + "step": 5350, + "unclipped_grad_norm": 0.7701666951179504 + }, + { + "bits_per_byte": 1.5047200679746828, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0429924726486206, + "step": 5360, + "unclipped_grad_norm": 0.7093538045883179 + }, + { + "bits_per_byte": 1.5323434054083145, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0621395111083984, + "step": 5370, + "unclipped_grad_norm": 0.8088909983634949 + }, + { + "bits_per_byte": 1.5949007187984543, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1055009365081787, + "step": 5380, + "unclipped_grad_norm": 0.7591733336448669 + }, + { + "bits_per_byte": 1.5063042001717801, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0440905094146729, + "step": 5390, + "unclipped_grad_norm": 0.7258779406547546 + }, + { + "bits_per_byte": 1.5763400071520381, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0926356315612793, + "step": 5400, + "unclipped_grad_norm": 0.7020314335823059 + }, + { + "bits_per_byte": 1.540318240928823, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0676672458648682, + "step": 5410, + "unclipped_grad_norm": 0.73995441198349 + }, + { + "bits_per_byte": 1.5024278832039832, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0414036512374878, + "step": 5420, + "unclipped_grad_norm": 0.7208534479141235 + }, + { + "bits_per_byte": 1.556975964543442, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0792135000228882, + "step": 5430, + "unclipped_grad_norm": 0.7300911545753479 + }, + { + "bits_per_byte": 1.5377587951186826, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0658931732177734, + "step": 5440, + "unclipped_grad_norm": 0.7053164839744568 + }, + { + "bits_per_byte": 1.5519281017579678, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0757145881652832, + "step": 5450, + "unclipped_grad_norm": 0.7570145726203918 + }, + { + "bits_per_byte": 1.5295854916190785, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.060227870941162, + "step": 5460, + "unclipped_grad_norm": 0.7293761968612671 + }, + { + "bits_per_byte": 1.5453655877663446, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0711658000946045, + "step": 5470, + "unclipped_grad_norm": 0.7067918181419373 + }, + { + "bits_per_byte": 1.5185650153344405, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0525890588760376, + "step": 5480, + "unclipped_grad_norm": 0.7616989612579346 + }, + { + "bits_per_byte": 1.5811864782533522, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0959949493408203, + "step": 5490, + "unclipped_grad_norm": 0.8041682243347168 + }, + { + "bits_per_byte": 1.5613092394145813, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0822170972824097, + "step": 5500, + "unclipped_grad_norm": 0.7645291686058044 + }, + { + "bits_per_byte": 1.5218208188979245, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0548458099365234, + "step": 5510, + "unclipped_grad_norm": 0.7549375891685486 + }, + { + "bits_per_byte": 1.4947696677437132, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.036095380783081, + "step": 5520, + "unclipped_grad_norm": 0.7179425358772278 + }, + { + "bits_per_byte": 1.5059726176209167, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.043860673904419, + "step": 5530, + "unclipped_grad_norm": 0.7216454744338989 + }, + { + "bits_per_byte": 1.625827155024557, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1269375085830688, + "step": 5540, + "unclipped_grad_norm": 0.7688449621200562 + }, + { + "bits_per_byte": 1.5739950237075238, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0910102128982544, + "step": 5550, + "unclipped_grad_norm": 0.7435398697853088 + }, + { + "bits_per_byte": 1.5977501273579493, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.107475996017456, + "step": 5560, + "unclipped_grad_norm": 0.8213235139846802 + }, + { + "bits_per_byte": 1.537488782356828, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0657060146331787, + "step": 5570, + "unclipped_grad_norm": 0.7458736896514893 + }, + { + "bits_per_byte": 1.569628556184768, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0879836082458496, + "step": 5580, + "unclipped_grad_norm": 0.745820939540863 + }, + { + "bits_per_byte": 1.5662062734152489, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0856114625930786, + "step": 5590, + "unclipped_grad_norm": 0.8023808002471924 + }, + { + "bits_per_byte": 1.5614218880508965, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0822951793670654, + "step": 5600, + "unclipped_grad_norm": 0.7724120020866394 + }, + { + "bits_per_byte": 1.5000096351502012, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0397274494171143, + "step": 5610, + "unclipped_grad_norm": 0.7330023050308228 + }, + { + "bits_per_byte": 1.564252034553495, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.084256887435913, + "step": 5620, + "unclipped_grad_norm": 0.7773064374923706 + }, + { + "bits_per_byte": 1.4778873347870674, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0243934392929077, + "step": 5630, + "unclipped_grad_norm": 0.7191233038902283 + }, + { + "bits_per_byte": 1.4838417181252395, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0285207033157349, + "step": 5640, + "unclipped_grad_norm": 0.7019065022468567 + }, + { + "bits_per_byte": 1.5419510442161144, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0687990188598633, + "step": 5650, + "unclipped_grad_norm": 0.7909054160118103 + }, + { + "bits_per_byte": 1.4908660055344292, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0333895683288574, + "step": 5660, + "unclipped_grad_norm": 0.7980607151985168 + }, + { + "bits_per_byte": 1.5729701790910324, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0902998447418213, + "step": 5670, + "unclipped_grad_norm": 0.796278178691864 + }, + { + "bits_per_byte": 1.5668854329035062, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0860822200775146, + "step": 5680, + "unclipped_grad_norm": 0.8280957341194153 + }, + { + "bits_per_byte": 1.592162067066115, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.103602647781372, + "step": 5690, + "unclipped_grad_norm": 0.7912419438362122 + }, + { + "bits_per_byte": 1.5918954939572776, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1034178733825684, + "step": 5700, + "unclipped_grad_norm": 0.7910536527633667 + }, + { + "bits_per_byte": 1.5647875885282816, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0846281051635742, + "step": 5710, + "unclipped_grad_norm": 0.8025376200675964 + }, + { + "bits_per_byte": 1.576262442976499, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.092581868171692, + "step": 5720, + "unclipped_grad_norm": 0.7657404541969299 + }, + { + "bits_per_byte": 1.5180726290050204, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0522477626800537, + "step": 5730, + "unclipped_grad_norm": 0.7721706032752991 + }, + { + "bits_per_byte": 1.6017487239905095, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1102476119995117, + "step": 5740, + "unclipped_grad_norm": 0.7985641956329346 + }, + { + "bits_per_byte": 1.4880221004197618, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0314183235168457, + "step": 5750, + "unclipped_grad_norm": 0.7275866866111755 + }, + { + "bits_per_byte": 1.5012069783655082, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0405573844909668, + "step": 5760, + "unclipped_grad_norm": 0.7711101174354553 + }, + { + "bits_per_byte": 1.5255566260399664, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0574352741241455, + "step": 5770, + "unclipped_grad_norm": 0.8030062913894653 + }, + { + "bits_per_byte": 1.6161017081009161, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1201963424682617, + "step": 5780, + "unclipped_grad_norm": 0.7891528606414795 + }, + { + "bits_per_byte": 1.436315000489906, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 0.9955776929855347, + "step": 5790, + "unclipped_grad_norm": 0.7522260546684265 + }, + { + "bits_per_byte": 1.5639025658069416, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.084014654159546, + "step": 5800, + "unclipped_grad_norm": 0.77913498878479 + }, + { + "bits_per_byte": 1.4787818165542048, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0250134468078613, + "step": 5810, + "unclipped_grad_norm": 0.744920551776886 + }, + { + "bits_per_byte": 1.485352929678371, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0295681953430176, + "step": 5820, + "unclipped_grad_norm": 0.8005070090293884 + }, + { + "bits_per_byte": 1.551830071646976, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0756466388702393, + "step": 5830, + "unclipped_grad_norm": 0.7964350581169128 + }, + { + "bits_per_byte": 1.5252578921754176, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0572282075881958, + "step": 5840, + "unclipped_grad_norm": 0.8099321722984314 + }, + { + "bits_per_byte": 1.4836750669365535, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0284051895141602, + "step": 5850, + "unclipped_grad_norm": 0.7372169494628906 + }, + { + "bits_per_byte": 1.511822779472666, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0479156970977783, + "step": 5860, + "unclipped_grad_norm": 0.7115837931632996 + }, + { + "bits_per_byte": 1.5625698722454058, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.083090901374817, + "step": 5870, + "unclipped_grad_norm": 0.7878124713897705 + }, + { + "bits_per_byte": 1.503004884997628, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0418035984039307, + "step": 5880, + "unclipped_grad_norm": 0.7451943159103394 + }, + { + "bits_per_byte": 1.5713764158654868, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0891951322555542, + "step": 5890, + "unclipped_grad_norm": 0.8092991709709167 + }, + { + "bits_per_byte": 1.5881084359852786, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1007928848266602, + "step": 5900, + "unclipped_grad_norm": 0.7848427891731262 + }, + { + "bits_per_byte": 1.5133455138634053, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.048971176147461, + "step": 5910, + "unclipped_grad_norm": 0.7443398237228394 + }, + { + "bits_per_byte": 1.4825475486774968, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0276236534118652, + "step": 5920, + "unclipped_grad_norm": 0.7709684371948242 + }, + { + "bits_per_byte": 1.5970936695796059, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1070209741592407, + "step": 5930, + "unclipped_grad_norm": 0.8097158074378967 + }, + { + "bits_per_byte": 1.5296348506398763, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0602620840072632, + "step": 5940, + "unclipped_grad_norm": 0.8011659383773804 + }, + { + "bits_per_byte": 1.5315722352018457, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0616049766540527, + "step": 5950, + "unclipped_grad_norm": 0.7703563570976257 + }, + { + "bits_per_byte": 1.4753576419755263, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0226399898529053, + "step": 5960, + "unclipped_grad_norm": 0.7793571949005127 + }, + { + "bits_per_byte": 1.514912963743369, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0500576496124268, + "step": 5970, + "unclipped_grad_norm": 0.7870919108390808 + }, + { + "bits_per_byte": 1.4934732625215095, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0351967811584473, + "step": 5980, + "unclipped_grad_norm": 0.8144509196281433 + }, + { + "bits_per_byte": 1.5080035607449556, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0452684164047241, + "step": 5990, + "unclipped_grad_norm": 0.7810203433036804 + }, + { + "bits_per_byte": 1.5782032671914858, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0939271450042725, + "step": 6000, + "unclipped_grad_norm": 0.790114164352417 + }, + { + "bits_per_byte": 1.5490276143511665, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0737041234970093, + "step": 6010, + "unclipped_grad_norm": 0.8144342303276062 + }, + { + "bits_per_byte": 1.515940732064925, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0507700443267822, + "step": 6020, + "unclipped_grad_norm": 0.7434252500534058 + }, + { + "bits_per_byte": 1.493858503659442, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.035463809967041, + "step": 6030, + "unclipped_grad_norm": 0.8062746524810791 + }, + { + "bits_per_byte": 1.5237423810560147, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0561777353286743, + "step": 6040, + "unclipped_grad_norm": 0.7702685594558716 + }, + { + "bits_per_byte": 1.5275683071071087, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.058829665184021, + "step": 6050, + "unclipped_grad_norm": 0.7716134786605835 + }, + { + "bits_per_byte": 1.5215145177967377, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0546334981918335, + "step": 6060, + "unclipped_grad_norm": 0.7933171987533569 + }, + { + "bits_per_byte": 1.415498478403448, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 0.9811487793922424, + "step": 6070, + "unclipped_grad_norm": 0.7639470100402832 + }, + { + "bits_per_byte": 1.4966260484771265, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0373821258544922, + "step": 6080, + "unclipped_grad_norm": 0.7900221943855286 + }, + { + "bits_per_byte": 1.4925770609278632, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0345755815505981, + "step": 6090, + "unclipped_grad_norm": 0.809319019317627 + }, + { + "bits_per_byte": 1.5936792980120267, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.104654312133789, + "step": 6100, + "unclipped_grad_norm": 0.824783205986023 + }, + { + "bits_per_byte": 1.5281914002511847, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0592615604400635, + "step": 6110, + "unclipped_grad_norm": 0.8245336413383484 + }, + { + "bits_per_byte": 1.48945093228313, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0324087142944336, + "step": 6120, + "unclipped_grad_norm": 0.8355178833007812 + }, + { + "bits_per_byte": 1.5475705773330568, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.072694182395935, + "step": 6130, + "unclipped_grad_norm": 0.7615814208984375 + }, + { + "bits_per_byte": 1.4980500648262705, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0383691787719727, + "step": 6140, + "unclipped_grad_norm": 0.7743073105812073 + }, + { + "bits_per_byte": 1.483834838819205, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0285159349441528, + "step": 6150, + "unclipped_grad_norm": 0.7909993529319763 + }, + { + "bits_per_byte": 1.5470265962083778, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.072317123413086, + "step": 6160, + "unclipped_grad_norm": 0.7707679867744446 + }, + { + "bits_per_byte": 1.478142213075646, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.024570107460022, + "step": 6170, + "unclipped_grad_norm": 0.7621827721595764 + }, + { + "bits_per_byte": 1.4841139666615555, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0287094116210938, + "step": 6180, + "unclipped_grad_norm": 0.7684755921363831 + }, + { + "bits_per_byte": 1.493858503659442, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.035463809967041, + "step": 6190, + "unclipped_grad_norm": 0.8198496699333191 + }, + { + "bits_per_byte": 1.4883674415826944, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0316576957702637, + "step": 6200, + "unclipped_grad_norm": 0.7631723284721375 + }, + { + "bits_per_byte": 1.5053304344025948, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0434155464172363, + "step": 6210, + "unclipped_grad_norm": 0.7891849875450134 + }, + { + "bits_per_byte": 1.5365181122753584, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.065033197402954, + "step": 6220, + "unclipped_grad_norm": 0.7962750196456909 + }, + { + "bits_per_byte": 1.4799394317771624, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0258158445358276, + "step": 6230, + "unclipped_grad_norm": 0.7991603016853333 + }, + { + "bits_per_byte": 1.5356217386990614, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0644118785858154, + "step": 6240, + "unclipped_grad_norm": 0.8105412721633911 + }, + { + "bits_per_byte": 1.4819163723488304, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0271861553192139, + "step": 6250, + "unclipped_grad_norm": 0.7951217293739319 + }, + { + "bits_per_byte": 1.4447946050906966, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.001455307006836, + "step": 6260, + "unclipped_grad_norm": 0.7625995874404907 + }, + { + "bits_per_byte": 1.5022859975170215, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0413053035736084, + "step": 6270, + "unclipped_grad_norm": 0.8279880881309509 + }, + { + "bits_per_byte": 1.5067209141348208, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0443793535232544, + "step": 6280, + "unclipped_grad_norm": 0.7734115719795227 + }, + { + "bits_per_byte": 1.5122324421470212, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0481996536254883, + "step": 6290, + "unclipped_grad_norm": 0.7805989980697632 + }, + { + "bits_per_byte": 1.476189006109797, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0232162475585938, + "step": 6300, + "unclipped_grad_norm": 0.8065475225448608 + }, + { + "bits_per_byte": 1.5409463215697738, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0681025981903076, + "step": 6310, + "unclipped_grad_norm": 0.8388196229934692 + }, + { + "bits_per_byte": 1.506346851869194, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0441200733184814, + "step": 6320, + "unclipped_grad_norm": 0.8094848990440369 + }, + { + "bits_per_byte": 1.5104558613636083, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0469682216644287, + "step": 6330, + "unclipped_grad_norm": 0.7866588234901428 + }, + { + "bits_per_byte": 1.427382135612766, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 0.9893859028816223, + "step": 6340, + "unclipped_grad_norm": 0.7612236738204956 + }, + { + "bits_per_byte": 1.541960503261912, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0688055753707886, + "step": 6350, + "unclipped_grad_norm": 0.783340573310852 + }, + { + "bits_per_byte": 1.5465876964833758, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0720129013061523, + "step": 6360, + "unclipped_grad_norm": 0.8162001371383667 + }, + { + "bits_per_byte": 1.4662069610710688, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0162972211837769, + "step": 6370, + "unclipped_grad_norm": 0.7845266461372375 + }, + { + "bits_per_byte": 1.4477447954835971, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.00350022315979, + "step": 6380, + "unclipped_grad_norm": 0.7937688231468201 + }, + { + "bits_per_byte": 1.5785561755910564, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0941717624664307, + "step": 6390, + "unclipped_grad_norm": 0.826732337474823 + }, + { + "bits_per_byte": 1.5478830698096744, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0729107856750488, + "step": 6400, + "unclipped_grad_norm": 0.8436270952224731 + }, + { + "bits_per_byte": 1.550731790438566, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.074885368347168, + "step": 6410, + "unclipped_grad_norm": 0.8814867734909058 + }, + { + "bits_per_byte": 1.4786806907554975, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0249433517456055, + "step": 6420, + "unclipped_grad_norm": 0.8324126601219177 + }, + { + "bits_per_byte": 1.5252604719151805, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.057229995727539, + "step": 6430, + "unclipped_grad_norm": 0.7683992981910706 + }, + { + "bits_per_byte": 1.4432244034883193, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0003669261932373, + "step": 6440, + "unclipped_grad_norm": 0.8131992816925049 + }, + { + "bits_per_byte": 1.465075831176344, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0155131816864014, + "step": 6450, + "unclipped_grad_norm": 0.758244514465332 + }, + { + "bits_per_byte": 1.4922360193312023, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.034339189529419, + "step": 6460, + "unclipped_grad_norm": 0.798247754573822 + }, + { + "bits_per_byte": 1.498638761440174, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.038777232170105, + "step": 6470, + "unclipped_grad_norm": 0.8465205430984497 + }, + { + "bits_per_byte": 1.456987831071568, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0099070072174072, + "step": 6480, + "unclipped_grad_norm": 0.7985926270484924 + }, + { + "bits_per_byte": 1.542977092711162, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0695102214813232, + "step": 6490, + "unclipped_grad_norm": 0.8188008666038513 + }, + { + "bits_per_byte": 1.582791420351204, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0971074104309082, + "step": 6500, + "unclipped_grad_norm": 0.8359130620956421 + }, + { + "bits_per_byte": 1.5352007251697493, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0641200542449951, + "step": 6510, + "unclipped_grad_norm": 0.854469895362854 + }, + { + "bits_per_byte": 1.5259941499037615, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0577385425567627, + "step": 6520, + "unclipped_grad_norm": 0.8313364386558533 + }, + { + "bits_per_byte": 1.4871191915027322, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.030792474746704, + "step": 6530, + "unclipped_grad_norm": 0.7717711329460144 + }, + { + "bits_per_byte": 1.4761642406080728, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0231990814208984, + "step": 6540, + "unclipped_grad_norm": 0.8019536733627319 + }, + { + "bits_per_byte": 1.5028273989019376, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0416805744171143, + "step": 6550, + "unclipped_grad_norm": 0.7671847343444824 + }, + { + "bits_per_byte": 1.5359399066031576, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0646324157714844, + "step": 6560, + "unclipped_grad_norm": 0.8164054751396179 + }, + { + "bits_per_byte": 1.504123116193538, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0425786972045898, + "step": 6570, + "unclipped_grad_norm": 0.7928623557090759 + }, + { + "bits_per_byte": 1.4852913598893622, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0295255184173584, + "step": 6580, + "unclipped_grad_norm": 0.8092948794364929 + }, + { + "bits_per_byte": 1.5248497773449203, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0569453239440918, + "step": 6590, + "unclipped_grad_norm": 0.8197707533836365 + }, + { + "bits_per_byte": 1.426588779644336, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9888359904289246, + "step": 6600, + "unclipped_grad_norm": 0.7768723964691162 + }, + { + "bits_per_byte": 1.4662528804388493, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.016329050064087, + "step": 6610, + "unclipped_grad_norm": 0.8190019726753235 + }, + { + "bits_per_byte": 1.5755777800434143, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0921072959899902, + "step": 6620, + "unclipped_grad_norm": 0.8630855083465576 + }, + { + "bits_per_byte": 1.5211641891369303, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0543906688690186, + "step": 6630, + "unclipped_grad_norm": 0.8092710971832275 + }, + { + "bits_per_byte": 1.4689426890983437, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0181934833526611, + "step": 6640, + "unclipped_grad_norm": 0.7870210409164429 + }, + { + "bits_per_byte": 1.4737494322073081, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.021525263786316, + "step": 6650, + "unclipped_grad_norm": 0.8340798020362854 + }, + { + "bits_per_byte": 1.5123179175245, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.048258900642395, + "step": 6660, + "unclipped_grad_norm": 0.8463515043258667 + }, + { + "bits_per_byte": 1.4946623505695749, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0360209941864014, + "step": 6670, + "unclipped_grad_norm": 0.8076959252357483 + }, + { + "bits_per_byte": 1.4999131528830674, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0396605730056763, + "step": 6680, + "unclipped_grad_norm": 0.8417643904685974 + }, + { + "bits_per_byte": 1.4927383806543726, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0346873998641968, + "step": 6690, + "unclipped_grad_norm": 0.8966479301452637 + }, + { + "bits_per_byte": 1.507631906236441, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0450108051300049, + "step": 6700, + "unclipped_grad_norm": 0.8065080642700195 + }, + { + "bits_per_byte": 1.526726795996437, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.058246374130249, + "step": 6710, + "unclipped_grad_norm": 0.8376452326774597 + }, + { + "bits_per_byte": 1.5822010039107919, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0966981649398804, + "step": 6720, + "unclipped_grad_norm": 0.8624657392501831 + }, + { + "bits_per_byte": 1.447227471669802, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0031416416168213, + "step": 6730, + "unclipped_grad_norm": 0.8040223717689514 + }, + { + "bits_per_byte": 1.4848161718250283, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0291961431503296, + "step": 6740, + "unclipped_grad_norm": 0.8131929636001587 + }, + { + "bits_per_byte": 1.4599282184533695, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.011945128440857, + "step": 6750, + "unclipped_grad_norm": 0.8242661356925964 + }, + { + "bits_per_byte": 1.4561461479782454, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0093235969543457, + "step": 6760, + "unclipped_grad_norm": 0.858147919178009 + }, + { + "bits_per_byte": 1.4679376224867013, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0174968242645264, + "step": 6770, + "unclipped_grad_norm": 0.8389740586280823 + }, + { + "bits_per_byte": 1.446337461451587, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.002524733543396, + "step": 6780, + "unclipped_grad_norm": 0.806380033493042 + }, + { + "bits_per_byte": 1.4957231395600967, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0367562770843506, + "step": 6790, + "unclipped_grad_norm": 0.8443998694419861 + }, + { + "bits_per_byte": 1.483685729860907, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0284125804901123, + "step": 6800, + "unclipped_grad_norm": 0.8614793419837952 + }, + { + "bits_per_byte": 1.4736859706091399, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0214812755584717, + "step": 6810, + "unclipped_grad_norm": 0.8073354959487915 + }, + { + "bits_per_byte": 1.4900140034820548, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0327990055084229, + "step": 6820, + "unclipped_grad_norm": 0.8066654801368713 + }, + { + "bits_per_byte": 1.4893948659389489, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.03236985206604, + "step": 6830, + "unclipped_grad_norm": 0.8152909278869629 + }, + { + "bits_per_byte": 1.4517927511369553, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0063060522079468, + "step": 6840, + "unclipped_grad_norm": 0.8145732283592224 + }, + { + "bits_per_byte": 1.5401706798143826, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0675649642944336, + "step": 6850, + "unclipped_grad_norm": 0.8944696187973022 + }, + { + "bits_per_byte": 1.5020013662298435, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0411080121994019, + "step": 6860, + "unclipped_grad_norm": 0.8296581506729126 + }, + { + "bits_per_byte": 1.4315734388056183, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9922910928726196, + "step": 6870, + "unclipped_grad_norm": 0.8298041224479675 + }, + { + "bits_per_byte": 1.4506086505857647, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0054852962493896, + "step": 6880, + "unclipped_grad_norm": 0.8620857000350952 + }, + { + "bits_per_byte": 1.4600797351687795, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0120501518249512, + "step": 6890, + "unclipped_grad_norm": 0.8483932614326477 + }, + { + "bits_per_byte": 1.5166142161257037, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.051236867904663, + "step": 6900, + "unclipped_grad_norm": 0.8512884378433228 + }, + { + "bits_per_byte": 1.5060785589338481, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0439341068267822, + "step": 6910, + "unclipped_grad_norm": 0.8712192177772522 + }, + { + "bits_per_byte": 1.4495721111490145, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.004766821861267, + "step": 6920, + "unclipped_grad_norm": 0.8044716715812683 + }, + { + "bits_per_byte": 1.4331765750856358, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9934023022651672, + "step": 6930, + "unclipped_grad_norm": 0.8494104743003845 + }, + { + "bits_per_byte": 1.5111775005666286, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0474684238433838, + "step": 6940, + "unclipped_grad_norm": 0.8407061696052551 + }, + { + "bits_per_byte": 1.4275953081085107, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9895336627960205, + "step": 6950, + "unclipped_grad_norm": 0.8335278630256653 + }, + { + "bits_per_byte": 1.4337008641967912, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 0.9937657117843628, + "step": 6960, + "unclipped_grad_norm": 0.8460164666175842 + }, + { + "bits_per_byte": 1.404613954404317, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 0.9736042022705078, + "step": 6970, + "unclipped_grad_norm": 0.8476742506027222 + }, + { + "bits_per_byte": 1.5285965913766173, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0595424175262451, + "step": 6980, + "unclipped_grad_norm": 0.9341830611228943 + }, + { + "bits_per_byte": 1.4927865357966141, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034720778465271, + "step": 6990, + "unclipped_grad_norm": 0.8535965085029602 + }, + { + "bits_per_byte": 1.5015640143486995, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0408048629760742, + "step": 7000, + "unclipped_grad_norm": 0.8839671015739441 + }, + { + "bits_per_byte": 1.409006391307353, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9766488075256348, + "step": 7010, + "unclipped_grad_norm": 0.8374313712120056 + }, + { + "bits_per_byte": 1.4918236049344333, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0340533256530762, + "step": 7020, + "unclipped_grad_norm": 0.8843756318092346 + }, + { + "bits_per_byte": 1.4765720114732686, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0234817266464233, + "step": 7030, + "unclipped_grad_norm": 0.8468295931816101 + }, + { + "bits_per_byte": 1.4469567709773439, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0029540061950684, + "step": 7040, + "unclipped_grad_norm": 0.8328400254249573 + }, + { + "bits_per_byte": 1.4604876780166263, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0123329162597656, + "step": 7050, + "unclipped_grad_norm": 0.8563967943191528 + }, + { + "bits_per_byte": 1.4275743262251053, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 0.9895191192626953, + "step": 7060, + "unclipped_grad_norm": 0.830858588218689 + }, + { + "bits_per_byte": 1.4821155282585297, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0273241996765137, + "step": 7070, + "unclipped_grad_norm": 0.8345243334770203 + }, + { + "bits_per_byte": 1.482162651504866, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0273568630218506, + "step": 7080, + "unclipped_grad_norm": 0.9049980044364929 + }, + { + "bits_per_byte": 1.4400559671114737, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.9981707334518433, + "step": 7090, + "unclipped_grad_norm": 0.86240553855896 + }, + { + "bits_per_byte": 1.4348600272636065, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9945691823959351, + "step": 7100, + "unclipped_grad_norm": 0.8350730538368225 + }, + { + "bits_per_byte": 1.4152483296377678, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 0.9809753894805908, + "step": 7110, + "unclipped_grad_norm": 0.8401304483413696 + }, + { + "bits_per_byte": 1.4797989219514076, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0257184505462646, + "step": 7120, + "unclipped_grad_norm": 0.8903806805610657 + }, + { + "bits_per_byte": 1.5240091261475028, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0563626289367676, + "step": 7130, + "unclipped_grad_norm": 0.8470056056976318 + }, + { + "bits_per_byte": 1.4628170110399121, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0139474868774414, + "step": 7140, + "unclipped_grad_norm": 0.8429069519042969 + }, + { + "bits_per_byte": 1.5096671489267515, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0464215278625488, + "step": 7150, + "unclipped_grad_norm": 0.8920191526412964 + }, + { + "bits_per_byte": 1.5009496923198173, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0403790473937988, + "step": 7160, + "unclipped_grad_norm": 0.8647960424423218 + }, + { + "bits_per_byte": 1.5036095759980617, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0422227382659912, + "step": 7170, + "unclipped_grad_norm": 0.9017168283462524 + }, + { + "bits_per_byte": 1.4616292988530535, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0131242275238037, + "step": 7180, + "unclipped_grad_norm": 0.8551055788993835 + }, + { + "bits_per_byte": 1.5122007973392624, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.048177719116211, + "step": 7190, + "unclipped_grad_norm": 0.8699921369552612 + }, + { + "bits_per_byte": 1.441016404225217, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 0.9988364577293396, + "step": 7200, + "unclipped_grad_norm": 0.8101796507835388 + }, + { + "bits_per_byte": 1.4373777672809125, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 0.9963143467903137, + "step": 7210, + "unclipped_grad_norm": 0.8248227834701538 + }, + { + "bits_per_byte": 1.4388484769197658, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9973337650299072, + "step": 7220, + "unclipped_grad_norm": 0.8497165441513062 + }, + { + "bits_per_byte": 1.4764554072359837, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.023400902748108, + "step": 7230, + "unclipped_grad_norm": 0.869898796081543 + }, + { + "bits_per_byte": 1.5094511387172678, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.046271800994873, + "step": 7240, + "unclipped_grad_norm": 0.8670166730880737 + }, + { + "bits_per_byte": 1.4896672864579155, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0325586795806885, + "step": 7250, + "unclipped_grad_norm": 0.8726136684417725 + }, + { + "bits_per_byte": 1.4660583680607233, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.016194224357605, + "step": 7260, + "unclipped_grad_norm": 0.8553854823112488 + }, + { + "bits_per_byte": 1.4854459722924878, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0296326875686646, + "step": 7270, + "unclipped_grad_norm": 0.8842402100563049 + }, + { + "bits_per_byte": 1.4706281190767991, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0193617343902588, + "step": 7280, + "unclipped_grad_norm": 0.8827332854270935 + }, + { + "bits_per_byte": 1.465586275684105, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.015866994857788, + "step": 7290, + "unclipped_grad_norm": 0.9209113121032715 + }, + { + "bits_per_byte": 1.4604543133823589, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0123097896575928, + "step": 7300, + "unclipped_grad_norm": 0.8443734645843506 + }, + { + "bits_per_byte": 1.44415740936925, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0010136365890503, + "step": 7310, + "unclipped_grad_norm": 0.8828385472297668 + }, + { + "bits_per_byte": 1.4200490533539523, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9843029975891113, + "step": 7320, + "unclipped_grad_norm": 0.878987193107605 + }, + { + "bits_per_byte": 1.4725175204791778, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0206713676452637, + "step": 7330, + "unclipped_grad_norm": 0.9326861500740051 + }, + { + "bits_per_byte": 1.4404713912046327, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9984586834907532, + "step": 7340, + "unclipped_grad_norm": 0.8777600526809692 + }, + { + "bits_per_byte": 1.448391966198794, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0039488077163696, + "step": 7350, + "unclipped_grad_norm": 0.8631419539451599 + }, + { + "bits_per_byte": 1.453881480431684, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.007753849029541, + "step": 7360, + "unclipped_grad_norm": 0.855156421661377 + }, + { + "bits_per_byte": 1.4104983408035878, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 0.9776829481124878, + "step": 7370, + "unclipped_grad_norm": 0.8616177439689636 + }, + { + "bits_per_byte": 1.406926175153842, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9752069115638733, + "step": 7380, + "unclipped_grad_norm": 0.8499169945716858 + }, + { + "bits_per_byte": 1.4229885808224993, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 0.9863405227661133, + "step": 7390, + "unclipped_grad_norm": 0.8316128849983215 + }, + { + "bits_per_byte": 1.4424061960268393, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9997997879981995, + "step": 7400, + "unclipped_grad_norm": 0.8269481062889099 + }, + { + "bits_per_byte": 1.4779136481326494, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.024411678314209, + "step": 7410, + "unclipped_grad_norm": 0.882456362247467 + }, + { + "bits_per_byte": 1.464608382331299, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0151891708374023, + "step": 7420, + "unclipped_grad_norm": 0.8758774995803833 + }, + { + "bits_per_byte": 1.3991048341492285, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9697855710983276, + "step": 7430, + "unclipped_grad_norm": 0.8292478919029236 + }, + { + "bits_per_byte": 1.401245502204518, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9712693691253662, + "step": 7440, + "unclipped_grad_norm": 0.875770092010498 + }, + { + "bits_per_byte": 1.5201309173705464, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0536744594573975, + "step": 7450, + "unclipped_grad_norm": 0.9047821760177612 + }, + { + "bits_per_byte": 1.4276059710328641, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9895410537719727, + "step": 7460, + "unclipped_grad_norm": 0.8659260272979736 + }, + { + "bits_per_byte": 1.4078822267099882, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9758695960044861, + "step": 7470, + "unclipped_grad_norm": 0.8766598105430603 + }, + { + "bits_per_byte": 1.486473568631393, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0303449630737305, + "step": 7480, + "unclipped_grad_norm": 0.8612073659896851 + }, + { + "bits_per_byte": 1.450941608997835, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.00571608543396, + "step": 7490, + "unclipped_grad_norm": 0.888404905796051 + }, + { + "bits_per_byte": 1.4814055838757678, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.026832103729248, + "step": 7500, + "unclipped_grad_norm": 0.9002490639686584 + }, + { + "bits_per_byte": 1.4606080658722302, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0124163627624512, + "step": 7510, + "unclipped_grad_norm": 0.8679240345954895 + }, + { + "bits_per_byte": 1.4798904167216664, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0257818698883057, + "step": 7520, + "unclipped_grad_norm": 0.9162602424621582 + }, + { + "bits_per_byte": 1.4597773896685629, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.011840581893921, + "step": 7530, + "unclipped_grad_norm": 0.8853410482406616 + }, + { + "bits_per_byte": 1.4621115382060728, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.013458490371704, + "step": 7540, + "unclipped_grad_norm": 0.869069516658783 + }, + { + "bits_per_byte": 1.4571945542179052, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0100502967834473, + "step": 7550, + "unclipped_grad_norm": 0.8861801028251648 + }, + { + "bits_per_byte": 1.38000641874489, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 0.9565475583076477, + "step": 7560, + "unclipped_grad_norm": 0.8688139915466309 + }, + { + "bits_per_byte": 1.4141796294453064, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9802346229553223, + "step": 7570, + "unclipped_grad_norm": 0.8542686700820923 + }, + { + "bits_per_byte": 1.4002238392670674, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9705612063407898, + "step": 7580, + "unclipped_grad_norm": 0.898608922958374 + }, + { + "bits_per_byte": 1.4730534184192665, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.021042823791504, + "step": 7590, + "unclipped_grad_norm": 0.8726481199264526 + }, + { + "bits_per_byte": 1.5103581752179183, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0469005107879639, + "step": 7600, + "unclipped_grad_norm": 0.8932017087936401 + }, + { + "bits_per_byte": 1.4571630813927972, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0100284814834595, + "step": 7610, + "unclipped_grad_norm": 0.8839172124862671 + }, + { + "bits_per_byte": 1.4709910024701196, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.019613265991211, + "step": 7620, + "unclipped_grad_norm": 0.8883293271064758 + }, + { + "bits_per_byte": 1.436151788954237, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.995464563369751, + "step": 7630, + "unclipped_grad_norm": 0.8680411577224731 + }, + { + "bits_per_byte": 1.4213451146108544, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.985201358795166, + "step": 7640, + "unclipped_grad_norm": 0.9127815961837769 + }, + { + "bits_per_byte": 1.3999687889958379, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 0.970384418964386, + "step": 7650, + "unclipped_grad_norm": 0.8844171166419983 + }, + { + "bits_per_byte": 1.4213168234647875, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9851817488670349, + "step": 7660, + "unclipped_grad_norm": 0.8884557485580444 + }, + { + "bits_per_byte": 1.467116749294133, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0169278383255005, + "step": 7670, + "unclipped_grad_norm": 0.9028065204620361 + }, + { + "bits_per_byte": 1.4497504571579594, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0048904418945312, + "step": 7680, + "unclipped_grad_norm": 0.8733177781105042 + }, + { + "bits_per_byte": 1.436977993608982, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9960372447967529, + "step": 7690, + "unclipped_grad_norm": 0.8574365973472595 + }, + { + "bits_per_byte": 1.4762694939904009, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0232720375061035, + "step": 7700, + "unclipped_grad_norm": 0.8926270008087158 + }, + { + "bits_per_byte": 1.4168368473924622, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9820764660835266, + "step": 7710, + "unclipped_grad_norm": 0.8520281314849854 + }, + { + "bits_per_byte": 1.4226952644114528, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9861372113227844, + "step": 7720, + "unclipped_grad_norm": 0.8500117063522339 + }, + { + "bits_per_byte": 1.4242657239878067, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9872257709503174, + "step": 7730, + "unclipped_grad_norm": 0.8397486209869385 + }, + { + "bits_per_byte": 1.4715379072998633, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0199923515319824, + "step": 7740, + "unclipped_grad_norm": 0.9084712266921997 + }, + { + "bits_per_byte": 1.4017008262726773, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9715849757194519, + "step": 7750, + "unclipped_grad_norm": 0.8779424428939819 + }, + { + "bits_per_byte": 1.4074879564828853, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9755963087081909, + "step": 7760, + "unclipped_grad_norm": 0.8754757642745972 + }, + { + "bits_per_byte": 1.4701193943955468, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0190091133117676, + "step": 7770, + "unclipped_grad_norm": 0.9014414548873901 + }, + { + "bits_per_byte": 1.4337202982363386, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 0.993779182434082, + "step": 7780, + "unclipped_grad_norm": 0.9682468175888062 + }, + { + "bits_per_byte": 1.461853220264477, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0132794380187988, + "step": 7790, + "unclipped_grad_norm": 0.8698559403419495 + }, + { + "bits_per_byte": 1.446109240473892, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.002366542816162, + "step": 7800, + "unclipped_grad_norm": 0.8869026303291321 + }, + { + "bits_per_byte": 1.4204974121247516, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9846137762069702, + "step": 7810, + "unclipped_grad_norm": 0.893868088722229 + }, + { + "bits_per_byte": 1.471287500560207, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0198187828063965, + "step": 7820, + "unclipped_grad_norm": 0.9250245690345764 + }, + { + "bits_per_byte": 1.457534563918661, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0102859735488892, + "step": 7830, + "unclipped_grad_norm": 0.902686595916748 + }, + { + "bits_per_byte": 1.4368130622468045, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9959229230880737, + "step": 7840, + "unclipped_grad_norm": 0.880992591381073 + }, + { + "bits_per_byte": 1.4160461571551204, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9815284013748169, + "step": 7850, + "unclipped_grad_norm": 0.8318398594856262 + }, + { + "bits_per_byte": 1.463385069735712, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0143412351608276, + "step": 7860, + "unclipped_grad_norm": 0.8814117908477783 + }, + { + "bits_per_byte": 1.4569142224969986, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0098559856414795, + "step": 7870, + "unclipped_grad_norm": 0.8954723477363586 + }, + { + "bits_per_byte": 1.4148711716844258, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.980713963508606, + "step": 7880, + "unclipped_grad_norm": 0.9006734490394592 + }, + { + "bits_per_byte": 1.44079876018055, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 0.9986855983734131, + "step": 7890, + "unclipped_grad_norm": 0.8986613154411316 + }, + { + "bits_per_byte": 1.4040539788931077, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 0.9732160568237305, + "step": 7900, + "unclipped_grad_norm": 0.8486817479133606 + }, + { + "bits_per_byte": 1.4565468675547557, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.009601354598999, + "step": 7910, + "unclipped_grad_norm": 0.960801362991333 + }, + { + "bits_per_byte": 1.4869515084181408, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.030676245689392, + "step": 7920, + "unclipped_grad_norm": 0.9004883766174316 + }, + { + "bits_per_byte": 1.426570807457321, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9888235330581665, + "step": 7930, + "unclipped_grad_norm": 0.9118951559066772 + }, + { + "bits_per_byte": 1.4575464307215704, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0102941989898682, + "step": 7940, + "unclipped_grad_norm": 0.9135404825210571 + }, + { + "bits_per_byte": 1.4061704833859507, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9746831059455872, + "step": 7950, + "unclipped_grad_norm": 0.8805200457572937 + }, + { + "bits_per_byte": 1.4792671515949398, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0253498554229736, + "step": 7960, + "unclipped_grad_norm": 0.9637572765350342 + }, + { + "bits_per_byte": 1.4296003678435947, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9909234642982483, + "step": 7970, + "unclipped_grad_norm": 0.8976891040802002 + }, + { + "bits_per_byte": 1.4664819613297986, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0164878368377686, + "step": 7980, + "unclipped_grad_norm": 0.9626050591468811 + }, + { + "bits_per_byte": 1.4209173077668331, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9849048256874084, + "step": 7990, + "unclipped_grad_norm": 0.9410001635551453 + }, + { + "bits_per_byte": 1.4482875727297202, + "learning_rate": 3e-05, + "loss_nats": 1.0038764476776123, + "step": 8000, + "unclipped_grad_norm": 0.9042860269546509 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073003.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073003.json new file mode 100644 index 0000000..8bbb7e5 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073003.json @@ -0,0 +1,8614 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "5f1f6749be2f106aaf597f0478348ce6e226aa739e557d119b2388ec4460aa16", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002897916710935533, + 0.0002662435290403664, + 0.00024211003619711846, + 0.00022891115804668516, + 0.00021424151782412082, + 0.00020374931045807898, + 0.00019545880786608905, + 0.00018862746946979314, + 0.00018024678865913302, + 0.00017406532424502075, + 0.0001698832056717947, + 0.00016633344057481736, + 0.00016165875422302634, + 0.0001582534605404362, + 0.00015452370280399919, + 0.0001515774056315422, + 0.00014964381989557296, + 0.00014728632231708616, + 0.00014459768135566264, + 0.00014173447561915964, + 0.00013920740457251668, + 0.00013723202573601156, + 0.00013503432273864746, + 0.0001331205276073888, + 0.00013141198724042624, + 0.00013009003305342048, + 0.0001286487968172878, + 0.00012732617324218154, + 0.00012632299331016839, + 0.00012548104859888554, + 0.00012477955897338688, + 0.00012385686568450183 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022864168749947567, + "first_to_last_ratio": 1.796936959436678, + "imbalance_abs_log_ratio": 0.5860835261825511, + "last_quartile_mean": 0.00012723968211503234, + "mean": 0.00016535780059712124, + "normalized": [ + 1.7525128542293782, + 1.6101056501655084, + 1.4641585417974736, + 1.3843384298779209, + 1.2956238958819981, + 1.2321723542664615, + 1.1820356049746095, + 1.1407231396924917, + 1.0900410383317047, + 1.052658680851196, + 1.0273673516358577, + 1.0059001751001344, + 0.9776300461137163, + 0.9570365593214796, + 0.9344808787127115, + 0.9166631697094613, + 0.9049698251621409, + 0.8907128770776013, + 0.874453341986335, + 0.8571381277892198, + 0.8418556854882369, + 0.8299095974937676, + 0.816619005883163, + 0.8050453448623477, + 0.7947129604160569, + 0.786718452855893, + 0.7780025880407573, + 0.7700040323613145, + 0.7639373096038118, + 0.7588456555769529, + 0.7546034025779078, + 0.7490234221623898 + ], + "population_cv": 0.25804396357350073 + }, + "activation_output_rms_by_block": [ + 0.028274519369006157, + 0.02845057100057602, + 0.02867918275296688, + 0.02882254309952259, + 0.0290658138692379, + 0.029274335131049156, + 0.029572302475571632, + 0.02980491891503334, + 0.03008633479475975, + 0.03044024296104908, + 0.030671589076519012, + 0.030981140211224556, + 0.0313686840236187, + 0.031716227531433105, + 0.03204340860247612, + 0.03237249702215195, + 0.032637808471918106, + 0.03308072313666344, + 0.03343126177787781, + 0.033881161361932755, + 0.03435666114091873, + 0.03482793644070625, + 0.03533180430531502, + 0.03546451777219772, + 0.036012887954711914, + 0.03628316894173622, + 0.03689942881464958, + 0.03742557764053345, + 0.03777860850095749, + 0.03856590390205383, + 0.038892220705747604, + 0.039198413491249084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02899302332662046, + "first_to_last_ratio": 0.7704348189669382, + "imbalance_abs_log_ratio": 0.2608002235603661, + "last_quartile_mean": 0.0376320262439549, + "mean": 0.032990387349855155, + "normalized": [ + 0.8570532703712039, + 0.8623897227657417, + 0.8693193701798957, + 0.8736648889225345, + 0.8810388784163674, + 0.887359545694382, + 0.8963914901017818, + 0.9034425270294439, + 0.9119727657545023, + 0.9227003805150147, + 0.9297129115597873, + 0.9390959821925394, + 0.9508431559460433, + 0.961377846070436, + 0.9712953128638184, + 0.9812705949417742, + 0.9893126784418129, + 1.002738245715344, + 1.0133637239037931, + 1.0270010170729782, + 1.0414142997647942, + 1.0556995306349188, + 1.070972702764284, + 1.0749954947817073, + 1.0916176149374623, + 1.099810334354729, + 1.1184903172957619, + 1.1344388668021435, + 1.1451398887901616, + 1.1690042767025335, + 1.1788955459450936, + 1.1881768187672155 + ], + "population_cv": 0.10167083190861884 + }, + "bits_per_byte": 8.0691508062416, + "branch_output_rms_by_sublayer": [ + 0.002122601494193077, + 0.002755566267296672, + 0.002245423151180148, + 0.0027348215226083994, + 0.0024954702239483595, + 0.002720654010772705, + 0.0023770928382873535, + 0.002733660163357854, + 0.0023630354553461075, + 0.0027867795433849096, + 0.0024109049700200558, + 0.0027537678834050894, + 0.002701795194298029, + 0.0027542903553694487, + 0.0029824089724570513, + 0.002726059639826417, + 0.0029706284403800964, + 0.0027311695739626884, + 0.003079906338825822, + 0.0027289879508316517, + 0.0028494365978986025, + 0.0027595392893999815, + 0.0032860173378139734, + 0.002710360335186124, + 0.0035252906382083893, + 0.0027479089330881834, + 0.003967344760894775, + 0.002725270576775074, + 0.0035556324291974306, + 0.002750981133431196, + 0.004272372927516699, + 0.0027044012676924467, + 0.004134721122682095, + 0.0027556782588362694, + 0.0041320133022964, + 0.0027166390791535378, + 0.0040069944225251675, + 0.0026826481334865093, + 0.004499548580497503, + 0.002729416359215975, + 0.0049178507179021835, + 0.002681323792785406, + 0.004439458716660738, + 0.002806412521749735, + 0.004581923596560955, + 0.0027696131728589535, + 0.004871132783591747, + 0.0027697959449142218, + 0.0049627418629825115, + 0.0028166540432721376, + 0.0051520452834665775, + 0.0027135310228914022, + 0.005508258938789368, + 0.002744151744991541, + 0.005101518705487251, + 0.002700615907087922, + 0.005637416150420904, + 0.0027939171995967627, + 0.005635067820549011, + 0.0027390308678150177, + 0.00545669998973608, + 0.002716719638556242, + 0.005571703426539898, + 0.002707480685785413 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006421208939522194, + 0.005746349410576503, + 0.005404164999391892, + 0.005101115831054325, + 0.00485388847333966, + 0.0045855144167447825, + 0.004662942770173713, + 0.005059104716899598, + 0.00461737020280362, + 0.004446534400320367, + 0.004114212128857249, + 0.004678028852169936, + 0.004404060169276539, + 0.004656150674369456, + 0.004513829588134647, + 0.004839518675960965, + 0.004348161380019098, + 0.0047201153472655015, + 0.004169246752947831, + 0.004575240937826365, + 0.0047925402340120975, + 0.004508705150890575, + 0.004155367099556682, + 0.004478773084339979, + 0.004234175381969994, + 0.004428191743895327, + 0.004153362780335295, + 0.00410615213012631, + 0.004175626130102292, + 0.004125604583408443, + 0.0038633837013440408, + 0.003947405979968418 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005229286194712833, + "first_to_last_ratio": 1.266404707857162, + "imbalance_abs_log_ratio": 0.236181947089728, + "last_quartile_mean": 0.004129237803893765, + "mean": 0.004590188958362615, + "normalized": [ + 1.3988986069568536, + 1.2518764396632391, + 1.1773295279154767, + 1.1113084618795224, + 1.057448509716931, + 0.9989816232707989, + 1.0158498511653973, + 1.1021560904769923, + 1.0059215959707901, + 0.9687039990411438, + 0.8963056131625672, + 1.0191364439686714, + 0.9594507348663767, + 1.0143701526462583, + 0.983364656461723, + 1.0543179637831923, + 0.9472728507390573, + 1.0283052375580706, + 0.9082952337620236, + 0.9967434847079623, + 1.044083430439357, + 0.9822482672911342, + 0.9052714686148694, + 0.9757273883421174, + 0.9224403222564466, + 0.9647079421050512, + 0.9048348157364001, + 0.894549694440255, + 0.9096850190654888, + 0.8987875272307099, + 0.8416611464993293, + 0.859965900265795 + ], + "population_cv": 0.11345956482154346 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028097067028284073, + 0.028159458190202713, + 0.028274519369006157, + 0.02833542972803116, + 0.02845057100057602, + 0.02851351536810398, + 0.02867918275296688, + 0.028714356943964958, + 0.02882254309952259, + 0.028916271403431892, + 0.0290658138692379, + 0.02915445901453495, + 0.029274335131049156, + 0.02944180741906166, + 0.029572302475571632, + 0.02970401756465435, + 0.02980491891503334, + 0.02993646264076233, + 0.03008633479475975, + 0.030266879126429558, + 0.03044024296104908, + 0.03057776764035225, + 0.030671589076519012, + 0.030867453664541245, + 0.030981140211224556, + 0.03126408904790878, + 0.0313686840236187, + 0.031607914716005325, + 0.031716227531433105, + 0.03191319853067398, + 0.03204340860247612, + 0.032251372933387756, + 0.03237249702215195, + 0.03250224143266678, + 0.032637808471918106, + 0.03294040635228157, + 0.03308072313666344, + 0.0333157554268837, + 0.03343126177787781, + 0.03377276286482811, + 0.033881161361932755, + 0.034277379512786865, + 0.03435666114091873, + 0.03472030907869339, + 0.03482793644070625, + 0.0352315790951252, + 0.03533180430531502, + 0.03532300516963005, + 0.03546451777219772, + 0.03583687171339989, + 0.036012887954711914, + 0.03619367629289627, + 0.03628316894173622, + 0.03677845373749733, + 0.03689942881464958, + 0.03733044117689133, + 0.03742557764053345, + 0.0377187617123127, + 0.03777860850095749, + 0.03849533945322037, + 0.03856590390205383, + 0.03877026215195656, + 0.038892220705747604, + 0.03908585384488106 + ], + "loss_nats": 5.593109130859375, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028159458190202713, + 0.028274519369006157, + 0.02833542972803116, + 0.02845057100057602, + 0.02851351536810398, + 0.02867918275296688, + 0.028714356943964958, + 0.02882254309952259, + 0.028916271403431892, + 0.0290658138692379, + 0.02915445901453495, + 0.029274335131049156, + 0.02944180741906166, + 0.029572302475571632, + 0.02970401756465435, + 0.02980491891503334, + 0.02993646264076233, + 0.03008633479475975, + 0.030266879126429558, + 0.03044024296104908, + 0.03057776764035225, + 0.030671589076519012, + 0.030867453664541245, + 0.030981140211224556, + 0.03126408904790878, + 0.0313686840236187, + 0.031607914716005325, + 0.031716227531433105, + 0.03191319853067398, + 0.03204340860247612, + 0.032251372933387756, + 0.03237249702215195, + 0.03250224143266678, + 0.032637808471918106, + 0.03294040635228157, + 0.03308072313666344, + 0.0333157554268837, + 0.03343126177787781, + 0.03377276286482811, + 0.033881161361932755, + 0.034277379512786865, + 0.03435666114091873, + 0.03472030907869339, + 0.03482793644070625, + 0.0352315790951252, + 0.03533180430531502, + 0.03532300516963005, + 0.03546451777219772, + 0.03583687171339989, + 0.036012887954711914, + 0.03619367629289627, + 0.03628316894173622, + 0.03677845373749733, + 0.03689942881464958, + 0.03733044117689133, + 0.03742557764053345, + 0.0377187617123127, + 0.03777860850095749, + 0.03849533945322037, + 0.03856590390205383, + 0.03877026215195656, + 0.038892220705747604, + 0.03908585384488106, + 0.039198413491249084 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.023746729828417e-05, + 6.667003617621958e-05, + 6.32333685643971e-05, + 6.012299854774028e-05, + 5.672227416653186e-05, + 5.335300011211075e-05, + 5.0173275667475536e-05, + 4.719031130662188e-05, + 4.408728636917658e-05, + 4.096491466043517e-05, + 3.824968007393181e-05, + 3.570610351744108e-05, + 3.324189310660586e-05, + 3.1057134037837386e-05, + 2.914190736191813e-05, + 2.7255719032837078e-05, + 2.56736693700077e-05, + 2.4158533051377162e-05, + 2.2910870029591024e-05, + 2.187793143093586e-05, + 2.0998755644541234e-05, + 2.0234831026755273e-05, + 1.963501563295722e-05, + 1.908456215460319e-05, + 1.8524928236729465e-05, + 1.8092870959662832e-05, + 1.7702994227875024e-05, + 1.74019587575458e-05, + 1.7127178580267355e-05, + 1.6885946024558507e-05, + 1.666023126745131e-05, + 1.649606565479189e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8462841479922645e-05, + "first_to_last_ratio": 3.3673800283354014, + "imbalance_abs_log_ratio": 1.2141350023382471, + "last_quartile_mean": 1.7361521713610273e-05, + "mean": 3.315230350153797e-05, + "normalized": [ + 2.1186300763392136, + 2.011022738529366, + 1.9073597272498315, + 1.8135390967614395, + 1.710960270495245, + 1.6093301061157226, + 1.5134174813870156, + 1.4234398917237432, + 1.3298408168570044, + 1.235658169530656, + 1.1537563316575383, + 1.0770323551057208, + 1.0027023644092703, + 0.9368016927208878, + 0.8790311466763148, + 0.8221365079977823, + 0.7744158522443748, + 0.7287135583280486, + 0.691079279861448, + 0.6599219094963015, + 0.6334026123876152, + 0.610359730382432, + 0.5922670088986827, + 0.5756632311754095, + 0.5587825363589011, + 0.5457500399277982, + 0.5339898697251538, + 0.5249094910324558, + 0.516621072181992, + 0.5093445776331997, + 0.5025361591142052, + 0.49758429769523016 + ], + "population_cv": 0.5033867184228629 + }, + "activation_output_rms_by_block": [ + 0.029364673420786858, + 0.03058232180774212, + 0.03253435716032982, + 0.035004038363695145, + 0.03789948299527168, + 0.041081421077251434, + 0.04464675113558769, + 0.04816310852766037, + 0.05221380665898323, + 0.056122854351997375, + 0.06031070277094841, + 0.06517986208200455, + 0.07041798532009125, + 0.07559357583522797, + 0.08127844333648682, + 0.08752159774303436, + 0.09348469227552414, + 0.10084687173366547, + 0.10794128477573395, + 0.11579020321369171, + 0.12420836836099625, + 0.13254153728485107, + 0.14211823046207428, + 0.151936337351799, + 0.16282601654529572, + 0.17539408802986145, + 0.19021794199943542, + 0.2045798897743225, + 0.2205442190170288, + 0.23628318309783936, + 0.2528626322746277, + 0.27021101117134094 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03740951931104064, + "first_to_last_ratio": 0.17471705179812935, + "imbalance_abs_log_ratio": 1.74458746043398, + "last_quartile_mean": 0.214114872738719, + "mean": 0.11030317156109959, + "normalized": [ + 0.2662178521722837, + 0.2772569580268309, + 0.29495395916434053, + 0.31734389744455865, + 0.34359377394944846, + 0.37244097785978386, + 0.40476398369793726, + 0.43664300714129206, + 0.47336632229165543, + 0.5088054455524842, + 0.5467721545752731, + 0.590915575314168, + 0.6384039944045038, + 0.685325496677626, + 0.736864064615447, + 0.793464018344695, + 0.8475249709727586, + 0.9142699190458359, + 0.9785873175545334, + 1.0497450034748341, + 1.126063435920283, + 1.2016112991949022, + 1.2884328569224464, + 1.3774430526472923, + 1.476168039783901, + 1.5901092012817277, + 1.7245011118657556, + 1.8547054167069055, + 1.9994367876798904, + 2.142125015570893, + 2.292433016167273, + 2.4497120739784397 + ], + "population_cv": 0.6367597776091559 + }, + "bits_per_byte": 5.184434896455056, + "branch_output_rms_by_sublayer": [ + 0.0026808311231434345, + 0.00477010989561677, + 0.0025234855711460114, + 0.004875619895756245, + 0.0027965111657977104, + 0.005397462286055088, + 0.002715951530262828, + 0.00578050222247839, + 0.0028346737381070852, + 0.005865047685801983, + 0.0025751085486263037, + 0.0059114922769367695, + 0.0034423002507537603, + 0.006014656741172075, + 0.0034009802620857954, + 0.005813058000057936, + 0.0035280236043035984, + 0.006192575208842754, + 0.003798100398853421, + 0.006026907358318567, + 0.0032950215972959995, + 0.006300882901996374, + 0.004497972317039967, + 0.006683246698230505, + 0.004819076508283615, + 0.006906723603606224, + 0.004967817571014166, + 0.006679704412817955, + 0.005188894923776388, + 0.006651155184954405, + 0.005964669398963451, + 0.006957803387194872, + 0.005442580208182335, + 0.006974066607654095, + 0.00629971781745553, + 0.007549752946943045, + 0.007111458573490381, + 0.007369632367044687, + 0.008486796170473099, + 0.0074090128764510155, + 0.008528640493750572, + 0.007484533358365297, + 0.007537186611443758, + 0.007801020983606577, + 0.00965037476271391, + 0.007585691753774881, + 0.009028015658259392, + 0.007420822512358427, + 0.010029759258031845, + 0.00862929131835699, + 0.01243194192647934, + 0.0086217625066638, + 0.0149146169424057, + 0.00852242112159729, + 0.014407102949917316, + 0.008064002729952335, + 0.01547334622591734, + 0.008129187859594822, + 0.015585782937705517, + 0.008083954453468323, + 0.014871877618134022, + 0.009126133285462856, + 0.016274450346827507, + 0.008773879148066044 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00026099601887828554, + 0.00022299818257146475, + 0.00021593606328411676, + 0.0001980902028289896, + 0.00019039206376599677, + 0.0001764249768268048, + 0.00019262760932023515, + 0.00017741019012756782, + 0.0001866267032514166, + 0.00018194420611625288, + 0.00016335812718032435, + 0.00018628060025801724, + 0.00018197064718745634, + 0.00018231380553041538, + 0.00016812738233283717, + 0.0001758928024359223, + 0.00015172497688937343, + 0.0001676233054310719, + 0.0001547068922126687, + 0.0001733627635453279, + 0.0001619137110647745, + 0.00014190991701657472, + 0.0001580141865100647, + 0.00014951069781870242, + 0.00014861801255322757, + 0.00015877260520542335, + 0.00016243531213177502, + 0.00014972906733594748, + 0.0001506252547372875, + 0.00014728724932094392, + 0.00014196626632039671, + 0.00013988030321371655 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00020435941345043265, + "first_to_last_ratio": 1.3631752910956885, + "imbalance_abs_log_ratio": 0.30981675126526836, + "last_quartile_mean": 0.00014991425885233975, + "mean": 0.00017248344072510563, + "normalized": [ + 1.513165656836857, + 1.2928671971871588, + 1.251923444803397, + 1.1484592491675452, + 1.1038280716433115, + 1.0228516783125923, + 1.1167890002103689, + 1.028563608087539, + 1.0819977991327974, + 1.0548502821567973, + 0.9470945529239257, + 1.0799912123442663, + 1.0550035784447904, + 1.0569930931571387, + 0.9747450632132802, + 1.0197663131978583, + 0.8796495260735443, + 0.9718225977311089, + 0.8969376513031868, + 1.0050980129833083, + 0.9387203222761739, + 0.8227451656808188, + 0.9161122125450802, + 0.8668118933050746, + 0.8616364094341472, + 0.9205092647616311, + 0.941744386875116, + 0.8680779250837024, + 0.8732737131406461, + 0.853921099334295, + 0.823071859672921, + 0.8109781589796199 + ], + "population_cv": 0.15033028761203962 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028610490262508392, + 0.028607122600078583, + 0.029364673420786858, + 0.029366537928581238, + 0.03058232180774212, + 0.03066444955766201, + 0.03253435716032982, + 0.03263945132493973, + 0.035004038363695145, + 0.035203028470277786, + 0.03789948299527168, + 0.03799305483698845, + 0.041081421077251434, + 0.04133254662156105, + 0.04464675113558769, + 0.04496326297521591, + 0.04816310852766037, + 0.048516932874917984, + 0.05221380665898323, + 0.05251964181661606, + 0.056122854351997375, + 0.056522201746702194, + 0.06031070277094841, + 0.060862768441438675, + 0.06517986208200455, + 0.06596556305885315, + 0.07041798532009125, + 0.0712207704782486, + 0.07559357583522797, + 0.07684765011072159, + 0.08127844333648682, + 0.08279602974653244, + 0.08752159774303436, + 0.08871174603700638, + 0.09348469227552414, + 0.09565582871437073, + 0.10084687173366547, + 0.10279740393161774, + 0.10794128477573395, + 0.11048463732004166, + 0.11579020321369171, + 0.11891037970781326, + 0.12420836836099625, + 0.1270110011100769, + 0.13254153728485107, + 0.13657133281230927, + 0.14211823046207428, + 0.14646445214748383, + 0.151936337351799, + 0.15653157234191895, + 0.16282601654529572, + 0.16900379955768585, + 0.17539408802986145, + 0.18384796380996704, + 0.19021794199943542, + 0.1985616832971573, + 0.2045798897743225, + 0.21447749435901642, + 0.2205442190170288, + 0.23017354309558868, + 0.23628318309783936, + 0.24575340747833252, + 0.2528626322746277, + 0.26334208250045776 + ], + "loss_nats": 3.593576431274414, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028607122600078583, + 0.029364673420786858, + 0.029366537928581238, + 0.03058232180774212, + 0.03066444955766201, + 0.03253435716032982, + 0.03263945132493973, + 0.035004038363695145, + 0.035203028470277786, + 0.03789948299527168, + 0.03799305483698845, + 0.041081421077251434, + 0.04133254662156105, + 0.04464675113558769, + 0.04496326297521591, + 0.04816310852766037, + 0.048516932874917984, + 0.05221380665898323, + 0.05251964181661606, + 0.056122854351997375, + 0.056522201746702194, + 0.06031070277094841, + 0.060862768441438675, + 0.06517986208200455, + 0.06596556305885315, + 0.07041798532009125, + 0.0712207704782486, + 0.07559357583522797, + 0.07684765011072159, + 0.08127844333648682, + 0.08279602974653244, + 0.08752159774303436, + 0.08871174603700638, + 0.09348469227552414, + 0.09565582871437073, + 0.10084687173366547, + 0.10279740393161774, + 0.10794128477573395, + 0.11048463732004166, + 0.11579020321369171, + 0.11891037970781326, + 0.12420836836099625, + 0.1270110011100769, + 0.13254153728485107, + 0.13657133281230927, + 0.14211823046207428, + 0.14646445214748383, + 0.151936337351799, + 0.15653157234191895, + 0.16282601654529572, + 0.16900379955768585, + 0.17539408802986145, + 0.18384796380996704, + 0.19021794199943542, + 0.1985616832971573, + 0.2045798897743225, + 0.21447749435901642, + 0.2205442190170288, + 0.23017354309558868, + 0.23628318309783936, + 0.24575340747833252, + 0.2528626322746277, + 0.26334208250045776, + 0.27021101117134094 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001506932603660971, + 0.00015340163372457027, + 0.00015237691695801914, + 0.000148982901009731, + 0.0001392244448652491, + 0.00012851000064983964, + 0.00011962141434196383, + 0.0001119004882639274, + 0.00010405662760604173, + 9.720238449517637e-05, + 9.135912114288658e-05, + 8.468296437058598e-05, + 7.889774860814214e-05, + 7.427640230162069e-05, + 6.881110311951488e-05, + 6.43495368422009e-05, + 5.969387711957097e-05, + 5.503925422090106e-05, + 5.034419518779032e-05, + 4.6255379857029766e-05, + 4.248250479577109e-05, + 3.87999571103137e-05, + 3.563905556802638e-05, + 3.2581570849288255e-05, + 3.028466016985476e-05, + 2.807797682180535e-05, + 2.6328352760174312e-05, + 2.5215304049197584e-05, + 2.4513139578630216e-05, + 2.4017937903408892e-05, + 2.3832226361264475e-05, + 2.3948901798576117e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013808888252242468, + "first_to_last_ratio": 5.356993010635397, + "imbalance_abs_log_ratio": 1.678402812181787, + "last_quartile_mean": 2.5777312430363963e-05, + "mean": 7.298128883803656e-05, + "normalized": [ + 2.0648204870774824, + 2.1019310040551114, + 2.087890189171378, + 2.041384899907163, + 1.907673145841839, + 1.7608623072556988, + 1.6390696334156716, + 1.533276406124618, + 1.425798711735127, + 1.3318808977310936, + 1.2518156721736573, + 1.1603380225103768, + 1.0810681732852876, + 1.0177458288857888, + 0.9428595221471582, + 0.881726506433839, + 0.8179339947263247, + 0.7541556897281809, + 0.6898233230645799, + 0.6337977938383883, + 0.5821013231220159, + 0.5316425309563984, + 0.488331408439747, + 0.44643731794864266, + 0.4149647211227507, + 0.3847284320247795, + 0.3607548342781861, + 0.3455036825282238, + 0.3358825250816124, + 0.32909720129375364, + 0.3265525553289974, + 0.3281512587661287 + ], + "population_cv": 0.6061250608697893 + }, + "activation_output_rms_by_block": [ + 0.048818644136190414, + 0.05689374357461929, + 0.06500593572854996, + 0.07087047398090363, + 0.07611831277608871, + 0.0806790366768837, + 0.08478696644306183, + 0.08861920237541199, + 0.0918315127491951, + 0.09433750063180923, + 0.09700070321559906, + 0.09972338378429413, + 0.10201811045408249, + 0.10424874722957611, + 0.10649008303880692, + 0.10806933790445328, + 0.11109450459480286, + 0.11401215940713882, + 0.11832662671804428, + 0.12293888628482819, + 0.12948566675186157, + 0.1387748122215271, + 0.14864373207092285, + 0.16069285571575165, + 0.1741981953382492, + 0.18964411318302155, + 0.20650440454483032, + 0.22468066215515137, + 0.24238750338554382, + 0.2638821303844452, + 0.2911032736301422, + 0.32429978251457214 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07147403946146369, + "first_to_last_ratio": 0.2983212272448851, + "imbalance_abs_log_ratio": 1.2095844292661684, + "last_quartile_mean": 0.23958750814199448, + "mean": 0.13550565636251122, + "normalized": [ + 0.3602701573253034, + 0.41986249948426085, + 0.4797285771941724, + 0.523007495652488, + 0.5617353165867361, + 0.5953923905659501, + 0.6257079499045831, + 0.6539889533344176, + 0.67769505136762, + 0.6961886548811884, + 0.7158424660598531, + 0.7359352108336301, + 0.7528697560872202, + 0.7693313329348029, + 0.7858718661449788, + 0.7975263970925391, + 0.8198514185828341, + 0.8413830275994386, + 0.8732227856340641, + 0.907260180755379, + 0.9555738869339544, + 1.024125604397429, + 1.0969559209636535, + 1.185875630799193, + 1.285541873412471, + 1.3995291287005507, + 1.523954129209041, + 1.6580906523494112, + 1.7887629925727762, + 1.9473883044298541, + 2.148273964677672, + 2.393256423532534 + ], + "population_cv": 0.5081939166956976 + }, + "bits_per_byte": 3.1154419867739596, + "branch_output_rms_by_sublayer": [ + 0.010533316060900688, + 0.01737932860851288, + 0.0117434561252594, + 0.012238939292728901, + 0.01211429014801979, + 0.011732303537428379, + 0.014587659388780594, + 0.009741203859448433, + 0.012628368102014065, + 0.009234351105988026, + 0.014413262717425823, + 0.00937508512288332, + 0.015033986419439316, + 0.00873780157417059, + 0.016592979431152344, + 0.008810308761894703, + 0.018142949789762497, + 0.00861456897109747, + 0.01592753641307354, + 0.008246068842709064, + 0.013729470781981945, + 0.008380995132029057, + 0.017536180093884468, + 0.008834624662995338, + 0.021067099645733833, + 0.009005429223179817, + 0.022672412917017937, + 0.009119905531406403, + 0.019366111606359482, + 0.0090357456356287, + 0.0166512168943882, + 0.009071468375623226, + 0.014951958321034908, + 0.01051315013319254, + 0.017114169895648956, + 0.010944797657430172, + 0.01693827658891678, + 0.012414379045367241, + 0.01647246815264225, + 0.013282178901135921, + 0.01672390289604664, + 0.014548475854098797, + 0.017381425946950912, + 0.017058325931429863, + 0.02400100603699684, + 0.018193069845438004, + 0.017901180312037468, + 0.02045683190226555, + 0.01391419768333435, + 0.021929442882537842, + 0.014953838661313057, + 0.02380906604230404, + 0.010647366754710674, + 0.02548205293715, + 0.005664720665663481, + 0.026819046586751938, + 0.008504491299390793, + 0.024410424754023552, + 0.008407330140471458, + 0.02826099656522274, + 0.01054995134472847, + 0.033158741891384125, + 0.02235519140958786, + 0.03220592811703682 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005931392614171548, + 0.0004012275990867258, + 0.00038098222807989436, + 0.0003649111519840816, + 0.00034656151495199636, + 0.00031005860528648927, + 0.00027821812913176474, + 0.0002561842642921938, + 0.0002305368063696604, + 0.00020943363497294808, + 0.0001942700943631853, + 0.0001900904574854747, + 0.00018678033884870243, + 0.00017357948080148642, + 0.00015815775362626997, + 0.00014490835987546657, + 0.0001375100278294081, + 0.0001295570650502436, + 0.00012744106699623042, + 0.00011906627060222178, + 0.00011368520556157287, + 0.0001091753996736031, + 0.00010941555658600093, + 9.636520203422524e-05, + 9.391483800802352e-05, + 9.101856383525677e-05, + 7.998410113295477e-05, + 7.102545922046693e-05, + 6.763159533414917e-05, + 6.887797791066819e-05, + 7.001610738319118e-05, + 7.664349539138475e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036641034427878756, + "first_to_last_ratio": 4.73465560322638, + "imbalance_abs_log_ratio": 1.5549089897216843, + "last_quartile_mean": 7.738901727701191e-05, + "mean": 0.00018688648791009675, + "normalized": [ + 3.173794253667441, + 2.146905341170195, + 2.038575567128047, + 1.9525817840807467, + 1.8543957823141959, + 1.6590744935805388, + 1.4887011481836172, + 1.3708014268823612, + 1.2335659412710565, + 1.1206462131906392, + 1.0395085088047697, + 1.017143933792149, + 0.9994320131830752, + 0.9287963123636216, + 0.8462770925544549, + 0.7753816848715992, + 0.7357943817509076, + 0.6932393374130293, + 0.681916966932018, + 0.6371047577259816, + 0.6083115308810451, + 0.5841802737826761, + 0.5854653153877886, + 0.5156349349375247, + 0.5025234250921448, + 0.4870259206703161, + 0.4279822582541749, + 0.38004598414110224, + 0.3618859559642637, + 0.3685551517443171, + 0.3746451023086966, + 0.4101072059755049 + ], + "population_cv": 0.6522632271008562 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04385505989193916, + 0.04404326155781746, + 0.048818644136190414, + 0.05170967057347298, + 0.05689374357461929, + 0.059983257204294205, + 0.06500593572854996, + 0.06660757958889008, + 0.07087047398090363, + 0.07236418128013611, + 0.07611831277608871, + 0.07718922197818756, + 0.0806790366768837, + 0.08160495012998581, + 0.08478696644306183, + 0.08585704118013382, + 0.08861920237541199, + 0.08891960233449936, + 0.0918315127491951, + 0.0918683186173439, + 0.09433750063180923, + 0.09460321068763733, + 0.09700070321559906, + 0.09746499359607697, + 0.09972338378429413, + 0.0994088351726532, + 0.10201811045408249, + 0.10191889107227325, + 0.10424874722957611, + 0.10386203229427338, + 0.10649008303880692, + 0.10544375330209732, + 0.10806933790445328, + 0.10778804868459702, + 0.11109450459480286, + 0.11027765274047852, + 0.11401215940713882, + 0.11356210708618164, + 0.11832662671804428, + 0.11750734597444534, + 0.12293888628482819, + 0.1225726455450058, + 0.12948566675186157, + 0.12988287210464478, + 0.1387748122215271, + 0.13870327174663544, + 0.14864373207092285, + 0.14834089577198029, + 0.16069285571575165, + 0.16013014316558838, + 0.1741981953382492, + 0.1741078794002533, + 0.18964411318302155, + 0.18942901492118835, + 0.20650440454483032, + 0.20662079751491547, + 0.22468066215515137, + 0.2257053554058075, + 0.24238750338554382, + 0.24448354542255402, + 0.2638821303844452, + 0.2678608000278473, + 0.2911032736301422, + 0.30181384086608887 + ], + "loss_nats": 2.1594598293304443, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04404326155781746, + 0.048818644136190414, + 0.05170967057347298, + 0.05689374357461929, + 0.059983257204294205, + 0.06500593572854996, + 0.06660757958889008, + 0.07087047398090363, + 0.07236418128013611, + 0.07611831277608871, + 0.07718922197818756, + 0.0806790366768837, + 0.08160495012998581, + 0.08478696644306183, + 0.08585704118013382, + 0.08861920237541199, + 0.08891960233449936, + 0.0918315127491951, + 0.0918683186173439, + 0.09433750063180923, + 0.09460321068763733, + 0.09700070321559906, + 0.09746499359607697, + 0.09972338378429413, + 0.0994088351726532, + 0.10201811045408249, + 0.10191889107227325, + 0.10424874722957611, + 0.10386203229427338, + 0.10649008303880692, + 0.10544375330209732, + 0.10806933790445328, + 0.10778804868459702, + 0.11109450459480286, + 0.11027765274047852, + 0.11401215940713882, + 0.11356210708618164, + 0.11832662671804428, + 0.11750734597444534, + 0.12293888628482819, + 0.1225726455450058, + 0.12948566675186157, + 0.12988287210464478, + 0.1387748122215271, + 0.13870327174663544, + 0.14864373207092285, + 0.14834089577198029, + 0.16069285571575165, + 0.16013014316558838, + 0.1741981953382492, + 0.1741078794002533, + 0.18964411318302155, + 0.18942901492118835, + 0.20650440454483032, + 0.20662079751491547, + 0.22468066215515137, + 0.2257053554058075, + 0.24238750338554382, + 0.24448354542255402, + 0.2638821303844452, + 0.2678608000278473, + 0.2911032736301422, + 0.30181384086608887, + 0.32429978251457214 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010946426482405514, + 0.00010930908320005983, + 0.00012598594184964895, + 0.00012082153261872008, + 0.00011562183499336243, + 0.00011443938274169341, + 0.00011117216490674764, + 0.00010893856233451515, + 0.00010224300058325753, + 9.949541708920151e-05, + 9.518271690467373e-05, + 9.17384895728901e-05, + 8.801270450931042e-05, + 8.554522355552763e-05, + 8.286484808195382e-05, + 8.027214062167332e-05, + 7.707033364567906e-05, + 7.422274211421609e-05, + 7.329088111873716e-05, + 7.053956505842507e-05, + 6.641655636485666e-05, + 6.314766505965963e-05, + 5.928793325438164e-05, + 5.485894507728517e-05, + 5.013645932194777e-05, + 4.561002060654573e-05, + 4.094795440323651e-05, + 3.615552486735396e-05, + 3.135877341264859e-05, + 2.6529152819421142e-05, + 2.2670756152365357e-05, + 2.0441382730496116e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011446909593360033, + "first_to_last_ratio": 3.3439937416939496, + "imbalance_abs_log_ratio": 1.2071658237176983, + "last_quartile_mean": 3.4231253039251897e-05, + "mean": 7.668099857482957e-05, + "normalized": [ + 1.427527899460436, + 1.4255041696332367, + 1.642987756956567, + 1.5756384875558933, + 1.5078290207779732, + 1.4924086131979242, + 1.4498006934307157, + 1.4206721920582042, + 1.3333551007878848, + 1.297523753451233, + 1.2412816561300926, + 1.196365348364192, + 1.1477772348442066, + 1.115598716050207, + 1.0806438312235815, + 1.046832228499728, + 1.0050773343864263, + 0.9679417781940518, + 0.955789341308797, + 0.91990931742482, + 0.866140994500009, + 0.8235112509396527, + 0.7731763325503017, + 0.7154177188205337, + 0.6538315913168741, + 0.5948021211804766, + 0.5340039274955088, + 0.47150566032432917, + 0.4089510308351939, + 0.3459677535828191, + 0.29565024678495777, + 0.26657689793317285 + ], + "population_cv": 0.3996524250074872 + }, + "activation_output_rms_by_block": [ + 0.09914769232273102, + 0.13720469176769257, + 0.1984461098909378, + 0.2206047922372818, + 0.22331808507442474, + 0.2258162647485733, + 0.22567619383335114, + 0.2251615822315216, + 0.22496475279331207, + 0.22249442338943481, + 0.21478813886642456, + 0.205490842461586, + 0.19144563376903534, + 0.188800647854805, + 0.18874111771583557, + 0.190367192029953, + 0.19439123570919037, + 0.198538675904274, + 0.20280276238918304, + 0.2084609866142273, + 0.21558508276939392, + 0.21985429525375366, + 0.22511647641658783, + 0.2311476618051529, + 0.24068132042884827, + 0.2547627091407776, + 0.2762547731399536, + 0.30550503730773926, + 0.34050679206848145, + 0.39206719398498535, + 0.46342816948890686, + 0.575099527835846 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19442192651331425, + "first_to_last_ratio": 0.5460704265504183, + "imbalance_abs_log_ratio": 0.6050073251991793, + "last_quartile_mean": 0.3560381904244423, + "mean": 0.2414584644138813, + "normalized": [ + 0.41062007316000754, + 0.5682331000519887, + 0.8218643747803495, + 0.9136345365766327, + 0.9248716362729685, + 0.9352178450099977, + 0.9346377414482436, + 0.932506477990246, + 0.9316913090596918, + 0.9214604421904198, + 0.8895448721907656, + 0.851040128000468, + 0.7928719096004871, + 0.7819177029602238, + 0.7816711589464782, + 0.7884055441669947, + 0.8050711172253067, + 0.8222477368362661, + 0.8399074469452479, + 0.8633409771748842, + 0.8928454146045669, + 0.9105263540353832, + 0.9323196723007322, + 0.9572978208332561, + 0.9967814589274413, + 1.0550995168431603, + 1.144108879390653, + 1.2652488205345183, + 1.410208554481745, + 1.6237459098263265, + 1.9192873217919157, + 2.381774145842633 + ], + "population_cv": 0.36935148393532197 + }, + "bits_per_byte": 2.073311222061611, + "branch_output_rms_by_sublayer": [ + 0.01824937015771866, + 0.06101369485259056, + 0.05977629870176315, + 0.0147639699280262, + 0.08494830876588821, + 0.003444893751293421, + 0.03928658738732338, + 0.004586480092257261, + 0.025128526613116264, + 0.007272258400917053, + 0.024004649370908737, + 0.006204464007169008, + 0.019449269399046898, + 0.007294370327144861, + 0.02213231660425663, + 0.010247048921883106, + 0.041971735656261444, + 0.012659681960940361, + 0.02208700217306614, + 0.013693951070308685, + 0.029153602197766304, + 0.01590428315103054, + 0.02919897623360157, + 0.017466455698013306, + 0.036670926958322525, + 0.021206999197602272, + 0.02140921540558338, + 0.021084947511553764, + 0.023707622662186623, + 0.022131068632006645, + 0.02047598734498024, + 0.022354252636432648, + 0.022473404183983803, + 0.02369093894958496, + 0.02454194240272045, + 0.02377566508948803, + 0.020227128639817238, + 0.025439493358135223, + 0.02723041921854019, + 0.027927791699767113, + 0.03427847847342491, + 0.028011852875351906, + 0.03502028062939644, + 0.030455492436885834, + 0.029505647718906403, + 0.032503895461559296, + 0.033919259905815125, + 0.03810396417975426, + 0.036724433302879333, + 0.04227631166577339, + 0.033869534730911255, + 0.04830062761902809, + 0.021475639194250107, + 0.057604189962148666, + 0.010480014607310295, + 0.06417365372180939, + 0.010398799553513527, + 0.06654922664165497, + 0.012935257516801357, + 0.0871838852763176, + 0.010905168019235134, + 0.11424379050731659, + 0.02395492233335972, + 0.16713054478168488 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00035888588058483766, + 0.00018018950310225553, + 0.0002078431087578172, + 0.0001449691382379752, + 0.00012177099939920197, + 0.00012306582379429542, + 0.00012204348242118167, + 0.00015411618529826362, + 0.00023712554584861813, + 0.0001667297606409996, + 0.00020124714337423614, + 0.00019657670547036952, + 0.00021589641757185544, + 0.00019691649893044806, + 0.0002141921139315525, + 0.00021492094770207205, + 0.00022724548428173296, + 0.00022961612017464663, + 0.00022356249125846063, + 0.00024496232741861765, + 0.0002428330999302831, + 0.0002472582804061272, + 0.00023198586071939664, + 0.00023918771709558322, + 0.0002286007324212131, + 0.00020791382420134994, + 0.00018978221308207498, + 0.00017514362262291224, + 0.0001585770655414503, + 0.0001491152323589239, + 0.00013960034115182495, + 0.00014044566543596396 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017661051519947854, + "first_to_last_ratio": 1.0170643451662862, + "imbalance_abs_log_ratio": 0.016920384648356147, + "last_quartile_mean": 0.00017364733710196415, + "mean": 0.0001978849791614544, + "normalized": [ + 1.813608501795493, + 0.9105769617573595, + 1.0503228170150194, + 0.7325929378383735, + 0.6153625197587583, + 0.6219058380064613, + 0.6167394965416064, + 0.7788169973857398, + 1.1982998752782916, + 0.8425589519099616, + 1.0169904973436037, + 0.9933887165330652, + 1.0910197352357174, + 0.9951058426207471, + 1.0824071379202214, + 1.0860902561316592, + 1.1483715704177997, + 1.160351438232726, + 1.1297597837178734, + 1.2379025859196358, + 1.227142661152444, + 1.2495050481036718, + 1.1723267814588358, + 1.2087209353087376, + 1.155220236472308, + 1.0506801733127658, + 0.959053152423619, + 0.8850779041698386, + 0.8013597909928638, + 0.7535449784556955, + 0.7054620403397319, + 0.7097338364493765 + ], + "population_cv": 0.24613765810690832 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0564301572740078, + 0.061963923275470734, + 0.09914769232273102, + 0.12944035232067108, + 0.13720469176769257, + 0.1976913958787918, + 0.1984461098909378, + 0.21992529928684235, + 0.2206047922372818, + 0.22257371246814728, + 0.22331808507442474, + 0.22564378380775452, + 0.2258162647485733, + 0.2261907458305359, + 0.22567619383335114, + 0.22598126530647278, + 0.2251615822315216, + 0.22548764944076538, + 0.22496475279331207, + 0.22340621054172516, + 0.22249442338943481, + 0.21654370427131653, + 0.21478813886642456, + 0.20714853703975677, + 0.205490842461586, + 0.19201314449310303, + 0.19144563376903534, + 0.18731488287448883, + 0.188800647854805, + 0.18594272434711456, + 0.18874111771583557, + 0.18583883345127106, + 0.190367192029953, + 0.18933312594890594, + 0.19439123570919037, + 0.1937587559223175, + 0.198538675904274, + 0.19723950326442719, + 0.20280276238918304, + 0.2025505006313324, + 0.2084609866142273, + 0.2101445198059082, + 0.21558508276939392, + 0.21427761018276215, + 0.21985429525375366, + 0.21906179189682007, + 0.22511647641658783, + 0.22344982624053955, + 0.2311476618051529, + 0.2303154021501541, + 0.24068132042884827, + 0.23970884084701538, + 0.2547627091407776, + 0.25386181473731995, + 0.2762547731399536, + 0.2756711542606354, + 0.30550503730773926, + 0.30545076727867126, + 0.34050679206848145, + 0.34091928601264954, + 0.39206719398498535, + 0.3925974369049072, + 0.46342816948890686, + 0.46681874990463257 + ], + "loss_nats": 1.4371098279953003, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.061963923275470734, + 0.09914769232273102, + 0.12944035232067108, + 0.13720469176769257, + 0.1976913958787918, + 0.1984461098909378, + 0.21992529928684235, + 0.2206047922372818, + 0.22257371246814728, + 0.22331808507442474, + 0.22564378380775452, + 0.2258162647485733, + 0.2261907458305359, + 0.22567619383335114, + 0.22598126530647278, + 0.2251615822315216, + 0.22548764944076538, + 0.22496475279331207, + 0.22340621054172516, + 0.22249442338943481, + 0.21654370427131653, + 0.21478813886642456, + 0.20714853703975677, + 0.205490842461586, + 0.19201314449310303, + 0.19144563376903534, + 0.18731488287448883, + 0.188800647854805, + 0.18594272434711456, + 0.18874111771583557, + 0.18583883345127106, + 0.190367192029953, + 0.18933312594890594, + 0.19439123570919037, + 0.1937587559223175, + 0.198538675904274, + 0.19723950326442719, + 0.20280276238918304, + 0.2025505006313324, + 0.2084609866142273, + 0.2101445198059082, + 0.21558508276939392, + 0.21427761018276215, + 0.21985429525375366, + 0.21906179189682007, + 0.22511647641658783, + 0.22344982624053955, + 0.2311476618051529, + 0.2303154021501541, + 0.24068132042884827, + 0.23970884084701538, + 0.2547627091407776, + 0.25386181473731995, + 0.2762547731399536, + 0.2756711542606354, + 0.30550503730773926, + 0.30545076727867126, + 0.34050679206848145, + 0.34091928601264954, + 0.39206719398498535, + 0.3925974369049072, + 0.46342816948890686, + 0.46681874990463257, + 0.575099527835846 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010021305934060365, + 0.00010125436529051512, + 0.000114263420982752, + 0.00011024689592886716, + 0.00010690757335396484, + 0.00010680041305022314, + 0.00010475025192135945, + 0.00010345032933400944, + 9.713075269246474e-05, + 9.490915545029566e-05, + 9.159108594758436e-05, + 8.836394408717752e-05, + 8.501022239215672e-05, + 8.242073090514168e-05, + 7.99787012510933e-05, + 7.76422893977724e-05, + 7.462384382961318e-05, + 7.162441761465743e-05, + 7.150838064262643e-05, + 6.863063026685268e-05, + 6.474268593592569e-05, + 6.149821274448186e-05, + 5.7751010899664834e-05, + 5.37736268597655e-05, + 4.920278661302291e-05, + 4.476245885598473e-05, + 4.037534017697908e-05, + 3.581849159672856e-05, + 3.1086256058188155e-05, + 2.5875511710182764e-05, + 2.1458168703247793e-05, + 1.863625948317349e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010598578865028685, + "first_to_last_ratio": 3.1730458332581706, + "imbalance_abs_log_ratio": 1.1546919572676668, + "last_quartile_mean": 3.3401909149688436e-05, + "mean": 7.300941479115863e-05, + "normalized": [ + 1.3726046103404645, + 1.3868672359606, + 1.565050498070684, + 1.5100367020366523, + 1.4642984560247596, + 1.4628306959550723, + 1.434749918500163, + 1.4169450560578547, + 1.330386676434875, + 1.2999577619103044, + 1.2545106163304838, + 1.2103088942698703, + 1.1643734254729483, + 1.12890551363689, + 1.0954573664214418, + 1.0634558518222064, + 1.0221126144220245, + 0.9810298825094963, + 0.9794405399245307, + 0.9400243853915095, + 0.8867717419886232, + 0.8423326350498187, + 0.7910077222897892, + 0.7365300353876748, + 0.6739238597346121, + 0.613105295858438, + 0.5530155294693376, + 0.4906009957645373, + 0.425784210804997, + 0.35441335592401246, + 0.29390961103616947, + 0.255258305199158 + ], + "population_cv": 0.383084243994172 + }, + "activation_output_rms_by_block": [ + 0.11517385393381119, + 0.17733007669448853, + 0.24607165157794952, + 0.2600470185279846, + 0.2621006965637207, + 0.26366734504699707, + 0.2635972201824188, + 0.26321178674697876, + 0.2646143436431885, + 0.2614729404449463, + 0.2522071599960327, + 0.23732660710811615, + 0.2133668214082718, + 0.2109174132347107, + 0.21198545396327972, + 0.21621736884117126, + 0.2226637899875641, + 0.22916270792484283, + 0.23635683953762054, + 0.24468488991260529, + 0.25434690713882446, + 0.26204079389572144, + 0.2696947753429413, + 0.27905911207199097, + 0.29096299409866333, + 0.30846187472343445, + 0.3323265314102173, + 0.36396095156669617, + 0.403833270072937, + 0.4673614203929901, + 0.5606572031974792, + 0.7190070152282715 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23139995615929365, + "first_to_last_ratio": 0.5371134119256165, + "imbalance_abs_log_ratio": 0.6215460113796702, + "last_quartile_mean": 0.43082140758633614, + "mean": 0.28637152607552707, + "normalized": [ + 0.4021833298588333, + 0.6192308262090269, + 0.859274156722729, + 0.9080756808880515, + 0.9152470573998156, + 0.9207177426482616, + 0.92047286891539, + 0.9191269479688416, + 0.9240246307637429, + 0.9130549535709983, + 0.8806991513867063, + 0.8287367475407597, + 0.7450699597557016, + 0.7365167065495323, + 0.740246269831209, + 0.7550239781316335, + 0.7775346698709116, + 0.8002286786864554, + 0.8253503509119278, + 0.8544316303572463, + 0.8881710784044343, + 0.9150378792429639, + 0.9417653320456605, + 0.9744652895357776, + 1.0160332561203216, + 1.0771387747610122, + 1.1604733751447416, + 1.2709397353656795, + 1.410172567109311, + 1.6320107896122644, + 1.9577966108600218, + 2.510748973830038 + ], + "population_cv": 0.3918463566300126 + }, + "bits_per_byte": 1.9262875534052675, + "branch_output_rms_by_sublayer": [ + 0.021954918280243874, + 0.07429475337266922, + 0.09672636538743973, + 0.017902610823512077, + 0.0940650999546051, + 0.0039193108677864075, + 0.036259762942790985, + 0.004245741758495569, + 0.02603325992822647, + 0.008932216092944145, + 0.02122427336871624, + 0.005480592139065266, + 0.02764410711824894, + 0.00817549042403698, + 0.02432539500296116, + 0.009956736117601395, + 0.05137033388018608, + 0.014394877478480339, + 0.02661585807800293, + 0.015817945823073387, + 0.030927525833249092, + 0.018779242411255836, + 0.03579767048358917, + 0.02186598815023899, + 0.05378979817032814, + 0.028890088200569153, + 0.025482553988695145, + 0.027309151366353035, + 0.0287014190107584, + 0.028864959254860878, + 0.027393119409680367, + 0.02912190742790699, + 0.029581313952803612, + 0.03136448562145233, + 0.0320778414607048, + 0.03278070688247681, + 0.028201179578900337, + 0.033851705491542816, + 0.035640619695186615, + 0.03695352375507355, + 0.04368399456143379, + 0.038013845682144165, + 0.04277802258729935, + 0.04267064854502678, + 0.03937495872378349, + 0.04597146809101105, + 0.04081331565976143, + 0.053922027349472046, + 0.05011079087853432, + 0.0602298267185688, + 0.04528728500008583, + 0.06855001300573349, + 0.028651293367147446, + 0.07740508019924164, + 0.014390980824828148, + 0.08345231413841248, + 0.015424116514623165, + 0.08889877796173096, + 0.01913117803633213, + 0.11831489205360413, + 0.01693635806441307, + 0.153398334980011, + 0.032383937388658524, + 0.24250756204128265 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003320807822012488, + 0.00017565239824668803, + 0.00017577232236880354, + 0.00012480421087066412, + 0.00013414970937401124, + 0.0001166490159724099, + 0.00011491447201106907, + 0.00015823331904477844, + 0.000249176250949333, + 0.0001712841712530518, + 0.00020010962025834788, + 0.00021729074089037623, + 0.0002489951018894741, + 0.00022387049947650086, + 0.00024825867897268754, + 0.000256508706958601, + 0.0002733879022225402, + 0.00028045470195987973, + 0.0002710271146389, + 0.0002966387758490912, + 0.00030658370753967526, + 0.0003196905817287839, + 0.0002949547776417074, + 0.00030211663360406435, + 0.0002930543360663917, + 0.00026176751657805116, + 0.0002415189744957938, + 0.00021641616631898222, + 0.00019981422740886822, + 0.0001881612684369175, + 0.00017039642606795488, + 0.0001626151127852574 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016653202876120915, + "first_to_last_ratio": 0.7684272928714567, + "imbalance_abs_log_ratio": 0.2634093296250088, + "last_quartile_mean": 0.0002167180035197771, + "mean": 0.00022582338200252827, + "normalized": [ + 1.4705332072191308, + 0.7778308724679426, + 0.7783619251917662, + 0.5526629251760424, + 0.5940470299594988, + 0.5165497697271398, + 0.5088687937982548, + 0.7006950194511165, + 1.103412094618897, + 0.758487317540635, + 0.8861333068517568, + 0.9622154223513644, + 1.1026099232128512, + 0.991352168634134, + 1.0993488662299287, + 1.1358819652950252, + 1.2106270829806252, + 1.2419205640837485, + 1.2001729503628886, + 1.313587517902686, + 1.3576260563498372, + 1.4156664331827458, + 1.306130370673508, + 1.3378447834984684, + 1.2977147603923083, + 1.159169233304284, + 1.069503841250105, + 0.9583425967668807, + 0.8848252365941058, + 0.8332231444253673, + 0.7545561693254915, + 0.7200986511814654 + ], + "population_cv": 0.2750960622596733 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06031418219208717, + 0.06744465231895447, + 0.11517385393381119, + 0.16882504522800446, + 0.17733007669448853, + 0.2454802393913269, + 0.24607165157794952, + 0.25892511010169983, + 0.2600470185279846, + 0.2610686123371124, + 0.2621006965637207, + 0.2635994553565979, + 0.26366734504699707, + 0.26440703868865967, + 0.2635972201824188, + 0.2638950049877167, + 0.26321178674697876, + 0.2655138075351715, + 0.2646143436431885, + 0.2625099718570709, + 0.2614729404449463, + 0.25542622804641724, + 0.2522071599960327, + 0.24079175293445587, + 0.23732660710811615, + 0.21542933583259583, + 0.2133668214082718, + 0.20939168334007263, + 0.2109174132347107, + 0.20824690163135529, + 0.21198545396327972, + 0.21042729914188385, + 0.21621736884117126, + 0.21581067144870758, + 0.2226637899875641, + 0.2222982496023178, + 0.22916270792484283, + 0.22995305061340332, + 0.23635683953762054, + 0.23728354275226593, + 0.24468488991260529, + 0.24780811369419098, + 0.25434690713882446, + 0.25493237376213074, + 0.26204079389572144, + 0.26210466027259827, + 0.2696947753429413, + 0.2699442505836487, + 0.27905911207199097, + 0.27968016266822815, + 0.29096299409866333, + 0.29178300499916077, + 0.30846187472343445, + 0.30848607420921326, + 0.3323265314102173, + 0.33218035101890564, + 0.36396095156669617, + 0.3638138175010681, + 0.403833270072937, + 0.40415704250335693, + 0.4673614203929901, + 0.4678023159503937, + 0.5606572031974792, + 0.5632834434509277 + ], + "loss_nats": 1.3352007865905762, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06744465231895447, + 0.11517385393381119, + 0.16882504522800446, + 0.17733007669448853, + 0.2454802393913269, + 0.24607165157794952, + 0.25892511010169983, + 0.2600470185279846, + 0.2610686123371124, + 0.2621006965637207, + 0.2635994553565979, + 0.26366734504699707, + 0.26440703868865967, + 0.2635972201824188, + 0.2638950049877167, + 0.26321178674697876, + 0.2655138075351715, + 0.2646143436431885, + 0.2625099718570709, + 0.2614729404449463, + 0.25542622804641724, + 0.2522071599960327, + 0.24079175293445587, + 0.23732660710811615, + 0.21542933583259583, + 0.2133668214082718, + 0.20939168334007263, + 0.2109174132347107, + 0.20824690163135529, + 0.21198545396327972, + 0.21042729914188385, + 0.21621736884117126, + 0.21581067144870758, + 0.2226637899875641, + 0.2222982496023178, + 0.22916270792484283, + 0.22995305061340332, + 0.23635683953762054, + 0.23728354275226593, + 0.24468488991260529, + 0.24780811369419098, + 0.25434690713882446, + 0.25493237376213074, + 0.26204079389572144, + 0.26210466027259827, + 0.2696947753429413, + 0.2699442505836487, + 0.27905911207199097, + 0.27968016266822815, + 0.29096299409866333, + 0.29178300499916077, + 0.30846187472343445, + 0.30848607420921326, + 0.3323265314102173, + 0.33218035101890564, + 0.36396095156669617, + 0.3638138175010681, + 0.403833270072937, + 0.40415704250335693, + 0.4673614203929901, + 0.4678023159503937, + 0.5606572031974792, + 0.5632834434509277, + 0.7190070152282715 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018638069741427898, + 0.00018956589337904006, + 0.00020923581905663013, + 0.00020349136320874095, + 0.00019885810615960509, + 0.0001991111203096807, + 0.00019671664631459862, + 0.00019502552459016442, + 0.00018589819956105202, + 0.00018270809960085899, + 0.00017816871695686132, + 0.00017320679035037756, + 0.0001670292258495465, + 0.00016241244156844914, + 0.00015707056445535272, + 0.0001517482742201537, + 0.0001448393304599449, + 0.00013789460354018956, + 0.00013790397497359663, + 0.00013200659304857254, + 0.00012367918679956347, + 0.00011618347343755886, + 0.00010830583050847054, + 9.980145841836929e-05, + 8.979889389593154e-05, + 8.034214988583699e-05, + 7.120905502233654e-05, + 6.192603905219585e-05, + 5.214559132582508e-05, + 4.129187436774373e-05, + 3.1231873435899615e-05, + 2.3505688659497537e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019729814630409237, + "first_to_last_ratio": 3.49624785701179, + "imbalance_abs_log_ratio": 1.2516903525947785, + "last_quartile_mean": 5.643139570565836e-05, + "mean": 0.00013714665936959136, + "normalized": [ + 1.3589882412812453, + 1.3822129846282736, + 1.525635549698432, + 1.4837500537316024, + 1.449966824373825, + 1.4518116680706283, + 1.434352445905914, + 1.4220216918634343, + 1.3554701253063843, + 1.3322096246506874, + 1.2991108771867432, + 1.2629311654147486, + 1.217887673074309, + 1.1842245543201309, + 1.1452744469121132, + 1.1064671565292232, + 1.0560908382727927, + 1.0054536083783316, + 1.0055219398524642, + 0.9625213888209478, + 0.9018024016630627, + 0.8471476736772742, + 0.7897081198062669, + 0.7276987924978771, + 0.6547654482340389, + 0.5858119348669365, + 0.519218297767195, + 0.451531516238494, + 0.38021772871113024, + 0.3010782366668357, + 0.22772609686200243, + 0.17139089473665517 + ], + "population_cv": 0.4031953558435787 + }, + "activation_output_rms_by_block": [ + 0.1026567816734314, + 0.17673343420028687, + 0.23246681690216064, + 0.24062344431877136, + 0.2416554093360901, + 0.24258241057395935, + 0.24218299984931946, + 0.24160927534103394, + 0.24130181968212128, + 0.23599836230278015, + 0.22310413420200348, + 0.20329499244689941, + 0.176778644323349, + 0.17310521006584167, + 0.1729687601327896, + 0.17606647312641144, + 0.1812635064125061, + 0.18683874607086182, + 0.19253391027450562, + 0.1985689103603363, + 0.20644955337047577, + 0.2135031372308731, + 0.22015690803527832, + 0.228074312210083, + 0.2383035272359848, + 0.2532499134540558, + 0.27322494983673096, + 0.2992607355117798, + 0.3339390158653259, + 0.3918353319168091, + 0.48234984278678894, + 0.651733934879303 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.21506382152438164, + "first_to_last_ratio": 0.5884305856918151, + "imbalance_abs_log_ratio": 0.5302963104844547, + "last_quartile_mean": 0.3654871564358473, + "mean": 0.2460754751227796, + "normalized": [ + 0.41717599700746566, + 0.7182082412402355, + 0.9446972185512233, + 0.9778440708027163, + 0.9820377638832796, + 0.9858049058034841, + 0.9841817829610285, + 0.9818502848383519, + 0.9806008484255633, + 0.9590486909962422, + 0.906649206267652, + 0.8261489380258848, + 0.7183919658598449, + 0.7034638863522286, + 0.7029093819548116, + 0.7154978489366439, + 0.7366175207913933, + 0.7592741453720182, + 0.7824181185810598, + 0.8069431147547724, + 0.838968423275288, + 0.8676327339176954, + 0.8946722867259763, + 0.9268469850410127, + 0.9684164060521798, + 1.0291554382967116, + 1.1103298681041054, + 1.2161339335521484, + 1.357059315637638, + 1.5923380081712835, + 1.9601703198830358, + 2.648512349937025 + ], + "population_cv": 0.40510862299171807 + }, + "bits_per_byte": 1.8611962476373152, + "branch_output_rms_by_sublayer": [ + 0.026356825605034828, + 0.057217150926589966, + 0.10627161711454391, + 0.01789308711886406, + 0.07707435637712479, + 0.0034156402107328176, + 0.02677960880100727, + 0.0038151287008076906, + 0.02153952419757843, + 0.007133069448173046, + 0.01632758229970932, + 0.004304513800889254, + 0.01752857305109501, + 0.006448030937463045, + 0.01845136471092701, + 0.007711905520409346, + 0.044129498302936554, + 0.011710400693118572, + 0.023254960775375366, + 0.01295715756714344, + 0.02913394197821617, + 0.01598021760582924, + 0.03491654247045517, + 0.01944364421069622, + 0.04857943207025528, + 0.02654583752155304, + 0.023438163101673126, + 0.02489112876355648, + 0.025785095989704132, + 0.0264157485216856, + 0.025251252576708794, + 0.02720639854669571, + 0.027512386441230774, + 0.029633820056915283, + 0.02885596640408039, + 0.031438712030649185, + 0.0244715828448534, + 0.03218274191021919, + 0.031743232160806656, + 0.03524785488843918, + 0.038931526243686676, + 0.037403788417577744, + 0.03771914169192314, + 0.041983578354120255, + 0.03549705073237419, + 0.04549537971615791, + 0.0359940230846405, + 0.052616480737924576, + 0.04251093044877052, + 0.059319522231817245, + 0.03818616643548012, + 0.06705058366060257, + 0.024311494082212448, + 0.07359735667705536, + 0.01473314967006445, + 0.07876058667898178, + 0.015392928384244442, + 0.0848938599228859, + 0.018577447161078453, + 0.11232978105545044, + 0.018359949812293053, + 0.1471690982580185, + 0.03667483106255531, + 0.24789714813232422 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005542142737822822, + 0.0003797328422683109, + 0.000340619215402376, + 0.0001793765938241596, + 0.00020684039436717747, + 0.00016374338231517116, + 0.00018705111879548875, + 0.0002256278252430331, + 0.000426778694633347, + 0.00027155262856832466, + 0.0003392540980461625, + 0.0003897844238340517, + 0.00045204461946776503, + 0.0004208641344247883, + 0.00047738343422760874, + 0.00048450285457785075, + 0.0005178591522769467, + 0.0005253914687520978, + 0.0005086607978429822, + 0.0005610796431247281, + 0.0005682688177114847, + 0.000574059459111366, + 0.0005453911942509562, + 0.0005508569019095279, + 0.0005214428582378581, + 0.000461966711188474, + 0.00041497592078977314, + 0.00037256322877604213, + 0.00033686502618798456, + 0.00030674152201740854, + 0.00025927963978097604, + 0.00021950976285567438 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002796507057497499, + "first_to_last_ratio": 0.7732247282264533, + "imbalance_abs_log_ratio": 0.25718555049038655, + "last_quartile_mean": 0.00036166808372927383, + "mean": 0.00039825883245600557, + "normalized": [ + 1.3915931766397287, + 0.9534825377922003, + 0.8552709636138532, + 0.4504020481302817, + 0.5193617253674505, + 0.41114815032572916, + 0.46967224214958675, + 0.5665356468094339, + 1.0716113739435822, + 0.681849607436697, + 0.8518432496626144, + 0.9787213542266133, + 1.1350523394046785, + 1.056760328024313, + 1.1986763263570401, + 1.2165526916000595, + 1.3003080159789122, + 1.319221134436827, + 1.2772115930389878, + 1.4088316376177517, + 1.4268831508570738, + 1.441423045337684, + 1.3694390426638032, + 1.383163051306287, + 1.3093064503358134, + 1.159966015918821, + 1.0419754365036318, + 0.9354801410893958, + 0.8458444577627716, + 0.7702064512311685, + 0.6510329932472191, + 0.5511736211899907 + ], + "population_cv": 0.3252608102570666 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.061885204166173935, + 0.07089564949274063, + 0.1026567816734314, + 0.16996370255947113, + 0.17673343420028687, + 0.23203739523887634, + 0.23246681690216064, + 0.23961465060710907, + 0.24062344431877136, + 0.24108263850212097, + 0.2416554093360901, + 0.24260269105434418, + 0.24258241057395935, + 0.2429492026567459, + 0.24218299984931946, + 0.24250134825706482, + 0.24160927534103394, + 0.242857426404953, + 0.24130181968212128, + 0.23791606724262238, + 0.23599836230278015, + 0.22686637938022614, + 0.22310413420200348, + 0.20835906267166138, + 0.20329499244689941, + 0.1809774935245514, + 0.176778644323349, + 0.17327234148979187, + 0.17310521006584167, + 0.17050857841968536, + 0.1729687601327896, + 0.17174123227596283, + 0.17606647312641144, + 0.1756037324666977, + 0.1812635064125061, + 0.1811160296201706, + 0.18683874607086182, + 0.1874236911535263, + 0.19253391027450562, + 0.19280622899532318, + 0.1985689103603363, + 0.20097090303897858, + 0.20644955337047577, + 0.2074177861213684, + 0.2135031372308731, + 0.21357181668281555, + 0.22015690803527832, + 0.22022640705108643, + 0.228074312210083, + 0.22874467074871063, + 0.2383035272359848, + 0.23907209932804108, + 0.2532499134540558, + 0.2534840703010559, + 0.27322494983673096, + 0.27303415536880493, + 0.2992607355117798, + 0.29903557896614075, + 0.3339390158653259, + 0.3341987431049347, + 0.3918353319168091, + 0.3921641707420349, + 0.48234984278678894, + 0.4832552373409271 + ], + "loss_nats": 1.2900829315185547, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07089564949274063, + 0.1026567816734314, + 0.16996370255947113, + 0.17673343420028687, + 0.23203739523887634, + 0.23246681690216064, + 0.23961465060710907, + 0.24062344431877136, + 0.24108263850212097, + 0.2416554093360901, + 0.24260269105434418, + 0.24258241057395935, + 0.2429492026567459, + 0.24218299984931946, + 0.24250134825706482, + 0.24160927534103394, + 0.242857426404953, + 0.24130181968212128, + 0.23791606724262238, + 0.23599836230278015, + 0.22686637938022614, + 0.22310413420200348, + 0.20835906267166138, + 0.20329499244689941, + 0.1809774935245514, + 0.176778644323349, + 0.17327234148979187, + 0.17310521006584167, + 0.17050857841968536, + 0.1729687601327896, + 0.17174123227596283, + 0.17606647312641144, + 0.1756037324666977, + 0.1812635064125061, + 0.1811160296201706, + 0.18683874607086182, + 0.1874236911535263, + 0.19253391027450562, + 0.19280622899532318, + 0.1985689103603363, + 0.20097090303897858, + 0.20644955337047577, + 0.2074177861213684, + 0.2135031372308731, + 0.21357181668281555, + 0.22015690803527832, + 0.22022640705108643, + 0.228074312210083, + 0.22874467074871063, + 0.2383035272359848, + 0.23907209932804108, + 0.2532499134540558, + 0.2534840703010559, + 0.27322494983673096, + 0.27303415536880493, + 0.2992607355117798, + 0.29903557896614075, + 0.3339390158653259, + 0.3341987431049347, + 0.3918353319168091, + 0.3921641707420349, + 0.48234984278678894, + 0.4832552373409271, + 0.651733934879303 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.06581270897968, + "cross_entropy_nats": 5.590795338153839, + "step": 0 + }, + { + "bits_per_byte": 5.193364149696528, + "cross_entropy_nats": 3.599765717983246, + "step": 100 + }, + { + "bits_per_byte": 3.118855971380574, + "cross_entropy_nats": 2.1618262231349945, + "step": 500 + }, + { + "bits_per_byte": 1.9576097217110056, + "cross_entropy_nats": 1.3569116592407227, + "step": 2000 + }, + { + "bits_per_byte": 1.792987261872353, + "cross_entropy_nats": 1.2428040653467178, + "step": 4000 + }, + { + "bits_per_byte": 1.709514213904039, + "cross_entropy_nats": 1.1849449574947357, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "final_optimizer_state": "e774625d2eec7b16bb949a8bd26f25b05b7288a59bf24a716a88b489cd22bcd6", + "final_public_parameters": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 42.39461278025134, + "measured_steps": 7980, + "median_ms": 42.272048493032344, + "p95_ms": 43.13050716154976, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.087939566883062, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.606132507324219, + "step": 1, + "unclipped_grad_norm": 19.928020477294922 + }, + { + "bits_per_byte": 7.3282798152711734, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.07957649230957, + "step": 10, + "unclipped_grad_norm": 11.731513977050781 + }, + { + "bits_per_byte": 6.791570117070589, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.707557678222656, + "step": 20, + "unclipped_grad_norm": 3.350315809249878 + }, + { + "bits_per_byte": 6.667928349712299, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.621855735778809, + "step": 30, + "unclipped_grad_norm": 3.0359153747558594 + }, + { + "bits_per_byte": 6.473404964696806, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.487022399902344, + "step": 40, + "unclipped_grad_norm": 3.083479404449463 + }, + { + "bits_per_byte": 6.159992036931666, + "learning_rate": 3.75e-05, + "loss_nats": 4.269781112670898, + "step": 50, + "unclipped_grad_norm": 2.770655393600464 + }, + { + "bits_per_byte": 5.940455495174687, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.117609977722168, + "step": 60, + "unclipped_grad_norm": 2.5750629901885986 + }, + { + "bits_per_byte": 5.762018679390299, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.993927001953125, + "step": 70, + "unclipped_grad_norm": 2.5019538402557373 + }, + { + "bits_per_byte": 5.5944951946990615, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8778085708618164, + "step": 80, + "unclipped_grad_norm": 2.5689163208007812 + }, + { + "bits_per_byte": 5.40584020169631, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7470428943634033, + "step": 90, + "unclipped_grad_norm": 2.3143606185913086 + }, + { + "bits_per_byte": 5.179311877251155, + "learning_rate": 7.5e-05, + "loss_nats": 3.5900254249572754, + "step": 100, + "unclipped_grad_norm": 2.227365255355835 + }, + { + "bits_per_byte": 4.996660110659421, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.463420867919922, + "step": 110, + "unclipped_grad_norm": 2.0371005535125732 + }, + { + "bits_per_byte": 4.800645980026251, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.327554225921631, + "step": 120, + "unclipped_grad_norm": 1.8128844499588013 + }, + { + "bits_per_byte": 4.634309863556569, + "learning_rate": 9.75e-05, + "loss_nats": 3.212258815765381, + "step": 130, + "unclipped_grad_norm": 1.5838756561279297 + }, + { + "bits_per_byte": 4.478764626532625, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.104443073272705, + "step": 140, + "unclipped_grad_norm": 1.4334924221038818 + }, + { + "bits_per_byte": 4.278149959988874, + "learning_rate": 0.0001125, + "loss_nats": 2.9653875827789307, + "step": 150, + "unclipped_grad_norm": 1.2058515548706055 + }, + { + "bits_per_byte": 4.130972366860008, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8633718490600586, + "step": 160, + "unclipped_grad_norm": 1.057314395904541 + }, + { + "bits_per_byte": 4.025422830407184, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.790210485458374, + "step": 170, + "unclipped_grad_norm": 0.9353183507919312 + }, + { + "bits_per_byte": 3.9235808959416594, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.719619035720825, + "step": 180, + "unclipped_grad_norm": 0.8658975958824158 + }, + { + "bits_per_byte": 3.837114554428572, + "learning_rate": 0.0001425, + "loss_nats": 2.6596851348876953, + "step": 190, + "unclipped_grad_norm": 0.8723611235618591 + }, + { + "bits_per_byte": 3.771249326731737, + "learning_rate": 0.00015, + "loss_nats": 2.6140308380126953, + "step": 200, + "unclipped_grad_norm": 0.9528701305389404 + }, + { + "bits_per_byte": 3.687100279456375, + "learning_rate": 0.0001575, + "loss_nats": 2.5557031631469727, + "step": 210, + "unclipped_grad_norm": 0.7721163034439087 + }, + { + "bits_per_byte": 3.664338719579984, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.539926052093506, + "step": 220, + "unclipped_grad_norm": 1.0300326347351074 + }, + { + "bits_per_byte": 3.616636923606072, + "learning_rate": 0.0001725, + "loss_nats": 2.506861686706543, + "step": 230, + "unclipped_grad_norm": 1.116415023803711 + }, + { + "bits_per_byte": 3.58821025521026, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4871578216552734, + "step": 240, + "unclipped_grad_norm": 0.7745823860168457 + }, + { + "bits_per_byte": 3.5119345736913976, + "learning_rate": 0.0001875, + "loss_nats": 2.4342875480651855, + "step": 250, + "unclipped_grad_norm": 0.9986598491668701 + }, + { + "bits_per_byte": 3.522710662629159, + "learning_rate": 0.000195, + "loss_nats": 2.4417569637298584, + "step": 260, + "unclipped_grad_norm": 0.8885983228683472 + }, + { + "bits_per_byte": 3.5021329384534243, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4274935722351074, + "step": 270, + "unclipped_grad_norm": 0.7707400321960449 + }, + { + "bits_per_byte": 3.4629779922967905, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.40035343170166, + "step": 280, + "unclipped_grad_norm": 1.055368423461914 + }, + { + "bits_per_byte": 3.4474741002868097, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3896069526672363, + "step": 290, + "unclipped_grad_norm": 1.053453803062439 + }, + { + "bits_per_byte": 3.4491093113312137, + "learning_rate": 0.000225, + "loss_nats": 2.390740394592285, + "step": 300, + "unclipped_grad_norm": 1.2985762357711792 + }, + { + "bits_per_byte": 3.334430591805289, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.311251163482666, + "step": 310, + "unclipped_grad_norm": 0.8886587619781494 + }, + { + "bits_per_byte": 3.4024717440008403, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3584136962890625, + "step": 320, + "unclipped_grad_norm": 1.0081987380981445 + }, + { + "bits_per_byte": 3.413793361907139, + "learning_rate": 0.0002475, + "loss_nats": 2.3662612438201904, + "step": 330, + "unclipped_grad_norm": 0.8511857986450195 + }, + { + "bits_per_byte": 3.3420762525320464, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3165507316589355, + "step": 340, + "unclipped_grad_norm": 0.9547706842422485 + }, + { + "bits_per_byte": 3.3350242759160675, + "learning_rate": 0.0002625, + "loss_nats": 2.3116626739501953, + "step": 350, + "unclipped_grad_norm": 1.1394811868667603 + }, + { + "bits_per_byte": 3.3268671387856448, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3060085773468018, + "step": 360, + "unclipped_grad_norm": 1.0605591535568237 + }, + { + "bits_per_byte": 3.320889365806955, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3018651008605957, + "step": 370, + "unclipped_grad_norm": 0.8489253520965576 + }, + { + "bits_per_byte": 3.2193453052927254, + "learning_rate": 0.000285, + "loss_nats": 2.231480121612549, + "step": 380, + "unclipped_grad_norm": 0.923150897026062 + }, + { + "bits_per_byte": 3.297119987596508, + "learning_rate": 0.0002925, + "loss_nats": 2.2853894233703613, + "step": 390, + "unclipped_grad_norm": 0.7830708026885986 + }, + { + "bits_per_byte": 3.2808776020837227, + "learning_rate": 0.0003, + "loss_nats": 2.2741310596466064, + "step": 400, + "unclipped_grad_norm": 0.7351507544517517 + }, + { + "bits_per_byte": 3.2985058237971607, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2863500118255615, + "step": 410, + "unclipped_grad_norm": 0.8769013285636902 + }, + { + "bits_per_byte": 3.225643997897925, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2358460426330566, + "step": 420, + "unclipped_grad_norm": 0.7473549842834473 + }, + { + "bits_per_byte": 3.2585050669635836, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2586236000061035, + "step": 430, + "unclipped_grad_norm": 0.9111914038658142 + }, + { + "bits_per_byte": 3.2092602426461303, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.224489688873291, + "step": 440, + "unclipped_grad_norm": 1.075931191444397 + }, + { + "bits_per_byte": 3.2640807445045557, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.26248836517334, + "step": 450, + "unclipped_grad_norm": 0.8512128591537476 + }, + { + "bits_per_byte": 3.207580316112502, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.223325252532959, + "step": 460, + "unclipped_grad_norm": 1.2196831703186035 + }, + { + "bits_per_byte": 3.1660169249131855, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.1945157051086426, + "step": 470, + "unclipped_grad_norm": 0.7675222158432007 + }, + { + "bits_per_byte": 3.1673281206433637, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1954245567321777, + "step": 480, + "unclipped_grad_norm": 0.9160394668579102 + }, + { + "bits_per_byte": 3.113679164602616, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.158237934112549, + "step": 490, + "unclipped_grad_norm": 0.6955189108848572 + }, + { + "bits_per_byte": 3.0709621137813112, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.128628730773926, + "step": 500, + "unclipped_grad_norm": 0.9366468191146851 + }, + { + "bits_per_byte": 3.1516054667014854, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1845264434814453, + "step": 510, + "unclipped_grad_norm": 1.3161629438400269 + }, + { + "bits_per_byte": 3.151479575401054, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.184439182281494, + "step": 520, + "unclipped_grad_norm": 0.8699809312820435 + }, + { + "bits_per_byte": 3.012768688174164, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.088292121887207, + "step": 530, + "unclipped_grad_norm": 1.1934181451797485 + }, + { + "bits_per_byte": 3.0200408025832473, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0933327674865723, + "step": 540, + "unclipped_grad_norm": 0.8321981430053711 + }, + { + "bits_per_byte": 3.02149439994834, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0943403244018555, + "step": 550, + "unclipped_grad_norm": 0.9657202959060669 + }, + { + "bits_per_byte": 3.0546815481200342, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.1173439025878906, + "step": 560, + "unclipped_grad_norm": 1.2880722284317017 + }, + { + "bits_per_byte": 2.973202359516667, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0608668327331543, + "step": 570, + "unclipped_grad_norm": 0.8365735411643982 + }, + { + "bits_per_byte": 2.986164347946895, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0698513984680176, + "step": 580, + "unclipped_grad_norm": 0.9132604002952576 + }, + { + "bits_per_byte": 2.926336399225948, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.028381824493408, + "step": 590, + "unclipped_grad_norm": 0.7623230814933777 + }, + { + "bits_per_byte": 2.903745446139213, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.012722969055176, + "step": 600, + "unclipped_grad_norm": 0.8222759366035461 + }, + { + "bits_per_byte": 2.915261748406288, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.020705461502075, + "step": 610, + "unclipped_grad_norm": 1.3526110649108887 + }, + { + "bits_per_byte": 2.8888483089214763, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.002397060394287, + "step": 620, + "unclipped_grad_norm": 0.9250975847244263 + }, + { + "bits_per_byte": 2.806001858243749, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.944972276687622, + "step": 630, + "unclipped_grad_norm": 0.7925068140029907 + }, + { + "bits_per_byte": 2.807606972324252, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9460848569869995, + "step": 640, + "unclipped_grad_norm": 0.8585688471794128 + }, + { + "bits_per_byte": 2.785753824809718, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.93093740940094, + "step": 650, + "unclipped_grad_norm": 0.7446964383125305 + }, + { + "bits_per_byte": 2.8113040833698495, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.9486474990844727, + "step": 660, + "unclipped_grad_norm": 1.026633381843567 + }, + { + "bits_per_byte": 2.7895744193986354, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.9335856437683105, + "step": 670, + "unclipped_grad_norm": 0.8819775581359863 + }, + { + "bits_per_byte": 2.7608092891459255, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.913647174835205, + "step": 680, + "unclipped_grad_norm": 0.845567524433136 + }, + { + "bits_per_byte": 2.7436651985946665, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.9017637968063354, + "step": 690, + "unclipped_grad_norm": 1.0123262405395508 + }, + { + "bits_per_byte": 2.7268093509662537, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.890080213546753, + "step": 700, + "unclipped_grad_norm": 0.8187565803527832 + }, + { + "bits_per_byte": 2.696082242632501, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8687818050384521, + "step": 710, + "unclipped_grad_norm": 0.9888712763786316 + }, + { + "bits_per_byte": 2.7032793726058073, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8737704753875732, + "step": 720, + "unclipped_grad_norm": 0.8442726731300354 + }, + { + "bits_per_byte": 2.680898066370474, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8582569360733032, + "step": 730, + "unclipped_grad_norm": 1.1784851551055908 + }, + { + "bits_per_byte": 2.619564409541223, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.8157436847686768, + "step": 740, + "unclipped_grad_norm": 0.8383970260620117 + }, + { + "bits_per_byte": 2.6352585143630582, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.8266220092773438, + "step": 750, + "unclipped_grad_norm": 1.062637448310852 + }, + { + "bits_per_byte": 2.547899926857294, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7660696506500244, + "step": 760, + "unclipped_grad_norm": 0.8228002786636353 + }, + { + "bits_per_byte": 2.573545291823353, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7838456630706787, + "step": 770, + "unclipped_grad_norm": 0.8357667326927185 + }, + { + "bits_per_byte": 2.523504187832405, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.749159812927246, + "step": 780, + "unclipped_grad_norm": 0.7918128967285156 + }, + { + "bits_per_byte": 2.5308719245953677, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7542667388916016, + "step": 790, + "unclipped_grad_norm": 1.3666422367095947 + }, + { + "bits_per_byte": 2.443318996694129, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6935796737670898, + "step": 800, + "unclipped_grad_norm": 0.7999250888824463 + }, + { + "bits_per_byte": 2.500851320991359, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7334580421447754, + "step": 810, + "unclipped_grad_norm": 1.1067378520965576 + }, + { + "bits_per_byte": 2.4739762800716316, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.714829683303833, + "step": 820, + "unclipped_grad_norm": 1.090567708015442 + }, + { + "bits_per_byte": 2.467352884221603, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7102386951446533, + "step": 830, + "unclipped_grad_norm": 0.9818849563598633 + }, + { + "bits_per_byte": 2.471093850843171, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.712831735610962, + "step": 840, + "unclipped_grad_norm": 0.9710109829902649 + }, + { + "bits_per_byte": 2.396449768733044, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6610924005508423, + "step": 850, + "unclipped_grad_norm": 1.0234057903289795 + }, + { + "bits_per_byte": 2.473427999380681, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.7144496440887451, + "step": 860, + "unclipped_grad_norm": 0.9857868552207947 + }, + { + "bits_per_byte": 2.465926632097998, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.7092500925064087, + "step": 870, + "unclipped_grad_norm": 1.1294745206832886 + }, + { + "bits_per_byte": 2.391602093753174, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6577322483062744, + "step": 880, + "unclipped_grad_norm": 0.8081791400909424 + }, + { + "bits_per_byte": 2.354239550783832, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6318345069885254, + "step": 890, + "unclipped_grad_norm": 0.8606147170066833 + }, + { + "bits_per_byte": 2.395512979233794, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6604430675506592, + "step": 900, + "unclipped_grad_norm": 0.8592826128005981 + }, + { + "bits_per_byte": 2.3758491708647442, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.646813154220581, + "step": 910, + "unclipped_grad_norm": 0.8169032335281372 + }, + { + "bits_per_byte": 2.353533905967342, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6313453912734985, + "step": 920, + "unclipped_grad_norm": 0.7358964085578918 + }, + { + "bits_per_byte": 2.2994974730142004, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5938901901245117, + "step": 930, + "unclipped_grad_norm": 0.8204946517944336 + }, + { + "bits_per_byte": 2.3548098452540933, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6322298049926758, + "step": 940, + "unclipped_grad_norm": 0.8031407594680786 + }, + { + "bits_per_byte": 2.362864480724602, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.637812852859497, + "step": 950, + "unclipped_grad_norm": 0.8955363035202026 + }, + { + "bits_per_byte": 2.311957100121258, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6025265455245972, + "step": 960, + "unclipped_grad_norm": 0.8827840089797974 + }, + { + "bits_per_byte": 2.2663949403067303, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5709452629089355, + "step": 970, + "unclipped_grad_norm": 1.4933069944381714 + }, + { + "bits_per_byte": 2.401434685868302, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6645476818084717, + "step": 980, + "unclipped_grad_norm": 1.0056283473968506 + }, + { + "bits_per_byte": 2.23815401317385, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5513701438903809, + "step": 990, + "unclipped_grad_norm": 0.7972294092178345 + }, + { + "bits_per_byte": 2.255509814368321, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5634002685546875, + "step": 1000, + "unclipped_grad_norm": 0.7948418855667114 + }, + { + "bits_per_byte": 2.249676162851055, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.559356689453125, + "step": 1010, + "unclipped_grad_norm": 0.9423647522926331 + }, + { + "bits_per_byte": 2.2157856056373317, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5358655452728271, + "step": 1020, + "unclipped_grad_norm": 1.3369964361190796 + }, + { + "bits_per_byte": 2.21131629248936, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.532767653465271, + "step": 1030, + "unclipped_grad_norm": 0.7792882919311523 + }, + { + "bits_per_byte": 2.300165797595453, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.594353437423706, + "step": 1040, + "unclipped_grad_norm": 0.9419035911560059 + }, + { + "bits_per_byte": 2.21666237319143, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.536473274230957, + "step": 1050, + "unclipped_grad_norm": 0.9680492281913757 + }, + { + "bits_per_byte": 2.242138163263738, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5541317462921143, + "step": 1060, + "unclipped_grad_norm": 0.9887726306915283 + }, + { + "bits_per_byte": 2.276496685270459, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5779472589492798, + "step": 1070, + "unclipped_grad_norm": 0.8514108657836914 + }, + { + "bits_per_byte": 2.2430374605450996, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5547550916671753, + "step": 1080, + "unclipped_grad_norm": 0.800697386264801 + }, + { + "bits_per_byte": 2.1823284447037823, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5126748085021973, + "step": 1090, + "unclipped_grad_norm": 0.9124817848205566 + }, + { + "bits_per_byte": 2.1899661942286, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5179688930511475, + "step": 1100, + "unclipped_grad_norm": 0.8022184371948242 + }, + { + "bits_per_byte": 2.145753238310091, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4873228073120117, + "step": 1110, + "unclipped_grad_norm": 1.139209508895874 + }, + { + "bits_per_byte": 2.188020210534087, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5166200399398804, + "step": 1120, + "unclipped_grad_norm": 0.8927217721939087 + }, + { + "bits_per_byte": 2.2131649340034842, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5340490341186523, + "step": 1130, + "unclipped_grad_norm": 0.8271659016609192 + }, + { + "bits_per_byte": 2.1401269978697646, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4834229946136475, + "step": 1140, + "unclipped_grad_norm": 0.9781836271286011 + }, + { + "bits_per_byte": 2.1955361963420934, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5218297243118286, + "step": 1150, + "unclipped_grad_norm": 0.8088869452476501 + }, + { + "bits_per_byte": 2.103003854750424, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4576911926269531, + "step": 1160, + "unclipped_grad_norm": 0.761218786239624 + }, + { + "bits_per_byte": 2.164645876472672, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5004181861877441, + "step": 1170, + "unclipped_grad_norm": 0.8793766498565674 + }, + { + "bits_per_byte": 2.159191274717907, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4966373443603516, + "step": 1180, + "unclipped_grad_norm": 0.8019644618034363 + }, + { + "bits_per_byte": 2.134348724766076, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4794178009033203, + "step": 1190, + "unclipped_grad_norm": 0.7809260487556458 + }, + { + "bits_per_byte": 2.1456660431061034, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4872623682022095, + "step": 1200, + "unclipped_grad_norm": 0.896268904209137 + }, + { + "bits_per_byte": 2.1662845271700926, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.501554012298584, + "step": 1210, + "unclipped_grad_norm": 0.8574862480163574 + }, + { + "bits_per_byte": 2.1378967268533757, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.481877088546753, + "step": 1220, + "unclipped_grad_norm": 0.8502950668334961 + }, + { + "bits_per_byte": 2.128576127089866, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4754165410995483, + "step": 1230, + "unclipped_grad_norm": 0.7498336434364319 + }, + { + "bits_per_byte": 2.0970609942498597, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4535719156265259, + "step": 1240, + "unclipped_grad_norm": 0.761203944683075 + }, + { + "bits_per_byte": 2.0385041693014396, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4129834175109863, + "step": 1250, + "unclipped_grad_norm": 0.7150603532791138 + }, + { + "bits_per_byte": 2.0885559362167414, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.447676658630371, + "step": 1260, + "unclipped_grad_norm": 0.7949564456939697 + }, + { + "bits_per_byte": 2.1179862953629884, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.468076229095459, + "step": 1270, + "unclipped_grad_norm": 0.862070620059967 + }, + { + "bits_per_byte": 2.1475409979658098, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.488561987876892, + "step": 1280, + "unclipped_grad_norm": 0.7539991140365601 + }, + { + "bits_per_byte": 2.1029943957046267, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4576846361160278, + "step": 1290, + "unclipped_grad_norm": 0.8074342608451843 + }, + { + "bits_per_byte": 2.049033979083166, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4202821254730225, + "step": 1300, + "unclipped_grad_norm": 0.7586180567741394 + }, + { + "bits_per_byte": 2.076369761524555, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4392298460006714, + "step": 1310, + "unclipped_grad_norm": 0.9284558892250061 + }, + { + "bits_per_byte": 2.051258058724124, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4218237400054932, + "step": 1320, + "unclipped_grad_norm": 0.7597900032997131 + }, + { + "bits_per_byte": 2.060438148661879, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4281868934631348, + "step": 1330, + "unclipped_grad_norm": 0.7334625720977783 + }, + { + "bits_per_byte": 2.04908454198252, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4203171730041504, + "step": 1340, + "unclipped_grad_norm": 0.7958364486694336 + }, + { + "bits_per_byte": 2.0721673654507224, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.436316967010498, + "step": 1350, + "unclipped_grad_norm": 0.7738369107246399 + }, + { + "bits_per_byte": 2.034246910761982, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4100325107574463, + "step": 1360, + "unclipped_grad_norm": 0.8466383218765259 + }, + { + "bits_per_byte": 2.0554857362476335, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4247541427612305, + "step": 1370, + "unclipped_grad_norm": 0.7919691205024719 + }, + { + "bits_per_byte": 2.020279167789508, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.40035080909729, + "step": 1380, + "unclipped_grad_norm": 0.8231279850006104 + }, + { + "bits_per_byte": 2.070644631059983, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4352614879608154, + "step": 1390, + "unclipped_grad_norm": 0.7695969343185425 + }, + { + "bits_per_byte": 2.174483628067326, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.507237195968628, + "step": 1400, + "unclipped_grad_norm": 0.8150069713592529 + }, + { + "bits_per_byte": 2.0594870846026074, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.427527666091919, + "step": 1410, + "unclipped_grad_norm": 0.7077280879020691 + }, + { + "bits_per_byte": 2.048123158964197, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4196507930755615, + "step": 1420, + "unclipped_grad_norm": 0.7647379040718079 + }, + { + "bits_per_byte": 2.0019392818668, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.3876385688781738, + "step": 1430, + "unclipped_grad_norm": 0.6688193082809448 + }, + { + "bits_per_byte": 2.0332329010524948, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4093296527862549, + "step": 1440, + "unclipped_grad_norm": 0.8427934646606445 + }, + { + "bits_per_byte": 2.0070671165849254, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.39119291305542, + "step": 1450, + "unclipped_grad_norm": 0.6920660138130188 + }, + { + "bits_per_byte": 2.1256969655317715, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4734208583831787, + "step": 1460, + "unclipped_grad_norm": 0.799986720085144 + }, + { + "bits_per_byte": 2.0930381480635276, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4507834911346436, + "step": 1470, + "unclipped_grad_norm": 0.6732411980628967 + }, + { + "bits_per_byte": 2.0451248134290543, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4175724983215332, + "step": 1480, + "unclipped_grad_norm": 0.6982382535934448 + }, + { + "bits_per_byte": 2.0566264971708064, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4255448579788208, + "step": 1490, + "unclipped_grad_norm": 0.7266837358474731 + }, + { + "bits_per_byte": 1.9341151717766374, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3406264781951904, + "step": 1500, + "unclipped_grad_norm": 0.6626343727111816 + }, + { + "bits_per_byte": 1.963616731740342, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3610754013061523, + "step": 1510, + "unclipped_grad_norm": 0.789225697517395 + }, + { + "bits_per_byte": 2.0151121210269856, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3967692852020264, + "step": 1520, + "unclipped_grad_norm": 0.7025586366653442 + }, + { + "bits_per_byte": 2.0371694119480934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4120582342147827, + "step": 1530, + "unclipped_grad_norm": 0.750766396522522 + }, + { + "bits_per_byte": 2.0209939276864937, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.400846242904663, + "step": 1540, + "unclipped_grad_norm": 0.7461745738983154 + }, + { + "bits_per_byte": 2.082936231117148, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4437813758850098, + "step": 1550, + "unclipped_grad_norm": 0.765493631362915 + }, + { + "bits_per_byte": 1.954267066908836, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3545947074890137, + "step": 1560, + "unclipped_grad_norm": 0.727699875831604 + }, + { + "bits_per_byte": 1.9578143810655324, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.357053518295288, + "step": 1570, + "unclipped_grad_norm": 0.7358320951461792 + }, + { + "bits_per_byte": 1.9655836973182597, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3624387979507446, + "step": 1580, + "unclipped_grad_norm": 0.6977834701538086 + }, + { + "bits_per_byte": 1.9752479184181935, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3691375255584717, + "step": 1590, + "unclipped_grad_norm": 0.6583591103553772 + }, + { + "bits_per_byte": 1.9806421542625054, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.372876524925232, + "step": 1600, + "unclipped_grad_norm": 0.727998673915863 + }, + { + "bits_per_byte": 1.9474555220387637, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3498733043670654, + "step": 1610, + "unclipped_grad_norm": 0.7535929083824158 + }, + { + "bits_per_byte": 1.942356580405983, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3463389873504639, + "step": 1620, + "unclipped_grad_norm": 0.6403960585594177 + }, + { + "bits_per_byte": 1.9826001767425787, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3742337226867676, + "step": 1630, + "unclipped_grad_norm": 0.7281255125999451 + }, + { + "bits_per_byte": 1.9377553565647996, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3431496620178223, + "step": 1640, + "unclipped_grad_norm": 0.7114111185073853 + }, + { + "bits_per_byte": 1.9278453722567828, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3362805843353271, + "step": 1650, + "unclipped_grad_norm": 0.6373128890991211 + }, + { + "bits_per_byte": 1.9498221752972869, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3515137434005737, + "step": 1660, + "unclipped_grad_norm": 0.6859769225120544 + }, + { + "bits_per_byte": 1.9095433225172644, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.323594570159912, + "step": 1670, + "unclipped_grad_norm": 0.7934040427207947 + }, + { + "bits_per_byte": 1.938967146322779, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.343989610671997, + "step": 1680, + "unclipped_grad_norm": 0.7104787230491638 + }, + { + "bits_per_byte": 1.9591448388526072, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.357975721359253, + "step": 1690, + "unclipped_grad_norm": 0.6892858743667603 + }, + { + "bits_per_byte": 1.9072468381802934, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3220027685165405, + "step": 1700, + "unclipped_grad_norm": 0.7498061656951904 + }, + { + "bits_per_byte": 1.9434417909329273, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3470911979675293, + "step": 1710, + "unclipped_grad_norm": 0.6478108763694763 + }, + { + "bits_per_byte": 1.8728540916260505, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.2981635332107544, + "step": 1720, + "unclipped_grad_norm": 0.7232697010040283 + }, + { + "bits_per_byte": 1.9589699324966796, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3578544855117798, + "step": 1730, + "unclipped_grad_norm": 0.7008518576622009 + }, + { + "bits_per_byte": 1.9590435410712488, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3579055070877075, + "step": 1740, + "unclipped_grad_norm": 0.6955974698066711 + }, + { + "bits_per_byte": 1.919930042733473, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.330794095993042, + "step": 1750, + "unclipped_grad_norm": 0.6556993126869202 + }, + { + "bits_per_byte": 1.9569465566092785, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3564519882202148, + "step": 1760, + "unclipped_grad_norm": 0.698861837387085 + }, + { + "bits_per_byte": 1.9512123110642108, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3524773120880127, + "step": 1770, + "unclipped_grad_norm": 0.6838504672050476 + }, + { + "bits_per_byte": 2.0216474617597724, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.401299238204956, + "step": 1780, + "unclipped_grad_norm": 0.7214158773422241 + }, + { + "bits_per_byte": 1.8826208143858987, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3049333095550537, + "step": 1790, + "unclipped_grad_norm": 0.7236432433128357 + }, + { + "bits_per_byte": 2.013814339943575, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3958697319030762, + "step": 1800, + "unclipped_grad_norm": 0.7153979539871216 + }, + { + "bits_per_byte": 1.8985195785622602, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3159534931182861, + "step": 1810, + "unclipped_grad_norm": 0.7165744304656982 + }, + { + "bits_per_byte": 1.956814645916067, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3563605546951294, + "step": 1820, + "unclipped_grad_norm": 0.7207518815994263 + }, + { + "bits_per_byte": 1.973846947744265, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.368166446685791, + "step": 1830, + "unclipped_grad_norm": 0.6764873266220093 + }, + { + "bits_per_byte": 1.953527541510126, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3540821075439453, + "step": 1840, + "unclipped_grad_norm": 0.6510032415390015 + }, + { + "bits_per_byte": 1.8838981295338566, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3058186769485474, + "step": 1850, + "unclipped_grad_norm": 0.6467540860176086 + }, + { + "bits_per_byte": 1.9116946534969073, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3250857591629028, + "step": 1860, + "unclipped_grad_norm": 0.6336910128593445 + }, + { + "bits_per_byte": 1.996913260877985, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3841547966003418, + "step": 1870, + "unclipped_grad_norm": 0.7375073432922363 + }, + { + "bits_per_byte": 1.8879835774051028, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3086504936218262, + "step": 1880, + "unclipped_grad_norm": 0.6048603057861328 + }, + { + "bits_per_byte": 1.9029378128629257, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3190159797668457, + "step": 1890, + "unclipped_grad_norm": 0.6331550478935242 + }, + { + "bits_per_byte": 1.9156196415548983, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3278063535690308, + "step": 1900, + "unclipped_grad_norm": 0.7952298521995544 + }, + { + "bits_per_byte": 1.9644835243006904, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3616762161254883, + "step": 1910, + "unclipped_grad_norm": 0.6600897312164307 + }, + { + "bits_per_byte": 1.8776484519841532, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3014867305755615, + "step": 1920, + "unclipped_grad_norm": 0.645132303237915 + }, + { + "bits_per_byte": 1.9490762865404947, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.350996732711792, + "step": 1930, + "unclipped_grad_norm": 0.6212714314460754 + }, + { + "bits_per_byte": 1.942025857768374, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3461097478866577, + "step": 1940, + "unclipped_grad_norm": 0.6636208295822144 + }, + { + "bits_per_byte": 1.8837596834999122, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.305722713470459, + "step": 1950, + "unclipped_grad_norm": 0.7458274960517883 + }, + { + "bits_per_byte": 1.8770238109962194, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.301053762435913, + "step": 1960, + "unclipped_grad_norm": 0.6796664595603943 + }, + { + "bits_per_byte": 1.873630765277347, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2987018823623657, + "step": 1970, + "unclipped_grad_norm": 0.7089046239852905 + }, + { + "bits_per_byte": 1.8950018454215123, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3135151863098145, + "step": 1980, + "unclipped_grad_norm": 0.6426982283592224 + }, + { + "bits_per_byte": 1.8681299001895, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.294888973236084, + "step": 1990, + "unclipped_grad_norm": 0.7099292278289795 + }, + { + "bits_per_byte": 1.964639340582372, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3617842197418213, + "step": 2000, + "unclipped_grad_norm": 0.7598643898963928 + }, + { + "bits_per_byte": 1.939805561745735, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3445707559585571, + "step": 2010, + "unclipped_grad_norm": 0.6349508166313171 + }, + { + "bits_per_byte": 1.851296754271001, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.2832211256027222, + "step": 2020, + "unclipped_grad_norm": 0.6008884310722351 + }, + { + "bits_per_byte": 1.890781391169339, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3105897903442383, + "step": 2030, + "unclipped_grad_norm": 0.637248158454895 + }, + { + "bits_per_byte": 1.882627693691933, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3049380779266357, + "step": 2040, + "unclipped_grad_norm": 0.6375738382339478 + }, + { + "bits_per_byte": 1.848167701921203, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2810522317886353, + "step": 2050, + "unclipped_grad_norm": 0.6515361070632935 + }, + { + "bits_per_byte": 1.833815233758749, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.271103858947754, + "step": 2060, + "unclipped_grad_norm": 0.6752470135688782 + }, + { + "bits_per_byte": 1.960125139962525, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3586552143096924, + "step": 2070, + "unclipped_grad_norm": 0.7093353867530823 + }, + { + "bits_per_byte": 1.93064937739645, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.338224172592163, + "step": 2080, + "unclipped_grad_norm": 0.7233569622039795 + }, + { + "bits_per_byte": 1.8411410067549012, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.276181697845459, + "step": 2090, + "unclipped_grad_norm": 0.6208991408348083 + }, + { + "bits_per_byte": 1.7919973082361553, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2421178817749023, + "step": 2100, + "unclipped_grad_norm": 0.6259884834289551 + }, + { + "bits_per_byte": 1.830389339353562, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2687292098999023, + "step": 2110, + "unclipped_grad_norm": 0.633368968963623 + }, + { + "bits_per_byte": 1.8446745622995284, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2786309719085693, + "step": 2120, + "unclipped_grad_norm": 0.6770470142364502 + }, + { + "bits_per_byte": 1.8875441617321482, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.308345913887024, + "step": 2130, + "unclipped_grad_norm": 0.7079503536224365 + }, + { + "bits_per_byte": 1.912643825747019, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3257436752319336, + "step": 2140, + "unclipped_grad_norm": 0.6220520734786987 + }, + { + "bits_per_byte": 1.8089860984536854, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2538936138153076, + "step": 2150, + "unclipped_grad_norm": 0.6506339907646179 + }, + { + "bits_per_byte": 1.9026210208200363, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3187963962554932, + "step": 2160, + "unclipped_grad_norm": 0.6963920593261719 + }, + { + "bits_per_byte": 1.8293292382936435, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2679944038391113, + "step": 2170, + "unclipped_grad_norm": 0.6530651450157166 + }, + { + "bits_per_byte": 1.90049290749826, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3173213005065918, + "step": 2180, + "unclipped_grad_norm": 0.6189562678337097 + }, + { + "bits_per_byte": 1.8805790363548553, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3035180568695068, + "step": 2190, + "unclipped_grad_norm": 0.6570361852645874 + }, + { + "bits_per_byte": 1.8344993807438814, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.271578073501587, + "step": 2200, + "unclipped_grad_norm": 0.5929612517356873 + }, + { + "bits_per_byte": 1.837753980428809, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.273833990097046, + "step": 2210, + "unclipped_grad_norm": 0.6836420893669128 + }, + { + "bits_per_byte": 1.8346952689832141, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2717138528823853, + "step": 2220, + "unclipped_grad_norm": 0.6392546892166138 + }, + { + "bits_per_byte": 1.7850062134785822, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.237272024154663, + "step": 2230, + "unclipped_grad_norm": 0.5976205468177795 + }, + { + "bits_per_byte": 1.8807291772090586, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3036221265792847, + "step": 2240, + "unclipped_grad_norm": 0.6432849168777466 + }, + { + "bits_per_byte": 1.861241479074492, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2901142835617065, + "step": 2250, + "unclipped_grad_norm": 0.6619172096252441 + }, + { + "bits_per_byte": 1.9088667427687702, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3231256008148193, + "step": 2260, + "unclipped_grad_norm": 0.6604200601577759 + }, + { + "bits_per_byte": 1.8085772956925845, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2536102533340454, + "step": 2270, + "unclipped_grad_norm": 0.6277530193328857 + }, + { + "bits_per_byte": 1.8640159031982109, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2920373678207397, + "step": 2280, + "unclipped_grad_norm": 0.6375154256820679 + }, + { + "bits_per_byte": 1.8154523021608253, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.258375644683838, + "step": 2290, + "unclipped_grad_norm": 0.6188551187515259 + }, + { + "bits_per_byte": 1.8127673090155552, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.256514549255371, + "step": 2300, + "unclipped_grad_norm": 0.6347740292549133 + }, + { + "bits_per_byte": 1.811916166876435, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2559245824813843, + "step": 2310, + "unclipped_grad_norm": 0.621824324131012 + }, + { + "bits_per_byte": 1.866308603916863, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2936265468597412, + "step": 2320, + "unclipped_grad_norm": 0.6393704414367676 + }, + { + "bits_per_byte": 1.8101679632304146, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2547128200531006, + "step": 2330, + "unclipped_grad_norm": 0.6721166372299194 + }, + { + "bits_per_byte": 1.82719372971788, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2665141820907593, + "step": 2340, + "unclipped_grad_norm": 0.646355152130127 + }, + { + "bits_per_byte": 1.8870428323048831, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3079984188079834, + "step": 2350, + "unclipped_grad_norm": 0.6942237019538879 + }, + { + "bits_per_byte": 1.804568896048925, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2508318424224854, + "step": 2360, + "unclipped_grad_norm": 0.6387670040130615 + }, + { + "bits_per_byte": 1.739788190948431, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2059292793273926, + "step": 2370, + "unclipped_grad_norm": 0.6083642840385437 + }, + { + "bits_per_byte": 1.8484354789085964, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2812378406524658, + "step": 2380, + "unclipped_grad_norm": 0.6751434803009033 + }, + { + "bits_per_byte": 1.8404750899307605, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2757201194763184, + "step": 2390, + "unclipped_grad_norm": 0.6037119626998901 + }, + { + "bits_per_byte": 1.8757652419572053, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3001813888549805, + "step": 2400, + "unclipped_grad_norm": 0.6957288980484009 + }, + { + "bits_per_byte": 1.801620425482533, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2487881183624268, + "step": 2410, + "unclipped_grad_norm": 0.626511812210083 + }, + { + "bits_per_byte": 1.8236299052267009, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.264043927192688, + "step": 2420, + "unclipped_grad_norm": 0.6481806635856628 + }, + { + "bits_per_byte": 1.8322460640522769, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2700161933898926, + "step": 2430, + "unclipped_grad_norm": 0.634190559387207 + }, + { + "bits_per_byte": 1.778641823500753, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2328605651855469, + "step": 2440, + "unclipped_grad_norm": 0.6495575308799744 + }, + { + "bits_per_byte": 1.9154751761281736, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.327706217765808, + "step": 2450, + "unclipped_grad_norm": 0.682089626789093 + }, + { + "bits_per_byte": 1.7848875454494868, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.237189769744873, + "step": 2460, + "unclipped_grad_norm": 0.6699327230453491 + }, + { + "bits_per_byte": 1.784353367335907, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2368195056915283, + "step": 2470, + "unclipped_grad_norm": 0.6535210609436035 + }, + { + "bits_per_byte": 1.865447314801342, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.293029546737671, + "step": 2480, + "unclipped_grad_norm": 0.6640622019767761 + }, + { + "bits_per_byte": 1.857090849778569, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.287237286567688, + "step": 2490, + "unclipped_grad_norm": 0.6223002076148987 + }, + { + "bits_per_byte": 1.8122928088818246, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2561856508255005, + "step": 2500, + "unclipped_grad_norm": 0.6052747368812561 + }, + { + "bits_per_byte": 1.76881026328153, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.226045846939087, + "step": 2510, + "unclipped_grad_norm": 0.6596174836158752 + }, + { + "bits_per_byte": 1.8006248179166882, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2480980157852173, + "step": 2520, + "unclipped_grad_norm": 0.6490770578384399 + }, + { + "bits_per_byte": 1.8232907554391995, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2638088464736938, + "step": 2530, + "unclipped_grad_norm": 0.6303134560585022 + }, + { + "bits_per_byte": 1.8125131186575798, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.256338357925415, + "step": 2540, + "unclipped_grad_norm": 0.6458812952041626 + }, + { + "bits_per_byte": 1.7674137641565237, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2250778675079346, + "step": 2550, + "unclipped_grad_norm": 0.6546228528022766 + }, + { + "bits_per_byte": 1.7342415784754543, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2020846605300903, + "step": 2560, + "unclipped_grad_norm": 0.602087676525116 + }, + { + "bits_per_byte": 1.774496353684356, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2299871444702148, + "step": 2570, + "unclipped_grad_norm": 0.6244005560874939 + }, + { + "bits_per_byte": 1.762712446412538, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2218191623687744, + "step": 2580, + "unclipped_grad_norm": 0.6090607643127441 + }, + { + "bits_per_byte": 1.7589357073995908, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2192013263702393, + "step": 2590, + "unclipped_grad_norm": 0.6631695032119751 + }, + { + "bits_per_byte": 1.7746397872151756, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2300865650177002, + "step": 2600, + "unclipped_grad_norm": 0.6302726864814758 + }, + { + "bits_per_byte": 1.8022297600145398, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2492104768753052, + "step": 2610, + "unclipped_grad_norm": 0.6297239065170288 + }, + { + "bits_per_byte": 1.7449476704743152, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.209505558013916, + "step": 2620, + "unclipped_grad_norm": 0.617965042591095 + }, + { + "bits_per_byte": 1.738682342503383, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.205162763595581, + "step": 2630, + "unclipped_grad_norm": 0.6048376560211182 + }, + { + "bits_per_byte": 1.7097366304672672, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1850991249084473, + "step": 2640, + "unclipped_grad_norm": 0.5914375185966492 + }, + { + "bits_per_byte": 1.790221415383346, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.240886926651001, + "step": 2650, + "unclipped_grad_norm": 0.6316050887107849 + }, + { + "bits_per_byte": 1.7080907564985102, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1839582920074463, + "step": 2660, + "unclipped_grad_norm": 0.6427422761917114 + }, + { + "bits_per_byte": 1.776176280217984, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2311515808105469, + "step": 2670, + "unclipped_grad_norm": 0.695833146572113 + }, + { + "bits_per_byte": 1.8751694940546166, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2997684478759766, + "step": 2680, + "unclipped_grad_norm": 0.7215304374694824 + }, + { + "bits_per_byte": 1.7935356930481232, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.243184208869934, + "step": 2690, + "unclipped_grad_norm": 0.6506277322769165 + }, + { + "bits_per_byte": 1.7071520751901008, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.1833076477050781, + "step": 2700, + "unclipped_grad_norm": 0.6119946241378784 + }, + { + "bits_per_byte": 1.7912357690581349, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2415900230407715, + "step": 2710, + "unclipped_grad_norm": 0.6407736539840698 + }, + { + "bits_per_byte": 1.8449512823847667, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2788227796554565, + "step": 2720, + "unclipped_grad_norm": 0.699180543422699 + }, + { + "bits_per_byte": 1.7188499911364874, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.1914160251617432, + "step": 2730, + "unclipped_grad_norm": 0.6055090427398682 + }, + { + "bits_per_byte": 1.7885469922945458, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2397263050079346, + "step": 2740, + "unclipped_grad_norm": 0.6273350715637207 + }, + { + "bits_per_byte": 1.7444217475279766, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2091410160064697, + "step": 2750, + "unclipped_grad_norm": 0.623899519443512 + }, + { + "bits_per_byte": 1.855314440977807, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.286005973815918, + "step": 2760, + "unclipped_grad_norm": 0.6717104315757751 + }, + { + "bits_per_byte": 1.79349716893433, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2431575059890747, + "step": 2770, + "unclipped_grad_norm": 0.6304712891578674 + }, + { + "bits_per_byte": 1.8011758503300526, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.248479962348938, + "step": 2780, + "unclipped_grad_norm": 0.6547600626945496 + }, + { + "bits_per_byte": 1.7772979510669111, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.231929063796997, + "step": 2790, + "unclipped_grad_norm": 0.6679553389549255 + }, + { + "bits_per_byte": 1.784543924113063, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2369515895843506, + "step": 2800, + "unclipped_grad_norm": 0.6488308310508728 + }, + { + "bits_per_byte": 1.723108969502454, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.1943681240081787, + "step": 2810, + "unclipped_grad_norm": 0.5890340209007263 + }, + { + "bits_per_byte": 1.7962786443467342, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2450854778289795, + "step": 2820, + "unclipped_grad_norm": 0.678447961807251 + }, + { + "bits_per_byte": 1.7361824026904413, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.203429937362671, + "step": 2830, + "unclipped_grad_norm": 0.5870603322982788 + }, + { + "bits_per_byte": 1.7695838412451108, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2265820503234863, + "step": 2840, + "unclipped_grad_norm": 0.6424716711044312 + }, + { + "bits_per_byte": 1.7215556221998611, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.193291425704956, + "step": 2850, + "unclipped_grad_norm": 0.621953010559082 + }, + { + "bits_per_byte": 1.750462466156882, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2133281230926514, + "step": 2860, + "unclipped_grad_norm": 0.6744791269302368 + }, + { + "bits_per_byte": 1.7458160108785215, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.210107445716858, + "step": 2870, + "unclipped_grad_norm": 0.6091592311859131 + }, + { + "bits_per_byte": 1.7563069525811528, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2173792123794556, + "step": 2880, + "unclipped_grad_norm": 0.6055625081062317 + }, + { + "bits_per_byte": 1.7055577960166026, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.1822025775909424, + "step": 2890, + "unclipped_grad_norm": 0.5587822794914246 + }, + { + "bits_per_byte": 1.7494334939567697, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.212614893913269, + "step": 2900, + "unclipped_grad_norm": 0.7476353049278259 + }, + { + "bits_per_byte": 1.7069099236176861, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.1831398010253906, + "step": 2910, + "unclipped_grad_norm": 0.6083102226257324 + }, + { + "bits_per_byte": 1.7573092674703812, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2180739641189575, + "step": 2920, + "unclipped_grad_norm": 0.594536542892456 + }, + { + "bits_per_byte": 1.7148656690639488, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.1886543035507202, + "step": 2930, + "unclipped_grad_norm": 0.6017132997512817 + }, + { + "bits_per_byte": 1.7206826382640814, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.1926863193511963, + "step": 2940, + "unclipped_grad_norm": 0.6191802620887756 + }, + { + "bits_per_byte": 1.7777722792179909, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2322578430175781, + "step": 2950, + "unclipped_grad_norm": 0.6343991756439209 + }, + { + "bits_per_byte": 1.7965679191654853, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.245285987854004, + "step": 2960, + "unclipped_grad_norm": 0.6511562466621399 + }, + { + "bits_per_byte": 1.750740734085978, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2135210037231445, + "step": 2970, + "unclipped_grad_norm": 0.6213991641998291 + }, + { + "bits_per_byte": 1.7817877301503355, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2350411415100098, + "step": 2980, + "unclipped_grad_norm": 0.6211501359939575 + }, + { + "bits_per_byte": 1.724867836072828, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.1955872774124146, + "step": 2990, + "unclipped_grad_norm": 0.642371416091919 + }, + { + "bits_per_byte": 1.795713509355999, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2446937561035156, + "step": 3000, + "unclipped_grad_norm": 0.6221276521682739 + }, + { + "bits_per_byte": 1.6848799779380654, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1678698062896729, + "step": 3010, + "unclipped_grad_norm": 0.6145718693733215 + }, + { + "bits_per_byte": 1.752584732068529, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.214799165725708, + "step": 3020, + "unclipped_grad_norm": 0.6433340907096863 + }, + { + "bits_per_byte": 1.6945199494842276, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.1745517253875732, + "step": 3030, + "unclipped_grad_norm": 0.6049435138702393 + }, + { + "bits_per_byte": 1.6808318503020565, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1650638580322266, + "step": 3040, + "unclipped_grad_norm": 0.5896543860435486 + }, + { + "bits_per_byte": 1.8340489581812716, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2712658643722534, + "step": 3050, + "unclipped_grad_norm": 0.6642063856124878 + }, + { + "bits_per_byte": 1.7718811854953362, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2281744480133057, + "step": 3060, + "unclipped_grad_norm": 0.6303152441978455 + }, + { + "bits_per_byte": 1.6793462641639036, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.164034128189087, + "step": 3070, + "unclipped_grad_norm": 0.6531770825386047 + }, + { + "bits_per_byte": 1.767122941493915, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2248762845993042, + "step": 3080, + "unclipped_grad_norm": 0.6228736042976379 + }, + { + "bits_per_byte": 1.7547854220689696, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2163245677947998, + "step": 3090, + "unclipped_grad_norm": 0.6808046102523804 + }, + { + "bits_per_byte": 1.7115178547822532, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1863337755203247, + "step": 3100, + "unclipped_grad_norm": 0.6307847499847412 + }, + { + "bits_per_byte": 1.8621731090942157, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2907600402832031, + "step": 3110, + "unclipped_grad_norm": 0.652417004108429 + }, + { + "bits_per_byte": 1.7445247651358435, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2092124223709106, + "step": 3120, + "unclipped_grad_norm": 0.6338313221931458 + }, + { + "bits_per_byte": 1.6854237870800937, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1682467460632324, + "step": 3130, + "unclipped_grad_norm": 0.6034438610076904 + }, + { + "bits_per_byte": 1.717318657613205, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.190354585647583, + "step": 3140, + "unclipped_grad_norm": 0.6048492193222046 + }, + { + "bits_per_byte": 1.7027365926118492, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1802470684051514, + "step": 3150, + "unclipped_grad_norm": 0.5864679217338562 + }, + { + "bits_per_byte": 1.705547477057551, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.1821954250335693, + "step": 3160, + "unclipped_grad_norm": 0.6051758527755737 + }, + { + "bits_per_byte": 1.7744621291368343, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2299634218215942, + "step": 3170, + "unclipped_grad_norm": 0.6531821489334106 + }, + { + "bits_per_byte": 1.7458908233316468, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2101593017578125, + "step": 3180, + "unclipped_grad_norm": 0.6303388476371765 + }, + { + "bits_per_byte": 1.7361681281304195, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2034200429916382, + "step": 3190, + "unclipped_grad_norm": 0.678484320640564 + }, + { + "bits_per_byte": 1.7289813171161645, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1984385251998901, + "step": 3200, + "unclipped_grad_norm": 0.6228032112121582 + }, + { + "bits_per_byte": 1.6986354943193747, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1774044036865234, + "step": 3210, + "unclipped_grad_norm": 0.6199705004692078 + }, + { + "bits_per_byte": 1.6704626723162361, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1578764915466309, + "step": 3220, + "unclipped_grad_norm": 0.6396132111549377 + }, + { + "bits_per_byte": 1.7446800654695727, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.209320068359375, + "step": 3230, + "unclipped_grad_norm": 0.6783336400985718 + }, + { + "bits_per_byte": 1.7503792265538642, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2132704257965088, + "step": 3240, + "unclipped_grad_norm": 0.6652441620826721 + }, + { + "bits_per_byte": 1.7183588086856232, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.1910755634307861, + "step": 3250, + "unclipped_grad_norm": 0.63749098777771 + }, + { + "bits_per_byte": 1.7194732562632142, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1918480396270752, + "step": 3260, + "unclipped_grad_norm": 0.6323399543762207 + }, + { + "bits_per_byte": 1.76014147776479, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2200371026992798, + "step": 3270, + "unclipped_grad_norm": 0.668753445148468 + }, + { + "bits_per_byte": 1.7466587258677493, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2106915712356567, + "step": 3280, + "unclipped_grad_norm": 0.5981830954551697 + }, + { + "bits_per_byte": 1.7077491989538964, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1837215423583984, + "step": 3290, + "unclipped_grad_norm": 0.6258406639099121 + }, + { + "bits_per_byte": 1.7122833495612435, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1868643760681152, + "step": 3300, + "unclipped_grad_norm": 0.6118162870407104 + }, + { + "bits_per_byte": 1.6971689982554674, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.176387906074524, + "step": 3310, + "unclipped_grad_norm": 0.614425003528595 + }, + { + "bits_per_byte": 1.7078183359795434, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1837694644927979, + "step": 3320, + "unclipped_grad_norm": 0.6319419741630554 + }, + { + "bits_per_byte": 1.7065344854908524, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1828795671463013, + "step": 3330, + "unclipped_grad_norm": 0.6474409699440002 + }, + { + "bits_per_byte": 1.6476438422171078, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1420596837997437, + "step": 3340, + "unclipped_grad_norm": 0.6128159165382385 + }, + { + "bits_per_byte": 1.693542744062025, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1738743782043457, + "step": 3350, + "unclipped_grad_norm": 0.6108464002609253 + }, + { + "bits_per_byte": 1.7204735073606323, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1925413608551025, + "step": 3360, + "unclipped_grad_norm": 0.6041978597640991 + }, + { + "bits_per_byte": 1.6622183399818258, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.152161955833435, + "step": 3370, + "unclipped_grad_norm": 0.5887104272842407 + }, + { + "bits_per_byte": 1.7058931621857851, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1824350357055664, + "step": 3380, + "unclipped_grad_norm": 0.6352498531341553 + }, + { + "bits_per_byte": 1.707757282138487, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1837271451950073, + "step": 3390, + "unclipped_grad_norm": 0.6421219706535339 + }, + { + "bits_per_byte": 1.6979798964542856, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1769499778747559, + "step": 3400, + "unclipped_grad_norm": 0.6395401954650879 + }, + { + "bits_per_byte": 1.6959995162296002, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.1755772829055786, + "step": 3410, + "unclipped_grad_norm": 0.6283812522888184 + }, + { + "bits_per_byte": 1.6799083034669233, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1644237041473389, + "step": 3420, + "unclipped_grad_norm": 0.6197249293327332 + }, + { + "bits_per_byte": 1.71363410130112, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.187800645828247, + "step": 3430, + "unclipped_grad_norm": 0.6102274060249329 + }, + { + "bits_per_byte": 1.7091876618457131, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1847186088562012, + "step": 3440, + "unclipped_grad_norm": 0.6624795198440552 + }, + { + "bits_per_byte": 1.7982115573597812, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2464252710342407, + "step": 3450, + "unclipped_grad_norm": 0.6986591815948486 + }, + { + "bits_per_byte": 1.7759950105039746, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2310259342193604, + "step": 3460, + "unclipped_grad_norm": 0.6196012496948242 + }, + { + "bits_per_byte": 1.7475674821949083, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.211321473121643, + "step": 3470, + "unclipped_grad_norm": 0.6346544623374939 + }, + { + "bits_per_byte": 1.7169992858305527, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1901332139968872, + "step": 3480, + "unclipped_grad_norm": 0.6380975246429443 + }, + { + "bits_per_byte": 1.6316707815355735, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1309880018234253, + "step": 3490, + "unclipped_grad_norm": 0.6554663181304932 + }, + { + "bits_per_byte": 1.641168867394774, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1375715732574463, + "step": 3500, + "unclipped_grad_norm": 0.6088140606880188 + }, + { + "bits_per_byte": 1.6880614849963764, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.1700750589370728, + "step": 3510, + "unclipped_grad_norm": 0.6358106732368469 + }, + { + "bits_per_byte": 1.6616972325497115, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1518007516860962, + "step": 3520, + "unclipped_grad_norm": 0.6153261661529541 + }, + { + "bits_per_byte": 1.6712895649015846, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.158449649810791, + "step": 3530, + "unclipped_grad_norm": 0.6411516666412354 + }, + { + "bits_per_byte": 1.6785265948498915, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.163465976715088, + "step": 3540, + "unclipped_grad_norm": 0.6034340858459473 + }, + { + "bits_per_byte": 1.7376134703282706, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.204421877861023, + "step": 3550, + "unclipped_grad_norm": 0.6545450687408447 + }, + { + "bits_per_byte": 1.6707187544833708, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.158053994178772, + "step": 3560, + "unclipped_grad_norm": 0.6340785026550293 + }, + { + "bits_per_byte": 1.7138805524398064, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1879714727401733, + "step": 3570, + "unclipped_grad_norm": 0.6533586382865906 + }, + { + "bits_per_byte": 1.7826314770354685, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.235625982284546, + "step": 3580, + "unclipped_grad_norm": 0.7123501896858215 + }, + { + "bits_per_byte": 1.7131767994324758, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1874836683273315, + "step": 3590, + "unclipped_grad_norm": 0.6637008786201477 + }, + { + "bits_per_byte": 1.6959864455481348, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1755682229995728, + "step": 3600, + "unclipped_grad_norm": 0.6217906475067139 + }, + { + "bits_per_byte": 1.6716725702650561, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1587151288986206, + "step": 3610, + "unclipped_grad_norm": 0.6237639784812927 + }, + { + "bits_per_byte": 1.6746752153664697, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1607964038848877, + "step": 3620, + "unclipped_grad_norm": 0.6187261343002319 + }, + { + "bits_per_byte": 1.6883108598401275, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1702479124069214, + "step": 3630, + "unclipped_grad_norm": 0.6150439381599426 + }, + { + "bits_per_byte": 1.6436062055228018, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.13926100730896, + "step": 3640, + "unclipped_grad_norm": 0.6298535466194153 + }, + { + "bits_per_byte": 1.7182858880416574, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1910250186920166, + "step": 3650, + "unclipped_grad_norm": 0.6331585049629211 + }, + { + "bits_per_byte": 1.6224437403341332, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1245923042297363, + "step": 3660, + "unclipped_grad_norm": 0.6383513808250427 + }, + { + "bits_per_byte": 1.644355533932611, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1397804021835327, + "step": 3670, + "unclipped_grad_norm": 0.6061356067657471 + }, + { + "bits_per_byte": 1.7035681287287707, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1808234453201294, + "step": 3680, + "unclipped_grad_norm": 0.6479222774505615 + }, + { + "bits_per_byte": 1.6706625161565387, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1580150127410889, + "step": 3690, + "unclipped_grad_norm": 0.6192170977592468 + }, + { + "bits_per_byte": 1.6109532354646872, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1166276931762695, + "step": 3700, + "unclipped_grad_norm": 0.6093165278434753 + }, + { + "bits_per_byte": 1.6773731072105547, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1626664400100708, + "step": 3710, + "unclipped_grad_norm": 0.6192633509635925 + }, + { + "bits_per_byte": 1.6614944650043442, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1516602039337158, + "step": 3720, + "unclipped_grad_norm": 0.6275249719619751 + }, + { + "bits_per_byte": 1.7267034069054867, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.19685959815979, + "step": 3730, + "unclipped_grad_norm": 0.6384285092353821 + }, + { + "bits_per_byte": 1.6506055554476162, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1441125869750977, + "step": 3740, + "unclipped_grad_norm": 0.6508066654205322 + }, + { + "bits_per_byte": 1.6398485565841001, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.136656403541565, + "step": 3750, + "unclipped_grad_norm": 0.6484293937683105 + }, + { + "bits_per_byte": 1.6392961483095287, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1362735033035278, + "step": 3760, + "unclipped_grad_norm": 0.6600143909454346 + }, + { + "bits_per_byte": 1.686776946577082, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.169184684753418, + "step": 3770, + "unclipped_grad_norm": 0.6462109684944153 + }, + { + "bits_per_byte": 1.6300704829692951, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.1298787593841553, + "step": 3780, + "unclipped_grad_norm": 0.6114524006843567 + }, + { + "bits_per_byte": 1.675651388892767, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.161473035812378, + "step": 3790, + "unclipped_grad_norm": 0.6200737953186035 + }, + { + "bits_per_byte": 1.617114685914498, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1208984851837158, + "step": 3800, + "unclipped_grad_norm": 0.6216682195663452 + }, + { + "bits_per_byte": 1.6234193979124778, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.125268578529358, + "step": 3810, + "unclipped_grad_norm": 0.6173998117446899 + }, + { + "bits_per_byte": 1.7138243141129743, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1879324913024902, + "step": 3820, + "unclipped_grad_norm": 0.6088318228721619 + }, + { + "bits_per_byte": 1.6341469877426964, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1327043771743774, + "step": 3830, + "unclipped_grad_norm": 0.6438922882080078 + }, + { + "bits_per_byte": 1.655027229401299, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1471774578094482, + "step": 3840, + "unclipped_grad_norm": 0.669468879699707 + }, + { + "bits_per_byte": 1.6818487837166085, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1657687425613403, + "step": 3850, + "unclipped_grad_norm": 0.6244412660598755 + }, + { + "bits_per_byte": 1.6164324307385254, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1204255819320679, + "step": 3860, + "unclipped_grad_norm": 0.6409146785736084 + }, + { + "bits_per_byte": 1.6824030838003392, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1661529541015625, + "step": 3870, + "unclipped_grad_norm": 0.6731106042861938 + }, + { + "bits_per_byte": 1.6375393455309653, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1350557804107666, + "step": 3880, + "unclipped_grad_norm": 0.6218310594558716 + }, + { + "bits_per_byte": 1.6796400105315774, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1642377376556396, + "step": 3890, + "unclipped_grad_norm": 0.635161280632019 + }, + { + "bits_per_byte": 1.666564169586478, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1551742553710938, + "step": 3900, + "unclipped_grad_norm": 0.7091010808944702 + }, + { + "bits_per_byte": 1.7026119051899735, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.180160641670227, + "step": 3910, + "unclipped_grad_norm": 0.6815565824508667 + }, + { + "bits_per_byte": 1.6485720325838145, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1427030563354492, + "step": 3920, + "unclipped_grad_norm": 0.628564178943634 + }, + { + "bits_per_byte": 1.6283515163739215, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1286872625350952, + "step": 3930, + "unclipped_grad_norm": 0.6302286982536316 + }, + { + "bits_per_byte": 1.6363571367889342, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1342363357543945, + "step": 3940, + "unclipped_grad_norm": 0.607494056224823 + }, + { + "bits_per_byte": 1.7146744243561893, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1885217428207397, + "step": 3950, + "unclipped_grad_norm": 0.70033860206604 + }, + { + "bits_per_byte": 1.7144123227962744, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1883400678634644, + "step": 3960, + "unclipped_grad_norm": 0.645106315612793 + }, + { + "bits_per_byte": 1.6837865122438798, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.167111873626709, + "step": 3970, + "unclipped_grad_norm": 0.6373703479766846 + }, + { + "bits_per_byte": 1.6677649524548022, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1560065746307373, + "step": 3980, + "unclipped_grad_norm": 0.6565650701522827 + }, + { + "bits_per_byte": 1.7059107044161732, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1824471950531006, + "step": 3990, + "unclipped_grad_norm": 0.6542640924453735 + }, + { + "bits_per_byte": 1.7121254694877515, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1867549419403076, + "step": 4000, + "unclipped_grad_norm": 0.6575196385383606 + }, + { + "bits_per_byte": 1.7220670986035271, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.19364595413208, + "step": 4010, + "unclipped_grad_norm": 0.6806800961494446 + }, + { + "bits_per_byte": 1.5958445595863893, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1061551570892334, + "step": 4020, + "unclipped_grad_norm": 0.6171497106552124 + }, + { + "bits_per_byte": 1.6345342926724393, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1329728364944458, + "step": 4030, + "unclipped_grad_norm": 0.6173087954521179 + }, + { + "bits_per_byte": 1.6363991005557448, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.134265422821045, + "step": 4040, + "unclipped_grad_norm": 0.634988009929657 + }, + { + "bits_per_byte": 1.6319382145576653, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1311733722686768, + "step": 4050, + "unclipped_grad_norm": 0.6698096990585327 + }, + { + "bits_per_byte": 1.6486795217406036, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1427775621414185, + "step": 4060, + "unclipped_grad_norm": 0.6841239929199219 + }, + { + "bits_per_byte": 1.6129987971140494, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1180455684661865, + "step": 4070, + "unclipped_grad_norm": 0.6276201605796814 + }, + { + "bits_per_byte": 1.6508999897458934, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1443166732788086, + "step": 4080, + "unclipped_grad_norm": 0.6372854113578796 + }, + { + "bits_per_byte": 1.6061886281051838, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1133251190185547, + "step": 4090, + "unclipped_grad_norm": 0.6243290305137634 + }, + { + "bits_per_byte": 1.578650766049031, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.0942373275756836, + "step": 4100, + "unclipped_grad_norm": 0.5791712403297424 + }, + { + "bits_per_byte": 1.6840439702722212, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1672903299331665, + "step": 4110, + "unclipped_grad_norm": 0.6848294734954834 + }, + { + "bits_per_byte": 1.6312047085517354, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1306649446487427, + "step": 4120, + "unclipped_grad_norm": 0.6449601054191589 + }, + { + "bits_per_byte": 1.594994965291127, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1055662631988525, + "step": 4130, + "unclipped_grad_norm": 0.6161995530128479 + }, + { + "bits_per_byte": 1.6232881751498696, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1251776218414307, + "step": 4140, + "unclipped_grad_norm": 0.6562530398368835 + }, + { + "bits_per_byte": 1.67032388231699, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1577802896499634, + "step": 4150, + "unclipped_grad_norm": 0.672451376914978 + }, + { + "bits_per_byte": 1.6420112384187, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138155460357666, + "step": 4160, + "unclipped_grad_norm": 0.6645690202713013 + }, + { + "bits_per_byte": 1.597898376402993, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1075787544250488, + "step": 4170, + "unclipped_grad_norm": 0.6766495108604431 + }, + { + "bits_per_byte": 1.5688259131531912, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.0874272584915161, + "step": 4180, + "unclipped_grad_norm": 0.6362206339836121 + }, + { + "bits_per_byte": 1.7152548658028512, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1889240741729736, + "step": 4190, + "unclipped_grad_norm": 0.6535307765007019 + }, + { + "bits_per_byte": 1.6086450563074575, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1150277853012085, + "step": 4200, + "unclipped_grad_norm": 0.6474415063858032 + }, + { + "bits_per_byte": 1.6237748860518113, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1255149841308594, + "step": 4210, + "unclipped_grad_norm": 0.6242862343788147 + }, + { + "bits_per_byte": 1.6672617312183775, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1556577682495117, + "step": 4220, + "unclipped_grad_norm": 0.6519601345062256 + }, + { + "bits_per_byte": 1.635159105643024, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1334059238433838, + "step": 4230, + "unclipped_grad_norm": 0.6224661469459534 + }, + { + "bits_per_byte": 1.6806368219759782, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.164928674697876, + "step": 4240, + "unclipped_grad_norm": 0.6462982296943665 + }, + { + "bits_per_byte": 1.7100040634893587, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1852844953536987, + "step": 4250, + "unclipped_grad_norm": 0.6697248220443726 + }, + { + "bits_per_byte": 1.6765044228410466, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1620643138885498, + "step": 4260, + "unclipped_grad_norm": 0.6539841294288635 + }, + { + "bits_per_byte": 1.6521546031839376, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1451863050460815, + "step": 4270, + "unclipped_grad_norm": 0.6337559819221497 + }, + { + "bits_per_byte": 1.688240002987972, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1701987981796265, + "step": 4280, + "unclipped_grad_norm": 0.6939600706100464 + }, + { + "bits_per_byte": 1.645464994013327, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1405494213104248, + "step": 4290, + "unclipped_grad_norm": 0.6979376673698425 + }, + { + "bits_per_byte": 1.5687516166480184, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0873757600784302, + "step": 4300, + "unclipped_grad_norm": 0.6490140557289124 + }, + { + "bits_per_byte": 1.720270739815265, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1924008131027222, + "step": 4310, + "unclipped_grad_norm": 0.7267009615898132 + }, + { + "bits_per_byte": 1.5912837516681586, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1029938459396362, + "step": 4320, + "unclipped_grad_norm": 0.6906614899635315 + }, + { + "bits_per_byte": 1.5899403951822693, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.102062702178955, + "step": 4330, + "unclipped_grad_norm": 0.6376214027404785 + }, + { + "bits_per_byte": 1.5963857889886546, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1065303087234497, + "step": 4340, + "unclipped_grad_norm": 0.6380553841590881 + }, + { + "bits_per_byte": 1.5301035033634773, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.060586929321289, + "step": 4350, + "unclipped_grad_norm": 0.6225847601890564 + }, + { + "bits_per_byte": 1.67761336697381, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1628329753875732, + "step": 4360, + "unclipped_grad_norm": 0.6995339393615723 + }, + { + "bits_per_byte": 1.6233086410853221, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1251918077468872, + "step": 4370, + "unclipped_grad_norm": 0.6690109372138977 + }, + { + "bits_per_byte": 1.5971554113512656, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1070637702941895, + "step": 4380, + "unclipped_grad_norm": 0.7015417814254761 + }, + { + "bits_per_byte": 1.6390723988807563, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1361184120178223, + "step": 4390, + "unclipped_grad_norm": 0.6816950440406799 + }, + { + "bits_per_byte": 1.5932933689434905, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.104386806488037, + "step": 4400, + "unclipped_grad_norm": 0.667495846748352 + }, + { + "bits_per_byte": 1.5991400911422224, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1084394454956055, + "step": 4410, + "unclipped_grad_norm": 0.6873564124107361 + }, + { + "bits_per_byte": 1.58255941575519, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.0969465970993042, + "step": 4420, + "unclipped_grad_norm": 0.6864304542541504 + }, + { + "bits_per_byte": 1.5926611607189187, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1039485931396484, + "step": 4430, + "unclipped_grad_norm": 0.6576697826385498 + }, + { + "bits_per_byte": 1.6095080652494869, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1156259775161743, + "step": 4440, + "unclipped_grad_norm": 0.6515920162200928 + }, + { + "bits_per_byte": 1.5838350110766395, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.0978307723999023, + "step": 4450, + "unclipped_grad_norm": 0.6458138227462769 + }, + { + "bits_per_byte": 1.5629618207067222, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0833625793457031, + "step": 4460, + "unclipped_grad_norm": 0.6665330529212952 + }, + { + "bits_per_byte": 1.6502230660320973, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1438474655151367, + "step": 4470, + "unclipped_grad_norm": 0.6849002838134766 + }, + { + "bits_per_byte": 1.5654218605446637, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.0850677490234375, + "step": 4480, + "unclipped_grad_norm": 0.630775511264801 + }, + { + "bits_per_byte": 1.533044750658533, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.0626256465911865, + "step": 4490, + "unclipped_grad_norm": 0.6286258697509766 + }, + { + "bits_per_byte": 1.6266820807819962, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1275300979614258, + "step": 4500, + "unclipped_grad_norm": 0.6595945954322815 + }, + { + "bits_per_byte": 1.5885456158837719, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1010959148406982, + "step": 4510, + "unclipped_grad_norm": 0.6231974363327026 + }, + { + "bits_per_byte": 1.5897587815029581, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1019368171691895, + "step": 4520, + "unclipped_grad_norm": 0.6408888697624207 + }, + { + "bits_per_byte": 1.6075569220754484, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1142735481262207, + "step": 4530, + "unclipped_grad_norm": 0.6721575260162354 + }, + { + "bits_per_byte": 1.5571181941957055, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.0793120861053467, + "step": 4540, + "unclipped_grad_norm": 0.6309816837310791 + }, + { + "bits_per_byte": 1.6337729254770696, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1324450969696045, + "step": 4550, + "unclipped_grad_norm": 0.6641245484352112 + }, + { + "bits_per_byte": 1.6370763682348426, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.134734869003296, + "step": 4560, + "unclipped_grad_norm": 0.6858609914779663 + }, + { + "bits_per_byte": 1.5844336826842929, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.0982457399368286, + "step": 4570, + "unclipped_grad_norm": 0.6500639319419861 + }, + { + "bits_per_byte": 1.5406408803818417, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0678908824920654, + "step": 4580, + "unclipped_grad_norm": 0.636735200881958 + }, + { + "bits_per_byte": 1.6489479866586005, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1429636478424072, + "step": 4590, + "unclipped_grad_norm": 0.6933116316795349 + }, + { + "bits_per_byte": 1.6320697812855753, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.131264567375183, + "step": 4600, + "unclipped_grad_norm": 0.6834596395492554 + }, + { + "bits_per_byte": 1.589729888417613, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.101916790008545, + "step": 4610, + "unclipped_grad_norm": 0.6632579565048218 + }, + { + "bits_per_byte": 1.5335797886853673, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.062996506690979, + "step": 4620, + "unclipped_grad_norm": 0.6395919322967529 + }, + { + "bits_per_byte": 1.6054043872172494, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1127815246582031, + "step": 4630, + "unclipped_grad_norm": 0.6490065455436707 + }, + { + "bits_per_byte": 1.5840957367753474, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.0980114936828613, + "step": 4640, + "unclipped_grad_norm": 0.6492030024528503 + }, + { + "bits_per_byte": 1.5576186637097162, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.0796589851379395, + "step": 4650, + "unclipped_grad_norm": 0.6488379836082458 + }, + { + "bits_per_byte": 1.593877078060519, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1047914028167725, + "step": 4660, + "unclipped_grad_norm": 0.6992486119270325 + }, + { + "bits_per_byte": 1.6105064245377456, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1163179874420166, + "step": 4670, + "unclipped_grad_norm": 0.6828540563583374 + }, + { + "bits_per_byte": 1.5810454524796447, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.0958971977233887, + "step": 4680, + "unclipped_grad_norm": 0.6880136728286743 + }, + { + "bits_per_byte": 1.5551754781715592, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.077965497970581, + "step": 4690, + "unclipped_grad_norm": 0.6890891194343567 + }, + { + "bits_per_byte": 1.6623815515174944, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1522750854492188, + "step": 4700, + "unclipped_grad_norm": 0.6787605285644531 + }, + { + "bits_per_byte": 1.574140349047503, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.0911109447479248, + "step": 4710, + "unclipped_grad_norm": 0.6684292554855347 + }, + { + "bits_per_byte": 1.5498508953008467, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.0742747783660889, + "step": 4720, + "unclipped_grad_norm": 0.6828400492668152 + }, + { + "bits_per_byte": 1.6704473658603094, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1578658819198608, + "step": 4730, + "unclipped_grad_norm": 0.6912305355072021 + }, + { + "bits_per_byte": 1.6014281483293014, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.110025405883789, + "step": 4740, + "unclipped_grad_norm": 0.6723529100418091 + }, + { + "bits_per_byte": 1.639597977861793, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1364827156066895, + "step": 4750, + "unclipped_grad_norm": 0.7447375059127808 + }, + { + "bits_per_byte": 1.523300557625948, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0558714866638184, + "step": 4760, + "unclipped_grad_norm": 0.6460341811180115 + }, + { + "bits_per_byte": 1.6314774730360038, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1308540105819702, + "step": 4770, + "unclipped_grad_norm": 0.6882368922233582 + }, + { + "bits_per_byte": 1.5726310293035308, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.0900647640228271, + "step": 4780, + "unclipped_grad_norm": 0.6284239888191223 + }, + { + "bits_per_byte": 1.60157536547844, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1101274490356445, + "step": 4790, + "unclipped_grad_norm": 0.6737696528434753 + }, + { + "bits_per_byte": 1.577140070443852, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0931901931762695, + "step": 4800, + "unclipped_grad_norm": 0.6438981890678406 + }, + { + "bits_per_byte": 1.5444138357764698, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.0705060958862305, + "step": 4810, + "unclipped_grad_norm": 0.690973699092865 + }, + { + "bits_per_byte": 1.6096637095485178, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1157338619232178, + "step": 4820, + "unclipped_grad_norm": 0.6775097250938416 + }, + { + "bits_per_byte": 1.6126245628657718, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.117786169052124, + "step": 4830, + "unclipped_grad_norm": 0.7250232696533203 + }, + { + "bits_per_byte": 1.6110075819823597, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1166653633117676, + "step": 4840, + "unclipped_grad_norm": 0.6946505308151245 + }, + { + "bits_per_byte": 1.560937584906067, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.0819594860076904, + "step": 4850, + "unclipped_grad_norm": 0.7250073552131653 + }, + { + "bits_per_byte": 1.5705242418304615, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.08860445022583, + "step": 4860, + "unclipped_grad_norm": 0.6968505382537842 + }, + { + "bits_per_byte": 1.5589560008028256, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.0805859565734863, + "step": 4870, + "unclipped_grad_norm": 0.659229040145874 + }, + { + "bits_per_byte": 1.7343542271117696, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.202162742614746, + "step": 4880, + "unclipped_grad_norm": 0.7630480527877808 + }, + { + "bits_per_byte": 1.5109135071975544, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0472854375839233, + "step": 4890, + "unclipped_grad_norm": 0.6373679041862488 + }, + { + "bits_per_byte": 1.6030399697331876, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.111142635345459, + "step": 4900, + "unclipped_grad_norm": 0.6882696151733398 + }, + { + "bits_per_byte": 1.6017267102111992, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1102323532104492, + "step": 4910, + "unclipped_grad_norm": 0.7222312688827515 + }, + { + "bits_per_byte": 1.533604038239139, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.0630133152008057, + "step": 4920, + "unclipped_grad_norm": 0.6319286227226257 + }, + { + "bits_per_byte": 1.5357895937663035, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.064528226852417, + "step": 4930, + "unclipped_grad_norm": 0.6290988922119141 + }, + { + "bits_per_byte": 1.5644580697692285, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.084399700164795, + "step": 4940, + "unclipped_grad_norm": 0.6635708808898926 + }, + { + "bits_per_byte": 1.596738869370876, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1067750453948975, + "step": 4950, + "unclipped_grad_norm": 0.7171201109886169 + }, + { + "bits_per_byte": 1.6163304450265636, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1203548908233643, + "step": 4960, + "unclipped_grad_norm": 0.739273726940155 + }, + { + "bits_per_byte": 1.566945798813959, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.086124062538147, + "step": 4970, + "unclipped_grad_norm": 0.6495019197463989 + }, + { + "bits_per_byte": 1.623716067985216, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.125474214553833, + "step": 4980, + "unclipped_grad_norm": 0.7210032939910889 + }, + { + "bits_per_byte": 1.574822088275523, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.091583490371704, + "step": 4990, + "unclipped_grad_norm": 0.6862797737121582 + }, + { + "bits_per_byte": 1.5840169687212522, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.097956895828247, + "step": 5000, + "unclipped_grad_norm": 0.6914816498756409 + }, + { + "bits_per_byte": 1.6270380848692823, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.127776861190796, + "step": 5010, + "unclipped_grad_norm": 0.7159916758537292 + }, + { + "bits_per_byte": 1.5917599716283977, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1033239364624023, + "step": 5020, + "unclipped_grad_norm": 0.7277734875679016 + }, + { + "bits_per_byte": 1.5241281381418998, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0564451217651367, + "step": 5030, + "unclipped_grad_norm": 0.6716130375862122 + }, + { + "bits_per_byte": 1.5837488477685573, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0977710485458374, + "step": 5040, + "unclipped_grad_norm": 0.6623762845993042 + }, + { + "bits_per_byte": 1.554003760371231, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0771533250808716, + "step": 5050, + "unclipped_grad_norm": 0.7006968855857849 + }, + { + "bits_per_byte": 1.621560093474, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1239798069000244, + "step": 5060, + "unclipped_grad_norm": 0.6823760271072388 + }, + { + "bits_per_byte": 1.5589410383122004, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.0805755853652954, + "step": 5070, + "unclipped_grad_norm": 0.685271680355072 + }, + { + "bits_per_byte": 1.6273230601217619, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1279743909835815, + "step": 5080, + "unclipped_grad_norm": 0.725353479385376 + }, + { + "bits_per_byte": 1.5689380458415538, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0875049829483032, + "step": 5090, + "unclipped_grad_norm": 0.6860542297363281 + }, + { + "bits_per_byte": 1.5571642855461367, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0793440341949463, + "step": 5100, + "unclipped_grad_norm": 0.7099642753601074 + }, + { + "bits_per_byte": 1.593192587110085, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1043169498443604, + "step": 5110, + "unclipped_grad_norm": 0.683228075504303 + }, + { + "bits_per_byte": 1.5025353723607726, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.041478157043457, + "step": 5120, + "unclipped_grad_norm": 0.6536243557929993 + }, + { + "bits_per_byte": 1.6259317204762818, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1270099878311157, + "step": 5130, + "unclipped_grad_norm": 0.7276868224143982 + }, + { + "bits_per_byte": 1.5937286570328242, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1046885251998901, + "step": 5140, + "unclipped_grad_norm": 0.7137117981910706 + }, + { + "bits_per_byte": 1.5363523209999266, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0649182796478271, + "step": 5150, + "unclipped_grad_norm": 0.6770874857902527 + }, + { + "bits_per_byte": 1.5548322008004372, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0777275562286377, + "step": 5160, + "unclipped_grad_norm": 0.6993823051452637 + }, + { + "bits_per_byte": 1.6209285731800318, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.123542070388794, + "step": 5170, + "unclipped_grad_norm": 0.7498713135719299 + }, + { + "bits_per_byte": 1.5887963665887297, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1012697219848633, + "step": 5180, + "unclipped_grad_norm": 0.7006720304489136 + }, + { + "bits_per_byte": 1.4814539110006604, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0268656015396118, + "step": 5190, + "unclipped_grad_norm": 0.6951001286506653 + }, + { + "bits_per_byte": 1.5501159205658264, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0744584798812866, + "step": 5200, + "unclipped_grad_norm": 0.696289598941803 + }, + { + "bits_per_byte": 1.519997458833477, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.053581953048706, + "step": 5210, + "unclipped_grad_norm": 0.6860307455062866 + }, + { + "bits_per_byte": 1.5330389032484038, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0626215934753418, + "step": 5220, + "unclipped_grad_norm": 0.6861689686775208 + }, + { + "bits_per_byte": 1.6136053799236425, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1184660196304321, + "step": 5230, + "unclipped_grad_norm": 0.6980286836624146 + }, + { + "bits_per_byte": 1.5805682006235005, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0955663919448853, + "step": 5240, + "unclipped_grad_norm": 0.7267829775810242 + }, + { + "bits_per_byte": 1.5736680846882336, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0907835960388184, + "step": 5250, + "unclipped_grad_norm": 0.6905968189239502 + }, + { + "bits_per_byte": 1.5160989561037188, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.050879716873169, + "step": 5260, + "unclipped_grad_norm": 0.7048349380493164 + }, + { + "bits_per_byte": 1.5706724908755052, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0887072086334229, + "step": 5270, + "unclipped_grad_norm": 0.7007996439933777 + }, + { + "bits_per_byte": 1.4955876172312168, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0366623401641846, + "step": 5280, + "unclipped_grad_norm": 0.7013132572174072 + }, + { + "bits_per_byte": 1.524417756925953, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.0566458702087402, + "step": 5290, + "unclipped_grad_norm": 0.6891999840736389 + }, + { + "bits_per_byte": 1.4950008124264729, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0362555980682373, + "step": 5300, + "unclipped_grad_norm": 0.6865445971488953 + }, + { + "bits_per_byte": 1.4986692023693766, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0387983322143555, + "step": 5310, + "unclipped_grad_norm": 0.6979682445526123 + }, + { + "bits_per_byte": 1.5140539104023092, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0494621992111206, + "step": 5320, + "unclipped_grad_norm": 0.7112694978713989 + }, + { + "bits_per_byte": 1.5563401446832021, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.078772783279419, + "step": 5330, + "unclipped_grad_norm": 0.7123240828514099 + }, + { + "bits_per_byte": 1.620746271570117, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1234157085418701, + "step": 5340, + "unclipped_grad_norm": 0.7256788611412048 + }, + { + "bits_per_byte": 1.4811187168141287, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0266332626342773, + "step": 5350, + "unclipped_grad_norm": 0.6512540578842163 + }, + { + "bits_per_byte": 1.5334795227999143, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.062927007675171, + "step": 5360, + "unclipped_grad_norm": 0.7001609206199646 + }, + { + "bits_per_byte": 1.6031871868823262, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1112446784973145, + "step": 5370, + "unclipped_grad_norm": 0.7270175814628601 + }, + { + "bits_per_byte": 1.5079560935333174, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.045235514640808, + "step": 5380, + "unclipped_grad_norm": 0.6764644980430603 + }, + { + "bits_per_byte": 1.5113635857948622, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0475974082946777, + "step": 5390, + "unclipped_grad_norm": 0.7015690803527832 + }, + { + "bits_per_byte": 1.567986121869028, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0868451595306396, + "step": 5400, + "unclipped_grad_norm": 0.7621549963951111 + }, + { + "bits_per_byte": 1.456367317667255, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.009476900100708, + "step": 5410, + "unclipped_grad_norm": 0.7006528377532959 + }, + { + "bits_per_byte": 1.6203708334432836, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1231554746627808, + "step": 5420, + "unclipped_grad_norm": 0.7240427136421204 + }, + { + "bits_per_byte": 1.543870886547696, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0701297521591187, + "step": 5430, + "unclipped_grad_norm": 0.7349725961685181 + }, + { + "bits_per_byte": 1.5391332804643783, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0668458938598633, + "step": 5440, + "unclipped_grad_norm": 0.6845248937606812 + }, + { + "bits_per_byte": 1.5639070373558641, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0840177536010742, + "step": 5450, + "unclipped_grad_norm": 0.7343961596488953 + }, + { + "bits_per_byte": 1.5791133993798518, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0945580005645752, + "step": 5460, + "unclipped_grad_norm": 0.6990759968757629 + }, + { + "bits_per_byte": 1.5450971228483479, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0709797143936157, + "step": 5470, + "unclipped_grad_norm": 0.73023521900177 + }, + { + "bits_per_byte": 1.4896457886265577, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0325437784194946, + "step": 5480, + "unclipped_grad_norm": 0.7134303450584412 + }, + { + "bits_per_byte": 1.5553481487530256, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.07808518409729, + "step": 5490, + "unclipped_grad_norm": 0.713401734828949 + }, + { + "bits_per_byte": 1.5796455137016214, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0949268341064453, + "step": 5500, + "unclipped_grad_norm": 0.7012893557548523 + }, + { + "bits_per_byte": 1.56096613402611, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0819792747497559, + "step": 5510, + "unclipped_grad_norm": 0.7354693412780762 + }, + { + "bits_per_byte": 1.558417523122974, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0802127122879028, + "step": 5520, + "unclipped_grad_norm": 0.8108682036399841 + }, + { + "bits_per_byte": 1.572987033390817, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0903115272521973, + "step": 5530, + "unclipped_grad_norm": 0.813023567199707 + }, + { + "bits_per_byte": 1.603627462468535, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1115498542785645, + "step": 5540, + "unclipped_grad_norm": 0.7556710243225098 + }, + { + "bits_per_byte": 1.5354511319094055, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.064293622970581, + "step": 5550, + "unclipped_grad_norm": 0.6996996998786926 + }, + { + "bits_per_byte": 1.531031521747533, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.061230182647705, + "step": 5560, + "unclipped_grad_norm": 0.7085531949996948 + }, + { + "bits_per_byte": 1.5727634559446952, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0901565551757812, + "step": 5570, + "unclipped_grad_norm": 0.7508514523506165 + }, + { + "bits_per_byte": 1.4695356852785186, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0186045169830322, + "step": 5580, + "unclipped_grad_norm": 0.6981281042098999 + }, + { + "bits_per_byte": 1.5444360215384312, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0705214738845825, + "step": 5590, + "unclipped_grad_norm": 0.7099512815475464 + }, + { + "bits_per_byte": 1.5326933901028204, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0623821020126343, + "step": 5600, + "unclipped_grad_norm": 0.7117766737937927 + }, + { + "bits_per_byte": 1.5419961036706404, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0688302516937256, + "step": 5610, + "unclipped_grad_norm": 0.7040424942970276 + }, + { + "bits_per_byte": 1.4775775940328635, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0241787433624268, + "step": 5620, + "unclipped_grad_norm": 0.7147260308265686 + }, + { + "bits_per_byte": 1.4921469323180554, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0342774391174316, + "step": 5630, + "unclipped_grad_norm": 0.7185394763946533 + }, + { + "bits_per_byte": 1.4712231790487844, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0197741985321045, + "step": 5640, + "unclipped_grad_norm": 0.7179238796234131 + }, + { + "bits_per_byte": 1.5902296700010203, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1022632122039795, + "step": 5650, + "unclipped_grad_norm": 0.7511428594589233 + }, + { + "bits_per_byte": 1.541337238135185, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0683735609054565, + "step": 5660, + "unclipped_grad_norm": 0.7174038887023926 + }, + { + "bits_per_byte": 1.5341244577406499, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0633740425109863, + "step": 5670, + "unclipped_grad_norm": 0.7094559669494629 + }, + { + "bits_per_byte": 1.5432753106277581, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0697169303894043, + "step": 5680, + "unclipped_grad_norm": 0.7527599334716797 + }, + { + "bits_per_byte": 1.5847210656938846, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.098444938659668, + "step": 5690, + "unclipped_grad_norm": 0.7476640939712524 + }, + { + "bits_per_byte": 1.5107903676195364, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.047200083732605, + "step": 5700, + "unclipped_grad_norm": 0.7240637540817261 + }, + { + "bits_per_byte": 1.5529789157547396, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0764429569244385, + "step": 5710, + "unclipped_grad_norm": 0.7409384250640869 + }, + { + "bits_per_byte": 1.5779322225337262, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0937392711639404, + "step": 5720, + "unclipped_grad_norm": 0.7523439526557922 + }, + { + "bits_per_byte": 1.4812139952027068, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0266993045806885, + "step": 5730, + "unclipped_grad_norm": 0.6902204155921936 + }, + { + "bits_per_byte": 1.5538140635073292, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.077021837234497, + "step": 5740, + "unclipped_grad_norm": 0.7577133178710938 + }, + { + "bits_per_byte": 1.5073047952345, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0447840690612793, + "step": 5750, + "unclipped_grad_norm": 0.7320472002029419 + }, + { + "bits_per_byte": 1.5094478710469015, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0462695360183716, + "step": 5760, + "unclipped_grad_norm": 0.7613655924797058 + }, + { + "bits_per_byte": 1.4626881960344158, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0138581991195679, + "step": 5770, + "unclipped_grad_norm": 0.7212797403335571 + }, + { + "bits_per_byte": 1.5541401426133652, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0772478580474854, + "step": 5780, + "unclipped_grad_norm": 0.7401144504547119 + }, + { + "bits_per_byte": 1.4976115090665703, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0380651950836182, + "step": 5790, + "unclipped_grad_norm": 0.6966454386711121 + }, + { + "bits_per_byte": 1.508392929466509, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.045538306236267, + "step": 5800, + "unclipped_grad_norm": 0.7519288063049316 + }, + { + "bits_per_byte": 1.536275616737642, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0648651123046875, + "step": 5810, + "unclipped_grad_norm": 0.7322137355804443 + }, + { + "bits_per_byte": 1.5318357126229676, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0617876052856445, + "step": 5820, + "unclipped_grad_norm": 0.7490501999855042 + }, + { + "bits_per_byte": 1.5069907549140245, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0445663928985596, + "step": 5830, + "unclipped_grad_norm": 0.7249871492385864 + }, + { + "bits_per_byte": 1.4694756633333674, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.018562912940979, + "step": 5840, + "unclipped_grad_norm": 0.7599647641181946 + }, + { + "bits_per_byte": 1.4762989030236986, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0232924222946167, + "step": 5850, + "unclipped_grad_norm": 0.7097748517990112 + }, + { + "bits_per_byte": 1.502543627528014, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0414838790893555, + "step": 5860, + "unclipped_grad_norm": 0.7764433026313782 + }, + { + "bits_per_byte": 1.5481047554466367, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0730644464492798, + "step": 5870, + "unclipped_grad_norm": 0.770169734954834 + }, + { + "bits_per_byte": 1.5544658777540994, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0774736404418945, + "step": 5880, + "unclipped_grad_norm": 0.731796145439148 + }, + { + "bits_per_byte": 1.528056737835559, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0591682195663452, + "step": 5890, + "unclipped_grad_norm": 0.7195150852203369 + }, + { + "bits_per_byte": 1.5162458292875556, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0509815216064453, + "step": 5900, + "unclipped_grad_norm": 0.8138624429702759 + }, + { + "bits_per_byte": 1.5072279189895643, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.04473078250885, + "step": 5910, + "unclipped_grad_norm": 0.7780150771141052 + }, + { + "bits_per_byte": 1.514708304388842, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0499157905578613, + "step": 5920, + "unclipped_grad_norm": 0.7354658246040344 + }, + { + "bits_per_byte": 1.5573187259666117, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.079451084136963, + "step": 5930, + "unclipped_grad_norm": 0.7978370785713196 + }, + { + "bits_per_byte": 1.5533937379086207, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.076730489730835, + "step": 5940, + "unclipped_grad_norm": 0.8458540439605713 + }, + { + "bits_per_byte": 1.5137727187681485, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.049267292022705, + "step": 5950, + "unclipped_grad_norm": 0.7532646059989929 + }, + { + "bits_per_byte": 1.5086940710881698, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0457470417022705, + "step": 5960, + "unclipped_grad_norm": 0.7333019375801086 + }, + { + "bits_per_byte": 1.5303000795334134, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0607231855392456, + "step": 5970, + "unclipped_grad_norm": 0.8049731254577637 + }, + { + "bits_per_byte": 1.4828514420215715, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0278342962265015, + "step": 5980, + "unclipped_grad_norm": 0.7353474497795105 + }, + { + "bits_per_byte": 1.4545697550004368, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0082309246063232, + "step": 5990, + "unclipped_grad_norm": 0.737419068813324 + }, + { + "bits_per_byte": 1.5419821730759207, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.068820595741272, + "step": 6000, + "unclipped_grad_norm": 0.7629180550575256 + }, + { + "bits_per_byte": 1.5277340983825405, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.058944582939148, + "step": 6010, + "unclipped_grad_norm": 0.7620755434036255 + }, + { + "bits_per_byte": 1.4955477172562166, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0366346836090088, + "step": 6020, + "unclipped_grad_norm": 0.7292536497116089 + }, + { + "bits_per_byte": 1.5533901262729524, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0767279863357544, + "step": 6030, + "unclipped_grad_norm": 0.767517626285553 + }, + { + "bits_per_byte": 1.495873108431649, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0368602275848389, + "step": 6040, + "unclipped_grad_norm": 0.7536795735359192 + }, + { + "bits_per_byte": 1.5050292927809341, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.043206810951233, + "step": 6050, + "unclipped_grad_norm": 0.7510635256767273 + }, + { + "bits_per_byte": 1.556982155918873, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.079217791557312, + "step": 6060, + "unclipped_grad_norm": 0.7961422801017761 + }, + { + "bits_per_byte": 1.5009287104364122, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0403645038604736, + "step": 6070, + "unclipped_grad_norm": 0.7310256958007812 + }, + { + "bits_per_byte": 1.5023740526342633, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0413663387298584, + "step": 6080, + "unclipped_grad_norm": 0.7653129696846008 + }, + { + "bits_per_byte": 1.5019286175685285, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0410575866699219, + "step": 6090, + "unclipped_grad_norm": 0.7687012553215027 + }, + { + "bits_per_byte": 1.5089062976793344, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0458941459655762, + "step": 6100, + "unclipped_grad_norm": 0.7417535185813904 + }, + { + "bits_per_byte": 1.475459455704837, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0227105617523193, + "step": 6110, + "unclipped_grad_norm": 0.7481279969215393 + }, + { + "bits_per_byte": 1.4820735644917191, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0272951126098633, + "step": 6120, + "unclipped_grad_norm": 0.7211357951164246 + }, + { + "bits_per_byte": 1.5036838725032344, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0422742366790771, + "step": 6130, + "unclipped_grad_norm": 0.7536711096763611 + }, + { + "bits_per_byte": 1.5397933498783898, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0673034191131592, + "step": 6140, + "unclipped_grad_norm": 0.7828236818313599 + }, + { + "bits_per_byte": 1.4767353949915882, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0235949754714966, + "step": 6150, + "unclipped_grad_norm": 0.7367910742759705 + }, + { + "bits_per_byte": 1.563501158299828, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0837364196777344, + "step": 6160, + "unclipped_grad_norm": 0.7933995723724365 + }, + { + "bits_per_byte": 1.433251129564785, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 0.9934539794921875, + "step": 6170, + "unclipped_grad_norm": 0.728959321975708 + }, + { + "bits_per_byte": 1.4945863342378936, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.03596830368042, + "step": 6180, + "unclipped_grad_norm": 0.7535345554351807 + }, + { + "bits_per_byte": 1.5074559679846085, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0448888540267944, + "step": 6190, + "unclipped_grad_norm": 0.7919867038726807 + }, + { + "bits_per_byte": 1.4628087558726706, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.013941764831543, + "step": 6200, + "unclipped_grad_norm": 0.7595178484916687 + }, + { + "bits_per_byte": 1.4945321597028718, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0359307527542114, + "step": 6210, + "unclipped_grad_norm": 0.7536692023277283 + }, + { + "bits_per_byte": 1.4821234394604692, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.027329683303833, + "step": 6220, + "unclipped_grad_norm": 0.7611709833145142 + }, + { + "bits_per_byte": 1.4502368240945993, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0052275657653809, + "step": 6230, + "unclipped_grad_norm": 0.7465225458145142 + }, + { + "bits_per_byte": 1.4792031740488187, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0253055095672607, + "step": 6240, + "unclipped_grad_norm": 0.8008144497871399 + }, + { + "bits_per_byte": 1.4709335602647315, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.019573450088501, + "step": 6250, + "unclipped_grad_norm": 0.7485248446464539 + }, + { + "bits_per_byte": 1.5311833824282448, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0613354444503784, + "step": 6260, + "unclipped_grad_norm": 0.7729804515838623 + }, + { + "bits_per_byte": 1.494225858601685, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0357184410095215, + "step": 6270, + "unclipped_grad_norm": 0.7719584703445435 + }, + { + "bits_per_byte": 1.606436799070379, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1134971380233765, + "step": 6280, + "unclipped_grad_norm": 0.814758837223053 + }, + { + "bits_per_byte": 1.469587624039079, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0186405181884766, + "step": 6290, + "unclipped_grad_norm": 0.7629152536392212 + }, + { + "bits_per_byte": 1.4932348945674137, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0350315570831299, + "step": 6300, + "unclipped_grad_norm": 0.7575296759605408 + }, + { + "bits_per_byte": 1.5423225267419782, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.069056510925293, + "step": 6310, + "unclipped_grad_norm": 0.7936079502105713 + }, + { + "bits_per_byte": 1.497591903044372, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0380516052246094, + "step": 6320, + "unclipped_grad_norm": 0.7546678781509399 + }, + { + "bits_per_byte": 1.438224007914483, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 0.9969009160995483, + "step": 6330, + "unclipped_grad_norm": 0.7539783716201782 + }, + { + "bits_per_byte": 1.5101449167308483, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.046752691268921, + "step": 6340, + "unclipped_grad_norm": 0.7760982513427734 + }, + { + "bits_per_byte": 1.5028686747381446, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0417091846466064, + "step": 6350, + "unclipped_grad_norm": 0.8006681203842163 + }, + { + "bits_per_byte": 1.4519638738745637, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0064246654510498, + "step": 6360, + "unclipped_grad_norm": 0.7430040240287781 + }, + { + "bits_per_byte": 1.5063248380898837, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.044104814529419, + "step": 6370, + "unclipped_grad_norm": 0.8066008687019348 + }, + { + "bits_per_byte": 1.461864399136783, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0132871866226196, + "step": 6380, + "unclipped_grad_norm": 0.739668071269989 + }, + { + "bits_per_byte": 1.518872692296834, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.052802324295044, + "step": 6390, + "unclipped_grad_norm": 0.8261381983757019 + }, + { + "bits_per_byte": 1.4734527621345699, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0213196277618408, + "step": 6400, + "unclipped_grad_norm": 0.7401824593544006 + }, + { + "bits_per_byte": 1.4482483606853236, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0038492679595947, + "step": 6410, + "unclipped_grad_norm": 0.7752094268798828 + }, + { + "bits_per_byte": 1.4492296936911466, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0045294761657715, + "step": 6420, + "unclipped_grad_norm": 0.7460422515869141 + }, + { + "bits_per_byte": 1.4798391658917094, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0257463455200195, + "step": 6430, + "unclipped_grad_norm": 0.7759443521499634 + }, + { + "bits_per_byte": 1.4925646781770012, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0345669984817505, + "step": 6440, + "unclipped_grad_norm": 0.7866507768630981 + }, + { + "bits_per_byte": 1.473837315341899, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0215861797332764, + "step": 6450, + "unclipped_grad_norm": 0.7885896563529968 + }, + { + "bits_per_byte": 1.452434934355277, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0067511796951294, + "step": 6460, + "unclipped_grad_norm": 0.7844802737236023 + }, + { + "bits_per_byte": 1.4767341911130323, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0235941410064697, + "step": 6470, + "unclipped_grad_norm": 0.7592610716819763 + }, + { + "bits_per_byte": 1.5086992305676956, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.045750617980957, + "step": 6480, + "unclipped_grad_norm": 0.8035558462142944 + }, + { + "bits_per_byte": 1.4908353926225757, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0333683490753174, + "step": 6490, + "unclipped_grad_norm": 0.8066015243530273 + }, + { + "bits_per_byte": 1.4884610001447638, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0317225456237793, + "step": 6500, + "unclipped_grad_norm": 0.7627172470092773 + }, + { + "bits_per_byte": 1.4363769142442164, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 0.995620608329773, + "step": 6510, + "unclipped_grad_norm": 0.7699123024940491 + }, + { + "bits_per_byte": 1.4159479550614777, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9814603328704834, + "step": 6520, + "unclipped_grad_norm": 0.7812637090682983 + }, + { + "bits_per_byte": 1.498887964301274, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.038949966430664, + "step": 6530, + "unclipped_grad_norm": 0.8116927742958069 + }, + { + "bits_per_byte": 1.4673561491441343, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0170937776565552, + "step": 6540, + "unclipped_grad_norm": 0.7371414303779602 + }, + { + "bits_per_byte": 1.5117546743429242, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0478684902191162, + "step": 6550, + "unclipped_grad_norm": 0.8187429308891296 + }, + { + "bits_per_byte": 1.4773364743563537, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0240116119384766, + "step": 6560, + "unclipped_grad_norm": 0.7700304388999939 + }, + { + "bits_per_byte": 1.4043874532531306, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 0.9734472036361694, + "step": 6570, + "unclipped_grad_norm": 0.7884607315063477 + }, + { + "bits_per_byte": 1.4975829599465271, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0380454063415527, + "step": 6580, + "unclipped_grad_norm": 0.778003454208374 + }, + { + "bits_per_byte": 1.4674713775202122, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0171736478805542, + "step": 6590, + "unclipped_grad_norm": 0.7916907668113708 + }, + { + "bits_per_byte": 1.3998400599816672, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9702951908111572, + "step": 6600, + "unclipped_grad_norm": 0.7509376406669617 + }, + { + "bits_per_byte": 1.5187203156681697, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.052696704864502, + "step": 6610, + "unclipped_grad_norm": 0.7783279418945312 + }, + { + "bits_per_byte": 1.511926828976438, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0479878187179565, + "step": 6620, + "unclipped_grad_norm": 0.8419784903526306 + }, + { + "bits_per_byte": 1.5028970518755371, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0417288541793823, + "step": 6630, + "unclipped_grad_norm": 0.8222247958183289 + }, + { + "bits_per_byte": 1.4773347545298452, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.024010419845581, + "step": 6640, + "unclipped_grad_norm": 0.8660594820976257 + }, + { + "bits_per_byte": 1.4614148364874275, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0129755735397339, + "step": 6650, + "unclipped_grad_norm": 0.7894479036331177 + }, + { + "bits_per_byte": 1.5265555012761776, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0581276416778564, + "step": 6660, + "unclipped_grad_norm": 0.810829758644104 + }, + { + "bits_per_byte": 1.4984096805492246, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0386184453964233, + "step": 6670, + "unclipped_grad_norm": 0.8341873288154602 + }, + { + "bits_per_byte": 1.5059076081788905, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0438156127929688, + "step": 6680, + "unclipped_grad_norm": 0.822468638420105 + }, + { + "bits_per_byte": 1.4780378196065724, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0244977474212646, + "step": 6690, + "unclipped_grad_norm": 0.810896635055542 + }, + { + "bits_per_byte": 1.524204498438883, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0564980506896973, + "step": 6700, + "unclipped_grad_norm": 0.8146765828132629 + }, + { + "bits_per_byte": 1.451385668202363, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.00602388381958, + "step": 6710, + "unclipped_grad_norm": 0.8321484327316284 + }, + { + "bits_per_byte": 1.4556473982907434, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0089778900146484, + "step": 6720, + "unclipped_grad_norm": 0.781524658203125 + }, + { + "bits_per_byte": 1.448410712307738, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0039618015289307, + "step": 6730, + "unclipped_grad_norm": 0.8034297227859497 + }, + { + "bits_per_byte": 1.4779710903380376, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.024451494216919, + "step": 6740, + "unclipped_grad_norm": 0.8218453526496887 + }, + { + "bits_per_byte": 1.4435792036970492, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0006128549575806, + "step": 6750, + "unclipped_grad_norm": 0.7989961504936218 + }, + { + "bits_per_byte": 1.4296412137231747, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 0.9909517765045166, + "step": 6760, + "unclipped_grad_norm": 0.7599804997444153 + }, + { + "bits_per_byte": 1.4469161830717403, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0029258728027344, + "step": 6770, + "unclipped_grad_norm": 0.7921444773674011 + }, + { + "bits_per_byte": 1.438630832875099, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 0.9971829056739807, + "step": 6780, + "unclipped_grad_norm": 0.7756091356277466 + }, + { + "bits_per_byte": 1.5384264317693321, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0663559436798096, + "step": 6790, + "unclipped_grad_norm": 0.8340134024620056 + }, + { + "bits_per_byte": 1.5144365718004789, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.049727439880371, + "step": 6800, + "unclipped_grad_norm": 0.8667078614234924 + }, + { + "bits_per_byte": 1.4971179188585941, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0377230644226074, + "step": 6810, + "unclipped_grad_norm": 0.8064154386520386 + }, + { + "bits_per_byte": 1.4405763866129846, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 0.9985314607620239, + "step": 6820, + "unclipped_grad_norm": 0.7713363170623779 + }, + { + "bits_per_byte": 1.4385394240961653, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 0.9971195459365845, + "step": 6830, + "unclipped_grad_norm": 0.8126972317695618 + }, + { + "bits_per_byte": 1.4959006256557872, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.036879301071167, + "step": 6840, + "unclipped_grad_norm": 0.7896748185157776 + }, + { + "bits_per_byte": 1.465242826330332, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0156289339065552, + "step": 6850, + "unclipped_grad_norm": 0.8369152545928955 + }, + { + "bits_per_byte": 1.4691382333723746, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0183290243148804, + "step": 6860, + "unclipped_grad_norm": 0.8024821877479553 + }, + { + "bits_per_byte": 1.4717907217966317, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.020167589187622, + "step": 6870, + "unclipped_grad_norm": 0.808424174785614 + }, + { + "bits_per_byte": 1.5114406340224489, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0476508140563965, + "step": 6880, + "unclipped_grad_norm": 0.8209573030471802 + }, + { + "bits_per_byte": 1.4045318326885299, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9735472798347473, + "step": 6890, + "unclipped_grad_norm": 0.7931460738182068 + }, + { + "bits_per_byte": 1.436284731543354, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 0.9955567121505737, + "step": 6900, + "unclipped_grad_norm": 0.7861348390579224 + }, + { + "bits_per_byte": 1.6150507221214936, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.119467854499817, + "step": 6910, + "unclipped_grad_norm": 0.8759272694587708 + }, + { + "bits_per_byte": 1.4220056999728183, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 0.9856592416763306, + "step": 6920, + "unclipped_grad_norm": 0.7773904204368591 + }, + { + "bits_per_byte": 1.4332720254568647, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9934684634208679, + "step": 6930, + "unclipped_grad_norm": 0.8008426427841187 + }, + { + "bits_per_byte": 1.46887819560427, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0181487798690796, + "step": 6940, + "unclipped_grad_norm": 0.8342399597167969 + }, + { + "bits_per_byte": 1.4565233059315876, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0095850229263306, + "step": 6950, + "unclipped_grad_norm": 0.8141571283340454 + }, + { + "bits_per_byte": 1.5009811651449254, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0404008626937866, + "step": 6960, + "unclipped_grad_norm": 0.8327651619911194 + }, + { + "bits_per_byte": 1.4767801104808127, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0236259698867798, + "step": 6970, + "unclipped_grad_norm": 0.801481306552887 + }, + { + "bits_per_byte": 1.517935386849632, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0521526336669922, + "step": 6980, + "unclipped_grad_norm": 0.8559384942054749 + }, + { + "bits_per_byte": 1.462364696668143, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0136339664459229, + "step": 6990, + "unclipped_grad_norm": 0.8250771164894104 + }, + { + "bits_per_byte": 1.4113358963132898, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 0.9782634973526001, + "step": 7000, + "unclipped_grad_norm": 0.8160101771354675 + }, + { + "bits_per_byte": 1.4572850170922589, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.010113000869751, + "step": 7010, + "unclipped_grad_norm": 0.8388007283210754 + }, + { + "bits_per_byte": 1.4311967968002286, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 0.9920300245285034, + "step": 7020, + "unclipped_grad_norm": 0.8063672780990601 + }, + { + "bits_per_byte": 1.4271649215247264, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 0.9892353415489197, + "step": 7030, + "unclipped_grad_norm": 0.7880839109420776 + }, + { + "bits_per_byte": 1.5230513547648479, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0556987524032593, + "step": 7040, + "unclipped_grad_norm": 0.8723810911178589 + }, + { + "bits_per_byte": 1.48768088684045, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.031181812286377, + "step": 7050, + "unclipped_grad_norm": 0.8570515513420105 + }, + { + "bits_per_byte": 1.4927676177050193, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0347076654434204, + "step": 7060, + "unclipped_grad_norm": 0.8982753157615662 + }, + { + "bits_per_byte": 1.4459658069430723, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0022671222686768, + "step": 7070, + "unclipped_grad_norm": 0.8039788603782654 + }, + { + "bits_per_byte": 1.364939964607379, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9461042881011963, + "step": 7080, + "unclipped_grad_norm": 0.7935104966163635 + }, + { + "bits_per_byte": 1.4353330655448047, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.9948970675468445, + "step": 7090, + "unclipped_grad_norm": 0.810722291469574 + }, + { + "bits_per_byte": 1.4301969756594377, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9913370013237, + "step": 7100, + "unclipped_grad_norm": 0.7980725169181824 + }, + { + "bits_per_byte": 1.4812796925753362, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0267448425292969, + "step": 7110, + "unclipped_grad_norm": 0.8589518666267395 + }, + { + "bits_per_byte": 1.491397775890897, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0337581634521484, + "step": 7120, + "unclipped_grad_norm": 0.881410539150238 + }, + { + "bits_per_byte": 1.4146181852050066, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 0.9805386066436768, + "step": 7130, + "unclipped_grad_norm": 0.8223039507865906 + }, + { + "bits_per_byte": 1.522814190689308, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0555343627929688, + "step": 7140, + "unclipped_grad_norm": 0.8451608419418335 + }, + { + "bits_per_byte": 1.5467134158011566, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.072100043296814, + "step": 7150, + "unclipped_grad_norm": 0.8498587012290955 + }, + { + "bits_per_byte": 1.4723551688567635, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0205588340759277, + "step": 7160, + "unclipped_grad_norm": 0.8513677716255188 + }, + { + "bits_per_byte": 1.4676500674944588, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0172975063323975, + "step": 7170, + "unclipped_grad_norm": 0.8321399092674255 + }, + { + "bits_per_byte": 1.3859237398218012, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 0.9606491327285767, + "step": 7180, + "unclipped_grad_norm": 0.7659841179847717 + }, + { + "bits_per_byte": 1.3902817801946648, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9636698961257935, + "step": 7190, + "unclipped_grad_norm": 0.7895154356956482 + }, + { + "bits_per_byte": 1.4776350362382518, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0242185592651367, + "step": 7200, + "unclipped_grad_norm": 0.8478621244430542 + }, + { + "bits_per_byte": 1.4778720283311406, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0243828296661377, + "step": 7210, + "unclipped_grad_norm": 0.8023228049278259 + }, + { + "bits_per_byte": 1.4423936412933265, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9997910857200623, + "step": 7220, + "unclipped_grad_norm": 0.8181896805763245 + }, + { + "bits_per_byte": 1.4464050506333759, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0025715827941895, + "step": 7230, + "unclipped_grad_norm": 0.8425759077072144 + }, + { + "bits_per_byte": 1.4082518174266925, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 0.9761257767677307, + "step": 7240, + "unclipped_grad_norm": 0.7972860336303711 + }, + { + "bits_per_byte": 1.4478307868090285, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0035598278045654, + "step": 7250, + "unclipped_grad_norm": 0.8181965947151184 + }, + { + "bits_per_byte": 1.4351106059859136, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9947428703308105, + "step": 7260, + "unclipped_grad_norm": 0.8426879048347473 + }, + { + "bits_per_byte": 1.5111732010003571, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.047465443611145, + "step": 7270, + "unclipped_grad_norm": 0.862413763999939 + }, + { + "bits_per_byte": 1.457399041589781, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0101920366287231, + "step": 7280, + "unclipped_grad_norm": 0.8488914370536804 + }, + { + "bits_per_byte": 1.4536087159474156, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0075647830963135, + "step": 7290, + "unclipped_grad_norm": 0.8383923172950745 + }, + { + "bits_per_byte": 1.4305855704590622, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 0.9916063547134399, + "step": 7300, + "unclipped_grad_norm": 0.8480672240257263 + }, + { + "bits_per_byte": 1.4800558640317965, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0258965492248535, + "step": 7310, + "unclipped_grad_norm": 0.8378974795341492 + }, + { + "bits_per_byte": 1.4690943777964045, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.018298625946045, + "step": 7320, + "unclipped_grad_norm": 0.8481025695800781 + }, + { + "bits_per_byte": 1.4814797083982898, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0268834829330444, + "step": 7330, + "unclipped_grad_norm": 0.9106674194335938 + }, + { + "bits_per_byte": 1.449738418372399, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0048820972442627, + "step": 7340, + "unclipped_grad_norm": 0.8405259847640991 + }, + { + "bits_per_byte": 1.486937577823421, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0306665897369385, + "step": 7350, + "unclipped_grad_norm": 0.8228537440299988 + }, + { + "bits_per_byte": 1.4739011209053692, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0216304063796997, + "step": 7360, + "unclipped_grad_norm": 0.8316923975944519 + }, + { + "bits_per_byte": 1.4767537971352307, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0236077308654785, + "step": 7370, + "unclipped_grad_norm": 0.8346049785614014 + }, + { + "bits_per_byte": 1.5318123229824503, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0617713928222656, + "step": 7380, + "unclipped_grad_norm": 0.9013696908950806 + }, + { + "bits_per_byte": 1.5039485538029123, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0424576997756958, + "step": 7390, + "unclipped_grad_norm": 0.8856703639030457 + }, + { + "bits_per_byte": 1.4314098833046476, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9921777248382568, + "step": 7400, + "unclipped_grad_norm": 0.8249487280845642 + }, + { + "bits_per_byte": 1.4394619390353935, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9977589845657349, + "step": 7410, + "unclipped_grad_norm": 0.8362482786178589 + }, + { + "bits_per_byte": 1.518494330464936, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0525400638580322, + "step": 7420, + "unclipped_grad_norm": 0.8421095609664917 + }, + { + "bits_per_byte": 1.4326227909498577, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9930184483528137, + "step": 7430, + "unclipped_grad_norm": 0.8718878626823425 + }, + { + "bits_per_byte": 1.4438445729273304, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0007967948913574, + "step": 7440, + "unclipped_grad_norm": 0.8476481437683105 + }, + { + "bits_per_byte": 1.3938656406646694, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 0.9661540389060974, + "step": 7450, + "unclipped_grad_norm": 0.8507251739501953 + }, + { + "bits_per_byte": 1.420028243453198, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9842885732650757, + "step": 7460, + "unclipped_grad_norm": 0.8349717259407043 + }, + { + "bits_per_byte": 1.4775118966602339, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0241332054138184, + "step": 7470, + "unclipped_grad_norm": 0.8582044243812561 + }, + { + "bits_per_byte": 1.3999588999934132, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 0.9703775644302368, + "step": 7480, + "unclipped_grad_norm": 0.84687340259552 + }, + { + "bits_per_byte": 1.4226986180731447, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 0.9861395359039307, + "step": 7490, + "unclipped_grad_norm": 0.831066370010376 + }, + { + "bits_per_byte": 1.4892197876003705, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0322484970092773, + "step": 7500, + "unclipped_grad_norm": 0.8232710361480713 + }, + { + "bits_per_byte": 1.461985990870943, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.013371467590332, + "step": 7510, + "unclipped_grad_norm": 0.8441055417060852 + }, + { + "bits_per_byte": 1.4446456681150495, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.001352071762085, + "step": 7520, + "unclipped_grad_norm": 0.8653393387794495 + }, + { + "bits_per_byte": 1.4775328785436392, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0241477489471436, + "step": 7530, + "unclipped_grad_norm": 0.8701198697090149 + }, + { + "bits_per_byte": 1.5027517265355579, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.041628122329712, + "step": 7540, + "unclipped_grad_norm": 0.8485988974571228 + }, + { + "bits_per_byte": 1.4225563884208812, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 0.9860409498214722, + "step": 7550, + "unclipped_grad_norm": 0.8343487977981567 + }, + { + "bits_per_byte": 1.4745732291049412, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0220962762832642, + "step": 7560, + "unclipped_grad_norm": 0.8472090363502502 + }, + { + "bits_per_byte": 1.480253472097638, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0260335206985474, + "step": 7570, + "unclipped_grad_norm": 0.8703269958496094 + }, + { + "bits_per_byte": 1.4382653697420156, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9969295859336853, + "step": 7580, + "unclipped_grad_norm": 0.847757875919342 + }, + { + "bits_per_byte": 1.4346089325933469, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9943951368331909, + "step": 7590, + "unclipped_grad_norm": 0.845406711101532 + }, + { + "bits_per_byte": 1.414465206637064, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9804325699806213, + "step": 7600, + "unclipped_grad_norm": 0.8379301428794861 + }, + { + "bits_per_byte": 1.3641118681434745, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9455302953720093, + "step": 7610, + "unclipped_grad_norm": 0.7969970107078552 + }, + { + "bits_per_byte": 1.4648639485504813, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0153663158416748, + "step": 7620, + "unclipped_grad_norm": 0.8826617002487183 + }, + { + "bits_per_byte": 1.4355323934371547, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9950352311134338, + "step": 7630, + "unclipped_grad_norm": 0.8636281490325928 + }, + { + "bits_per_byte": 1.4410591419139565, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.998866081237793, + "step": 7640, + "unclipped_grad_norm": 0.8592743277549744 + }, + { + "bits_per_byte": 1.5176450801349755, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0519514083862305, + "step": 7650, + "unclipped_grad_norm": 0.9256166219711304 + }, + { + "bits_per_byte": 1.4474489853241133, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0032951831817627, + "step": 7660, + "unclipped_grad_norm": 0.8606430292129517 + }, + { + "bits_per_byte": 1.4195236463555665, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 0.9839388132095337, + "step": 7670, + "unclipped_grad_norm": 0.8635953664779663 + }, + { + "bits_per_byte": 1.3989710316468573, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9696928262710571, + "step": 7680, + "unclipped_grad_norm": 0.8459085822105408 + }, + { + "bits_per_byte": 1.446349844202449, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0025333166122437, + "step": 7690, + "unclipped_grad_norm": 0.8500027656555176 + }, + { + "bits_per_byte": 1.456846289349908, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.009808897972107, + "step": 7700, + "unclipped_grad_norm": 0.896350085735321 + }, + { + "bits_per_byte": 1.3794107568336267, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9561346769332886, + "step": 7710, + "unclipped_grad_norm": 0.8459534049034119 + }, + { + "bits_per_byte": 1.3797531742914946, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9563720226287842, + "step": 7720, + "unclipped_grad_norm": 0.8360687494277954 + }, + { + "bits_per_byte": 1.4376540574095236, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.996505856513977, + "step": 7730, + "unclipped_grad_norm": 0.82816481590271 + }, + { + "bits_per_byte": 1.4181697129366488, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9830003380775452, + "step": 7740, + "unclipped_grad_norm": 0.83390212059021 + }, + { + "bits_per_byte": 1.4407892151434272, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.998678982257843, + "step": 7750, + "unclipped_grad_norm": 0.8495392203330994 + }, + { + "bits_per_byte": 1.4190407190719436, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9836040735244751, + "step": 7760, + "unclipped_grad_norm": 0.850292444229126 + }, + { + "bits_per_byte": 1.4212888762840223, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9851623773574829, + "step": 7770, + "unclipped_grad_norm": 0.888531506061554 + }, + { + "bits_per_byte": 1.5506865590013892, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0748540163040161, + "step": 7780, + "unclipped_grad_norm": 0.9023171067237854 + }, + { + "bits_per_byte": 1.4346873566821403, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9944494962692261, + "step": 7790, + "unclipped_grad_norm": 0.8470334410667419 + }, + { + "bits_per_byte": 1.4727918328073042, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0208615064620972, + "step": 7800, + "unclipped_grad_norm": 0.9052314758300781 + }, + { + "bits_per_byte": 1.3950921349392975, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9670041799545288, + "step": 7810, + "unclipped_grad_norm": 0.841830849647522 + }, + { + "bits_per_byte": 1.4118079886899082, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.978590726852417, + "step": 7820, + "unclipped_grad_norm": 0.8733639121055603 + }, + { + "bits_per_byte": 1.4649716096899212, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0154409408569336, + "step": 7830, + "unclipped_grad_norm": 0.851719081401825 + }, + { + "bits_per_byte": 1.4172854641372379, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9823874235153198, + "step": 7840, + "unclipped_grad_norm": 0.8457031846046448 + }, + { + "bits_per_byte": 1.3998510668713222, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9703028202056885, + "step": 7850, + "unclipped_grad_norm": 0.8359724283218384 + }, + { + "bits_per_byte": 1.413704183406996, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 0.9799050688743591, + "step": 7860, + "unclipped_grad_norm": 0.8466342091560364 + }, + { + "bits_per_byte": 1.4588555626599382, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0112016201019287, + "step": 7870, + "unclipped_grad_norm": 0.8837466239929199 + }, + { + "bits_per_byte": 1.4292317230314702, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9906679391860962, + "step": 7880, + "unclipped_grad_norm": 0.8551597595214844 + }, + { + "bits_per_byte": 1.5078522160121963, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0451635122299194, + "step": 7890, + "unclipped_grad_norm": 0.8927572965621948 + }, + { + "bits_per_byte": 1.4757206973514978, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.022891640663147, + "step": 7900, + "unclipped_grad_norm": 0.8807728290557861 + }, + { + "bits_per_byte": 1.4780766876856672, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0245246887207031, + "step": 7910, + "unclipped_grad_norm": 0.9293215870857239 + }, + { + "bits_per_byte": 1.3870213330996077, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 0.9614099264144897, + "step": 7920, + "unclipped_grad_norm": 0.8720042109489441 + }, + { + "bits_per_byte": 1.3890644869918578, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.962826132774353, + "step": 7930, + "unclipped_grad_norm": 0.8701853156089783 + }, + { + "bits_per_byte": 1.4495222361802644, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0047322511672974, + "step": 7940, + "unclipped_grad_norm": 0.8989008069038391 + }, + { + "bits_per_byte": 1.4133007980993975, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9796254634857178, + "step": 7950, + "unclipped_grad_norm": 0.839148759841919 + }, + { + "bits_per_byte": 1.4208300265715204, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9848443269729614, + "step": 7960, + "unclipped_grad_norm": 0.8453875184059143 + }, + { + "bits_per_byte": 1.41087549875693, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9779443740844727, + "step": 7970, + "unclipped_grad_norm": 0.8587523102760315 + }, + { + "bits_per_byte": 1.446222061092858, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0024447441101074, + "step": 7980, + "unclipped_grad_norm": 0.8559814691543579 + }, + { + "bits_per_byte": 1.4566966644436572, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0097051858901978, + "step": 7990, + "unclipped_grad_norm": 0.860873818397522 + }, + { + "bits_per_byte": 1.3805011268400968, + "learning_rate": 3e-05, + "loss_nats": 0.9568904638290405, + "step": 8000, + "unclipped_grad_norm": 0.8578172326087952 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073001.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073001.json new file mode 100644 index 0000000..ccdb217 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073001.json @@ -0,0 +1,13072 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "c7554d9beea6fe9e63617aafd88403f7fc93b4b305a0c6d603a98a8572e5b5f1", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "population_cv": 0.5072011380661324 + }, + "activation_output_rms_by_block": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013632480811793357, + "first_to_last_ratio": 0.32005493218522185, + "imbalance_abs_log_ratio": 1.1392626348419668, + "last_quartile_mean": 0.04259419068694115, + "mean": 0.025403792154975235, + "normalized": [ + 0.23092935972382672, + 0.3992639012357397, + 0.5833875213879504, + 0.7700561688360058, + 0.2642123711098415, + 0.4687595594389148, + 0.6729013896370168, + 0.9035435305603466, + 0.2950160146386087, + 0.5418894364067922, + 0.8025599005701267, + 1.0558320747576517, + 0.36189473544610173, + 0.6606003816028687, + 0.9564482474371727, + 1.2361787191078082, + 0.389077999650742, + 0.7376930678145923, + 1.1018562250488806, + 1.4948807199009364, + 0.4969232792693754, + 0.9255949835565679, + 1.3737978229029388, + 1.8632120600111322, + 0.6868422339014575, + 1.1618129342705898, + 1.7580201402629136, + 2.3690874226600394, + 0.7468506348084945, + 1.4805931380763324, + 2.2249315569778956, + 2.985352468990339 + ], + "population_cv": 0.6518934572295025 + }, + "bits_per_byte": 5.150682269327325, + "branch_output_rms_by_sublayer": [ + 0.002919268561527133, + 0.005079356953501701, + 0.0026904908008873463, + 0.0052208867855370045, + 0.0028405722696334124, + 0.005689430516213179, + 0.0029144894797354937, + 0.005868426524102688, + 0.0030634324066340923, + 0.00590060418471694, + 0.0032732728868722916, + 0.0059102908708155155, + 0.003848863998427987, + 0.0058776624500751495, + 0.003392934100702405, + 0.006663382984697819, + 0.0037359180860221386, + 0.006277414970099926, + 0.004302032291889191, + 0.006413500756025314, + 0.004344160668551922, + 0.006856496445834637, + 0.004835603758692741, + 0.006499431096017361, + 0.005531937349587679, + 0.006718362681567669, + 0.005483300890773535, + 0.0070298356004059315, + 0.005397035740315914, + 0.006969383917748928, + 0.005739198997616768, + 0.0068367342464625835, + 0.005659298039972782, + 0.007033169735223055, + 0.007230010814964771, + 0.007157569285482168, + 0.006938732694834471, + 0.0075136395171284676, + 0.007343437522649765, + 0.007708695251494646, + 0.007882392033934593, + 0.007887008599936962, + 0.00830139685422182, + 0.008205465972423553, + 0.008992243558168411, + 0.00814912747591734, + 0.009393857792019844, + 0.008287571370601654, + 0.012831038795411587, + 0.008026733994483948, + 0.00905293133109808, + 0.008120592683553696, + 0.012243949808180332, + 0.008479002863168716, + 0.012258118949830532, + 0.00907627958804369, + 0.013431218452751637, + 0.00841403380036354, + 0.01483316533267498, + 0.008773921057581902, + 0.015388047322630882, + 0.008821850642561913, + 0.014634119346737862, + 0.009164483286440372 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00039664652254938686, + 0.00033413090452743513, + 0.0003161152806455274, + 0.0003097753114302562, + 0.00028318594507881776, + 0.00028174654548060826, + 0.00029366070958977545, + 0.0002617199768643518, + 0.00023850476284541366, + 0.0002578321956415913, + 0.00023580491915923377, + 0.00024129375767696618, + 0.00024677521358199467, + 0.00023282043614108414, + 0.00020802360655182162, + 0.00020816264493602737, + 0.00019546204845585216, + 0.00020244473639840523, + 0.00019508742079131332, + 0.00018509966437476374, + 0.00018490270602469487, + 0.0001750081453781441, + 0.00017332318168420213, + 0.00016951527760705314, + 0.0001849276373753642, + 0.00014904354377519617, + 0.00017214209100244258, + 0.0001582098123018746, + 0.0001606899226932654, + 0.0001683991686835421, + 0.00016911992547040899, + 0.00016138660409180947 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030962264952076986, + "first_to_last_ratio": 1.8709465967014842, + "imbalance_abs_log_ratio": 0.6264445042344079, + "last_quartile_mean": 0.00016548983817423794, + "mean": 0.0002234675193377695, + "normalized": [ + 1.774962749507496, + 1.4952101563466982, + 1.4145916220053505, + 1.3862207463001954, + 1.2672353723620313, + 1.2607941696204648, + 1.314109139708619, + 1.1711768119140573, + 1.0672905107292818, + 1.1537792892929548, + 1.0552089174213117, + 1.0797710485712801, + 1.104300143095945, + 1.0418535849461763, + 0.9308896754583627, + 0.9315118615577912, + 0.8746776669606858, + 0.9059246596477921, + 0.8730012369110346, + 0.828306793413619, + 0.8274254199117672, + 0.7831480203331949, + 0.775607937107967, + 0.7585678585836542, + 0.827536985792765, + 0.6669584206996899, + 0.7703226469447271, + 0.7079767689314242, + 0.719075072607683, + 0.7535733568018358, + 0.7567986881117407, + 0.7221926684024036 + ], + "population_cv": 0.268339881652204 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6930912733078003, + "mean_weights": [ + 0.5023526549339294, + 0.49764734506607056 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692903995513916, + "mean_weights": [ + 0.5103387832641602, + 0.48966121673583984 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928629875183105, + "mean_weights": [ + 0.5053091049194336, + 0.4946908950805664 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927823424339294, + "mean_weights": [ + 0.5126834511756897, + 0.4873165488243103 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928155422210693, + "mean_weights": [ + 0.502953290939331, + 0.49704670906066895 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692751407623291, + "mean_weights": [ + 0.5129156708717346, + 0.48708435893058777 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925709247589111, + "mean_weights": [ + 0.5112693905830383, + 0.4887305796146393 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929247379302979, + "mean_weights": [ + 0.5095131397247314, + 0.49048691987991333 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0983518362045288, + "mean_weights": [ + 0.3398353159427643, + 0.32833361625671387, + 0.33183106780052185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098447322845459, + "mean_weights": [ + 0.3408135175704956, + 0.3303682208061218, + 0.3288182020187378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980432033538818, + "mean_weights": [ + 0.3472674489021301, + 0.3313220739364624, + 0.32141047716140747 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985240936279297, + "mean_weights": [ + 0.3373557925224304, + 0.33420175313949585, + 0.32844245433807373 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098268985748291, + "mean_weights": [ + 0.3445510268211365, + 0.33061009645462036, + 0.32483887672424316 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985302925109863, + "mean_weights": [ + 0.3371829390525818, + 0.3332410156726837, + 0.3295760750770569 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983161926269531, + "mean_weights": [ + 0.3423102796077728, + 0.3327076733112335, + 0.32498204708099365 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098489761352539, + "mean_weights": [ + 0.33793729543685913, + 0.3344462811946869, + 0.327616423368454 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3861088752746582, + "mean_weights": [ + 0.2561652958393097, + 0.2513129711151123, + 0.2458309531211853, + 0.2466908097267151 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861844539642334, + "mean_weights": [ + 0.2536832094192505, + 0.25214672088623047, + 0.24721932411193848, + 0.24695077538490295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3860987424850464, + "mean_weights": [ + 0.25705933570861816, + 0.2498370260000229, + 0.2462591528892517, + 0.24684447050094604 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862221240997314, + "mean_weights": [ + 0.2514326870441437, + 0.2528519928455353, + 0.24950753152370453, + 0.24620777368545532 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386016845703125, + "mean_weights": [ + 0.25482749938964844, + 0.2536851465702057, + 0.24858617782592773, + 0.24290117621421814 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862277269363403, + "mean_weights": [ + 0.25171467661857605, + 0.2530350983142853, + 0.24868467450141907, + 0.24656552076339722 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859888315200806, + "mean_weights": [ + 0.25458407402038574, + 0.2546530067920685, + 0.248786062002182, + 0.24197682738304138 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862541913986206, + "mean_weights": [ + 0.2501963675022125, + 0.2521768808364868, + 0.24995486438274384, + 0.24767187237739563 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6090729236602783, + "mean_weights": [ + 0.20360878109931946, + 0.20631682872772217, + 0.19986985623836517, + 0.19557830691337585, + 0.19462622702121735 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093083620071411, + "mean_weights": [ + 0.2017374038696289, + 0.20362424850463867, + 0.20011676847934723, + 0.1968517303466797, + 0.1976698487997055 + ], + "sources": 5 + }, + { + "entropy_mean": 1.608644723892212, + "mean_weights": [ + 0.20431089401245117, + 0.20831988751888275, + 0.20422273874282837, + 0.19545811414718628, + 0.18768835067749023 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609345555305481, + "mean_weights": [ + 0.2002599537372589, + 0.20386311411857605, + 0.2001732438802719, + 0.19764204323291779, + 0.19806164503097534 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6086196899414062, + "mean_weights": [ + 0.20515945553779602, + 0.21025606989860535, + 0.20179016888141632, + 0.1938556730747223, + 0.18893863260746002 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609370470046997, + "mean_weights": [ + 0.2002836912870407, + 0.20340535044670105, + 0.20010054111480713, + 0.1980183869600296, + 0.1981920301914215 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6083989143371582, + "mean_weights": [ + 0.20363344252109528, + 0.21095217764377594, + 0.20392383635044098, + 0.19413790106773376, + 0.18735264241695404 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093392372131348, + "mean_weights": [ + 0.20084109902381897, + 0.20331618189811707, + 0.2011999487876892, + 0.19900763034820557, + 0.19563516974449158 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7908520698547363, + "mean_weights": [ + 0.17160072922706604, + 0.17730243504047394, + 0.16903036832809448, + 0.16099891066551208, + 0.15854957699775696, + 0.1625179797410965 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916775941848755, + "mean_weights": [ + 0.16706335544586182, + 0.16981256008148193, + 0.16858771443367004, + 0.1652718484401703, + 0.16462719440460205, + 0.16463729739189148 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898297309875488, + "mean_weights": [ + 0.17284834384918213, + 0.18101516366004944, + 0.1735592484474182, + 0.16231873631477356, + 0.15689018368721008, + 0.153368279337883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917213439941406, + "mean_weights": [ + 0.16615921258926392, + 0.16885727643966675, + 0.16764026880264282, + 0.16661542654037476, + 0.16609220206737518, + 0.16463561356067657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890949249267578, + "mean_weights": [ + 0.17241932451725006, + 0.18413974344730377, + 0.17484498023986816, + 0.1621202528476715, + 0.15553176403045654, + 0.15094394981861115 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917208671569824, + "mean_weights": [ + 0.16663745045661926, + 0.168935626745224, + 0.16597260534763336, + 0.16514870524406433, + 0.16661295294761658, + 0.16669265925884247 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789101004600525, + "mean_weights": [ + 0.17229264974594116, + 0.18365857005119324, + 0.1747075319290161, + 0.1640234887599945, + 0.155386283993721, + 0.14993149042129517 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917447090148926, + "mean_weights": [ + 0.1658121943473816, + 0.167683407664299, + 0.16629494726657867, + 0.1669958382844925, + 0.16702724993228912, + 0.1661863625049591 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9436898231506348, + "mean_weights": [ + 0.14778709411621094, + 0.15856821835041046, + 0.1500256359577179, + 0.1408001184463501, + 0.13533659279346466, + 0.130964457988739, + 0.13651786744594574 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458942413330078, + "mean_weights": [ + 0.142832413315773, + 0.1429906189441681, + 0.14157448709011078, + 0.14285019040107727, + 0.14392998814582825, + 0.14335544407367706, + 0.14246684312820435 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9425597190856934, + "mean_weights": [ + 0.1483290046453476, + 0.1615946888923645, + 0.15280696749687195, + 0.14239417016506195, + 0.13462956249713898, + 0.1307675987482071, + 0.12947802245616913 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458987712860107, + "mean_weights": [ + 0.1426449418067932, + 0.14291533827781677, + 0.1416938602924347, + 0.14274153113365173, + 0.14325542747974396, + 0.1432831734418869, + 0.14346569776535034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941068410873413, + "mean_weights": [ + 0.14912807941436768, + 0.16456490755081177, + 0.15460123121738434, + 0.14255177974700928, + 0.13539987802505493, + 0.1283380389213562, + 0.1254161298274994 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458861351013184, + "mean_weights": [ + 0.14264336228370667, + 0.1430070698261261, + 0.14314037561416626, + 0.14221979677677155, + 0.14132913947105408, + 0.1431397795677185, + 0.14452049136161804 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415628910064697, + "mean_weights": [ + 0.14895881712436676, + 0.16342055797576904, + 0.15406624972820282, + 0.14293311536312103, + 0.1363803744316101, + 0.12938986718654633, + 0.1248510330915451 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458723068237305, + "mean_weights": [ + 0.14191991090774536, + 0.1418565809726715, + 0.14164230227470398, + 0.14289972186088562, + 0.14397987723350525, + 0.14332248270511627, + 0.14437910914421082 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073223114013672, + "mean_weights": [ + 0.13280923664569855, + 0.14827430248260498, + 0.13796664774417877, + 0.12756125628948212, + 0.12019773572683334, + 0.11310221254825592, + 0.10922321677207947, + 0.11086538434028625 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079413890838623, + "mean_weights": [ + 0.12475188821554184, + 0.12460128962993622, + 0.12424701452255249, + 0.12420951575040817, + 0.12484367191791534, + 0.12469655275344849, + 0.12630701065063477, + 0.12634305655956268 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072925567626953, + "mean_weights": [ + 0.13228744268417358, + 0.14843900501728058, + 0.13848665356636047, + 0.12843599915504456, + 0.12169650197029114, + 0.11442162096500397, + 0.11023421585559845, + 0.10599856078624725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793263912200928, + "mean_weights": [ + 0.12420415878295898, + 0.12273682653903961, + 0.12360996007919312, + 0.1238352432847023, + 0.1241140365600586, + 0.12607696652412415, + 0.1275939792394638, + 0.12782883644104004 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073427200317383, + "mean_weights": [ + 0.13135744631290436, + 0.1465209424495697, + 0.13772746920585632, + 0.12966200709342957, + 0.12215200066566467, + 0.11654195189476013, + 0.11090746521949768, + 0.10513069480657578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079422950744629, + "mean_weights": [ + 0.12477636337280273, + 0.12493626773357391, + 0.1242394968867302, + 0.12470392882823944, + 0.12560302019119263, + 0.12461308389902115, + 0.12591606378555298, + 0.12521176040172577 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0725789070129395, + "mean_weights": [ + 0.13186156749725342, + 0.14827941358089447, + 0.13866659998893738, + 0.12988534569740295, + 0.12183371186256409, + 0.11524046957492828, + 0.1099783331155777, + 0.10425455868244171 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793380737304688, + "mean_weights": [ + 0.12417487055063248, + 0.1225123256444931, + 0.12277349829673767, + 0.1243998259305954, + 0.12567630410194397, + 0.1261572688817978, + 0.12747105956077576, + 0.12683483958244324 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1906280517578125, + "mean_weights": [ + 0.11854428052902222, + 0.13291707634925842, + 0.12378416955471039, + 0.11745279282331467, + 0.11140251904726028, + 0.10409101843833923, + 0.09838292002677917, + 0.09486069530248642, + 0.09856453537940979 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197185516357422, + "mean_weights": [ + 0.11084984987974167, + 0.11021925508975983, + 0.11018812656402588, + 0.11046157777309418, + 0.11131840199232101, + 0.11167057603597641, + 0.11249847710132599, + 0.1118280440568924, + 0.11096569895744324 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1899101734161377, + "mean_weights": [ + 0.1190595030784607, + 0.13352498412132263, + 0.1239522397518158, + 0.11779220402240753, + 0.11234381794929504, + 0.10513705015182495, + 0.09944089502096176, + 0.0946362093091011, + 0.09411308169364929 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197190284729004, + "mean_weights": [ + 0.11067462712526321, + 0.11021950840950012, + 0.11038416624069214, + 0.11103623360395432, + 0.11016053706407547, + 0.11144962906837463, + 0.11242866516113281, + 0.11139006912708282, + 0.11225657165050507 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1896841526031494, + "mean_weights": [ + 0.11968795955181122, + 0.13428261876106262, + 0.12373492866754532, + 0.11790731549263, + 0.11133000254631042, + 0.10504785925149918, + 0.09994535893201828, + 0.09480959177017212, + 0.09325435757637024 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971938610076904, + "mean_weights": [ + 0.11116788536310196, + 0.1117495447397232, + 0.11143069714307785, + 0.11117202788591385, + 0.11217275261878967, + 0.11135637760162354, + 0.11160033196210861, + 0.10969291627407074, + 0.10965746641159058 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188610792160034, + "mean_weights": [ + 0.11987313628196716, + 0.1360916942358017, + 0.12462760508060455, + 0.11755925416946411, + 0.11237476021051407, + 0.10438287258148193, + 0.09920360147953033, + 0.09431086480617523, + 0.09157621115446091 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028571506962180138, + 0.014360599219799042, + 0.01502692885696888, + 0.014914286322891712, + 0.015751885250210762, + 0.015590262599289417, + 0.016715344041585922, + 0.016720104962587357, + 0.01783851347863674, + 0.01203431747853756, + 0.012985615991055965, + 0.01319553330540657, + 0.014028912410140038, + 0.014193638227880001, + 0.015184907242655754, + 0.015358141623437405, + 0.016659293323755264, + 0.012656479142606258, + 0.013609340414404869, + 0.013699201866984367, + 0.014725575223565102, + 0.01495310477912426, + 0.016002535820007324, + 0.01618308201432228, + 0.017315881326794624, + 0.01409733947366476, + 0.014967667870223522, + 0.01521329116076231, + 0.016154561191797256, + 0.016374915838241577, + 0.01737469807267189, + 0.017517324537038803, + 0.018479224294424057, + 0.015613679774105549, + 0.01647813245654106, + 0.016780734062194824, + 0.017671043053269386, + 0.017834074795246124, + 0.01892261393368244, + 0.01904122717678547, + 0.020303834229707718, + 0.017550688236951828, + 0.01861351728439331, + 0.018688000738620758, + 0.01984807848930359, + 0.019847052171826363, + 0.021230308338999748, + 0.021104788407683372, + 0.022779423743486404, + 0.019968591630458832, + 0.02138628251850605, + 0.021149026229977608, + 0.02279111184179783, + 0.02249406836926937, + 0.024287279695272446, + 0.02368815243244171, + 0.02611272968351841, + 0.02278727851808071, + 0.024725768715143204, + 0.024227207526564598, + 0.026407435536384583, + 0.02562388963997364, + 0.028016721829771996, + 0.0269329734146595 + ], + "loss_nats": 3.570180892944336, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1930694580078125, + "mean_weights": [ + 0.10071488469839096, + 0.09411877393722534, + 0.10181896388530731, + 0.10989432036876678, + 0.1145806610584259, + 0.11861918121576309, + 0.11984598636627197, + 0.12094368040561676, + 0.11946355551481247 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.002919268561527133, + 0.005866481456905603, + 0.0065401275642216206, + 0.010142817161977291, + 0.010618876665830612, + 0.01482025533914566, + 0.015120964497327805, + 0.01956234686076641, + 0.0030634324066340923, + 0.0067119961604475975, + 0.007598300464451313, + 0.011908270418643951, + 0.012669830583035946, + 0.017094247043132782, + 0.017725100740790367, + 0.02295343205332756, + 0.0037359180860221386, + 0.007494525518268347, + 0.009108707308769226, + 0.013766046613454819, + 0.015153576619923115, + 0.02038806490600109, + 0.021781234070658684, + 0.02682213857769966, + 0.005531937349587679, + 0.009193498641252518, + 0.0119093032553792, + 0.016781754791736603, + 0.01904802769422531, + 0.024297412484884262, + 0.02626260370016098, + 0.03140362724661827, + 0.005659298039972782, + 0.009884056635200977, + 0.013823030516505241, + 0.018740201368927956, + 0.02223494090139866, + 0.02799132652580738, + 0.0319429449737072, + 0.03797563910484314, + 0.007882392033934593, + 0.012623735703527927, + 0.01784195750951767, + 0.02351362258195877, + 0.028768405318260193, + 0.034899674355983734, + 0.040818072855472565, + 0.047332651913166046, + 0.012831038795411587, + 0.01744839735329151, + 0.023826664313673973, + 0.029514454305171967, + 0.03824474662542343, + 0.044660378247499466, + 0.053081437945365906, + 0.0601838044822216, + 0.013431218452751637, + 0.018972838297486305, + 0.03125866502523422, + 0.0376126803457737, + 0.04969283565878868, + 0.0565216988325119, + 0.06860727071762085, + 0.07583927363157272 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "population_cv": 0.5868596491376813 + }, + "activation_output_rms_by_block": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.036073930095881224, + "first_to_last_ratio": 0.5177916953272103, + "imbalance_abs_log_ratio": 0.6581822501725932, + "last_quartile_mean": 0.0696688077878207, + "mean": 0.04647241474594921, + "normalized": [ + 0.46329906589537145, + 0.8300990481085756, + 1.06603234209322, + 1.2414755802942754, + 0.4606131805713009, + 0.5586764120892566, + 0.7186330436105184, + 0.8711228081179162, + 0.4630334112674251, + 0.7541501214633191, + 0.8015850238486076, + 0.9304351337283648, + 0.4571868851775181, + 0.6517799422454194, + 0.8562272308804426, + 1.1237390363826774, + 0.8427218914862018, + 0.8798161433899512, + 1.0235622313138022, + 1.1926338466716728, + 0.4420161942758587, + 0.7919574083446129, + 1.1031381372276345, + 1.482919724824311, + 0.6253151219931478, + 1.1530933116414785, + 1.6957169376190813, + 2.3138065984257823, + 0.6272181919070765, + 1.163956934601208, + 1.8566997944110037, + 2.5573392660929675 + ], + "population_cv": 0.5056023373965206 + }, + "bits_per_byte": 3.0609520355704922, + "branch_output_rms_by_sublayer": [ + 0.010549778118729591, + 0.017611408606171608, + 0.013795096427202225, + 0.014558257535099983, + 0.012347182258963585, + 0.01080919150263071, + 0.012820021249353886, + 0.010327119380235672, + 0.01913420483469963, + 0.008616750128567219, + 0.018582463264465332, + 0.007770510856062174, + 0.013614931143820286, + 0.007606723811477423, + 0.013612493872642517, + 0.008487755432724953, + 0.019129201769828796, + 0.00865182839334011, + 0.02525762841105461, + 0.009020541794598103, + 0.027986859902739525, + 0.008682658895850182, + 0.014458909630775452, + 0.008706990629434586, + 0.018231205642223358, + 0.009003226645290852, + 0.016914349049329758, + 0.009712434373795986, + 0.022478405386209488, + 0.00981286820024252, + 0.03932488337159157, + 0.011337134055793285, + 0.03611915931105614, + 0.010840619914233685, + 0.0266753900796175, + 0.010956712998449802, + 0.018954744562506676, + 0.012231985107064247, + 0.014813725836575031, + 0.014082341454923153, + 0.013802599161863327, + 0.014436324127018452, + 0.016533223912119865, + 0.016183899715542793, + 0.016679473221302032, + 0.017385432496666908, + 0.01826235093176365, + 0.02254617027938366, + 0.015529165975749493, + 0.022119678556919098, + 0.011229476891458035, + 0.022372635081410408, + 0.008122270926833153, + 0.024847008287906647, + 0.008894859813153744, + 0.026897024363279343, + 0.009738311171531677, + 0.026162855327129364, + 0.007465044502168894, + 0.024674534797668457, + 0.008605006150901318, + 0.030845578759908676, + 0.009693782776594162, + 0.029862264171242714 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006151943096300597, + 0.0003715258289130363, + 0.0003398511463891951, + 0.0003030496529751275, + 0.00028984271802693696, + 0.0002579116693919001, + 0.00022400787473702318, + 0.00028477639847268295, + 0.000278695861793727, + 0.00022792247112618847, + 0.00023417680672174036, + 0.00019296669269337194, + 0.00020274366236211453, + 0.00017465616055496573, + 0.00015881266908702134, + 0.00017808900330691153, + 0.00016226510475368574, + 0.00014508607560593053, + 0.00013101353173913878, + 0.00011930204103094746, + 0.00012786456583361604, + 0.00012508100764064805, + 0.00012354146515800217, + 0.00012514897623388166, + 0.00011397604830293932, + 9.556324385288696e-05, + 9.289013346881659e-05, + 9.744025747256133e-05, + 8.302691520063822e-05, + 7.404706874242776e-05, + 8.914255818565412e-05, + 8.643783624226864e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003357699498169952, + "first_to_last_ratio": 3.666991625028822, + "imbalance_abs_log_ratio": 1.2993716052111575, + "last_quartile_mean": 9.156550768352412e-05, + "mean": 0.00019143905486393895, + "normalized": [ + 3.2135256312631473, + 1.940700288021637, + 1.7752445896201103, + 1.5830085098911153, + 1.514020836724963, + 1.3472259857070703, + 1.1701263093689624, + 1.487556437609251, + 1.4557941794677367, + 1.1905745736583337, + 1.223244686870066, + 1.0079797606111183, + 1.0590506859021536, + 0.9123329650739171, + 0.8295729896906037, + 0.9302647436986373, + 0.8476071137574935, + 0.7578708310540252, + 0.6843615678747147, + 0.6231854890619823, + 0.6679126467924371, + 0.6533724674390322, + 0.645330521746498, + 0.6537275078109245, + 0.5953646625760102, + 0.49918363795102527, + 0.4852203931681348, + 0.5089883960292994, + 0.4336989401647838, + 0.386791863316788, + 0.46564457941463516, + 0.45151620866339115 + ], + "population_cv": 0.578773657781326 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.658259391784668, + "mean_weights": [ + 0.3887256383895874, + 0.6112743616104126 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6694332361221313, + "mean_weights": [ + 0.42782124876976013, + 0.5721787214279175 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6890689730644226, + "mean_weights": [ + 0.504529595375061, + 0.49547040462493896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740086078643799, + "mean_weights": [ + 0.45225459337234497, + 0.547745406627655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.679557204246521, + "mean_weights": [ + 0.4461178183555603, + 0.5538821816444397 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740646362304688, + "mean_weights": [ + 0.44189155101776123, + 0.5581084489822388 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6792672276496887, + "mean_weights": [ + 0.4455937147140503, + 0.5544062852859497 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674970269203186, + "mean_weights": [ + 0.44674086570739746, + 0.5532591342926025 + ], + "sources": 2 + }, + { + "entropy_mean": 1.093062162399292, + "mean_weights": [ + 0.3082374036312103, + 0.33985745906829834, + 0.35190510749816895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0853323936462402, + "mean_weights": [ + 0.30180174112319946, + 0.3268374502658844, + 0.3713608384132385 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0921990871429443, + "mean_weights": [ + 0.3034760355949402, + 0.3359980285167694, + 0.3605259656906128 + ], + "sources": 3 + }, + { + "entropy_mean": 1.076724648475647, + "mean_weights": [ + 0.29931002855300903, + 0.30481812357902527, + 0.3958718776702881 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856893062591553, + "mean_weights": [ + 0.2801036834716797, + 0.3310527801513672, + 0.3888435363769531 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0752873420715332, + "mean_weights": [ + 0.29463040828704834, + 0.31052330136299133, + 0.3948462903499603 + ], + "sources": 3 + }, + { + "entropy_mean": 1.088913917541504, + "mean_weights": [ + 0.29400017857551575, + 0.32166045904159546, + 0.3843393623828888 + ], + "sources": 3 + }, + { + "entropy_mean": 1.054490327835083, + "mean_weights": [ + 0.2850497364997864, + 0.2890807092189789, + 0.42586958408355713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3700100183486938, + "mean_weights": [ + 0.21610887348651886, + 0.2282472848892212, + 0.2728016972541809, + 0.28284209966659546 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3620216846466064, + "mean_weights": [ + 0.22791308164596558, + 0.2214205265045166, + 0.2562018632888794, + 0.2944645285606384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3784711360931396, + "mean_weights": [ + 0.23347607254981995, + 0.23307976126670837, + 0.2570483684539795, + 0.2763957977294922 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3619427680969238, + "mean_weights": [ + 0.24715131521224976, + 0.22003202140331268, + 0.23014183342456818, + 0.3026748299598694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804662227630615, + "mean_weights": [ + 0.24302782118320465, + 0.23108476400375366, + 0.24228686094284058, + 0.2836005389690399 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3502743244171143, + "mean_weights": [ + 0.2455797642469406, + 0.21441739797592163, + 0.2208172082901001, + 0.31918564438819885 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3820215463638306, + "mean_weights": [ + 0.24522465467453003, + 0.22846508026123047, + 0.25027886033058167, + 0.27603137493133545 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3491160869598389, + "mean_weights": [ + 0.24498313665390015, + 0.21178750693798065, + 0.2205154001712799, + 0.3227139711380005 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6032840013504028, + "mean_weights": [ + 0.20074918866157532, + 0.17959122359752655, + 0.18619459867477417, + 0.2124267816543579, + 0.22103822231292725 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5699318647384644, + "mean_weights": [ + 0.1938496232032776, + 0.17019830644130707, + 0.16257275640964508, + 0.20397180318832397, + 0.2694075107574463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6068798303604126, + "mean_weights": [ + 0.19717518985271454, + 0.1942940056324005, + 0.19104912877082825, + 0.20343774557113647, + 0.21404394507408142 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5761134624481201, + "mean_weights": [ + 0.1940101534128189, + 0.18250694870948792, + 0.16849923133850098, + 0.18937936425209045, + 0.26560431718826294 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6027262210845947, + "mean_weights": [ + 0.20452985167503357, + 0.18702910840511322, + 0.17977532744407654, + 0.20095866918563843, + 0.22770702838897705 + ], + "sources": 5 + }, + { + "entropy_mean": 1.577937126159668, + "mean_weights": [ + 0.1962222307920456, + 0.1818992793560028, + 0.17167484760284424, + 0.18149182200431824, + 0.2687118649482727 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5977532863616943, + "mean_weights": [ + 0.2153339385986328, + 0.18402469158172607, + 0.17019470036029816, + 0.1919252574443817, + 0.23852142691612244 + ], + "sources": 5 + }, + { + "entropy_mean": 1.578121542930603, + "mean_weights": [ + 0.20422281324863434, + 0.1835813820362091, + 0.16642534732818604, + 0.17722904682159424, + 0.26854145526885986 + ], + "sources": 5 + }, + { + "entropy_mean": 1.782798409461975, + "mean_weights": [ + 0.15961389243602753, + 0.16475999355316162, + 0.14714640378952026, + 0.15768738090991974, + 0.1760568618774414, + 0.19473549723625183 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7707297801971436, + "mean_weights": [ + 0.16306114196777344, + 0.1578725278377533, + 0.139655202627182, + 0.14299748837947845, + 0.18966570496559143, + 0.20674791932106018 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781312108039856, + "mean_weights": [ + 0.15919369459152222, + 0.16844786703586578, + 0.14742718636989594, + 0.1539241373538971, + 0.1669306755065918, + 0.20407646894454956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7736209630966187, + "mean_weights": [ + 0.15695160627365112, + 0.16567477583885193, + 0.1478763222694397, + 0.14320015907287598, + 0.17375096678733826, + 0.2125461995601654 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781569480895996, + "mean_weights": [ + 0.15368077158927917, + 0.1725173443555832, + 0.14978237450122833, + 0.15674246847629547, + 0.16688688099384308, + 0.20039016008377075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7793617248535156, + "mean_weights": [ + 0.1678246557712555, + 0.1683025360107422, + 0.14878368377685547, + 0.14351718127727509, + 0.16739681363105774, + 0.20417512953281403 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7772380113601685, + "mean_weights": [ + 0.1536770462989807, + 0.18459689617156982, + 0.14832255244255066, + 0.15016385912895203, + 0.1652815043926239, + 0.19795814156532288 + ], + "sources": 6 + }, + { + "entropy_mean": 1.780207633972168, + "mean_weights": [ + 0.16951784491539001, + 0.17200809717178345, + 0.15038040280342102, + 0.14669056236743927, + 0.16062122583389282, + 0.20078186690807343 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937079668045044, + "mean_weights": [ + 0.13008031249046326, + 0.15163566172122955, + 0.1323973685503006, + 0.13641127943992615, + 0.13806134462356567, + 0.1444631665945053, + 0.16695083677768707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939245343208313, + "mean_weights": [ + 0.14692582190036774, + 0.14671875536441803, + 0.13336658477783203, + 0.1307726800441742, + 0.131200909614563, + 0.147438645362854, + 0.16357660293579102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9312366247177124, + "mean_weights": [ + 0.12334205210208893, + 0.16950437426567078, + 0.13870283961296082, + 0.1300538182258606, + 0.12885476648807526, + 0.14288470149040222, + 0.1666574478149414 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9345850944519043, + "mean_weights": [ + 0.1465124636888504, + 0.153631791472435, + 0.13159149885177612, + 0.13120220601558685, + 0.1253916621208191, + 0.14393125474452972, + 0.1677391082048416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9284589290618896, + "mean_weights": [ + 0.12412485480308533, + 0.1698783040046692, + 0.13807369768619537, + 0.13034150004386902, + 0.13028821349143982, + 0.1419651210308075, + 0.16532832384109497 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9329564571380615, + "mean_weights": [ + 0.1454196274280548, + 0.1566689908504486, + 0.1286500096321106, + 0.12666445970535278, + 0.12782412767410278, + 0.14222374558448792, + 0.17254900932312012 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9259672164916992, + "mean_weights": [ + 0.12082186341285706, + 0.17791953682899475, + 0.1424643099308014, + 0.13298648595809937, + 0.12929362058639526, + 0.1344166398048401, + 0.16209754347801208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9201786518096924, + "mean_weights": [ + 0.13756656646728516, + 0.17355048656463623, + 0.14306491613388062, + 0.13006147742271423, + 0.11658851057291031, + 0.139857679605484, + 0.15931037068367004 + ], + "sources": 7 + }, + { + "entropy_mean": 2.043663501739502, + "mean_weights": [ + 0.10298450291156769, + 0.1751018464565277, + 0.1414714753627777, + 0.12628036737442017, + 0.11767096072435379, + 0.12069195508956909, + 0.12702946364879608, + 0.08876943588256836 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0607244968414307, + "mean_weights": [ + 0.1220984011888504, + 0.1498527079820633, + 0.12913095951080322, + 0.1179434061050415, + 0.10775719583034515, + 0.12020602822303772, + 0.13844498991966248, + 0.11456632614135742 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0388383865356445, + "mean_weights": [ + 0.1025313287973404, + 0.18260905146598816, + 0.14208021759986877, + 0.12606360018253326, + 0.11357533931732178, + 0.11596501618623734, + 0.12797299027442932, + 0.08920245617628098 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0634398460388184, + "mean_weights": [ + 0.12216806411743164, + 0.15090107917785645, + 0.12766355276107788, + 0.1134132519364357, + 0.10988184064626694, + 0.11335282772779465, + 0.13999530673027039, + 0.12262405455112457 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0386834144592285, + "mean_weights": [ + 0.10029368102550507, + 0.17931804060935974, + 0.14185777306556702, + 0.13114282488822937, + 0.11872631311416626, + 0.11815309524536133, + 0.12660562992095947, + 0.08390261977910995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0626261234283447, + "mean_weights": [ + 0.11644256860017776, + 0.15398627519607544, + 0.1359611600637436, + 0.12036904692649841, + 0.10964122414588928, + 0.11886263638734818, + 0.13317911326885223, + 0.1115579903125763 + ], + "sources": 8 + }, + { + "entropy_mean": 2.020051956176758, + "mean_weights": [ + 0.09590239822864532, + 0.1913527548313141, + 0.14906750619411469, + 0.1341754049062729, + 0.11847006529569626, + 0.11603336781263351, + 0.12013453245162964, + 0.07486395537853241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0452136993408203, + "mean_weights": [ + 0.11420649290084839, + 0.15673109889030457, + 0.13583534955978394, + 0.12758757174015045, + 0.1112617775797844, + 0.13417255878448486, + 0.12519711256027222, + 0.09500804543495178 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1403071880340576, + "mean_weights": [ + 0.08488726615905762, + 0.1673685610294342, + 0.13726134598255157, + 0.12220048159360886, + 0.10616928339004517, + 0.1037440225481987, + 0.10401669889688492, + 0.06658264249563217, + 0.10776971280574799 + ], + "sources": 9 + }, + { + "entropy_mean": 2.18704891204834, + "mean_weights": [ + 0.10885682702064514, + 0.12684625387191772, + 0.11236055195331573, + 0.10673345625400543, + 0.10181289166212082, + 0.11441421508789062, + 0.11840946972370148, + 0.10466625541448593, + 0.1059001013636589 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1283018589019775, + "mean_weights": [ + 0.08667369186878204, + 0.1812124401330948, + 0.14178472757339478, + 0.12440058588981628, + 0.11160320788621902, + 0.10330478847026825, + 0.11197598278522491, + 0.06831350177526474, + 0.07073107361793518 + ], + "sources": 9 + }, + { + "entropy_mean": 2.173035144805908, + "mean_weights": [ + 0.10752685368061066, + 0.13639645278453827, + 0.11839058995246887, + 0.11235916614532471, + 0.10138258337974548, + 0.11953261494636536, + 0.11256720125675201, + 0.09452059864997864, + 0.097323939204216 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1028335094451904, + "mean_weights": [ + 0.08200174570083618, + 0.19083696603775024, + 0.15512439608573914, + 0.1322997808456421, + 0.11209174990653992, + 0.10435720533132553, + 0.1024213582277298, + 0.06072331964969635, + 0.06014348566532135 + ], + "sources": 9 + }, + { + "entropy_mean": 2.147671937942505, + "mean_weights": [ + 0.09926989674568176, + 0.14984291791915894, + 0.12505242228507996, + 0.13004425168037415, + 0.1122550517320633, + 0.12226949632167816, + 0.10689133405685425, + 0.07532045990228653, + 0.07905416935682297 + ], + "sources": 9 + }, + { + "entropy_mean": 2.109609603881836, + "mean_weights": [ + 0.08386656641960144, + 0.18956288695335388, + 0.1487923562526703, + 0.12990039587020874, + 0.11159682273864746, + 0.10738210380077362, + 0.10535432398319244, + 0.0630122721195221, + 0.06053228676319122 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04393457621335983, + 0.018067941069602966, + 0.024121688678860664, + 0.027584951370954514, + 0.030756350606679916, + 0.03231721371412277, + 0.03544071316719055, + 0.036101218312978745, + 0.03840360417962074, + 0.025345133617520332, + 0.026969486847519875, + 0.027166159823536873, + 0.027978016063570976, + 0.028526438400149345, + 0.029923241585493088, + 0.02979987859725952, + 0.03152087330818176, + 0.02248864620923996, + 0.024161184206604958, + 0.023894935846328735, + 0.025710709393024445, + 0.02451697364449501, + 0.02641460672020912, + 0.025183402001857758, + 0.02742084674537182, + 0.020499059930443764, + 0.021717891097068787, + 0.02105838619172573, + 0.0224718376994133, + 0.0216353852301836, + 0.023445574566721916, + 0.022715488448739052, + 0.024849263951182365, + 0.01881815865635872, + 0.01992032676935196, + 0.0193756353110075, + 0.02058842033147812, + 0.02001863345503807, + 0.020979218184947968, + 0.021088071167469025, + 0.02214338816702366, + 0.018417155370116234, + 0.01903437450528145, + 0.019219009205698967, + 0.02008410543203354, + 0.020342160016298294, + 0.021635910496115685, + 0.02150510996580124, + 0.023218298330903053, + 0.02099662832915783, + 0.022257626056671143, + 0.02211105264723301, + 0.024358727037906647, + 0.02328667603433132, + 0.025900650769472122, + 0.024160878732800484, + 0.027197737246751785, + 0.02260657213628292, + 0.02823900803923607, + 0.024989914149045944, + 0.02913021855056286, + 0.02536177635192871, + 0.029048049822449684, + 0.027097374200820923 + ], + "loss_nats": 2.121690273284912, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1622862815856934, + "mean_weights": [ + 0.12356589734554291, + 0.09235234558582306, + 0.08736804127693176, + 0.08735492080450058, + 0.08889827132225037, + 0.10504580289125443, + 0.14004790782928467, + 0.14956718683242798, + 0.12579965591430664 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.010549778118729591, + 0.021530626341700554, + 0.02910016104578972, + 0.03857670724391937, + 0.04255657270550728, + 0.04954109713435173, + 0.05177471786737442, + 0.057694368064403534, + 0.01913420483469963, + 0.02140580676496029, + 0.02246284671127796, + 0.025963041931390762, + 0.029843268916010857, + 0.03339661285281181, + 0.036163609474897385, + 0.040483180433511734, + 0.019129201769828796, + 0.02151828072965145, + 0.0314205139875412, + 0.035047177225351334, + 0.03302976116538048, + 0.03725159168243408, + 0.03871368616819382, + 0.043239567428827286, + 0.018231205642223358, + 0.02124657854437828, + 0.02580643631517887, + 0.03028978779911995, + 0.0349617674946785, + 0.03979094699025154, + 0.04659247398376465, + 0.05222286656498909, + 0.03611915931105614, + 0.03916332125663757, + 0.03664228320121765, + 0.04088718071579933, + 0.04177439957857132, + 0.04756740853190422, + 0.04776441678404808, + 0.05542457476258278, + 0.013802599161863327, + 0.020541559904813766, + 0.02693362347781658, + 0.03680417314171791, + 0.03932841867208481, + 0.05126549303531647, + 0.0514267161488533, + 0.06891486048698425, + 0.015529165975749493, + 0.02905990369617939, + 0.03558400645852089, + 0.05358703061938286, + 0.05707564577460289, + 0.07880406081676483, + 0.08332383632659912, + 0.10752817988395691, + 0.009738311171531677, + 0.029148343950510025, + 0.03187238425016403, + 0.054091889411211014, + 0.05786185711622238, + 0.08628532290458679, + 0.09122034162282944, + 0.11884573101997375 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "population_cv": 0.5384894543542103 + }, + "activation_output_rms_by_block": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14976411033421755, + "first_to_last_ratio": 1.036636701844262, + "imbalance_abs_log_ratio": 0.035981532131895505, + "last_quartile_mean": 0.14447116339579225, + "mean": 0.12951817945577204, + "normalized": [ + 0.7297249643070838, + 1.5686735699681784, + 1.7647660354854744, + 1.8765703797812938, + 0.636469098825333, + 0.749653246535227, + 0.8711234054002422, + 1.0535576234146717, + 0.42020993720306404, + 0.6840666215997374, + 0.9431112804194469, + 1.192130706187048, + 0.4699981697046636, + 0.7312780916470571, + 0.9733494312076132, + 1.2651958587237788, + 0.6835788065012903, + 0.9366911620158862, + 1.2091399212742004, + 1.6196823469097898, + 0.34397948019882096, + 0.5722161994862464, + 0.7727653276358534, + 1.008461533628578, + 0.3829837776042769, + 0.608007158744095, + 0.8769750006177236, + 1.2205488119398642, + 0.5370088925855727, + 1.0271058497013745, + 1.7339584411053826, + 2.5370188696411313 + ], + "population_cv": 0.4929365839545065 + }, + "bits_per_byte": 2.073383110809672, + "branch_output_rms_by_sublayer": [ + 0.04804428666830063, + 0.07808538526296616, + 0.12356939166784286, + 0.03881331905722618, + 0.05923242121934891, + 0.01480295043438673, + 0.04508936405181885, + 0.02500341460108757, + 0.0799286738038063, + 0.015180142596364021, + 0.03290615975856781, + 0.013721407391130924, + 0.03170505166053772, + 0.016902483999729156, + 0.03961387276649475, + 0.03325798362493515, + 0.04508088901638985, + 0.026172807440161705, + 0.0312613770365715, + 0.027887089177966118, + 0.029994890093803406, + 0.03240044787526131, + 0.03058313950896263, + 0.03682815283536911, + 0.0497574545443058, + 0.029301028698682785, + 0.03759258985519409, + 0.0338764488697052, + 0.030690157786011696, + 0.0393521711230278, + 0.03000541403889656, + 0.046694882214069366, + 0.079553984105587, + 0.029503758996725082, + 0.0414140485227108, + 0.03615153953433037, + 0.04122982919216156, + 0.030687781050801277, + 0.05830265209078789, + 0.05279085040092468, + 0.03440643846988678, + 0.024021649733185768, + 0.033897366374731064, + 0.03336174041032791, + 0.02957344427704811, + 0.04255227372050285, + 0.03406442329287529, + 0.0520954355597496, + 0.025225287303328514, + 0.03984410688281059, + 0.01915411651134491, + 0.04249191656708717, + 0.014526215381920338, + 0.05309459939599037, + 0.011566666886210442, + 0.06231515854597092, + 0.0242468249052763, + 0.06388300657272339, + 0.013597902841866016, + 0.0749494656920433, + 0.020146220922470093, + 0.10156872868537903, + 0.02710210345685482, + 0.11044161766767502 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002879944983597963, + 0.00022853159560210997, + 0.00014071406888316784, + 0.00014318671869519129, + 0.00013886536811003923, + 0.00010286844600842235, + 9.677924556041764e-05, + 0.0001337288949114584, + 0.00010758657043361882, + 9.13787469069405e-05, + 9.373485859236601e-05, + 9.381017142422257e-05, + 0.00010148426099734976, + 9.552468088000852e-05, + 0.00010200246393672037, + 0.00010753255944705701, + 0.00013288008273706816, + 0.00012966970691820486, + 0.0001395588859415414, + 0.00019785772756123382, + 0.0004949688435464942, + 0.0005919211037074493, + 0.000516172452953512, + 0.0004666614668327092, + 0.0003441119880144902, + 0.00027788329882105144, + 0.00022798541204965527, + 0.00021216050695355764, + 0.00012431994828037904, + 0.00011340006302052245, + 0.00011536177905162644, + 0.00011797314476595472 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015908360451632537, + "first_to_last_ratio": 0.8300756844691923, + "imbalance_abs_log_ratio": 0.18623839624123317, + "last_quartile_mean": 0.00019164951761965464, + "mean": 0.00019589404874701053, + "normalized": [ + 1.4701544033720488, + 1.1666081591751138, + 0.7183172219024098, + 0.730939605419618, + 0.7088799736299207, + 0.5251228746681984, + 0.4940387223575344, + 0.682659303673712, + 0.5492079576779928, + 0.4664702551783621, + 0.4784977348312449, + 0.4788821917983569, + 0.5180568866000248, + 0.4876344201930039, + 0.5207022091235275, + 0.5489322423772612, + 0.6783262870260932, + 0.661937959563377, + 0.7124202436684344, + 1.01002418821186, + 2.526717104028653, + 3.0216390313719645, + 2.6349572958192744, + 2.3822135987162336, + 1.7566229817369148, + 1.4185387488719823, + 1.1638200012093756, + 1.0830370208313709, + 0.6346285100316312, + 0.5788846764149236, + 0.5888988450109153, + 0.6022293455086652 + ], + "population_cv": 0.7015245062301059 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4738388657569885, + "mean_weights": [ + 0.8067666292190552, + 0.19323334097862244 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6344536542892456, + "mean_weights": [ + 0.4895974397659302, + 0.5104025602340698 + ], + "sources": 2 + }, + { + "entropy_mean": 0.41773349046707153, + "mean_weights": [ + 0.8461704254150391, + 0.15382957458496094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868985891342163, + "mean_weights": [ + 0.5008196830749512, + 0.49918028712272644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5841774344444275, + "mean_weights": [ + 0.684928297996521, + 0.315071702003479 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6896515488624573, + "mean_weights": [ + 0.5092520117759705, + 0.49074798822402954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611414909362793, + "mean_weights": [ + 0.7280503511428833, + 0.2719496488571167 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6817618608474731, + "mean_weights": [ + 0.5313050746917725, + 0.46869492530822754 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0218309164047241, + "mean_weights": [ + 0.4974328279495239, + 0.24236315488815308, + 0.2602040767669678 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0932960510253906, + "mean_weights": [ + 0.3533782958984375, + 0.29999595880508423, + 0.34662577509880066 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0456881523132324, + "mean_weights": [ + 0.4338536858558655, + 0.23105540871620178, + 0.3350909352302551 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0891339778900146, + "mean_weights": [ + 0.33998236060142517, + 0.2821658253669739, + 0.37785181403160095 + ], + "sources": 3 + }, + { + "entropy_mean": 1.038641333580017, + "mean_weights": [ + 0.34165239334106445, + 0.23904743790626526, + 0.4193001389503479 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0754427909851074, + "mean_weights": [ + 0.3781457543373108, + 0.24021697044372559, + 0.3816372752189636 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0433809757232666, + "mean_weights": [ + 0.37802231311798096, + 0.21331989765167236, + 0.4086577892303467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0922105312347412, + "mean_weights": [ + 0.3460429906845093, + 0.3074926435947418, + 0.3464643955230713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3404325246810913, + "mean_weights": [ + 0.2880150079727173, + 0.16260021924972534, + 0.2385387420654297, + 0.3108460605144501 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765960931777954, + "mean_weights": [ + 0.28703436255455017, + 0.2552259564399719, + 0.2101576328277588, + 0.24758204817771912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3274900913238525, + "mean_weights": [ + 0.33539247512817383, + 0.1987970769405365, + 0.20543015003204346, + 0.2603802978992462 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3747133016586304, + "mean_weights": [ + 0.27988845109939575, + 0.23608103394508362, + 0.20066317915916443, + 0.2833673357963562 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3239198923110962, + "mean_weights": [ + 0.3361120820045471, + 0.20296622812747955, + 0.1885247528553009, + 0.2723969519138336 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3692121505737305, + "mean_weights": [ + 0.28167375922203064, + 0.21946553885936737, + 0.20140981674194336, + 0.2974509000778198 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3342361450195312, + "mean_weights": [ + 0.36468517780303955, + 0.20593920350074768, + 0.18943950533866882, + 0.23993611335754395 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3645737171173096, + "mean_weights": [ + 0.2597482204437256, + 0.22607669234275818, + 0.209133118391037, + 0.30504196882247925 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5910131931304932, + "mean_weights": [ + 0.2617476284503937, + 0.18299485743045807, + 0.17935843765735626, + 0.17898105084896088, + 0.19691801071166992 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5884685516357422, + "mean_weights": [ + 0.2046908289194107, + 0.17294223606586456, + 0.15044087171554565, + 0.20923088490962982, + 0.26269516348838806 + ], + "sources": 5 + }, + { + "entropy_mean": 1.599259853363037, + "mean_weights": [ + 0.21657392382621765, + 0.20580387115478516, + 0.19366897642612457, + 0.1731838881969452, + 0.21076934039592743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5786192417144775, + "mean_weights": [ + 0.20599734783172607, + 0.20593789219856262, + 0.13473063707351685, + 0.17552703619003296, + 0.2778070569038391 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580681324005127, + "mean_weights": [ + 0.21277281641960144, + 0.18522506952285767, + 0.18489867448806763, + 0.17205308377742767, + 0.2450503408908844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5705549716949463, + "mean_weights": [ + 0.2125053107738495, + 0.20224159955978394, + 0.12915469706058502, + 0.1714920997619629, + 0.28460633754730225 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5599372386932373, + "mean_weights": [ + 0.2665032148361206, + 0.181569442152977, + 0.15844659507274628, + 0.14596089720726013, + 0.2475198209285736 + ], + "sources": 5 + }, + { + "entropy_mean": 1.562828540802002, + "mean_weights": [ + 0.21806953847408295, + 0.22324788570404053, + 0.11591021716594696, + 0.17848318815231323, + 0.26428914070129395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.761409044265747, + "mean_weights": [ + 0.1612042486667633, + 0.1651904582977295, + 0.1752963364124298, + 0.14996090531349182, + 0.2072838991880417, + 0.14106415212154388 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7366840839385986, + "mean_weights": [ + 0.1851401925086975, + 0.1663871705532074, + 0.09063514322042465, + 0.11971023678779602, + 0.18503433465957642, + 0.2530929446220398 + ], + "sources": 6 + }, + { + "entropy_mean": 1.777672290802002, + "mean_weights": [ + 0.15483248233795166, + 0.16576308012008667, + 0.16957050561904907, + 0.14225167036056519, + 0.16980668902397156, + 0.19777557253837585 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7414222955703735, + "mean_weights": [ + 0.18020270764827728, + 0.1900298297405243, + 0.09944288432598114, + 0.11960913985967636, + 0.15577411651611328, + 0.25494131445884705 + ], + "sources": 6 + }, + { + "entropy_mean": 1.733978509902954, + "mean_weights": [ + 0.15204599499702454, + 0.14166373014450073, + 0.1616208702325821, + 0.1256796270608902, + 0.1571422517299652, + 0.26184749603271484 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7344707250595093, + "mean_weights": [ + 0.1829337775707245, + 0.18416841328144073, + 0.09014187753200531, + 0.12335793673992157, + 0.15779981017112732, + 0.2615981996059418 + ], + "sources": 6 + }, + { + "entropy_mean": 1.735896348953247, + "mean_weights": [ + 0.17276549339294434, + 0.15101772546768188, + 0.13220663368701935, + 0.11932596564292908, + 0.15458261966705322, + 0.27010154724121094 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7644665241241455, + "mean_weights": [ + 0.19160008430480957, + 0.1915181279182434, + 0.11520625650882721, + 0.1341419219970703, + 0.1580837219953537, + 0.2094498574733734 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6835086345672607, + "mean_weights": [ + 0.10065847635269165, + 0.09339895844459534, + 0.09682720899581909, + 0.08155890554189682, + 0.09126688539981842, + 0.10349369794130325, + 0.4327958822250366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7937740087509155, + "mean_weights": [ + 0.13723835349082947, + 0.12937892973423004, + 0.07450413703918457, + 0.08537043631076813, + 0.09880411624908447, + 0.12944120168685913, + 0.3452628552913666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5790722370147705, + "mean_weights": [ + 0.09500400722026825, + 0.08892714232206345, + 0.09081116318702698, + 0.07538282871246338, + 0.07685283571481705, + 0.08106325566768646, + 0.49195876717567444 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8034920692443848, + "mean_weights": [ + 0.142653226852417, + 0.13419847190380096, + 0.07437048852443695, + 0.09070360660552979, + 0.1013825386762619, + 0.12073953449726105, + 0.33595213294029236 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5278494358062744, + "mean_weights": [ + 0.10045769065618515, + 0.08844951540231705, + 0.08561251312494278, + 0.06598107516765594, + 0.06767018139362335, + 0.07361933588981628, + 0.51820969581604 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8435695171356201, + "mean_weights": [ + 0.14839640259742737, + 0.1482572704553604, + 0.08113027364015579, + 0.09534022957086563, + 0.10612697899341583, + 0.12550383806228638, + 0.2952450215816498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6421003341674805, + "mean_weights": [ + 0.11422763019800186, + 0.10543526709079742, + 0.09285669028759003, + 0.07746599614620209, + 0.07571054995059967, + 0.08126918226480484, + 0.4530346691608429 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8733552694320679, + "mean_weights": [ + 0.14798465371131897, + 0.16052685678005219, + 0.09668852388858795, + 0.10477185249328613, + 0.10857058316469193, + 0.11694246530532837, + 0.26451510190963745 + ], + "sources": 7 + }, + { + "entropy_mean": 1.784506916999817, + "mean_weights": [ + 0.08589775860309601, + 0.08786045759916306, + 0.07714233547449112, + 0.06314891576766968, + 0.05684572085738182, + 0.056072384119033813, + 0.29875272512435913, + 0.2742796838283539 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9675533771514893, + "mean_weights": [ + 0.11549945175647736, + 0.12592051923274994, + 0.07530814409255981, + 0.08277833461761475, + 0.08296641707420349, + 0.09016790241003036, + 0.17876628041267395, + 0.24859298765659332 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7119876146316528, + "mean_weights": [ + 0.08804735541343689, + 0.07527872174978256, + 0.05666808784008026, + 0.05041901767253876, + 0.05142981559038162, + 0.05313319340348244, + 0.2591783106327057, + 0.3658455014228821 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9094492197036743, + "mean_weights": [ + 0.10337945818901062, + 0.10583682358264923, + 0.08642245084047318, + 0.08043928444385529, + 0.0717030018568039, + 0.0645938515663147, + 0.21096080541610718, + 0.2766643166542053 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7860023975372314, + "mean_weights": [ + 0.09247160702943802, + 0.0784597396850586, + 0.07236532866954803, + 0.05986309051513672, + 0.05389959365129471, + 0.06059400737285614, + 0.2579019069671631, + 0.3244447112083435 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9232951402664185, + "mean_weights": [ + 0.1160542219877243, + 0.11346358060836792, + 0.08816123008728027, + 0.08138670027256012, + 0.07161372900009155, + 0.06013647839426994, + 0.21822679042816162, + 0.2509572505950928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294003009796143, + "mean_weights": [ + 0.08831655979156494, + 0.10072161257266998, + 0.0818098857998848, + 0.07514279335737228, + 0.06651484966278076, + 0.06491033732891083, + 0.31431835889816284, + 0.20826560258865356 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9393219947814941, + "mean_weights": [ + 0.11041727662086487, + 0.12080886960029602, + 0.09995828568935394, + 0.09934850037097931, + 0.08031868189573288, + 0.075287364423275, + 0.2604207992553711, + 0.15344023704528809 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9925593137741089, + "mean_weights": [ + 0.08698172867298126, + 0.08857265114784241, + 0.07299254834651947, + 0.060478877276182175, + 0.054115522652864456, + 0.056821923702955246, + 0.2503892779350281, + 0.14502599835395813, + 0.18462148308753967 + ], + "sources": 9 + }, + { + "entropy_mean": 2.068819999694824, + "mean_weights": [ + 0.09429103136062622, + 0.09028280526399612, + 0.09188265353441238, + 0.0735517293214798, + 0.06746295094490051, + 0.060166649520397186, + 0.21415996551513672, + 0.15325874090194702, + 0.15494348108768463 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0139355659484863, + "mean_weights": [ + 0.0956040769815445, + 0.09831400215625763, + 0.0714857280254364, + 0.0677608773112297, + 0.05654233694076538, + 0.05388312786817551, + 0.24272149801254272, + 0.17641445994377136, + 0.137273907661438 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0349321365356445, + "mean_weights": [ + 0.10256819427013397, + 0.0938093364238739, + 0.0912778377532959, + 0.08394120633602142, + 0.06161238253116608, + 0.053358763456344604, + 0.2261163890361786, + 0.15197426080703735, + 0.13534162938594818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.046657085418701, + "mean_weights": [ + 0.08380690217018127, + 0.1217331811785698, + 0.10242430865764618, + 0.09225839376449585, + 0.06788390129804611, + 0.06336672604084015, + 0.2572976052761078, + 0.11757771670818329, + 0.09365126490592957 + ], + "sources": 9 + }, + { + "entropy_mean": 2.094015121459961, + "mean_weights": [ + 0.10284869372844696, + 0.13057856261730194, + 0.10152992606163025, + 0.10079468786716461, + 0.0796234980225563, + 0.07438632845878601, + 0.2018478661775589, + 0.1111527606844902, + 0.09723767638206482 + ], + "sources": 9 + }, + { + "entropy_mean": 2.057201385498047, + "mean_weights": [ + 0.09265108406543732, + 0.11481083929538727, + 0.09007447212934494, + 0.08496075868606567, + 0.06770835071802139, + 0.05925413966178894, + 0.24551761150360107, + 0.15022093057632446, + 0.09480181336402893 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05595625191926956, + 0.047033343464136124, + 0.0632273480296135, + 0.060984931886196136, + 0.11160287261009216, + 0.09079429507255554, + 0.12218338996171951, + 0.08531255275011063, + 0.12365243583917618, + 0.0786692351102829, + 0.0926642045378685, + 0.08076046407222748, + 0.093480683863163, + 0.08904207497835159, + 0.08749273419380188, + 0.08403915911912918, + 0.09927791357040405, + 0.06012389436364174, + 0.07820390164852142, + 0.06841155141592026, + 0.07669360190629959, + 0.07131828367710114, + 0.07840896397829056, + 0.07358616590499878, + 0.08920575678348541, + 0.06597626209259033, + 0.06850729882717133, + 0.07235056161880493, + 0.07440610229969025, + 0.07261744141578674, + 0.07867594808340073, + 0.07352335751056671, + 0.09058429300785065, + 0.07520031183958054, + 0.07234266400337219, + 0.07378771901130676, + 0.07771008461713791, + 0.07715436071157455, + 0.08416424691677094, + 0.08649503439664841, + 0.08947562426328659, + 0.04556623473763466, + 0.056279782205820084, + 0.04546353965997696, + 0.057985786348581314, + 0.05167657509446144, + 0.06238175556063652, + 0.058827146887779236, + 0.06515856087207794, + 0.04798740893602371, + 0.05041429027915001, + 0.046526361256837845, + 0.05083387345075607, + 0.051699329167604446, + 0.05600840970873833, + 0.059578344225883484, + 0.06128782033920288, + 0.05090610682964325, + 0.056301049888134, + 0.05821637064218521, + 0.06424970924854279, + 0.06188955530524254, + 0.06844019144773483, + 0.06796324998140335 + ], + "loss_nats": 1.4371596574783325, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.8435449600219727, + "mean_weights": [ + 0.1112452894449234, + 0.06898629665374756, + 0.05835210159420967, + 0.042265649884939194, + 0.041034020483493805, + 0.04826357960700989, + 0.10493715852499008, + 0.31774088740348816, + 0.20717504620552063 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04804428666830063, + 0.09451264888048172, + 0.18500657379627228, + 0.2031717449426651, + 0.22299833595752716, + 0.22856928408145905, + 0.2331935465335846, + 0.2430499792098999, + 0.0799286738038063, + 0.08243431895971298, + 0.09402555972337723, + 0.09709372371435165, + 0.10843002796173096, + 0.11282631754875183, + 0.12457633018493652, + 0.13645486533641815, + 0.04508088901638985, + 0.05442482605576515, + 0.0760042667388916, + 0.0885990634560585, + 0.10498681664466858, + 0.12215005606412888, + 0.13159531354904175, + 0.15440259873867035, + 0.0497574545443058, + 0.060873307287693024, + 0.07735268771648407, + 0.09471380710601807, + 0.10224150121212006, + 0.1260664463043213, + 0.13362281024456024, + 0.163865864276886, + 0.079553984105587, + 0.08853588253259659, + 0.10536550730466843, + 0.12131853401660919, + 0.14260481297969818, + 0.15660560131072998, + 0.19080305099487305, + 0.2097783088684082, + 0.03440643846988678, + 0.04455159604549408, + 0.060867179185152054, + 0.07411240041255951, + 0.08364810794591904, + 0.10008715838193893, + 0.11119203269481659, + 0.13061410188674927, + 0.025225287303328514, + 0.04960336163640022, + 0.05530283972620964, + 0.07874798029661179, + 0.08037404716014862, + 0.11358420550823212, + 0.11502843350172043, + 0.1580832600593567, + 0.0242468249052763, + 0.06955241411924362, + 0.07164724916219711, + 0.13302887976169586, + 0.1371813863515854, + 0.22457914054393768, + 0.23144805431365967, + 0.32859006524086 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "population_cv": 0.6027314599912318 + }, + "activation_output_rms_by_block": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22688884753733873, + "first_to_last_ratio": 1.3228372499356764, + "imbalance_abs_log_ratio": 0.2797788616441042, + "last_quartile_mean": 0.17151682684198022, + "mean": 0.17372163233812898, + "normalized": [ + 0.9268367409782202, + 1.919387740182172, + 2.055289136269615, + 2.170611306021289, + 0.6406640960204442, + 0.7450635822219737, + 0.8704616111421293, + 1.1200724150644035, + 0.4345583639666029, + 0.677281348405711, + 0.9287495476341928, + 1.2372530952113672, + 0.4794804996179952, + 0.7087278040538177, + 0.9675973641757485, + 1.32977546738468, + 0.6641820538556276, + 0.930123337383164, + 1.1558946518222766, + 1.5614450026347633, + 0.3131140120506511, + 0.5373702635317168, + 0.742418848247821, + 0.9851745133746388, + 0.3553571676266622, + 0.5579738073870804, + 0.7927337262517246, + 1.0584297759573136, + 0.4595608055861626, + 0.8937492173058054, + 1.5039930398498347, + 2.276669658784396 + ], + "population_cv": 0.5173518438224235 + }, + "bits_per_byte": 1.9433676664104056, + "branch_output_rms_by_sublayer": [ + 0.08528144657611847, + 0.13266965746879578, + 0.2063225358724594, + 0.045500293374061584, + 0.07479097694158554, + 0.018729008734226227, + 0.053203217685222626, + 0.03815881907939911, + 0.1070452407002449, + 0.02209169790148735, + 0.03943018242716789, + 0.01640710048377514, + 0.041509829461574554, + 0.02340921014547348, + 0.06257687509059906, + 0.05481670796871185, + 0.061002109199762344, + 0.037687551230192184, + 0.04113767296075821, + 0.04149388521909714, + 0.040399398654699326, + 0.048693906515836716, + 0.03907898813486099, + 0.06190986558794975, + 0.06714949756860733, + 0.04065176844596863, + 0.04437125846743584, + 0.049332331866025925, + 0.0412614606320858, + 0.0589165985584259, + 0.046277787536382675, + 0.08173932880163193, + 0.10073095560073853, + 0.043543290346860886, + 0.05776162073016167, + 0.05228494107723236, + 0.05185067281126976, + 0.04403071105480194, + 0.07827401161193848, + 0.0687810555100441, + 0.04170985147356987, + 0.030229199677705765, + 0.041845910251140594, + 0.04699210822582245, + 0.038240060210227966, + 0.06082639843225479, + 0.046402644366025925, + 0.07667754590511322, + 0.030182205140590668, + 0.050203632563352585, + 0.025695007294416428, + 0.05539015680551529, + 0.024306239560246468, + 0.06946423649787903, + 0.017157377675175667, + 0.07630576938390732, + 0.030125634744763374, + 0.07239200174808502, + 0.01829664409160614, + 0.09628339111804962, + 0.024464696645736694, + 0.12468792498111725, + 0.032671861350536346, + 0.15056027472019196 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002458031044227025, + 0.00017276993155813553, + 0.00010939491852011586, + 0.000125258738441463, + 0.00011215544691093282, + 7.31087164016768e-05, + 9.049562402296399e-05, + 0.00013630654451635742, + 9.25970647966133e-05, + 7.580293890923953e-05, + 7.957502622146584e-05, + 8.776523713070475e-05, + 0.00010318467043462154, + 0.00010632361913360551, + 0.00011315585331317908, + 0.00012408116098017353, + 0.00012849590289080713, + 0.00014012462333747852, + 0.00013373629199672077, + 0.00019202792118308513, + 0.0005640243510171648, + 0.0006705621295490249, + 0.000579819507581129, + 0.0005486985299520294, + 0.0004559472160958734, + 0.00038131537611027277, + 0.00030455316817913544, + 0.000254991251417659, + 0.00017923215935365346, + 0.0001463414879582755, + 0.00013146832622599984, + 0.0001387805380572583 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0001331616280992935, + "first_to_last_ratio": 0.5346167023449758, + "imbalance_abs_log_ratio": 0.6262052330699333, + "last_quartile_mean": 0.000249078690424766, + "mean": 0.00021243429301935995, + "normalized": [ + 1.1570782707870122, + 0.8132864477883067, + 0.5149588466403882, + 0.5896352074852985, + 0.5279535865742339, + 0.34414743195447317, + 0.4259934812630124, + 0.6416409637964399, + 0.4358856730733892, + 0.3568300476907056, + 0.37458653727914765, + 0.4131406275478619, + 0.4857251104237578, + 0.5005012026185237, + 0.5326628375526298, + 0.584091952464876, + 0.6048736344046712, + 0.6596139509580424, + 0.6295419166835406, + 0.9039403123373536, + 2.655053207279312, + 3.156562530551101, + 2.7294063464992813, + 2.5829093888434693, + 2.146297613325176, + 1.7949803239890372, + 1.433634672869791, + 1.200329983419488, + 0.8437063376453977, + 0.6888788334421074, + 0.6188658354422025, + 0.6532868873699723 + ], + "population_cv": 0.7868568919954686 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.24809812009334564, + "mean_weights": [ + 0.9288002252578735, + 0.07119979709386826 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6067080497741699, + "mean_weights": [ + 0.5604246854782104, + 0.43957534432411194 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3847596049308777, + "mean_weights": [ + 0.8648147583007812, + 0.13518527150154114 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6843481659889221, + "mean_weights": [ + 0.49438929557800293, + 0.5056107044219971 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5728601813316345, + "mean_weights": [ + 0.6954652070999146, + 0.3045347332954407 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686518132686615, + "mean_weights": [ + 0.5263072848320007, + 0.4736926555633545 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044870376586914, + "mean_weights": [ + 0.777843713760376, + 0.2221563160419464 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6724032163619995, + "mean_weights": [ + 0.5694063901901245, + 0.4305936098098755 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9718273282051086, + "mean_weights": [ + 0.5544795989990234, + 0.20597517490386963, + 0.23954525589942932 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856361389160156, + "mean_weights": [ + 0.3703838884830475, + 0.26754841208457947, + 0.36206769943237305 + ], + "sources": 3 + }, + { + "entropy_mean": 1.022249460220337, + "mean_weights": [ + 0.46441033482551575, + 0.21309645473957062, + 0.32249322533607483 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0742855072021484, + "mean_weights": [ + 0.38770636916160583, + 0.24731171131134033, + 0.36498188972473145 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0207322835922241, + "mean_weights": [ + 0.3404029607772827, + 0.22699853777885437, + 0.4325985312461853 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0628657341003418, + "mean_weights": [ + 0.4176652133464813, + 0.22035521268844604, + 0.361979603767395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.029329538345337, + "mean_weights": [ + 0.42153143882751465, + 0.19259268045425415, + 0.3858758807182312 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0849454402923584, + "mean_weights": [ + 0.37970077991485596, + 0.2838303744792938, + 0.3364688754081726 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3144235610961914, + "mean_weights": [ + 0.3205832540988922, + 0.13048580288887024, + 0.2396528720855713, + 0.30927807092666626 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647149801254272, + "mean_weights": [ + 0.3188267648220062, + 0.2379944771528244, + 0.19448311626911163, + 0.24869562685489655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2778496742248535, + "mean_weights": [ + 0.37896889448165894, + 0.16353651881217957, + 0.1984466314315796, + 0.2590479254722595 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3567583560943604, + "mean_weights": [ + 0.29770219326019287, + 0.21492166817188263, + 0.17857596278190613, + 0.3088001608848572 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2773678302764893, + "mean_weights": [ + 0.3856483995914459, + 0.18032220005989075, + 0.1928499937057495, + 0.24117937684059143 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655540943145752, + "mean_weights": [ + 0.29819539189338684, + 0.20063453912734985, + 0.20929867029190063, + 0.29187142848968506 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2754871845245361, + "mean_weights": [ + 0.42651012539863586, + 0.1713070571422577, + 0.1726072132587433, + 0.22957557439804077 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3596348762512207, + "mean_weights": [ + 0.2750997543334961, + 0.22746169567108154, + 0.18702520430088043, + 0.31041333079338074 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5752925872802734, + "mean_weights": [ + 0.28211507201194763, + 0.1675817370414734, + 0.1759152114391327, + 0.16564446687698364, + 0.20874354243278503 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5704567432403564, + "mean_weights": [ + 0.21000899374485016, + 0.18079039454460144, + 0.13544103503227234, + 0.1749160885810852, + 0.29884350299835205 + ], + "sources": 5 + }, + { + "entropy_mean": 1.579707384109497, + "mean_weights": [ + 0.21335941553115845, + 0.18788449466228485, + 0.20250459015369415, + 0.1511598825454712, + 0.24509160220623016 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5663042068481445, + "mean_weights": [ + 0.21676909923553467, + 0.19467836618423462, + 0.13713446259498596, + 0.15313217043876648, + 0.29828590154647827 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5384478569030762, + "mean_weights": [ + 0.21904908120632172, + 0.15864285826683044, + 0.18178997933864594, + 0.14981383085250854, + 0.29070425033569336 + ], + "sources": 5 + }, + { + "entropy_mean": 1.552668571472168, + "mean_weights": [ + 0.22710487246513367, + 0.1922147125005722, + 0.13047492504119873, + 0.14686013758182526, + 0.30334532260894775 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5210195779800415, + "mean_weights": [ + 0.2812899947166443, + 0.14511609077453613, + 0.15909329056739807, + 0.13197475671768188, + 0.282525897026062 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5376527309417725, + "mean_weights": [ + 0.23905543982982635, + 0.21673071384429932, + 0.0950920507311821, + 0.16199833154678345, + 0.2871234714984894 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7209875583648682, + "mean_weights": [ + 0.1703755110502243, + 0.14151470363140106, + 0.17087554931640625, + 0.12742051482200623, + 0.25328749418258667, + 0.1365262269973755 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7200093269348145, + "mean_weights": [ + 0.18992580473423004, + 0.16036251187324524, + 0.08439569175243378, + 0.11250905692577362, + 0.1775888204574585, + 0.27521812915802 + ], + "sources": 6 + }, + { + "entropy_mean": 1.753357172012329, + "mean_weights": [ + 0.15539541840553284, + 0.1388990879058838, + 0.15745995938777924, + 0.12427005916833878, + 0.17903099954128265, + 0.24494445323944092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7199265956878662, + "mean_weights": [ + 0.19207912683486938, + 0.17904852330684662, + 0.08410340547561646, + 0.11219910532236099, + 0.15951165556907654, + 0.2730581760406494 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6756346225738525, + "mean_weights": [ + 0.15863081812858582, + 0.11242729425430298, + 0.15970338881015778, + 0.09774985909461975, + 0.15905189514160156, + 0.3124367296695709 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7271867990493774, + "mean_weights": [ + 0.19560003280639648, + 0.18333327770233154, + 0.08475087583065033, + 0.11598999053239822, + 0.15889793634414673, + 0.2614278793334961 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6916961669921875, + "mean_weights": [ + 0.19065657258033752, + 0.12983909249305725, + 0.11857748031616211, + 0.09883742034435272, + 0.1529499739408493, + 0.3091394901275635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7504222393035889, + "mean_weights": [ + 0.20809361338615417, + 0.19103100895881653, + 0.10743936896324158, + 0.1194620355963707, + 0.15873855352401733, + 0.2152354121208191 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5379457473754883, + "mean_weights": [ + 0.09571269154548645, + 0.07078731805086136, + 0.0824701339006424, + 0.060012347996234894, + 0.08145605027675629, + 0.09452696144580841, + 0.5150344967842102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7319592237472534, + "mean_weights": [ + 0.13678354024887085, + 0.11740625649690628, + 0.06668869405984879, + 0.07360264658927917, + 0.09287553280591965, + 0.12232495844364166, + 0.390318363904953 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4663677215576172, + "mean_weights": [ + 0.09485072642564774, + 0.07167200744152069, + 0.0841417908668518, + 0.05784624442458153, + 0.07052354514598846, + 0.0724712610244751, + 0.5484944581985474 + ], + "sources": 7 + }, + { + "entropy_mean": 1.749481201171875, + "mean_weights": [ + 0.14530614018440247, + 0.12230368703603745, + 0.0660901889204979, + 0.07630561292171478, + 0.09453228116035461, + 0.12125740200281143, + 0.3742046654224396 + ], + "sources": 7 + }, + { + "entropy_mean": 1.472255825996399, + "mean_weights": [ + 0.10688216984272003, + 0.07465942203998566, + 0.08327433466911316, + 0.054589346051216125, + 0.0637175589799881, + 0.07299347221851349, + 0.5438836812973022 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7924723625183105, + "mean_weights": [ + 0.15522946417331696, + 0.13724805414676666, + 0.06889298558235168, + 0.08198928833007812, + 0.09816981852054596, + 0.123870849609375, + 0.3345995247364044 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755163431167603, + "mean_weights": [ + 0.12274101376533508, + 0.08900541067123413, + 0.08551642298698425, + 0.06285787373781204, + 0.07429315149784088, + 0.07904880493879318, + 0.4865373373031616 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8390922546386719, + "mean_weights": [ + 0.15884171426296234, + 0.14726482331752777, + 0.08053631335496902, + 0.09097073972225189, + 0.10738551616668701, + 0.12150256335735321, + 0.29349833726882935 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6550344228744507, + "mean_weights": [ + 0.0844201147556305, + 0.06695197522640228, + 0.061310961842536926, + 0.0447731651365757, + 0.04748314991593361, + 0.045562684535980225, + 0.2905467748641968, + 0.3589511513710022 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8679088354110718, + "mean_weights": [ + 0.11674155294895172, + 0.1088850349187851, + 0.05779466778039932, + 0.06273136287927628, + 0.07066237181425095, + 0.08129343390464783, + 0.17753683030605316, + 0.32435476779937744 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5923268795013428, + "mean_weights": [ + 0.08656083047389984, + 0.05770102143287659, + 0.05105586349964142, + 0.039375390857458115, + 0.04410157352685928, + 0.04500962793827057, + 0.2357216477394104, + 0.4404740333557129 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8469831943511963, + "mean_weights": [ + 0.10824646055698395, + 0.09116974472999573, + 0.0701109915971756, + 0.0629766583442688, + 0.07049155235290527, + 0.07297641038894653, + 0.18795298039913177, + 0.33607521653175354 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6590619087219238, + "mean_weights": [ + 0.0944472923874855, + 0.0654030293226242, + 0.060137998312711716, + 0.0449567511677742, + 0.044405773282051086, + 0.05244143307209015, + 0.2202538251876831, + 0.4179539084434509 + ], + "sources": 8 + }, + { + "entropy_mean": 1.884438157081604, + "mean_weights": [ + 0.11613816022872925, + 0.09775452315807343, + 0.08394932001829147, + 0.07057420164346695, + 0.06920696794986725, + 0.06340071558952332, + 0.21605131030082703, + 0.2829247713088989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7424441576004028, + "mean_weights": [ + 0.08763163536787033, + 0.07771545648574829, + 0.07502467930316925, + 0.06006643921136856, + 0.06314834952354431, + 0.054570816457271576, + 0.3030073344707489, + 0.278835266828537 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8953055143356323, + "mean_weights": [ + 0.1135263741016388, + 0.10274903476238251, + 0.0858360230922699, + 0.08475605398416519, + 0.08078759908676147, + 0.07204630225896835, + 0.2779633104801178, + 0.18233531713485718 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801485300064087, + "mean_weights": [ + 0.07824120670557022, + 0.06023171544075012, + 0.054100219160318375, + 0.039871927350759506, + 0.04207881540060043, + 0.045095670968294144, + 0.2047339826822281, + 0.15412676334381104, + 0.321519672870636 + ], + "sources": 9 + }, + { + "entropy_mean": 2.008213520050049, + "mean_weights": [ + 0.09438058733940125, + 0.07153058052062988, + 0.07423336803913116, + 0.06403130292892456, + 0.0645662397146225, + 0.05577655881643295, + 0.21496367454528809, + 0.1609123945236206, + 0.1996053010225296 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8966764211654663, + "mean_weights": [ + 0.09610478579998016, + 0.07267358899116516, + 0.060228414833545685, + 0.046177469193935394, + 0.04206448048353195, + 0.04096413776278496, + 0.21562573313713074, + 0.21000833809375763, + 0.21615305542945862 + ], + "sources": 9 + }, + { + "entropy_mean": 1.981720209121704, + "mean_weights": [ + 0.10501760244369507, + 0.06946457922458649, + 0.07816961407661438, + 0.07385893911123276, + 0.06364607810974121, + 0.05328303575515747, + 0.23084522783756256, + 0.15472577512264252, + 0.17098914086818695 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0028607845306396, + "mean_weights": [ + 0.0842779204249382, + 0.10298711061477661, + 0.09207960218191147, + 0.07771934568881989, + 0.06265503168106079, + 0.0552704893052578, + 0.2620389461517334, + 0.1382768750190735, + 0.12469469010829926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0655057430267334, + "mean_weights": [ + 0.10472239553928375, + 0.11792458593845367, + 0.0928182601928711, + 0.10183010995388031, + 0.08287280797958374, + 0.06570011377334595, + 0.21871674060821533, + 0.11616810411214828, + 0.09924688935279846 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0258567333221436, + "mean_weights": [ + 0.09674200415611267, + 0.09482249617576599, + 0.08996410667896271, + 0.06836152076721191, + 0.06293784081935883, + 0.05016916245222092, + 0.22868779301643372, + 0.1848844438791275, + 0.12343062460422516 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059412047266960144, + 0.056020382791757584, + 0.08935670554637909, + 0.0742802768945694, + 0.17747226357460022, + 0.1285802572965622, + 0.17946027219295502, + 0.10598119348287582, + 0.17260213196277618, + 0.10133238881826401, + 0.12674333155155182, + 0.10985831171274185, + 0.12412907183170319, + 0.12611564993858337, + 0.11767174303531647, + 0.1125042662024498, + 0.13582420349121094, + 0.07848236709833145, + 0.10855290293693542, + 0.0899009183049202, + 0.10545514523983002, + 0.09790628403425217, + 0.11005345731973648, + 0.09713682532310486, + 0.13049271702766418, + 0.09147685021162033, + 0.09666567295789719, + 0.10080859810113907, + 0.10358717292547226, + 0.09896469116210938, + 0.11041814833879471, + 0.10007023066282272, + 0.13209649920463562, + 0.10974904894828796, + 0.10239993035793304, + 0.10291008651256561, + 0.11153820157051086, + 0.10674569010734558, + 0.11912636458873749, + 0.12148815393447876, + 0.127323716878891, + 0.055056143552064896, + 0.07468672841787338, + 0.05981758236885071, + 0.07975661009550095, + 0.07101839780807495, + 0.08746611326932907, + 0.08339047431945801, + 0.09389390796422958, + 0.06143829599022865, + 0.06575146317481995, + 0.06018613278865814, + 0.0681111291050911, + 0.06816941499710083, + 0.07568862289190292, + 0.08121132105588913, + 0.08424623310565948, + 0.05916989967226982, + 0.0719028189778328, + 0.07458337396383286, + 0.08439797908067703, + 0.08333029597997665, + 0.08961156010627747, + 0.09104464948177338 + ], + "loss_nats": 1.347039818763733, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7394249439239502, + "mean_weights": [ + 0.10602211207151413, + 0.05902107432484627, + 0.049884356558322906, + 0.034865088760852814, + 0.03282830864191055, + 0.043302543461322784, + 0.08693542331457138, + 0.3314756751060486, + 0.2556654214859009 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08528144657611847, + 0.16101159155368805, + 0.3133305609226227, + 0.3334391713142395, + 0.3510810136795044, + 0.3570481836795807, + 0.36267346143722534, + 0.37708213925361633, + 0.1070452407002449, + 0.11129721254110336, + 0.1260577291250229, + 0.12943366169929504, + 0.14502130448818207, + 0.1512180119752884, + 0.17329032719135284, + 0.1945808082818985, + 0.061002109199762344, + 0.07549218833446503, + 0.0987050011754036, + 0.11765842139720917, + 0.13416865468025208, + 0.16134388744831085, + 0.17569874227046967, + 0.21493762731552124, + 0.06714949756860733, + 0.08329613506793976, + 0.09735818952322006, + 0.12312135100364685, + 0.13312570750713348, + 0.16809259355068207, + 0.17941679060459137, + 0.23101076483726501, + 0.10073095560073853, + 0.11538279056549072, + 0.13780468702316284, + 0.16158254444599152, + 0.17965665459632874, + 0.20080390572547913, + 0.24453195929527283, + 0.27125677466392517, + 0.04170985147356987, + 0.0543946772813797, + 0.07352267950773239, + 0.09335283935070038, + 0.10462787002325058, + 0.12897421419620514, + 0.14256280660629272, + 0.17114612460136414, + 0.030182205140590668, + 0.06173322722315788, + 0.06915032118558884, + 0.09693212062120438, + 0.10093247890472412, + 0.13771499693393707, + 0.1392749696969986, + 0.18387214839458466, + 0.030125634744763374, + 0.07983565330505371, + 0.08267941325902939, + 0.15526357293128967, + 0.15948866307735443, + 0.26127612590789795, + 0.2670789062976837, + 0.39550676941871643 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "population_cv": 0.6420341358971389 + }, + "activation_output_rms_by_block": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23796202801167965, + "first_to_last_ratio": 1.7210004980371292, + "imbalance_abs_log_ratio": 0.542905806617582, + "last_quartile_mean": 0.13826958695426583, + "mean": 0.15974999987520278, + "normalized": [ + 1.0588703589137207, + 2.3439518181213, + 2.5018026404306735, + 2.625406629974646, + 0.6192476728139908, + 0.7301872144203714, + 0.8733231516990753, + 1.1639318005158055, + 0.41362918444307983, + 0.6526334563206251, + 0.9108721190988135, + 1.2435622068278698, + 0.40590548584340497, + 0.6529420199592338, + 0.9214830517691831, + 1.2794736788600816, + 0.6292358348878756, + 0.8817240486108118, + 1.0928942572485663, + 1.5151234871428112, + 0.30958389010160414, + 0.5326764501933169, + 0.737604494775539, + 0.9796364773984299, + 0.3464072479049888, + 0.5325333617345424, + 0.7475662125351719, + 0.9760454607844148, + 0.41495429177381643, + 0.7752670281456541, + 1.247487625087717, + 1.8840373416628649 + ], + "population_cv": 0.591409330949146 + }, + "bits_per_byte": 1.8740129107275643, + "branch_output_rms_by_sublayer": [ + 0.0933690071105957, + 0.13583384454250336, + 0.24537226557731628, + 0.0415157787501812, + 0.07698935270309448, + 0.018140379339456558, + 0.0523640513420105, + 0.039518751204013824, + 0.09380833059549332, + 0.022877270355820656, + 0.03647654876112938, + 0.015532293356955051, + 0.039120711386203766, + 0.02314678393304348, + 0.06371714919805527, + 0.05432315170764923, + 0.05156813934445381, + 0.03743875026702881, + 0.03728010877966881, + 0.04006272554397583, + 0.03983496502041817, + 0.045583780854940414, + 0.040284499526023865, + 0.05995568633079529, + 0.046871740370988846, + 0.038651175796985626, + 0.04314620420336723, + 0.04559725895524025, + 0.04033593460917473, + 0.05632574483752251, + 0.04471700266003609, + 0.07952121645212173, + 0.08397205173969269, + 0.042337141931056976, + 0.0528092160820961, + 0.05044656619429588, + 0.04443390294909477, + 0.04418695345520973, + 0.07560808956623077, + 0.06310147047042847, + 0.03708760067820549, + 0.029010163620114326, + 0.03599191829562187, + 0.04586305841803551, + 0.033304885029792786, + 0.06033914536237717, + 0.04061548039317131, + 0.07656749337911606, + 0.025487789884209633, + 0.045884523540735245, + 0.021565141156315804, + 0.051604680716991425, + 0.02342291921377182, + 0.06502768397331238, + 0.017244853079319, + 0.06792232394218445, + 0.024166326969861984, + 0.060306549072265625, + 0.016819356009364128, + 0.08025261014699936, + 0.021590016782283783, + 0.09663844108581543, + 0.026201345026493073, + 0.12230555713176727 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003065916678101079, + 0.00023586292031870838, + 0.0001322871105193671, + 0.00016391009158296018, + 0.0001640597280926294, + 0.00011452252186811768, + 0.00012670462239247783, + 0.00021513822087305528, + 0.00015124888511926465, + 0.00012797922905715274, + 0.0001317390201682035, + 0.00014536342393434786, + 0.00017054401801644127, + 0.00018163687748006386, + 0.00019727429589294548, + 0.0002118946661868627, + 0.0002209032617907253, + 0.0002444386191807451, + 0.00023590577825104205, + 0.00032383840230939837, + 0.0010316902394974224, + 0.0010821684520459678, + 0.0009765210521856827, + 0.0008761640307174725, + 0.0008326776627851861, + 0.0006767392019523396, + 0.00052552098450176, + 0.0004066875433884613, + 0.0003364842674135882, + 0.00025378360531775436, + 0.000205309661176443, + 0.00020659565641729775 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018238461043217796, + "first_to_last_ratio": 0.42368241008054586, + "imbalance_abs_log_ratio": 0.8587711372443304, + "last_quartile_mean": 0.0004304748228691038, + "mean": 0.00035038080369512474, + "normalized": [ + 0.875024158220954, + 0.6731616510701846, + 0.37755239192405493, + 0.46780556998089007, + 0.46823263821091626, + 0.3268515873596963, + 0.3616197607181892, + 0.6140125788975943, + 0.43167001024082974, + 0.3652575361078021, + 0.37598812143496585, + 0.4148726825252455, + 0.486739045683667, + 0.5183984840622454, + 0.5630282647122382, + 0.6047553517550511, + 0.630466222638552, + 0.69763701836085, + 0.6732839692219831, + 0.9242469875466649, + 2.944482770223686, + 3.0885494885375917, + 2.7870278333951726, + 2.500605117282182, + 2.3764933866346176, + 1.9314391508194257, + 1.499856667259172, + 1.160701554136312, + 0.9603387624693381, + 0.7243079604856947, + 0.5859614996348036, + 0.5896317784494322 + ], + "population_cv": 0.8248093894360681 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22886337339878082, + "mean_weights": [ + 0.9364937543869019, + 0.06350618600845337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.603729784488678, + "mean_weights": [ + 0.5866844654083252, + 0.4133155643939972 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3967982828617096, + "mean_weights": [ + 0.858893632888794, + 0.14110638201236725 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6852496862411499, + "mean_weights": [ + 0.48102498054504395, + 0.518975019454956 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5890734195709229, + "mean_weights": [ + 0.6876576542854309, + 0.3123423457145691 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873323321342468, + "mean_weights": [ + 0.5168514847755432, + 0.4831485152244568 + ], + "sources": 2 + }, + { + "entropy_mean": 0.487301766872406, + "mean_weights": [ + 0.7939615249633789, + 0.2060384750366211 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674697756767273, + "mean_weights": [ + 0.5704737305641174, + 0.42952626943588257 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9691112041473389, + "mean_weights": [ + 0.5506306886672974, + 0.18125343322753906, + 0.2681158185005188 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0745112895965576, + "mean_weights": [ + 0.342629998922348, + 0.24457570910453796, + 0.412794291973114 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0074045658111572, + "mean_weights": [ + 0.4399106502532959, + 0.18251104652881622, + 0.3775783181190491 + ], + "sources": 3 + }, + { + "entropy_mean": 1.060497522354126, + "mean_weights": [ + 0.3786153495311737, + 0.2188822329044342, + 0.4025024175643921 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9941019415855408, + "mean_weights": [ + 0.32296958565711975, + 0.19686520099639893, + 0.4801652133464813 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0503522157669067, + "mean_weights": [ + 0.40130966901779175, + 0.19853851199150085, + 0.4001518487930298 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0129402875900269, + "mean_weights": [ + 0.4093267321586609, + 0.16906049847602844, + 0.42161279916763306 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0774019956588745, + "mean_weights": [ + 0.38510820269584656, + 0.25884830951690674, + 0.3560434877872467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2740377187728882, + "mean_weights": [ + 0.29550763964653015, + 0.09541245549917221, + 0.24569378793239594, + 0.3633860945701599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.360302448272705, + "mean_weights": [ + 0.3109527826309204, + 0.20984822511672974, + 0.1935039907693863, + 0.28569501638412476 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2722704410552979, + "mean_weights": [ + 0.350364625453949, + 0.12631991505622864, + 0.22418734431266785, + 0.29912811517715454 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3558036088943481, + "mean_weights": [ + 0.2989768385887146, + 0.1941717267036438, + 0.1926535815000534, + 0.314197838306427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2696614265441895, + "mean_weights": [ + 0.3667730689048767, + 0.15310388803482056, + 0.19384080171585083, + 0.2862822115421295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3606936931610107, + "mean_weights": [ + 0.2982274293899536, + 0.18611131608486176, + 0.21284276247024536, + 0.3028184771537781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.269316554069519, + "mean_weights": [ + 0.4081852436065674, + 0.14420126378536224, + 0.18228203058242798, + 0.2653314471244812 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3636136054992676, + "mean_weights": [ + 0.27427661418914795, + 0.21341991424560547, + 0.19867971539497375, + 0.3136237859725952 + ], + "sources": 4 + }, + { + "entropy_mean": 1.570773959159851, + "mean_weights": [ + 0.26587405800819397, + 0.13419361412525177, + 0.1907704770565033, + 0.17973750829696655, + 0.2294243425130844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5560038089752197, + "mean_weights": [ + 0.20477107167243958, + 0.15721666812896729, + 0.13811248540878296, + 0.1752161681652069, + 0.32468360662460327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5659973621368408, + "mean_weights": [ + 0.20531430840492249, + 0.15986156463623047, + 0.21557101607322693, + 0.15222224593162537, + 0.26703086495399475 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5552496910095215, + "mean_weights": [ + 0.2162160575389862, + 0.17982497811317444, + 0.1307673156261444, + 0.1554446816444397, + 0.31774696707725525 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5068919658660889, + "mean_weights": [ + 0.20646695792675018, + 0.1197461187839508, + 0.19215428829193115, + 0.15179987251758575, + 0.3298327624797821 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5435223579406738, + "mean_weights": [ + 0.22410975396633148, + 0.16986176371574402, + 0.13080401718616486, + 0.15279728174209595, + 0.3224271535873413 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4949920177459717, + "mean_weights": [ + 0.27174079418182373, + 0.12178683280944824, + 0.1529826819896698, + 0.1350194811820984, + 0.31847020983695984 + ], + "sources": 5 + }, + { + "entropy_mean": 1.544264793395996, + "mean_weights": [ + 0.24016115069389343, + 0.20730282366275787, + 0.09692574292421341, + 0.1695065051317215, + 0.2861037850379944 + ], + "sources": 5 + }, + { + "entropy_mean": 1.686682939529419, + "mean_weights": [ + 0.16755425930023193, + 0.10962115228176117, + 0.1783454418182373, + 0.13310004770755768, + 0.2885603904724121, + 0.12281867861747742 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7070834636688232, + "mean_weights": [ + 0.181177020072937, + 0.14557164907455444, + 0.08318156003952026, + 0.11424209922552109, + 0.17698991298675537, + 0.2988377511501312 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7419235706329346, + "mean_weights": [ + 0.1532188206911087, + 0.11420352011919022, + 0.1659742295742035, + 0.1250365972518921, + 0.18140509724617004, + 0.26016172766685486 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7127439975738525, + "mean_weights": [ + 0.18731769919395447, + 0.16715484857559204, + 0.08496320992708206, + 0.11362075805664062, + 0.1561521738767624, + 0.2907913327217102 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6076834201812744, + "mean_weights": [ + 0.15032559633255005, + 0.08624927699565887, + 0.15736976265907288, + 0.08584291487932205, + 0.14632999897003174, + 0.3738824725151062 + ], + "sources": 6 + }, + { + "entropy_mean": 1.709290862083435, + "mean_weights": [ + 0.18991519510746002, + 0.17091524600982666, + 0.07822999358177185, + 0.116765096783638, + 0.1540035903453827, + 0.2901708781719208 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6690887212753296, + "mean_weights": [ + 0.19029225409030914, + 0.11002412438392639, + 0.11471512168645859, + 0.0999990925192833, + 0.1507222205400467, + 0.3342471718788147 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7458693981170654, + "mean_weights": [ + 0.20528709888458252, + 0.1722964346408844, + 0.10435444861650467, + 0.12075909972190857, + 0.16116347908973694, + 0.2361394464969635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4612125158309937, + "mean_weights": [ + 0.08934466540813446, + 0.054379623383283615, + 0.07567165046930313, + 0.05758609622716904, + 0.07884950935840607, + 0.08982288837432861, + 0.5543456077575684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7241814136505127, + "mean_weights": [ + 0.13360008597373962, + 0.10609637945890427, + 0.06363911926746368, + 0.07426095008850098, + 0.09383077174425125, + 0.13373199105262756, + 0.39484071731567383 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3979300260543823, + "mean_weights": [ + 0.08959861099720001, + 0.05414731800556183, + 0.07668881118297577, + 0.05603286623954773, + 0.06979228556156158, + 0.07073447853326797, + 0.5830056071281433 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7459560632705688, + "mean_weights": [ + 0.14318783581256866, + 0.1094653457403183, + 0.06349228322505951, + 0.07859592139720917, + 0.09567482024431229, + 0.13266830146312714, + 0.37691545486450195 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4055393934249878, + "mean_weights": [ + 0.10154954344034195, + 0.05619275942444801, + 0.07791627943515778, + 0.053915880620479584, + 0.06398355215787888, + 0.06915920972824097, + 0.5772827863693237 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7853620052337646, + "mean_weights": [ + 0.15284112095832825, + 0.12206703424453735, + 0.06508444249629974, + 0.08347312361001968, + 0.09811970591545105, + 0.13767032325267792, + 0.3407442569732666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.509842872619629, + "mean_weights": [ + 0.11788781732320786, + 0.06822571903467178, + 0.08556114137172699, + 0.06067090481519699, + 0.07228855788707733, + 0.07258446514606476, + 0.5227814316749573 + ], + "sources": 7 + }, + { + "entropy_mean": 1.839367389678955, + "mean_weights": [ + 0.15906387567520142, + 0.13139380514621735, + 0.07717745006084442, + 0.09607303887605667, + 0.11078508198261261, + 0.12967455387115479, + 0.29583218693733215 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5841834545135498, + "mean_weights": [ + 0.07896879315376282, + 0.048752427101135254, + 0.055707160383462906, + 0.04139826446771622, + 0.04449019208550453, + 0.04285799711942673, + 0.2801477909088135, + 0.40767738223075867 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8125314712524414, + "mean_weights": [ + 0.10889512300491333, + 0.08656036853790283, + 0.048123255372047424, + 0.06561170518398285, + 0.06917259097099304, + 0.08286689221858978, + 0.16869932413101196, + 0.3700707256793976 + ], + "sources": 8 + }, + { + "entropy_mean": 1.50089693069458, + "mean_weights": [ + 0.07974998652935028, + 0.04166398197412491, + 0.04607272148132324, + 0.03586764261126518, + 0.04223340004682541, + 0.04095568507909775, + 0.22249138355255127, + 0.49096518754959106 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8059929609298706, + "mean_weights": [ + 0.10711923241615295, + 0.07651708275079727, + 0.058284737169742584, + 0.06609588116407394, + 0.07139427959918976, + 0.07493151724338531, + 0.17425751686096191, + 0.37139976024627686 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5622732639312744, + "mean_weights": [ + 0.08775827288627625, + 0.047765426337718964, + 0.053928621113300323, + 0.04293380677700043, + 0.04174989461898804, + 0.04734005779027939, + 0.2016349583864212, + 0.4768889844417572 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8757455348968506, + "mean_weights": [ + 0.11563669890165329, + 0.08108697831630707, + 0.07280858606100082, + 0.07553227990865707, + 0.07606013864278793, + 0.06935927271842957, + 0.19856664538383484, + 0.3109493851661682 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6924852132797241, + "mean_weights": [ + 0.08638577163219452, + 0.059912849217653275, + 0.06609142571687698, + 0.056745607405900955, + 0.06277773529291153, + 0.05352145433425903, + 0.2830069661140442, + 0.3315581977367401 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8980522155761719, + "mean_weights": [ + 0.11603894829750061, + 0.08804173022508621, + 0.07975024729967117, + 0.09026999771595001, + 0.08430293202400208, + 0.07433462888002396, + 0.26602888107299805, + 0.2012326419353485 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7512003183364868, + "mean_weights": [ + 0.07744461297988892, + 0.046086352318525314, + 0.05025433003902435, + 0.037484876811504364, + 0.04005061089992523, + 0.041623204946517944, + 0.198423832654953, + 0.17380405962467194, + 0.33482813835144043 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0160694122314453, + "mean_weights": [ + 0.0938527062535286, + 0.06208505481481552, + 0.06560768187046051, + 0.07029052078723907, + 0.06935097277164459, + 0.056832704693078995, + 0.1942581683397293, + 0.1706070452928543, + 0.21711516380310059 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7837426662445068, + "mean_weights": [ + 0.08836328238248825, + 0.04953558370471001, + 0.04813014343380928, + 0.0373263955116272, + 0.037015400826931, + 0.03390160948038101, + 0.1840541511774063, + 0.23140482604503632, + 0.29026859998703003 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9973840713500977, + "mean_weights": [ + 0.10261696577072144, + 0.06039813905954361, + 0.06671924889087677, + 0.08377155661582947, + 0.07002593576908112, + 0.05768897384405136, + 0.20831166207790375, + 0.1594475358724594, + 0.19101998209953308 + ], + "sources": 9 + }, + { + "entropy_mean": 1.969987154006958, + "mean_weights": [ + 0.08120635896921158, + 0.07894632965326309, + 0.08615730702877045, + 0.07609778642654419, + 0.058728985488414764, + 0.05076971277594566, + 0.24503237009048462, + 0.1544169783592224, + 0.16864418983459473 + ], + "sources": 9 + }, + { + "entropy_mean": 2.071648359298706, + "mean_weights": [ + 0.10569609701633453, + 0.09922641515731812, + 0.08180510997772217, + 0.10646798461675644, + 0.0826798751950264, + 0.06615597754716873, + 0.20955613255500793, + 0.13320739567279816, + 0.11520501971244812 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781701564788818, + "mean_weights": [ + 0.09446848928928375, + 0.07056106626987457, + 0.08033449202775955, + 0.0658978745341301, + 0.058234505355358124, + 0.04316519945859909, + 0.2036968618631363, + 0.22251945734024048, + 0.16112206876277924 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0626930519938469, + 0.059514161199331284, + 0.09030644595623016, + 0.08157797157764435, + 0.20229491591453552, + 0.14292451739311218, + 0.20300470292568207, + 0.10997853428125381, + 0.19049881398677826, + 0.0991024449467659, + 0.12695962190628052, + 0.1065012738108635, + 0.12132706493139267, + 0.12155740708112717, + 0.11585728824138641, + 0.1095810979604721, + 0.13427142798900604, + 0.06964563578367233, + 0.10485932976007462, + 0.08311934769153595, + 0.10419609397649765, + 0.09363401681184769, + 0.10852954536676407, + 0.0941167026758194, + 0.1286265105009079, + 0.08564268052577972, + 0.09096142649650574, + 0.09493856877088547, + 0.10080574452877045, + 0.09063401818275452, + 0.10566592961549759, + 0.09473779052495956, + 0.1287592649459839, + 0.10505300015211105, + 0.09650235623121262, + 0.09355910867452621, + 0.1065765842795372, + 0.09618847072124481, + 0.11446042358875275, + 0.11416846513748169, + 0.12395420670509338, + 0.047205954790115356, + 0.07221997529268265, + 0.053352247923612595, + 0.07677769660949707, + 0.06512638926506042, + 0.08416415005922318, + 0.07829347252845764, + 0.09094838052988052, + 0.05368548259139061, + 0.059439923614263535, + 0.05310077592730522, + 0.062182098627090454, + 0.061235878616571426, + 0.06896103918552399, + 0.07320034503936768, + 0.0776243805885315, + 0.05304349213838577, + 0.06285123527050018, + 0.0639304369688034, + 0.07231374084949493, + 0.07404901832342148, + 0.08016443997621536, + 0.08157160878181458 + ], + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.6036605834960938, + "mean_weights": [ + 0.09759747982025146, + 0.04215940088033676, + 0.041145484894514084, + 0.029554029926657677, + 0.02899969555437565, + 0.03647579252719879, + 0.06642461568117142, + 0.341248482465744, + 0.31639501452445984 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0933690071105957, + 0.16915453970432281, + 0.35791492462158203, + 0.374446302652359, + 0.3940892517566681, + 0.39966297149658203, + 0.4057151973247528, + 0.4194087088108063, + 0.09380833059549332, + 0.09892481565475464, + 0.11286329478025436, + 0.11664740741252899, + 0.13231655955314636, + 0.13951337337493896, + 0.1621972620487213, + 0.18593810498714447, + 0.05156813934445381, + 0.06607726216316223, + 0.08533531427383423, + 0.10425819456577301, + 0.12003090232610703, + 0.14551182091236115, + 0.16063299775123596, + 0.19865906238555908, + 0.046871740370988846, + 0.06484340131282806, + 0.079384446144104, + 0.10430748760700226, + 0.11459959298372269, + 0.14720691740512848, + 0.15625107288360596, + 0.20439592003822327, + 0.08397205173969269, + 0.10052042454481125, + 0.11654024571180344, + 0.14085541665554047, + 0.15294702351093292, + 0.1745898574590683, + 0.21572710573673248, + 0.2420409768819809, + 0.03708760067820549, + 0.049456026405096054, + 0.06553087383508682, + 0.08509506285190582, + 0.09390194714069366, + 0.11783231794834137, + 0.12852425873279572, + 0.15649692714214325, + 0.025487789884209633, + 0.05533855780959129, + 0.06059965491294861, + 0.08507220447063446, + 0.08860157430171967, + 0.11942370235919952, + 0.12100546807050705, + 0.1559232622385025, + 0.024166326969861984, + 0.06628894805908203, + 0.06877554953098297, + 0.12384890764951706, + 0.12682415544986725, + 0.19928614795207977, + 0.2025238573551178, + 0.30097496509552 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.14386622172245, + "measured_steps": 7980, + "median_ms": 109.04977201425936, + "p95_ms": 109.64786536205793, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + }, + { + "bits_per_byte": 6.611855126224988, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.582988739013672, + "step": 30, + "unclipped_grad_norm": 3.075686454772949 + }, + { + "bits_per_byte": 6.3915955694037825, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43031644821167, + "step": 40, + "unclipped_grad_norm": 3.2318925857543945 + }, + { + "bits_per_byte": 6.093941755902502, + "learning_rate": 3.75e-05, + "loss_nats": 4.223998546600342, + "step": 50, + "unclipped_grad_norm": 2.8704335689544678 + }, + { + "bits_per_byte": 5.8858000966610895, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.079725742340088, + "step": 60, + "unclipped_grad_norm": 2.669306755065918 + }, + { + "bits_per_byte": 5.715495996470702, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9616799354553223, + "step": 70, + "unclipped_grad_norm": 3.1737496852874756 + }, + { + "bits_per_byte": 5.565897919513593, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8579864501953125, + "step": 80, + "unclipped_grad_norm": 2.5544967651367188 + }, + { + "bits_per_byte": 5.354669171689194, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.711573839187622, + "step": 90, + "unclipped_grad_norm": 2.425950050354004 + }, + { + "bits_per_byte": 5.180876231443403, + "learning_rate": 7.5e-05, + "loss_nats": 3.5911097526550293, + "step": 100, + "unclipped_grad_norm": 2.2783501148223877 + }, + { + "bits_per_byte": 4.992272489270609, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4603796005249023, + "step": 110, + "unclipped_grad_norm": 2.1105899810791016 + }, + { + "bits_per_byte": 4.775783480086919, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3103208541870117, + "step": 120, + "unclipped_grad_norm": 1.8969556093215942 + }, + { + "bits_per_byte": 4.582375918510225, + "learning_rate": 9.75e-05, + "loss_nats": 3.1762609481811523, + "step": 130, + "unclipped_grad_norm": 1.7015109062194824 + }, + { + "bits_per_byte": 4.427903165297061, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0691885948181152, + "step": 140, + "unclipped_grad_norm": 1.446854829788208 + }, + { + "bits_per_byte": 4.271120341117507, + "learning_rate": 0.0001125, + "loss_nats": 2.960515022277832, + "step": 150, + "unclipped_grad_norm": 1.2307006120681763 + }, + { + "bits_per_byte": 4.166711049639903, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.888144016265869, + "step": 160, + "unclipped_grad_norm": 1.0598198175430298 + }, + { + "bits_per_byte": 4.026318516052877, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7908313274383545, + "step": 170, + "unclipped_grad_norm": 0.9654788374900818 + }, + { + "bits_per_byte": 3.94755837315965, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.736238956451416, + "step": 180, + "unclipped_grad_norm": 0.8419675230979919 + }, + { + "bits_per_byte": 3.838975750676209, + "learning_rate": 0.0001425, + "loss_nats": 2.660975217819214, + "step": 190, + "unclipped_grad_norm": 0.817121684551239 + }, + { + "bits_per_byte": 3.774983758012572, + "learning_rate": 0.00015, + "loss_nats": 2.616619348526001, + "step": 200, + "unclipped_grad_norm": 0.955824613571167 + }, + { + "bits_per_byte": 3.7344016998190783, + "learning_rate": 0.0001575, + "loss_nats": 2.5884900093078613, + "step": 210, + "unclipped_grad_norm": 0.8557147979736328 + }, + { + "bits_per_byte": 3.63166201591605, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5172762870788574, + "step": 220, + "unclipped_grad_norm": 0.6891153454780579 + }, + { + "bits_per_byte": 3.6208855830129867, + "learning_rate": 0.0001725, + "loss_nats": 2.5098066329956055, + "step": 230, + "unclipped_grad_norm": 1.036860466003418 + }, + { + "bits_per_byte": 3.4987607026353063, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4251561164855957, + "step": 240, + "unclipped_grad_norm": 0.6406477689743042 + }, + { + "bits_per_byte": 3.5387748741156506, + "learning_rate": 0.0001875, + "loss_nats": 2.4528918266296387, + "step": 250, + "unclipped_grad_norm": 1.106186866760254 + }, + { + "bits_per_byte": 3.5260464381252943, + "learning_rate": 0.000195, + "loss_nats": 2.4440691471099854, + "step": 260, + "unclipped_grad_norm": 0.8556521534919739 + }, + { + "bits_per_byte": 3.5219934949750615, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4412598609924316, + "step": 270, + "unclipped_grad_norm": 0.9917709827423096 + }, + { + "bits_per_byte": 3.474740573719901, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4085066318511963, + "step": 280, + "unclipped_grad_norm": 1.2901222705841064 + }, + { + "bits_per_byte": 3.363479149466621, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.331386089324951, + "step": 290, + "unclipped_grad_norm": 0.7895627617835999 + }, + { + "bits_per_byte": 3.4037392561376993, + "learning_rate": 0.000225, + "loss_nats": 2.3592922687530518, + "step": 300, + "unclipped_grad_norm": 1.5281883478164673 + }, + { + "bits_per_byte": 3.3541442751430894, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.324915647506714, + "step": 310, + "unclipped_grad_norm": 0.8648361563682556 + }, + { + "bits_per_byte": 3.3434108379027414, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174757957458496, + "step": 320, + "unclipped_grad_norm": 1.4613573551177979 + }, + { + "bits_per_byte": 3.3325662998699364, + "learning_rate": 0.0002475, + "loss_nats": 2.3099589347839355, + "step": 330, + "unclipped_grad_norm": 1.1215400695800781 + }, + { + "bits_per_byte": 3.4180712583647006, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3692264556884766, + "step": 340, + "unclipped_grad_norm": 1.1166281700134277 + }, + { + "bits_per_byte": 3.3577222022116393, + "learning_rate": 0.0002625, + "loss_nats": 2.3273956775665283, + "step": 350, + "unclipped_grad_norm": 1.0810235738754272 + }, + { + "bits_per_byte": 3.2923828974611427, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2821059226989746, + "step": 360, + "unclipped_grad_norm": 1.128420114517212 + }, + { + "bits_per_byte": 3.291989401155969, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2818331718444824, + "step": 370, + "unclipped_grad_norm": 1.0846341848373413 + }, + { + "bits_per_byte": 3.3389695579268603, + "learning_rate": 0.000285, + "loss_nats": 2.3143973350524902, + "step": 380, + "unclipped_grad_norm": 1.0868512392044067 + }, + { + "bits_per_byte": 3.256548592327368, + "learning_rate": 0.0002925, + "loss_nats": 2.257267475128174, + "step": 390, + "unclipped_grad_norm": 1.1476690769195557 + }, + { + "bits_per_byte": 3.2058776878689605, + "learning_rate": 0.0003, + "loss_nats": 2.2221450805664062, + "step": 400, + "unclipped_grad_norm": 1.1263198852539062 + }, + { + "bits_per_byte": 3.195324144481415, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.214829921722412, + "step": 410, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 3.227808915506986, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.237346649169922, + "step": 420, + "unclipped_grad_norm": 1.2929860353469849 + }, + { + "bits_per_byte": 3.1876103786249166, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2094831466674805, + "step": 430, + "unclipped_grad_norm": 1.071694016456604 + }, + { + "bits_per_byte": 3.2374495749837515, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2440290451049805, + "step": 440, + "unclipped_grad_norm": 0.9089367985725403 + }, + { + "bits_per_byte": 3.155426921203657, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1871752738952637, + "step": 450, + "unclipped_grad_norm": 1.1129212379455566 + }, + { + "bits_per_byte": 3.188086770567806, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.209813356399536, + "step": 460, + "unclipped_grad_norm": 1.3905333280563354 + }, + { + "bits_per_byte": 3.128408790718412, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.168447732925415, + "step": 470, + "unclipped_grad_norm": 1.059533953666687 + }, + { + "bits_per_byte": 3.136691131218663, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1741886138916016, + "step": 480, + "unclipped_grad_norm": 1.0161854028701782 + }, + { + "bits_per_byte": 3.0887341129908705, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1409473419189453, + "step": 490, + "unclipped_grad_norm": 1.385905385017395 + }, + { + "bits_per_byte": 3.033835874974255, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1028947830200195, + "step": 500, + "unclipped_grad_norm": 1.1716443300247192 + }, + { + "bits_per_byte": 3.0607071322756636, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.121520519256592, + "step": 510, + "unclipped_grad_norm": 0.947066605091095 + }, + { + "bits_per_byte": 2.997100036784657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0774314403533936, + "step": 520, + "unclipped_grad_norm": 1.352820634841919 + }, + { + "bits_per_byte": 3.0643050093317137, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.124014377593994, + "step": 530, + "unclipped_grad_norm": 1.2646857500076294 + }, + { + "bits_per_byte": 3.008989197438803, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.085672378540039, + "step": 540, + "unclipped_grad_norm": 1.0451104640960693 + }, + { + "bits_per_byte": 2.978105584892765, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.064265489578247, + "step": 550, + "unclipped_grad_norm": 1.2619587182998657 + }, + { + "bits_per_byte": 2.968300166036473, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.057468891143799, + "step": 560, + "unclipped_grad_norm": 0.9861598610877991 + }, + { + "bits_per_byte": 2.9221661639078262, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.025491237640381, + "step": 570, + "unclipped_grad_norm": 1.1261875629425049 + }, + { + "bits_per_byte": 2.96999178739036, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0586414337158203, + "step": 580, + "unclipped_grad_norm": 0.8640331625938416 + }, + { + "bits_per_byte": 2.883698804389342, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9988276958465576, + "step": 590, + "unclipped_grad_norm": 1.0034456253051758 + }, + { + "bits_per_byte": 2.9404706214044567, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0381789207458496, + "step": 600, + "unclipped_grad_norm": 0.8714175224304199 + }, + { + "bits_per_byte": 2.8380133330141457, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.967160940170288, + "step": 610, + "unclipped_grad_norm": 0.9427422881126404 + }, + { + "bits_per_byte": 2.894644124255553, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0064144134521484, + "step": 620, + "unclipped_grad_norm": 1.0238139629364014 + }, + { + "bits_per_byte": 2.81883675149499, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9538687467575073, + "step": 630, + "unclipped_grad_norm": 0.9875481128692627 + }, + { + "bits_per_byte": 2.7787291934352267, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9260683059692383, + "step": 640, + "unclipped_grad_norm": 1.0125378370285034 + }, + { + "bits_per_byte": 2.728423580127252, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8911991119384766, + "step": 650, + "unclipped_grad_norm": 1.5507988929748535 + }, + { + "bits_per_byte": 2.649404087396524, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8364269733428955, + "step": 660, + "unclipped_grad_norm": 0.9040125012397766 + }, + { + "bits_per_byte": 2.7024973674923345, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8732284307479858, + "step": 670, + "unclipped_grad_norm": 1.205861210823059 + }, + { + "bits_per_byte": 2.641403626461037, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8308814764022827, + "step": 680, + "unclipped_grad_norm": 1.3370649814605713 + }, + { + "bits_per_byte": 2.7325929555321187, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8940891027450562, + "step": 690, + "unclipped_grad_norm": 1.6124995946884155 + }, + { + "bits_per_byte": 2.6130706886099455, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8112425804138184, + "step": 700, + "unclipped_grad_norm": 1.08383309841156 + }, + { + "bits_per_byte": 2.628106443844277, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.821664571762085, + "step": 710, + "unclipped_grad_norm": 1.0767779350280762 + }, + { + "bits_per_byte": 2.6725746220166666, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.852487564086914, + "step": 720, + "unclipped_grad_norm": 1.296352505683899 + }, + { + "bits_per_byte": 2.608218370098502, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8078792095184326, + "step": 730, + "unclipped_grad_norm": 1.0864766836166382 + }, + { + "bits_per_byte": 2.555699340073923, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7714757919311523, + "step": 740, + "unclipped_grad_norm": 1.0339913368225098 + }, + { + "bits_per_byte": 2.515936435246489, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743914246559143, + "step": 750, + "unclipped_grad_norm": 1.1017143726348877 + }, + { + "bits_per_byte": 2.4657125136976736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.709101676940918, + "step": 760, + "unclipped_grad_norm": 1.7173939943313599 + }, + { + "bits_per_byte": 2.601046349592221, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.802907943725586, + "step": 770, + "unclipped_grad_norm": 0.9487742781639099 + }, + { + "bits_per_byte": 2.4794518637098015, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7186250686645508, + "step": 780, + "unclipped_grad_norm": 0.9247332811355591 + }, + { + "bits_per_byte": 2.466098098800908, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7093689441680908, + "step": 790, + "unclipped_grad_norm": 1.9121034145355225 + }, + { + "bits_per_byte": 2.4025570446478333, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.66532564163208, + "step": 800, + "unclipped_grad_norm": 1.1767630577087402 + }, + { + "bits_per_byte": 2.455517898102479, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7020353078842163, + "step": 810, + "unclipped_grad_norm": 1.1299240589141846 + }, + { + "bits_per_byte": 2.400107839716896, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6636279821395874, + "step": 820, + "unclipped_grad_norm": 1.0006046295166016 + }, + { + "bits_per_byte": 2.4500607166079513, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6982526779174805, + "step": 830, + "unclipped_grad_norm": 1.0554603338241577 + }, + { + "bits_per_byte": 2.5153210813217015, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7434877157211304, + "step": 840, + "unclipped_grad_norm": 1.0736713409423828 + }, + { + "bits_per_byte": 2.399556119372928, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6632455587387085, + "step": 850, + "unclipped_grad_norm": 0.8279580473899841 + }, + { + "bits_per_byte": 2.414819407654351, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6738252639770508, + "step": 860, + "unclipped_grad_norm": 1.2342168092727661 + }, + { + "bits_per_byte": 2.3222158652452247, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6096373796463013, + "step": 870, + "unclipped_grad_norm": 1.0648512840270996 + }, + { + "bits_per_byte": 2.3629336177502487, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6378607749938965, + "step": 880, + "unclipped_grad_norm": 0.9258757829666138 + }, + { + "bits_per_byte": 2.2961121665146167, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5915436744689941, + "step": 890, + "unclipped_grad_norm": 1.0717874765396118 + }, + { + "bits_per_byte": 2.3158979105831285, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6052581071853638, + "step": 900, + "unclipped_grad_norm": 0.9598665833473206 + }, + { + "bits_per_byte": 2.3301258633063577, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6151201725006104, + "step": 910, + "unclipped_grad_norm": 1.0995051860809326 + }, + { + "bits_per_byte": 2.3123376977276173, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.602790355682373, + "step": 920, + "unclipped_grad_norm": 1.1837356090545654 + }, + { + "bits_per_byte": 2.3340250539667196, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6178228855133057, + "step": 930, + "unclipped_grad_norm": 1.1502224206924438 + }, + { + "bits_per_byte": 2.330418921743428, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6153233051300049, + "step": 940, + "unclipped_grad_norm": 1.0088714361190796 + }, + { + "bits_per_byte": 2.2995397807463123, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5939195156097412, + "step": 950, + "unclipped_grad_norm": 0.9483612179756165 + }, + { + "bits_per_byte": 2.3536873144919115, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6314517259597778, + "step": 960, + "unclipped_grad_norm": 0.9582105278968811 + }, + { + "bits_per_byte": 2.2526236015215417, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5613996982574463, + "step": 970, + "unclipped_grad_norm": 1.2002545595169067 + }, + { + "bits_per_byte": 2.3230659754884395, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6102266311645508, + "step": 980, + "unclipped_grad_norm": 0.9191399812698364 + }, + { + "bits_per_byte": 2.285470911975131, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.584167718887329, + "step": 990, + "unclipped_grad_norm": 1.053741216659546 + }, + { + "bits_per_byte": 2.248061417742104, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5582374334335327, + "step": 1000, + "unclipped_grad_norm": 0.9024814963340759 + }, + { + "bits_per_byte": 2.2402749032242903, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.552840232849121, + "step": 1010, + "unclipped_grad_norm": 1.048907995223999 + }, + { + "bits_per_byte": 2.2041726491204714, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278160572052002, + "step": 1020, + "unclipped_grad_norm": 1.0486118793487549 + }, + { + "bits_per_byte": 2.216881995036582, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5366255044937134, + "step": 1030, + "unclipped_grad_norm": 0.9243784546852112 + }, + { + "bits_per_byte": 2.2610579746851553, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5672459602355957, + "step": 1040, + "unclipped_grad_norm": 1.102870225906372 + }, + { + "bits_per_byte": 2.2528270569975124, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5615407228469849, + "step": 1050, + "unclipped_grad_norm": 1.0764544010162354 + }, + { + "bits_per_byte": 2.14889054582713, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.489497423171997, + "step": 1060, + "unclipped_grad_norm": 0.971049427986145 + }, + { + "bits_per_byte": 2.2350196293618754, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5491975545883179, + "step": 1070, + "unclipped_grad_norm": 1.0203380584716797 + }, + { + "bits_per_byte": 2.217042282867186, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5367366075515747, + "step": 1080, + "unclipped_grad_norm": 1.023221492767334 + }, + { + "bits_per_byte": 2.186512266651322, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5155748128890991, + "step": 1090, + "unclipped_grad_norm": 1.111006498336792 + }, + { + "bits_per_byte": 2.2323391077655277, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5473395586013794, + "step": 1100, + "unclipped_grad_norm": 0.9547863602638245 + }, + { + "bits_per_byte": 2.116467344590568, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4670233726501465, + "step": 1110, + "unclipped_grad_norm": 1.1085364818572998 + }, + { + "bits_per_byte": 2.143238164023873, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.485579490661621, + "step": 1120, + "unclipped_grad_norm": 0.901618242263794 + }, + { + "bits_per_byte": 2.077939275196329, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4403177499771118, + "step": 1130, + "unclipped_grad_norm": 0.8683149814605713 + }, + { + "bits_per_byte": 2.224320932617002, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5417817831039429, + "step": 1140, + "unclipped_grad_norm": 1.030465006828308 + }, + { + "bits_per_byte": 2.0642131678483175, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4308035373687744, + "step": 1150, + "unclipped_grad_norm": 0.827274739742279 + }, + { + "bits_per_byte": 2.103388407957753, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4579577445983887, + "step": 1160, + "unclipped_grad_norm": 0.9158841967582703 + }, + { + "bits_per_byte": 2.1566975262803965, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4949088096618652, + "step": 1170, + "unclipped_grad_norm": 0.9397322535514832 + }, + { + "bits_per_byte": 2.1322636071070153, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4779725074768066, + "step": 1180, + "unclipped_grad_norm": 0.8510220050811768 + }, + { + "bits_per_byte": 2.068760561119781, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4339555501937866, + "step": 1190, + "unclipped_grad_norm": 0.9270830154418945 + }, + { + "bits_per_byte": 2.0310528489701576, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4078185558319092, + "step": 1200, + "unclipped_grad_norm": 0.8815417885780334 + }, + { + "bits_per_byte": 2.1008361854189492, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.456188678741455, + "step": 1210, + "unclipped_grad_norm": 1.2037686109542847 + }, + { + "bits_per_byte": 2.1123913557651197, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.464198112487793, + "step": 1220, + "unclipped_grad_norm": 0.8964646458625793 + }, + { + "bits_per_byte": 2.0711254945517954, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4355947971343994, + "step": 1230, + "unclipped_grad_norm": 0.8661414980888367 + }, + { + "bits_per_byte": 2.0159510523978943, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.397350788116455, + "step": 1240, + "unclipped_grad_norm": 0.9392534494400024 + }, + { + "bits_per_byte": 2.104896867788471, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4590033292770386, + "step": 1250, + "unclipped_grad_norm": 0.8430818915367126 + }, + { + "bits_per_byte": 2.08180441329182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.442996859550476, + "step": 1260, + "unclipped_grad_norm": 0.8917548656463623 + }, + { + "bits_per_byte": 2.087352401626003, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4468424320220947, + "step": 1270, + "unclipped_grad_norm": 0.9866762161254883 + }, + { + "bits_per_byte": 1.9607164163161914, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.359065055847168, + "step": 1280, + "unclipped_grad_norm": 0.8596939444541931 + }, + { + "bits_per_byte": 2.088993460080536, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4479799270629883, + "step": 1290, + "unclipped_grad_norm": 1.007126808166504 + }, + { + "bits_per_byte": 2.034349412421896, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4101035594940186, + "step": 1300, + "unclipped_grad_norm": 0.8474438786506653 + }, + { + "bits_per_byte": 2.0247273270714237, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4034340381622314, + "step": 1310, + "unclipped_grad_norm": 0.8226730823516846 + }, + { + "bits_per_byte": 2.020511000402871, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4005115032196045, + "step": 1320, + "unclipped_grad_norm": 0.9414357542991638 + }, + { + "bits_per_byte": 2.085571349293668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4456079006195068, + "step": 1330, + "unclipped_grad_norm": 0.9917200207710266 + }, + { + "bits_per_byte": 2.0422444479924042, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4155759811401367, + "step": 1340, + "unclipped_grad_norm": 0.8823419213294983 + }, + { + "bits_per_byte": 2.0447034559344406, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4172804355621338, + "step": 1350, + "unclipped_grad_norm": 0.8861486315727234 + }, + { + "bits_per_byte": 2.0988803987133373, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548330307006836, + "step": 1360, + "unclipped_grad_norm": 0.9090827107429504 + }, + { + "bits_per_byte": 2.082760980795919, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4436599016189575, + "step": 1370, + "unclipped_grad_norm": 0.8366561532020569 + }, + { + "bits_per_byte": 2.0317923743688677, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4083311557769775, + "step": 1380, + "unclipped_grad_norm": 0.8379060626029968 + }, + { + "bits_per_byte": 2.0432778917414387, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4162923097610474, + "step": 1390, + "unclipped_grad_norm": 0.8665340542793274 + }, + { + "bits_per_byte": 1.9746810636009497, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3687446117401123, + "step": 1400, + "unclipped_grad_norm": 0.8112426996231079 + }, + { + "bits_per_byte": 2.0034977886489185, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.388718843460083, + "step": 1410, + "unclipped_grad_norm": 0.8745980262756348 + }, + { + "bits_per_byte": 2.019574210903621, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.3998621702194214, + "step": 1420, + "unclipped_grad_norm": 0.8596643209457397 + }, + { + "bits_per_byte": 2.0775151659793014, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4400237798690796, + "step": 1430, + "unclipped_grad_norm": 0.913244903087616 + }, + { + "bits_per_byte": 2.027370700415185, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.405266284942627, + "step": 1440, + "unclipped_grad_norm": 1.0067864656448364 + }, + { + "bits_per_byte": 1.9631205617926026, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3607314825057983, + "step": 1450, + "unclipped_grad_norm": 0.8125852942466736 + }, + { + "bits_per_byte": 1.9959419028659118, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.383481502532959, + "step": 1460, + "unclipped_grad_norm": 0.9857028126716614 + }, + { + "bits_per_byte": 1.9364156117145783, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3422210216522217, + "step": 1470, + "unclipped_grad_norm": 0.8698804974555969 + }, + { + "bits_per_byte": 1.996789089404062, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3840687274932861, + "step": 1480, + "unclipped_grad_norm": 0.8397032022476196 + }, + { + "bits_per_byte": 2.017818268038312, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.398645043373108, + "step": 1490, + "unclipped_grad_norm": 0.8181726932525635 + }, + { + "bits_per_byte": 1.996446843928845, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.38383150100708, + "step": 1500, + "unclipped_grad_norm": 0.8242432475090027 + }, + { + "bits_per_byte": 2.0162594172908914, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3975645303726196, + "step": 1510, + "unclipped_grad_norm": 0.9299814701080322 + }, + { + "bits_per_byte": 1.980743108078562, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3729465007781982, + "step": 1520, + "unclipped_grad_norm": 0.8446369171142578 + }, + { + "bits_per_byte": 2.047233664693934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4190342426300049, + "step": 1530, + "unclipped_grad_norm": 0.9717140197753906 + }, + { + "bits_per_byte": 1.993557363411699, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3818286657333374, + "step": 1540, + "unclipped_grad_norm": 0.8417568802833557 + }, + { + "bits_per_byte": 2.049996737962696, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209494590759277, + "step": 1550, + "unclipped_grad_norm": 0.8546327352523804 + }, + { + "bits_per_byte": 1.966114779744124, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3628069162368774, + "step": 1560, + "unclipped_grad_norm": 0.9258830547332764 + }, + { + "bits_per_byte": 2.0427641795633114, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4159362316131592, + "step": 1570, + "unclipped_grad_norm": 0.8567822575569153 + }, + { + "bits_per_byte": 1.9830621221427962, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.374553918838501, + "step": 1580, + "unclipped_grad_norm": 0.8522606492042542 + }, + { + "bits_per_byte": 1.9876789964052082, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3777540922164917, + "step": 1590, + "unclipped_grad_norm": 0.8731296062469482 + }, + { + "bits_per_byte": 2.0122188568915207, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3947638273239136, + "step": 1600, + "unclipped_grad_norm": 0.8627029657363892 + }, + { + "bits_per_byte": 1.9498011934138815, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3514991998672485, + "step": 1610, + "unclipped_grad_norm": 0.7740172147750854 + }, + { + "bits_per_byte": 1.9095543294069197, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3236021995544434, + "step": 1620, + "unclipped_grad_norm": 0.7910023331642151 + }, + { + "bits_per_byte": 1.9407452749500496, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3452221155166626, + "step": 1630, + "unclipped_grad_norm": 0.8484107851982117 + }, + { + "bits_per_byte": 1.9911346438089947, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3801493644714355, + "step": 1640, + "unclipped_grad_norm": 0.8722788691520691 + }, + { + "bits_per_byte": 2.0368601871418424, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4118438959121704, + "step": 1650, + "unclipped_grad_norm": 0.8482462763786316 + }, + { + "bits_per_byte": 1.9665676100638458, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3631207942962646, + "step": 1660, + "unclipped_grad_norm": 0.8444085121154785 + }, + { + "bits_per_byte": 1.9945896032821775, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3825441598892212, + "step": 1670, + "unclipped_grad_norm": 0.8560684323310852 + }, + { + "bits_per_byte": 2.0061820938635853, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3905794620513916, + "step": 1680, + "unclipped_grad_norm": 0.8526912927627563 + }, + { + "bits_per_byte": 2.0514603103215387, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4219639301300049, + "step": 1690, + "unclipped_grad_norm": 0.8917502760887146 + }, + { + "bits_per_byte": 1.9422809080396035, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3462865352630615, + "step": 1700, + "unclipped_grad_norm": 0.7474871873855591 + }, + { + "bits_per_byte": 1.878015978909047, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3017414808273315, + "step": 1710, + "unclipped_grad_norm": 0.8072315454483032 + }, + { + "bits_per_byte": 1.8905051870320533, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3103983402252197, + "step": 1720, + "unclipped_grad_norm": 0.845556914806366 + }, + { + "bits_per_byte": 1.8762399140735866, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3005104064941406, + "step": 1730, + "unclipped_grad_norm": 0.9344940185546875 + }, + { + "bits_per_byte": 1.9632232354351677, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3608026504516602, + "step": 1740, + "unclipped_grad_norm": 0.7860697507858276 + }, + { + "bits_per_byte": 1.989613629244764, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3790950775146484, + "step": 1750, + "unclipped_grad_norm": 0.8348115086555481 + }, + { + "bits_per_byte": 1.9869124697303127, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3772227764129639, + "step": 1760, + "unclipped_grad_norm": 0.8862095475196838 + }, + { + "bits_per_byte": 1.9928130224987648, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3813127279281616, + "step": 1770, + "unclipped_grad_norm": 0.816246509552002 + }, + { + "bits_per_byte": 1.8514405317671223, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.2833207845687866, + "step": 1780, + "unclipped_grad_norm": 0.8118871450424194 + }, + { + "bits_per_byte": 1.9588221993995885, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3577520847320557, + "step": 1790, + "unclipped_grad_norm": 0.839783251285553 + }, + { + "bits_per_byte": 1.8827697513615458, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3050365447998047, + "step": 1800, + "unclipped_grad_norm": 0.8949951529502869 + }, + { + "bits_per_byte": 1.92937567388416, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.33734130859375, + "step": 1810, + "unclipped_grad_norm": 0.7295374274253845 + }, + { + "bits_per_byte": 1.8974312723476003, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3151991367340088, + "step": 1820, + "unclipped_grad_norm": 0.7755441069602966 + }, + { + "bits_per_byte": 1.8886371114783813, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3091034889221191, + "step": 1830, + "unclipped_grad_norm": 0.7666956186294556 + }, + { + "bits_per_byte": 1.8876932706904463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3084492683410645, + "step": 1840, + "unclipped_grad_norm": 0.8784642219543457 + }, + { + "bits_per_byte": 1.8964055678178544, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.314488172531128, + "step": 1850, + "unclipped_grad_norm": 0.8603965044021606 + }, + { + "bits_per_byte": 1.8927124123732264, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3119282722473145, + "step": 1860, + "unclipped_grad_norm": 0.8056252002716064 + }, + { + "bits_per_byte": 1.8818489562488263, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3043982982635498, + "step": 1870, + "unclipped_grad_norm": 0.8151036500930786 + }, + { + "bits_per_byte": 1.8811881989042114, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3039402961730957, + "step": 1880, + "unclipped_grad_norm": 0.79389888048172 + }, + { + "bits_per_byte": 1.8949089747900463, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.313450813293457, + "step": 1890, + "unclipped_grad_norm": 0.8192400932312012 + }, + { + "bits_per_byte": 1.9233574849825177, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3331698179244995, + "step": 1900, + "unclipped_grad_norm": 0.7950363159179688 + }, + { + "bits_per_byte": 1.8611783614416253, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2900705337524414, + "step": 1910, + "unclipped_grad_norm": 0.8037525415420532 + }, + { + "bits_per_byte": 1.855479888287937, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2861206531524658, + "step": 1920, + "unclipped_grad_norm": 0.7507341504096985 + }, + { + "bits_per_byte": 1.9602173226633874, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3587191104888916, + "step": 1930, + "unclipped_grad_norm": 0.8329893350601196 + }, + { + "bits_per_byte": 1.8066889261861108, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2523013353347778, + "step": 1940, + "unclipped_grad_norm": 0.7595474123954773 + }, + { + "bits_per_byte": 1.9033319970987033, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.319289207458496, + "step": 1950, + "unclipped_grad_norm": 0.7690061330795288 + }, + { + "bits_per_byte": 1.816397862775269, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259031057357788, + "step": 1960, + "unclipped_grad_norm": 0.7758741974830627 + }, + { + "bits_per_byte": 1.909067618504978, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3232648372650146, + "step": 1970, + "unclipped_grad_norm": 0.7503845691680908 + }, + { + "bits_per_byte": 1.9405849871194454, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3451110124588013, + "step": 1980, + "unclipped_grad_norm": 0.8655099868774414 + }, + { + "bits_per_byte": 1.9436471382180576, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.347233533859253, + "step": 1990, + "unclipped_grad_norm": 0.8436980247497559 + }, + { + "bits_per_byte": 1.9015801818170146, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3180749416351318, + "step": 2000, + "unclipped_grad_norm": 0.8021923303604126 + }, + { + "bits_per_byte": 1.8746163978494417, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.2993850708007812, + "step": 2010, + "unclipped_grad_norm": 0.7083144187927246 + }, + { + "bits_per_byte": 1.9072580170525995, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3220105171203613, + "step": 2020, + "unclipped_grad_norm": 0.7678324580192566 + }, + { + "bits_per_byte": 1.983610230851096, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3749338388442993, + "step": 2030, + "unclipped_grad_norm": 0.8317387104034424 + }, + { + "bits_per_byte": 1.909348466173837, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.323459506034851, + "step": 2040, + "unclipped_grad_norm": 0.8302199840545654 + }, + { + "bits_per_byte": 1.961244747019642, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.359431266784668, + "step": 2050, + "unclipped_grad_norm": 0.7990598082542419 + }, + { + "bits_per_byte": 1.9220292629699043, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3322491645812988, + "step": 2060, + "unclipped_grad_norm": 0.8404503464698792 + }, + { + "bits_per_byte": 1.8663830724046866, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2936781644821167, + "step": 2070, + "unclipped_grad_norm": 0.7720745801925659 + }, + { + "bits_per_byte": 1.8672120287818452, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2942527532577515, + "step": 2080, + "unclipped_grad_norm": 0.8028935790061951 + }, + { + "bits_per_byte": 1.857879218250124, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2877837419509888, + "step": 2090, + "unclipped_grad_norm": 0.7713416814804077 + }, + { + "bits_per_byte": 1.8603779541845098, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.289515733718872, + "step": 2100, + "unclipped_grad_norm": 0.7443708181381226 + }, + { + "bits_per_byte": 1.897931741861611, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3155460357666016, + "step": 2110, + "unclipped_grad_norm": 0.7534405589103699 + }, + { + "bits_per_byte": 1.8710701155886507, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969269752502441, + "step": 2120, + "unclipped_grad_norm": 0.7505779266357422 + }, + { + "bits_per_byte": 1.8349618420920515, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.271898627281189, + "step": 2130, + "unclipped_grad_norm": 0.7835680246353149 + }, + { + "bits_per_byte": 1.9616896661374241, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3597396612167358, + "step": 2140, + "unclipped_grad_norm": 0.7786996960639954 + }, + { + "bits_per_byte": 1.8168395142226847, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2593371868133545, + "step": 2150, + "unclipped_grad_norm": 0.7194197773933411 + }, + { + "bits_per_byte": 1.8739360344826286, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.2989134788513184, + "step": 2160, + "unclipped_grad_norm": 0.8012002110481262 + }, + { + "bits_per_byte": 1.9634492206384013, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3609592914581299, + "step": 2170, + "unclipped_grad_norm": 0.8270084261894226 + }, + { + "bits_per_byte": 1.8175344961148212, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2598189115524292, + "step": 2180, + "unclipped_grad_norm": 0.7824540734291077 + }, + { + "bits_per_byte": 1.8458584908680682, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.279451608657837, + "step": 2190, + "unclipped_grad_norm": 0.7973959445953369 + }, + { + "bits_per_byte": 1.8557901449900935, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2863357067108154, + "step": 2200, + "unclipped_grad_norm": 0.7451788783073425 + }, + { + "bits_per_byte": 1.8380872828061812, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2740650177001953, + "step": 2210, + "unclipped_grad_norm": 0.7804975509643555 + }, + { + "bits_per_byte": 1.8298283319464475, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2683403491973877, + "step": 2220, + "unclipped_grad_norm": 0.7694668173789978 + }, + { + "bits_per_byte": 1.9031531351418058, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3191652297973633, + "step": 2230, + "unclipped_grad_norm": 0.8418469429016113 + }, + { + "bits_per_byte": 1.8400618156207371, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2754336595535278, + "step": 2240, + "unclipped_grad_norm": 0.7987614870071411 + }, + { + "bits_per_byte": 1.852736936989326, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2842193841934204, + "step": 2250, + "unclipped_grad_norm": 0.7795421481132507 + }, + { + "bits_per_byte": 1.8339105121473271, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.271169900894165, + "step": 2260, + "unclipped_grad_norm": 0.7681040167808533 + }, + { + "bits_per_byte": 1.825916414569922, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2656288146972656, + "step": 2270, + "unclipped_grad_norm": 0.7933830618858337 + }, + { + "bits_per_byte": 1.8451832869807807, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2789835929870605, + "step": 2280, + "unclipped_grad_norm": 0.7935850024223328 + }, + { + "bits_per_byte": 1.8711039961708706, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2969504594802856, + "step": 2290, + "unclipped_grad_norm": 0.7818295955657959 + }, + { + "bits_per_byte": 1.7944778140095496, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2438372373580933, + "step": 2300, + "unclipped_grad_norm": 0.7245032787322998 + }, + { + "bits_per_byte": 1.8221434591752936, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2630136013031006, + "step": 2310, + "unclipped_grad_norm": 0.7396021485328674 + }, + { + "bits_per_byte": 1.7928696042413317, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.242722511291504, + "step": 2320, + "unclipped_grad_norm": 0.7618010640144348 + }, + { + "bits_per_byte": 1.8702191454321815, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2963371276855469, + "step": 2330, + "unclipped_grad_norm": 0.7263991236686707 + }, + { + "bits_per_byte": 1.7994940319872652, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.247314214706421, + "step": 2340, + "unclipped_grad_norm": 0.7788637280464172 + }, + { + "bits_per_byte": 1.849551646312696, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2820115089416504, + "step": 2350, + "unclipped_grad_norm": 0.7732409238815308 + }, + { + "bits_per_byte": 1.8613248906601605, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2901721000671387, + "step": 2360, + "unclipped_grad_norm": 0.8177725672721863 + }, + { + "bits_per_byte": 1.8822589628884834, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3046824932098389, + "step": 2370, + "unclipped_grad_norm": 0.7931574583053589 + }, + { + "bits_per_byte": 1.8382422391746087, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2741724252700806, + "step": 2380, + "unclipped_grad_norm": 0.8604201674461365 + }, + { + "bits_per_byte": 1.8797817247854554, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3029654026031494, + "step": 2390, + "unclipped_grad_norm": 0.7636454105377197 + }, + { + "bits_per_byte": 1.817860919186159, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2600451707839966, + "step": 2400, + "unclipped_grad_norm": 0.779283881187439 + }, + { + "bits_per_byte": 1.8354538844561699, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2722396850585938, + "step": 2410, + "unclipped_grad_norm": 0.7781111001968384 + }, + { + "bits_per_byte": 1.8069649583407457, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2524926662445068, + "step": 2420, + "unclipped_grad_norm": 0.7548633813858032 + }, + { + "bits_per_byte": 1.8169932667125561, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.259443759918213, + "step": 2430, + "unclipped_grad_norm": 0.7562755942344666 + }, + { + "bits_per_byte": 1.8760644917697067, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3003888130187988, + "step": 2440, + "unclipped_grad_norm": 0.7478408813476562 + }, + { + "bits_per_byte": 1.838183937055966, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2741320133209229, + "step": 2450, + "unclipped_grad_norm": 0.760445237159729 + }, + { + "bits_per_byte": 1.8195946762895068, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.261246919631958, + "step": 2460, + "unclipped_grad_norm": 0.7331485748291016 + }, + { + "bits_per_byte": 1.8163858239897086, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2590227127075195, + "step": 2470, + "unclipped_grad_norm": 0.7682369351387024 + }, + { + "bits_per_byte": 1.8266683227194942, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2661499977111816, + "step": 2480, + "unclipped_grad_norm": 0.7132375240325928 + }, + { + "bits_per_byte": 1.8042156436840528, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.250586986541748, + "step": 2490, + "unclipped_grad_norm": 0.7770085334777832 + }, + { + "bits_per_byte": 1.8313916542427904, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2694239616394043, + "step": 2500, + "unclipped_grad_norm": 0.7582318782806396 + }, + { + "bits_per_byte": 1.7788244690759694, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2329871654510498, + "step": 2510, + "unclipped_grad_norm": 0.8006963133811951 + }, + { + "bits_per_byte": 1.8448090527325032, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.278724193572998, + "step": 2520, + "unclipped_grad_norm": 0.7802237272262573 + }, + { + "bits_per_byte": 1.855150369528884, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2858922481536865, + "step": 2530, + "unclipped_grad_norm": 0.7603861689567566 + }, + { + "bits_per_byte": 1.7974587892969547, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.245903491973877, + "step": 2540, + "unclipped_grad_norm": 0.7829954624176025 + }, + { + "bits_per_byte": 1.8048358131230642, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2510168552398682, + "step": 2550, + "unclipped_grad_norm": 0.8207260966300964 + }, + { + "bits_per_byte": 1.8410618947355044, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2761268615722656, + "step": 2560, + "unclipped_grad_norm": 0.8064310550689697 + }, + { + "bits_per_byte": 1.8072459779922554, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2526874542236328, + "step": 2570, + "unclipped_grad_norm": 0.7300699353218079 + }, + { + "bits_per_byte": 1.8352736466380657, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2721147537231445, + "step": 2580, + "unclipped_grad_norm": 0.7476873993873596 + }, + { + "bits_per_byte": 1.793617556789934, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2432409524917603, + "step": 2590, + "unclipped_grad_norm": 0.7210070490837097 + }, + { + "bits_per_byte": 1.797604974550188, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2460048198699951, + "step": 2600, + "unclipped_grad_norm": 0.7853913903236389 + }, + { + "bits_per_byte": 1.7385629865436842, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2050800323486328, + "step": 2610, + "unclipped_grad_norm": 0.7092626094818115 + }, + { + "bits_per_byte": 1.8079149905041119, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2531511783599854, + "step": 2620, + "unclipped_grad_norm": 0.7522015571594238 + }, + { + "bits_per_byte": 1.8198622812942493, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.261432409286499, + "step": 2630, + "unclipped_grad_norm": 0.7675716876983643 + }, + { + "bits_per_byte": 1.680096624469618, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.164554238319397, + "step": 2640, + "unclipped_grad_norm": 0.6825466156005859 + }, + { + "bits_per_byte": 1.7638260340768748, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2225910425186157, + "step": 2650, + "unclipped_grad_norm": 0.7630164623260498 + }, + { + "bits_per_byte": 1.8201589513669878, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2616380453109741, + "step": 2660, + "unclipped_grad_norm": 0.7772946953773499 + }, + { + "bits_per_byte": 1.811244230659514, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2554588317871094, + "step": 2670, + "unclipped_grad_norm": 0.7582975625991821 + }, + { + "bits_per_byte": 1.74257860945868, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.207863450050354, + "step": 2680, + "unclipped_grad_norm": 0.7818799614906311 + }, + { + "bits_per_byte": 1.9030688636428832, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3191068172454834, + "step": 2690, + "unclipped_grad_norm": 0.7075229287147522 + }, + { + "bits_per_byte": 1.8200497423836899, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2615623474121094, + "step": 2700, + "unclipped_grad_norm": 0.8272333741188049 + }, + { + "bits_per_byte": 1.7354167359288, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2028992176055908, + "step": 2710, + "unclipped_grad_norm": 0.7310962677001953 + }, + { + "bits_per_byte": 1.728331394678554, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1979880332946777, + "step": 2720, + "unclipped_grad_norm": 0.7193791270256042 + }, + { + "bits_per_byte": 1.76859700479446, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.225898027420044, + "step": 2730, + "unclipped_grad_norm": 0.773657500743866 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2331104278564453, + "step": 2740, + "unclipped_grad_norm": 0.7759174704551697 + }, + { + "bits_per_byte": 1.7909771071512373, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.241410732269287, + "step": 2750, + "unclipped_grad_norm": 0.770866870880127 + }, + { + "bits_per_byte": 1.81886375002334, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2607402801513672, + "step": 2760, + "unclipped_grad_norm": 0.7793042659759521 + }, + { + "bits_per_byte": 1.704546881994831, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.181501865386963, + "step": 2770, + "unclipped_grad_norm": 0.7179723978042603 + }, + { + "bits_per_byte": 1.7222445846992174, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.1937689781188965, + "step": 2780, + "unclipped_grad_norm": 0.711125910282135 + }, + { + "bits_per_byte": 1.8073385046584196, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2527515888214111, + "step": 2790, + "unclipped_grad_norm": 0.7564381957054138 + }, + { + "bits_per_byte": 1.8564130661515186, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2867674827575684, + "step": 2800, + "unclipped_grad_norm": 0.7982874512672424 + }, + { + "bits_per_byte": 1.7924368958917607, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2424225807189941, + "step": 2810, + "unclipped_grad_norm": 0.827074408531189 + }, + { + "bits_per_byte": 1.781587886310033, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2349026203155518, + "step": 2820, + "unclipped_grad_norm": 0.6945767998695374 + }, + { + "bits_per_byte": 1.862601517877515, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2910569906234741, + "step": 2830, + "unclipped_grad_norm": 0.7473636269569397 + }, + { + "bits_per_byte": 1.7520161574247766, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2144050598144531, + "step": 2840, + "unclipped_grad_norm": 0.7269325852394104 + }, + { + "bits_per_byte": 1.7726036846116109, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2286752462387085, + "step": 2850, + "unclipped_grad_norm": 0.7537488341331482 + }, + { + "bits_per_byte": 1.7375121725469125, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2043516635894775, + "step": 2860, + "unclipped_grad_norm": 0.7839256525039673 + }, + { + "bits_per_byte": 1.836773851301542, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.273154616355896, + "step": 2870, + "unclipped_grad_norm": 0.7615643739700317 + }, + { + "bits_per_byte": 1.8235200083127996, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.263967752456665, + "step": 2880, + "unclipped_grad_norm": 0.7500733137130737 + }, + { + "bits_per_byte": 1.7346435019305209, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2023632526397705, + "step": 2890, + "unclipped_grad_norm": 0.7335168123245239 + }, + { + "bits_per_byte": 1.710140445731493, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1853790283203125, + "step": 2900, + "unclipped_grad_norm": 0.7237004041671753 + }, + { + "bits_per_byte": 1.7640040361205178, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2227144241333008, + "step": 2910, + "unclipped_grad_norm": 0.7653093338012695 + }, + { + "bits_per_byte": 1.7372325287566095, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.204157829284668, + "step": 2920, + "unclipped_grad_norm": 0.7592934966087341 + }, + { + "bits_per_byte": 1.7605243111456106, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2203024625778198, + "step": 2930, + "unclipped_grad_norm": 0.7871323227882385 + }, + { + "bits_per_byte": 1.7773966691085064, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2319974899291992, + "step": 2940, + "unclipped_grad_norm": 0.7994735240936279 + }, + { + "bits_per_byte": 1.761262116717812, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208138704299927, + "step": 2950, + "unclipped_grad_norm": 0.7672474384307861 + }, + { + "bits_per_byte": 1.8009139207527887, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2482984066009521, + "step": 2960, + "unclipped_grad_norm": 0.7721540927886963 + }, + { + "bits_per_byte": 1.7577087831683356, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218350887298584, + "step": 2970, + "unclipped_grad_norm": 0.7659915685653687 + }, + { + "bits_per_byte": 1.7511909846659368, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2138330936431885, + "step": 2980, + "unclipped_grad_norm": 0.7457980513572693 + }, + { + "bits_per_byte": 1.7677185174138528, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2252891063690186, + "step": 2990, + "unclipped_grad_norm": 0.7551211714744568 + }, + { + "bits_per_byte": 1.7804380103063644, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2341055870056152, + "step": 3000, + "unclipped_grad_norm": 0.7369211316108704 + }, + { + "bits_per_byte": 1.77081558099059, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.227435827255249, + "step": 3010, + "unclipped_grad_norm": 0.7526711225509644 + }, + { + "bits_per_byte": 1.726040929734363, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1964004039764404, + "step": 3020, + "unclipped_grad_norm": 0.8113403916358948 + }, + { + "bits_per_byte": 1.8045116258261877, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.250792145729065, + "step": 3030, + "unclipped_grad_norm": 0.7493385672569275 + }, + { + "bits_per_byte": 1.7229166928987893, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.194234848022461, + "step": 3040, + "unclipped_grad_norm": 0.7606903314590454 + }, + { + "bits_per_byte": 1.7403560776615798, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2063229084014893, + "step": 3050, + "unclipped_grad_norm": 0.7690193057060242 + }, + { + "bits_per_byte": 1.759874388708, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2198519706726074, + "step": 3060, + "unclipped_grad_norm": 0.7480750679969788 + }, + { + "bits_per_byte": 1.6873952242069339, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.169613242149353, + "step": 3070, + "unclipped_grad_norm": 0.7001752853393555 + }, + { + "bits_per_byte": 1.737272084766308, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2041852474212646, + "step": 3080, + "unclipped_grad_norm": 0.8322204351425171 + }, + { + "bits_per_byte": 1.8015143121869506, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.248714566230774, + "step": 3090, + "unclipped_grad_norm": 0.8183287382125854 + }, + { + "bits_per_byte": 1.7001176408045102, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.178431749343872, + "step": 3100, + "unclipped_grad_norm": 0.7750749588012695 + }, + { + "bits_per_byte": 1.7770231227908324, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.231738567352295, + "step": 3110, + "unclipped_grad_norm": 0.7082787752151489 + }, + { + "bits_per_byte": 1.749435729731231, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2126164436340332, + "step": 3120, + "unclipped_grad_norm": 0.736611545085907 + }, + { + "bits_per_byte": 1.7966217497352055, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2453233003616333, + "step": 3130, + "unclipped_grad_norm": 0.7366204857826233 + }, + { + "bits_per_byte": 1.7636881039908827, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.222495436668396, + "step": 3140, + "unclipped_grad_norm": 0.7583194375038147 + }, + { + "bits_per_byte": 1.7359698321339747, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2032825946807861, + "step": 3150, + "unclipped_grad_norm": 0.7600930333137512 + }, + { + "bits_per_byte": 1.7415977924008093, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.207183599472046, + "step": 3160, + "unclipped_grad_norm": 0.7639689445495605 + }, + { + "bits_per_byte": 1.654884311818432, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1470783948898315, + "step": 3170, + "unclipped_grad_norm": 0.7011478543281555 + }, + { + "bits_per_byte": 1.7584833930278216, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2188878059387207, + "step": 3180, + "unclipped_grad_norm": 0.7645173072814941 + }, + { + "bits_per_byte": 1.7305350083840592, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.199515461921692, + "step": 3190, + "unclipped_grad_norm": 0.7665988206863403 + }, + { + "bits_per_byte": 1.7003714871971838, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178607702255249, + "step": 3200, + "unclipped_grad_norm": 0.7379894852638245 + }, + { + "bits_per_byte": 1.6959066455981344, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755129098892212, + "step": 3210, + "unclipped_grad_norm": 0.8013631701469421 + }, + { + "bits_per_byte": 1.7098007799960389, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1851435899734497, + "step": 3220, + "unclipped_grad_norm": 0.7296364903450012 + }, + { + "bits_per_byte": 1.7987765203678656, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.246816873550415, + "step": 3230, + "unclipped_grad_norm": 0.7610434889793396 + }, + { + "bits_per_byte": 1.6862874839527267, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1688454151153564, + "step": 3240, + "unclipped_grad_norm": 0.723595917224884 + }, + { + "bits_per_byte": 1.8345631863073515, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2716223001480103, + "step": 3250, + "unclipped_grad_norm": 0.8353017568588257 + }, + { + "bits_per_byte": 1.6872093109613513, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1694843769073486, + "step": 3260, + "unclipped_grad_norm": 0.7602720260620117 + }, + { + "bits_per_byte": 1.7198954736710823, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1921406984329224, + "step": 3270, + "unclipped_grad_norm": 0.7516167163848877 + }, + { + "bits_per_byte": 1.7692164863028679, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2263274192810059, + "step": 3280, + "unclipped_grad_norm": 0.7487898468971252 + }, + { + "bits_per_byte": 1.7624494849393688, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2216368913650513, + "step": 3290, + "unclipped_grad_norm": 0.7708044052124023 + }, + { + "bits_per_byte": 1.7002387165907176, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1785156726837158, + "step": 3300, + "unclipped_grad_norm": 0.7703046202659607 + }, + { + "bits_per_byte": 1.7332599015043295, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2014042139053345, + "step": 3310, + "unclipped_grad_norm": 0.8132672905921936 + }, + { + "bits_per_byte": 1.7262779218272521, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1965646743774414, + "step": 3320, + "unclipped_grad_norm": 0.7545798420906067 + }, + { + "bits_per_byte": 1.7272033604715449, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1972061395645142, + "step": 3330, + "unclipped_grad_norm": 0.7775130867958069 + }, + { + "bits_per_byte": 1.6975052243379043, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1766209602355957, + "step": 3340, + "unclipped_grad_norm": 0.7105540633201599 + }, + { + "bits_per_byte": 1.7083408192728646, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1841316223144531, + "step": 3350, + "unclipped_grad_norm": 0.7163253426551819 + }, + { + "bits_per_byte": 1.732425613664994, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2008259296417236, + "step": 3360, + "unclipped_grad_norm": 0.7626740336418152 + }, + { + "bits_per_byte": 1.6756070173688444, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1614422798156738, + "step": 3370, + "unclipped_grad_norm": 0.8061453700065613 + }, + { + "bits_per_byte": 1.7092863798873084, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1847870349884033, + "step": 3380, + "unclipped_grad_norm": 0.7406735420227051 + }, + { + "bits_per_byte": 1.7539413312185348, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2157394886016846, + "step": 3390, + "unclipped_grad_norm": 0.7695116400718689 + }, + { + "bits_per_byte": 1.8092177590843976, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2540541887283325, + "step": 3400, + "unclipped_grad_norm": 0.7858016490936279 + }, + { + "bits_per_byte": 1.8255297975707825, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2653608322143555, + "step": 3410, + "unclipped_grad_norm": 0.7603964805603027 + }, + { + "bits_per_byte": 1.6857877023693193, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1684989929199219, + "step": 3420, + "unclipped_grad_norm": 0.7168062329292297 + }, + { + "bits_per_byte": 1.6176447364444573, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1212658882141113, + "step": 3430, + "unclipped_grad_norm": 0.7431134581565857 + }, + { + "bits_per_byte": 1.711189711884407, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1861063241958618, + "step": 3440, + "unclipped_grad_norm": 0.7753933668136597 + }, + { + "bits_per_byte": 1.77535626693867, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2305831909179688, + "step": 3450, + "unclipped_grad_norm": 0.8042635321617126 + }, + { + "bits_per_byte": 1.7083845028661837, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.184161901473999, + "step": 3460, + "unclipped_grad_norm": 0.7639697194099426 + }, + { + "bits_per_byte": 1.6937178224006033, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1739957332611084, + "step": 3470, + "unclipped_grad_norm": 0.748127281665802 + }, + { + "bits_per_byte": 1.6501064617948125, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1437666416168213, + "step": 3480, + "unclipped_grad_norm": 0.7506517767906189 + }, + { + "bits_per_byte": 1.8358463488654388, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2725117206573486, + "step": 3490, + "unclipped_grad_norm": 0.836082398891449 + }, + { + "bits_per_byte": 1.714104301868579, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.188126564025879, + "step": 3500, + "unclipped_grad_norm": 0.7544166445732117 + }, + { + "bits_per_byte": 1.744907598516664, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2094777822494507, + "step": 3510, + "unclipped_grad_norm": 0.7650617361068726 + }, + { + "bits_per_byte": 1.738600994709525, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2051063776016235, + "step": 3520, + "unclipped_grad_norm": 0.7794367671012878 + }, + { + "bits_per_byte": 1.7051804660806096, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.181941032409668, + "step": 3530, + "unclipped_grad_norm": 0.7445787787437439 + }, + { + "bits_per_byte": 1.7158172490711725, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1893138885498047, + "step": 3540, + "unclipped_grad_norm": 0.7629107236862183 + }, + { + "bits_per_byte": 1.7173026632266746, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1903434991836548, + "step": 3550, + "unclipped_grad_norm": 0.7763495445251465 + }, + { + "bits_per_byte": 1.6560372835098163, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.14787757396698, + "step": 3560, + "unclipped_grad_norm": 0.7450886964797974 + }, + { + "bits_per_byte": 1.6838996768281473, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1671903133392334, + "step": 3570, + "unclipped_grad_norm": 0.7487124800682068 + }, + { + "bits_per_byte": 1.6946102403759304, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1746143102645874, + "step": 3580, + "unclipped_grad_norm": 0.757170557975769 + }, + { + "bits_per_byte": 1.7307594457434352, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1996710300445557, + "step": 3590, + "unclipped_grad_norm": 0.7700981497764587 + }, + { + "bits_per_byte": 1.7491381997452384, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2124102115631104, + "step": 3600, + "unclipped_grad_norm": 0.7877120971679688 + }, + { + "bits_per_byte": 1.688025540622346, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1700501441955566, + "step": 3610, + "unclipped_grad_norm": 0.780867874622345 + }, + { + "bits_per_byte": 1.6436277033541595, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1392759084701538, + "step": 3620, + "unclipped_grad_norm": 0.7423334717750549 + }, + { + "bits_per_byte": 1.6950637586262558, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1749286651611328, + "step": 3630, + "unclipped_grad_norm": 0.7513464689254761 + }, + { + "bits_per_byte": 1.6758522646289749, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1616122722625732, + "step": 3640, + "unclipped_grad_norm": 0.7453336715698242 + }, + { + "bits_per_byte": 1.736063734661346, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2033476829528809, + "step": 3650, + "unclipped_grad_norm": 0.7682862877845764 + }, + { + "bits_per_byte": 1.6360068081291268, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1339935064315796, + "step": 3660, + "unclipped_grad_norm": 0.7531437277793884 + }, + { + "bits_per_byte": 1.7183529612754938, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1910715103149414, + "step": 3670, + "unclipped_grad_norm": 0.7578480243682861 + }, + { + "bits_per_byte": 1.6583936178092877, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1495108604431152, + "step": 3680, + "unclipped_grad_norm": 0.7224898934364319 + }, + { + "bits_per_byte": 1.6372521345040243, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1348567008972168, + "step": 3690, + "unclipped_grad_norm": 0.7246464490890503 + }, + { + "bits_per_byte": 1.6659130432703113, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1547229290008545, + "step": 3700, + "unclipped_grad_norm": 0.7878638505935669 + }, + { + "bits_per_byte": 1.694639821391879, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1746348142623901, + "step": 3710, + "unclipped_grad_norm": 0.7455682754516602 + }, + { + "bits_per_byte": 1.6891824679147003, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1708520650863647, + "step": 3720, + "unclipped_grad_norm": 0.7515487670898438 + }, + { + "bits_per_byte": 1.6091924770851538, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1154072284698486, + "step": 3730, + "unclipped_grad_norm": 0.7353127598762512 + }, + { + "bits_per_byte": 1.6633680440028435, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.152958869934082, + "step": 3740, + "unclipped_grad_norm": 0.7253232002258301 + }, + { + "bits_per_byte": 1.6922951819126664, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1730096340179443, + "step": 3750, + "unclipped_grad_norm": 0.7686370611190796 + }, + { + "bits_per_byte": 1.7713081393026613, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2277772426605225, + "step": 3760, + "unclipped_grad_norm": 0.7601476907730103 + }, + { + "bits_per_byte": 1.7305630415561497, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1995348930358887, + "step": 3770, + "unclipped_grad_norm": 0.7877833247184753 + }, + { + "bits_per_byte": 1.5644979697442287, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0844273567199707, + "step": 3780, + "unclipped_grad_norm": 0.7386898994445801 + }, + { + "bits_per_byte": 1.6644766441703052, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1537272930145264, + "step": 3790, + "unclipped_grad_norm": 0.7420179843902588 + }, + { + "bits_per_byte": 1.6653939996300076, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1543631553649902, + "step": 3800, + "unclipped_grad_norm": 0.7838358283042908 + }, + { + "bits_per_byte": 1.6952983429620325, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17509126663208, + "step": 3810, + "unclipped_grad_norm": 0.8048396110534668 + }, + { + "bits_per_byte": 1.6744323758634516, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.160628080368042, + "step": 3820, + "unclipped_grad_norm": 0.7709245681762695 + }, + { + "bits_per_byte": 1.720191799778519, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1923460960388184, + "step": 3830, + "unclipped_grad_norm": 0.7672807574272156 + }, + { + "bits_per_byte": 1.6467005173771254, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1414058208465576, + "step": 3840, + "unclipped_grad_norm": 0.7319741249084473 + }, + { + "bits_per_byte": 1.5964396195583748, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106567621231079, + "step": 3850, + "unclipped_grad_norm": 0.7756319642066956 + }, + { + "bits_per_byte": 1.6360811046342996, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1340450048446655, + "step": 3860, + "unclipped_grad_norm": 0.766754150390625 + }, + { + "bits_per_byte": 1.7433616464680584, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2084062099456787, + "step": 3870, + "unclipped_grad_norm": 0.7916337847709656 + }, + { + "bits_per_byte": 1.6358671582166262, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1338967084884644, + "step": 3880, + "unclipped_grad_norm": 0.7497678399085999 + }, + { + "bits_per_byte": 1.600699113872294, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1095200777053833, + "step": 3890, + "unclipped_grad_norm": 0.7555440664291382 + }, + { + "bits_per_byte": 1.6778517349279058, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1629981994628906, + "step": 3900, + "unclipped_grad_norm": 0.826478123664856 + }, + { + "bits_per_byte": 1.6805711246033486, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1648831367492676, + "step": 3910, + "unclipped_grad_norm": 0.7849931120872498 + }, + { + "bits_per_byte": 1.6375147520118918, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1350387334823608, + "step": 3920, + "unclipped_grad_norm": 0.7588120102882385 + }, + { + "bits_per_byte": 1.673005091843941, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.15963876247406, + "step": 3930, + "unclipped_grad_norm": 0.742434024810791 + }, + { + "bits_per_byte": 1.6996766772876981, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1781260967254639, + "step": 3940, + "unclipped_grad_norm": 0.7588658928871155 + }, + { + "bits_per_byte": 1.7339400928884918, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2018756866455078, + "step": 3950, + "unclipped_grad_norm": 0.7937244176864624 + }, + { + "bits_per_byte": 1.713467450112434, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1876851320266724, + "step": 3960, + "unclipped_grad_norm": 0.7884036302566528 + }, + { + "bits_per_byte": 1.6516962694193882, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1448686122894287, + "step": 3970, + "unclipped_grad_norm": 0.8004661798477173 + }, + { + "bits_per_byte": 1.6740500584305835, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1603630781173706, + "step": 3980, + "unclipped_grad_norm": 0.7692103385925293 + }, + { + "bits_per_byte": 1.691941929547794, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.172764778137207, + "step": 3990, + "unclipped_grad_norm": 0.7891059517860413 + }, + { + "bits_per_byte": 1.5813677479673616, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0961205959320068, + "step": 4000, + "unclipped_grad_norm": 0.7436221837997437 + }, + { + "bits_per_byte": 1.6793110077204767, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.164009690284729, + "step": 4010, + "unclipped_grad_norm": 0.8044730424880981 + }, + { + "bits_per_byte": 1.6584589712166156, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1495561599731445, + "step": 4020, + "unclipped_grad_norm": 0.8183668851852417 + }, + { + "bits_per_byte": 1.6850787898824628, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1680076122283936, + "step": 4030, + "unclipped_grad_norm": 0.793128252029419 + }, + { + "bits_per_byte": 1.617821878574846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1213886737823486, + "step": 4040, + "unclipped_grad_norm": 0.7582541108131409 + }, + { + "bits_per_byte": 1.6450140555027646, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1402368545532227, + "step": 4050, + "unclipped_grad_norm": 0.7559582591056824 + }, + { + "bits_per_byte": 1.6581645369183384, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1493520736694336, + "step": 4060, + "unclipped_grad_norm": 0.743245005607605 + }, + { + "bits_per_byte": 1.680131880913045, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1645786762237549, + "step": 4070, + "unclipped_grad_norm": 0.7624887824058533 + }, + { + "bits_per_byte": 1.7564567494700543, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2174830436706543, + "step": 4080, + "unclipped_grad_norm": 0.8132302761077881 + }, + { + "bits_per_byte": 1.6669108866106175, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1554145812988281, + "step": 4090, + "unclipped_grad_norm": 0.8276432752609253 + }, + { + "bits_per_byte": 1.638365894151012, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1356287002563477, + "step": 4100, + "unclipped_grad_norm": 0.7751885652542114 + }, + { + "bits_per_byte": 1.6303674970073352, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1300846338272095, + "step": 4110, + "unclipped_grad_norm": 0.7898787260055542 + }, + { + "bits_per_byte": 1.6896222275529564, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.171156883239746, + "step": 4120, + "unclipped_grad_norm": 0.8148247599601746 + }, + { + "bits_per_byte": 1.6213521664491068, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1238356828689575, + "step": 4130, + "unclipped_grad_norm": 0.8133904933929443 + }, + { + "bits_per_byte": 1.6881359534842002, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1701266765594482, + "step": 4140, + "unclipped_grad_norm": 0.8217440247535706 + }, + { + "bits_per_byte": 1.6129334437067215, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1180002689361572, + "step": 4150, + "unclipped_grad_norm": 0.7947918176651001 + }, + { + "bits_per_byte": 1.651558683298698, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.144773244857788, + "step": 4160, + "unclipped_grad_norm": 0.7860805988311768 + }, + { + "bits_per_byte": 1.6828089628563754, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1664342880249023, + "step": 4170, + "unclipped_grad_norm": 0.8207855224609375 + }, + { + "bits_per_byte": 1.7311515661874024, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1999428272247314, + "step": 4180, + "unclipped_grad_norm": 0.8373809456825256 + }, + { + "bits_per_byte": 1.6433404923272186, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.139076828956604, + "step": 4190, + "unclipped_grad_norm": 0.7621071338653564 + }, + { + "bits_per_byte": 1.682679287937625, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.166344404220581, + "step": 4200, + "unclipped_grad_norm": 0.7672834396362305 + }, + { + "bits_per_byte": 1.661693964879345, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1517984867095947, + "step": 4210, + "unclipped_grad_norm": 0.7330533862113953 + }, + { + "bits_per_byte": 1.6972419188994332, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1764384508132935, + "step": 4220, + "unclipped_grad_norm": 0.7866858243942261 + }, + { + "bits_per_byte": 1.6670219874030747, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.155491590499878, + "step": 4230, + "unclipped_grad_norm": 0.7902593612670898 + }, + { + "bits_per_byte": 1.6114956687455084, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1170036792755127, + "step": 4240, + "unclipped_grad_norm": 0.740251362323761 + }, + { + "bits_per_byte": 1.6335233786506678, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1322721242904663, + "step": 4250, + "unclipped_grad_norm": 0.7777352333068848 + }, + { + "bits_per_byte": 1.6115895712728796, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1170687675476074, + "step": 4260, + "unclipped_grad_norm": 0.774424135684967 + }, + { + "bits_per_byte": 1.619611874005026, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1226294040679932, + "step": 4270, + "unclipped_grad_norm": 0.7803477048873901 + }, + { + "bits_per_byte": 1.6299772683725275, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1298141479492188, + "step": 4280, + "unclipped_grad_norm": 0.7580373287200928 + }, + { + "bits_per_byte": 1.6795242662075467, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.164157509803772, + "step": 4290, + "unclipped_grad_norm": 0.7638103365898132 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.109092354774475, + "step": 4300, + "unclipped_grad_norm": 0.7825598120689392 + }, + { + "bits_per_byte": 1.6771565810531184, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1625163555145264, + "step": 4310, + "unclipped_grad_norm": 0.7940689921379089 + }, + { + "bits_per_byte": 1.6577068910843924, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.149034857749939, + "step": 4320, + "unclipped_grad_norm": 0.7976187467575073 + }, + { + "bits_per_byte": 1.6195226150092283, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1225675344467163, + "step": 4330, + "unclipped_grad_norm": 0.7892199158668518 + }, + { + "bits_per_byte": 1.655279183984813, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1473520994186401, + "step": 4340, + "unclipped_grad_norm": 0.8144169449806213 + }, + { + "bits_per_byte": 1.6394932404274176, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.136410117149353, + "step": 4350, + "unclipped_grad_norm": 0.8119622468948364 + }, + { + "bits_per_byte": 1.6287904160989233, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1289914846420288, + "step": 4360, + "unclipped_grad_norm": 0.7554351091384888 + }, + { + "bits_per_byte": 1.6321829458698431, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1313430070877075, + "step": 4370, + "unclipped_grad_norm": 0.808085024356842 + }, + { + "bits_per_byte": 1.6820371047193032, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1658992767333984, + "step": 4380, + "unclipped_grad_norm": 0.7708722949028015 + }, + { + "bits_per_byte": 1.6522423143358775, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1452471017837524, + "step": 4390, + "unclipped_grad_norm": 0.8123121857643127 + }, + { + "bits_per_byte": 1.6025224739367414, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1107839345932007, + "step": 4400, + "unclipped_grad_norm": 0.7717524766921997 + }, + { + "bits_per_byte": 1.6853694405624209, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1682090759277344, + "step": 4410, + "unclipped_grad_norm": 0.81525057554245 + }, + { + "bits_per_byte": 1.6142167782474597, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1188898086547852, + "step": 4420, + "unclipped_grad_norm": 0.7956154346466064 + }, + { + "bits_per_byte": 1.6755413199962148, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1613967418670654, + "step": 4430, + "unclipped_grad_norm": 0.7907513380050659 + }, + { + "bits_per_byte": 1.6011154838700328, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1098086833953857, + "step": 4440, + "unclipped_grad_norm": 0.7607184648513794 + }, + { + "bits_per_byte": 1.5990663105850023, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1083883047103882, + "step": 4450, + "unclipped_grad_norm": 0.7442869544029236 + }, + { + "bits_per_byte": 1.5859970049806358, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0993293523788452, + "step": 4460, + "unclipped_grad_norm": 0.7537500262260437 + }, + { + "bits_per_byte": 1.6451918855637568, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1403601169586182, + "step": 4470, + "unclipped_grad_norm": 0.7931647896766663 + }, + { + "bits_per_byte": 1.6505943765753102, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1441048383712769, + "step": 4480, + "unclipped_grad_norm": 0.9106901288032532 + }, + { + "bits_per_byte": 1.6301040195862135, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1299020051956177, + "step": 4490, + "unclipped_grad_norm": 0.7690432667732239 + }, + { + "bits_per_byte": 1.6520481459230536, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1451125144958496, + "step": 4500, + "unclipped_grad_norm": 0.7990603446960449 + }, + { + "bits_per_byte": 1.5968599451570833, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1068589687347412, + "step": 4510, + "unclipped_grad_norm": 0.8130239248275757 + }, + { + "bits_per_byte": 1.6430763269754933, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.138893723487854, + "step": 4520, + "unclipped_grad_norm": 0.7745712399482727 + }, + { + "bits_per_byte": 1.660709880151108, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1511163711547852, + "step": 4530, + "unclipped_grad_norm": 0.7695059776306152 + }, + { + "bits_per_byte": 1.707295852686222, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1834073066711426, + "step": 4540, + "unclipped_grad_norm": 0.8436031937599182 + }, + { + "bits_per_byte": 1.5623325361872151, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.0829263925552368, + "step": 4550, + "unclipped_grad_norm": 0.7549967169761658 + }, + { + "bits_per_byte": 1.6257084869954619, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1268552541732788, + "step": 4560, + "unclipped_grad_norm": 0.8449703454971313 + }, + { + "bits_per_byte": 1.6175644205465043, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1212102174758911, + "step": 4570, + "unclipped_grad_norm": 0.7528937458992004 + }, + { + "bits_per_byte": 1.570274179056107, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0884311199188232, + "step": 4580, + "unclipped_grad_norm": 0.790579617023468 + }, + { + "bits_per_byte": 1.609637052237634, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1157153844833374, + "step": 4590, + "unclipped_grad_norm": 0.7935193181037903 + }, + { + "bits_per_byte": 1.6088013885370918, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1151361465454102, + "step": 4600, + "unclipped_grad_norm": 0.7915687561035156 + }, + { + "bits_per_byte": 1.6615916352020816, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.151727557182312, + "step": 4610, + "unclipped_grad_norm": 0.8493216633796692 + }, + { + "bits_per_byte": 1.6675298521710726, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1558436155319214, + "step": 4620, + "unclipped_grad_norm": 0.8179065585136414 + }, + { + "bits_per_byte": 1.6286588493710132, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1289002895355225, + "step": 4630, + "unclipped_grad_norm": 0.8027536273002625 + }, + { + "bits_per_byte": 1.7031861552612044, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.180558681488037, + "step": 4640, + "unclipped_grad_norm": 0.8179929256439209 + }, + { + "bits_per_byte": 1.6059254946493637, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.113142728805542, + "step": 4650, + "unclipped_grad_norm": 0.8228299021720886 + }, + { + "bits_per_byte": 1.5739184914278899, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0909571647644043, + "step": 4660, + "unclipped_grad_norm": 0.820624589920044 + }, + { + "bits_per_byte": 1.7237126286069824, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.194786548614502, + "step": 4670, + "unclipped_grad_norm": 0.8582949638366699 + }, + { + "bits_per_byte": 1.7009104808249877, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1789813041687012, + "step": 4680, + "unclipped_grad_norm": 0.8158925175666809 + }, + { + "bits_per_byte": 1.6679322915740917, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1561225652694702, + "step": 4690, + "unclipped_grad_norm": 0.849236249923706 + }, + { + "bits_per_byte": 1.589747430648001, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.101928949356079, + "step": 4700, + "unclipped_grad_norm": 0.7403190732002258 + }, + { + "bits_per_byte": 1.6742342518496576, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1604907512664795, + "step": 4710, + "unclipped_grad_norm": 0.8582251071929932 + }, + { + "bits_per_byte": 1.607032031025015, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1139097213745117, + "step": 4720, + "unclipped_grad_norm": 0.8400667905807495 + }, + { + "bits_per_byte": 1.5176990826873464, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0519888401031494, + "step": 4730, + "unclipped_grad_norm": 0.7827285528182983 + }, + { + "bits_per_byte": 1.5868581221135059, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.099926233291626, + "step": 4740, + "unclipped_grad_norm": 0.7815871238708496 + }, + { + "bits_per_byte": 1.6377919880450826, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1352308988571167, + "step": 4750, + "unclipped_grad_norm": 0.7846289873123169 + }, + { + "bits_per_byte": 1.6125860387519786, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1177594661712646, + "step": 4760, + "unclipped_grad_norm": 0.8307337164878845 + }, + { + "bits_per_byte": 1.5936545325103024, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1046371459960938, + "step": 4770, + "unclipped_grad_norm": 0.8213374614715576 + }, + { + "bits_per_byte": 1.6121588338472355, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1174633502960205, + "step": 4780, + "unclipped_grad_norm": 0.8224621415138245 + }, + { + "bits_per_byte": 1.5705015401205475, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0885887145996094, + "step": 4790, + "unclipped_grad_norm": 0.8122685551643372 + }, + { + "bits_per_byte": 1.5722774329733569, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0898196697235107, + "step": 4800, + "unclipped_grad_norm": 0.8455460667610168 + }, + { + "bits_per_byte": 1.6480244398234671, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1423234939575195, + "step": 4810, + "unclipped_grad_norm": 0.8817458748817444 + }, + { + "bits_per_byte": 1.574972745077679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0916879177093506, + "step": 4820, + "unclipped_grad_norm": 0.7954387664794922 + }, + { + "bits_per_byte": 1.608869665649484, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1151834726333618, + "step": 4830, + "unclipped_grad_norm": 0.8687117695808411 + }, + { + "bits_per_byte": 1.5985519104762718, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1080317497253418, + "step": 4840, + "unclipped_grad_norm": 0.8505465388298035 + }, + { + "bits_per_byte": 1.6228387844831649, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1248661279678345, + "step": 4850, + "unclipped_grad_norm": 0.7912812232971191 + }, + { + "bits_per_byte": 1.5432041098103009, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0696675777435303, + "step": 4860, + "unclipped_grad_norm": 0.7933070063591003 + }, + { + "bits_per_byte": 1.635918581029234, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.13393235206604, + "step": 4870, + "unclipped_grad_norm": 0.8805916905403137 + }, + { + "bits_per_byte": 1.4766798445953597, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0235564708709717, + "step": 4880, + "unclipped_grad_norm": 0.8066675066947937 + }, + { + "bits_per_byte": 1.5344280071194227, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0635844469070435, + "step": 4890, + "unclipped_grad_norm": 0.8088182806968689 + }, + { + "bits_per_byte": 1.6101516243290155, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1160720586776733, + "step": 4900, + "unclipped_grad_norm": 0.8653903603553772 + }, + { + "bits_per_byte": 1.6779198400576476, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1630454063415527, + "step": 4910, + "unclipped_grad_norm": 0.8391038179397583 + }, + { + "bits_per_byte": 1.6284904783558185, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1287835836410522, + "step": 4920, + "unclipped_grad_norm": 0.8723531365394592 + }, + { + "bits_per_byte": 1.5831391692712484, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0973484516143799, + "step": 4930, + "unclipped_grad_norm": 0.7553638815879822 + }, + { + "bits_per_byte": 1.5755332365368409, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0920764207839966, + "step": 4940, + "unclipped_grad_norm": 0.8158645033836365 + }, + { + "bits_per_byte": 1.530142715407874, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0606141090393066, + "step": 4950, + "unclipped_grad_norm": 0.7773033380508423 + }, + { + "bits_per_byte": 1.5220387209165676, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0549968481063843, + "step": 4960, + "unclipped_grad_norm": 0.8325244784355164 + }, + { + "bits_per_byte": 1.6031579498316795, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1112244129180908, + "step": 4970, + "unclipped_grad_norm": 0.8591333627700806 + }, + { + "bits_per_byte": 1.6532000857185325, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1459109783172607, + "step": 4980, + "unclipped_grad_norm": 0.8351151347160339 + }, + { + "bits_per_byte": 1.585272098107249, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.0988268852233887, + "step": 4990, + "unclipped_grad_norm": 0.8779996037483215 + }, + { + "bits_per_byte": 1.5394700224947675, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670793056488037, + "step": 5000, + "unclipped_grad_norm": 0.7928086519241333 + }, + { + "bits_per_byte": 1.5552827953456976, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0780398845672607, + "step": 5010, + "unclipped_grad_norm": 0.8201308846473694 + }, + { + "bits_per_byte": 1.5898205232746179, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1019796133041382, + "step": 5020, + "unclipped_grad_norm": 0.7774789929389954 + }, + { + "bits_per_byte": 1.5697566832596608, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.088072419166565, + "step": 5030, + "unclipped_grad_norm": 0.7855319380760193 + }, + { + "bits_per_byte": 1.539816567536256, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0673195123672485, + "step": 5040, + "unclipped_grad_norm": 0.7920348048210144 + }, + { + "bits_per_byte": 1.5092143186070297, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0461076498031616, + "step": 5050, + "unclipped_grad_norm": 0.8045132160186768 + }, + { + "bits_per_byte": 1.5339026001210367, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0632202625274658, + "step": 5060, + "unclipped_grad_norm": 0.8044583797454834 + }, + { + "bits_per_byte": 1.6138325690054323, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1186234951019287, + "step": 5070, + "unclipped_grad_norm": 0.8723475337028503 + }, + { + "bits_per_byte": 1.574335549356232, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.091246247291565, + "step": 5080, + "unclipped_grad_norm": 0.8741123080253601 + }, + { + "bits_per_byte": 1.5594206979254568, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0809080600738525, + "step": 5090, + "unclipped_grad_norm": 0.8465037941932678 + }, + { + "bits_per_byte": 1.5640659493252613, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0841279029846191, + "step": 5100, + "unclipped_grad_norm": 0.783299446105957 + }, + { + "bits_per_byte": 1.488738924108558, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0319151878356934, + "step": 5110, + "unclipped_grad_norm": 0.8348572254180908 + }, + { + "bits_per_byte": 1.5847128105266433, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0984392166137695, + "step": 5120, + "unclipped_grad_norm": 0.8110998868942261 + }, + { + "bits_per_byte": 1.590178763136365, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1022279262542725, + "step": 5130, + "unclipped_grad_norm": 0.8208310604095459 + }, + { + "bits_per_byte": 1.5494388248693793, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0739891529083252, + "step": 5140, + "unclipped_grad_norm": 0.8678956627845764 + }, + { + "bits_per_byte": 1.6273239200350162, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1279749870300293, + "step": 5150, + "unclipped_grad_norm": 0.848126232624054 + }, + { + "bits_per_byte": 1.5085432423033631, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0456424951553345, + "step": 5160, + "unclipped_grad_norm": 0.7922812104225159 + }, + { + "bits_per_byte": 1.6700107019097687, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1575632095336914, + "step": 5170, + "unclipped_grad_norm": 0.8759046196937561 + }, + { + "bits_per_byte": 1.5869253673299932, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0999728441238403, + "step": 5180, + "unclipped_grad_norm": 0.8647724390029907 + }, + { + "bits_per_byte": 1.5237559676854329, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0561871528625488, + "step": 5190, + "unclipped_grad_norm": 0.813337504863739 + }, + { + "bits_per_byte": 1.556985939537192, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0792204141616821, + "step": 5200, + "unclipped_grad_norm": 0.8388134241104126 + }, + { + "bits_per_byte": 1.5602121620847276, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0814566612243652, + "step": 5210, + "unclipped_grad_norm": 0.8162683248519897 + }, + { + "bits_per_byte": 1.6432956048553435, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1390457153320312, + "step": 5220, + "unclipped_grad_norm": 0.8667425513267517 + }, + { + "bits_per_byte": 1.5501931407760636, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.074512004852295, + "step": 5230, + "unclipped_grad_norm": 0.8668702840805054 + }, + { + "bits_per_byte": 1.5510165937083948, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.075082778930664, + "step": 5240, + "unclipped_grad_norm": 0.8049055337905884 + }, + { + "bits_per_byte": 1.5393878147876552, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0670223236083984, + "step": 5250, + "unclipped_grad_norm": 0.8209428191184998 + }, + { + "bits_per_byte": 1.540701762240247, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0679330825805664, + "step": 5260, + "unclipped_grad_norm": 0.8524174690246582 + }, + { + "bits_per_byte": 1.5376201771020872, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0657970905303955, + "step": 5270, + "unclipped_grad_norm": 0.8064137697219849 + }, + { + "bits_per_byte": 1.5550475230793173, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0778768062591553, + "step": 5280, + "unclipped_grad_norm": 0.8169121146202087 + }, + { + "bits_per_byte": 1.5423493560355128, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.069075107574463, + "step": 5290, + "unclipped_grad_norm": 0.8263237476348877 + }, + { + "bits_per_byte": 1.5542627662434303, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.077332854270935, + "step": 5300, + "unclipped_grad_norm": 0.8492859601974487 + }, + { + "bits_per_byte": 1.5509104804128124, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0750092267990112, + "step": 5310, + "unclipped_grad_norm": 0.8885292410850525 + }, + { + "bits_per_byte": 1.5517654061702515, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0756018161773682, + "step": 5320, + "unclipped_grad_norm": 0.8326203227043152 + }, + { + "bits_per_byte": 1.5477100552629064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0727908611297607, + "step": 5330, + "unclipped_grad_norm": 0.8806997537612915 + }, + { + "bits_per_byte": 1.503639328996661, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0422433614730835, + "step": 5340, + "unclipped_grad_norm": 0.8234722018241882 + }, + { + "bits_per_byte": 1.5869049013945407, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0999586582183838, + "step": 5350, + "unclipped_grad_norm": 0.8706242442131042 + }, + { + "bits_per_byte": 1.5072631754329913, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.044755220413208, + "step": 5360, + "unclipped_grad_norm": 0.8708282709121704 + }, + { + "bits_per_byte": 1.471025742965594, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0196373462677002, + "step": 5370, + "unclipped_grad_norm": 0.8303158283233643 + }, + { + "bits_per_byte": 1.5968417149960918, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1068463325500488, + "step": 5380, + "unclipped_grad_norm": 0.883955180644989 + }, + { + "bits_per_byte": 1.531507741707772, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0615602731704712, + "step": 5390, + "unclipped_grad_norm": 0.8507125377655029 + }, + { + "bits_per_byte": 1.5483677169198058, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073246717453003, + "step": 5400, + "unclipped_grad_norm": 0.8522453904151917 + }, + { + "bits_per_byte": 1.571479089508052, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.089266300201416, + "step": 5410, + "unclipped_grad_norm": 0.8313266038894653 + }, + { + "bits_per_byte": 1.5408727129952047, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0680515766143799, + "step": 5420, + "unclipped_grad_norm": 0.8902517557144165 + }, + { + "bits_per_byte": 1.5778336764747818, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0936709642410278, + "step": 5430, + "unclipped_grad_norm": 0.8956695795059204 + }, + { + "bits_per_byte": 1.513927503153925, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0493745803833008, + "step": 5440, + "unclipped_grad_norm": 0.897573709487915 + }, + { + "bits_per_byte": 1.5078418970531444, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0451563596725464, + "step": 5450, + "unclipped_grad_norm": 0.8440007567405701 + }, + { + "bits_per_byte": 1.5030907043404085, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0418630838394165, + "step": 5460, + "unclipped_grad_norm": 0.8436890244483948 + }, + { + "bits_per_byte": 1.5464133060754008, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0718920230865479, + "step": 5470, + "unclipped_grad_norm": 0.8681005239486694 + }, + { + "bits_per_byte": 1.553998600891705, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.077149748802185, + "step": 5480, + "unclipped_grad_norm": 0.8523774147033691 + }, + { + "bits_per_byte": 1.543775436176467, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.070063591003418, + "step": 5490, + "unclipped_grad_norm": 0.8906207084655762 + }, + { + "bits_per_byte": 1.5444891641775478, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0705583095550537, + "step": 5500, + "unclipped_grad_norm": 0.8658390045166016 + }, + { + "bits_per_byte": 1.6370650173798855, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1347270011901855, + "step": 5510, + "unclipped_grad_norm": 0.9161565899848938 + }, + { + "bits_per_byte": 1.5520985365649729, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.075832724571228, + "step": 5520, + "unclipped_grad_norm": 0.845907986164093 + }, + { + "bits_per_byte": 1.5215033389244317, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0546257495880127, + "step": 5530, + "unclipped_grad_norm": 0.8585748672485352 + }, + { + "bits_per_byte": 1.5943249208833656, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1051018238067627, + "step": 5540, + "unclipped_grad_norm": 0.9360165596008301 + }, + { + "bits_per_byte": 1.5176287417831436, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0519400835037231, + "step": 5550, + "unclipped_grad_norm": 0.8308430910110474 + }, + { + "bits_per_byte": 1.5164790377621256, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0511431694030762, + "step": 5560, + "unclipped_grad_norm": 0.894058346748352 + }, + { + "bits_per_byte": 1.4905304673825959, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0331569910049438, + "step": 5570, + "unclipped_grad_norm": 0.8399901390075684 + }, + { + "bits_per_byte": 1.5655759569998369, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.085174560546875, + "step": 5580, + "unclipped_grad_norm": 0.8579773902893066 + }, + { + "bits_per_byte": 1.482571282283316, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0276401042938232, + "step": 5590, + "unclipped_grad_norm": 0.8161733746528625 + }, + { + "bits_per_byte": 1.5496305855250914, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0741220712661743, + "step": 5600, + "unclipped_grad_norm": 0.8440633416175842 + }, + { + "bits_per_byte": 1.5466313800766949, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0720431804656982, + "step": 5610, + "unclipped_grad_norm": 0.8384022116661072 + }, + { + "bits_per_byte": 1.55366271877457, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0769169330596924, + "step": 5620, + "unclipped_grad_norm": 0.8643615245819092 + }, + { + "bits_per_byte": 1.5359743031333302, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0646562576293945, + "step": 5630, + "unclipped_grad_norm": 0.8559057712554932 + }, + { + "bits_per_byte": 1.5154779267514533, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.050449252128601, + "step": 5640, + "unclipped_grad_norm": 0.8564271330833435 + }, + { + "bits_per_byte": 1.5399061704973558, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0673816204071045, + "step": 5650, + "unclipped_grad_norm": 0.899230420589447 + }, + { + "bits_per_byte": 1.6017691899259623, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1102617979049683, + "step": 5660, + "unclipped_grad_norm": 0.8796955943107605 + }, + { + "bits_per_byte": 1.5203328250026595, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.05381441116333, + "step": 5670, + "unclipped_grad_norm": 0.8509314060211182 + }, + { + "bits_per_byte": 1.5301421994599214, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.060613751411438, + "step": 5680, + "unclipped_grad_norm": 0.8624072074890137 + }, + { + "bits_per_byte": 1.6028342784827556, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1110000610351562, + "step": 5690, + "unclipped_grad_norm": 0.8914096355438232 + }, + { + "bits_per_byte": 1.526354281574668, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.057988166809082, + "step": 5700, + "unclipped_grad_norm": 0.8579604625701904 + }, + { + "bits_per_byte": 1.5347007716036911, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.063773512840271, + "step": 5710, + "unclipped_grad_norm": 0.8810272216796875 + }, + { + "bits_per_byte": 1.5491378552303696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0737805366516113, + "step": 5720, + "unclipped_grad_norm": 0.8394830226898193 + }, + { + "bits_per_byte": 1.5456280332915613, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.071347713470459, + "step": 5730, + "unclipped_grad_norm": 0.865943968296051 + }, + { + "bits_per_byte": 1.5433466834278662, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0697664022445679, + "step": 5740, + "unclipped_grad_norm": 0.886944591999054 + }, + { + "bits_per_byte": 1.5488365416260579, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0735716819763184, + "step": 5750, + "unclipped_grad_norm": 0.9145897626876831 + }, + { + "bits_per_byte": 1.5636783004302166, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0838592052459717, + "step": 5760, + "unclipped_grad_norm": 0.9009037017822266 + }, + { + "bits_per_byte": 1.511110599315443, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0474220514297485, + "step": 5770, + "unclipped_grad_norm": 0.844070315361023 + }, + { + "bits_per_byte": 1.494012772097266, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.035570740699768, + "step": 5780, + "unclipped_grad_norm": 0.8943845629692078 + }, + { + "bits_per_byte": 1.5135893852623288, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.049140214920044, + "step": 5790, + "unclipped_grad_norm": 0.8841454386711121 + }, + { + "bits_per_byte": 1.4905120652389536, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033144235610962, + "step": 5800, + "unclipped_grad_norm": 0.8580894470214844 + }, + { + "bits_per_byte": 1.5735937881830608, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0907320976257324, + "step": 5810, + "unclipped_grad_norm": 0.933175265789032 + }, + { + "bits_per_byte": 1.5042643139498963, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.042676568031311, + "step": 5820, + "unclipped_grad_norm": 0.828717052936554 + }, + { + "bits_per_byte": 1.5156406223391694, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0505620241165161, + "step": 5830, + "unclipped_grad_norm": 0.9511673450469971 + }, + { + "bits_per_byte": 1.5092205099824607, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0461119413375854, + "step": 5840, + "unclipped_grad_norm": 0.8395604491233826 + }, + { + "bits_per_byte": 1.477936693807865, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0244276523590088, + "step": 5850, + "unclipped_grad_norm": 0.9143134355545044 + }, + { + "bits_per_byte": 1.5336375748560571, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.063036561012268, + "step": 5860, + "unclipped_grad_norm": 0.9014923572540283 + }, + { + "bits_per_byte": 1.5402322496033916, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0676076412200928, + "step": 5870, + "unclipped_grad_norm": 0.8728002905845642 + }, + { + "bits_per_byte": 1.555285031120159, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.078041434288025, + "step": 5880, + "unclipped_grad_norm": 0.8936865925788879 + }, + { + "bits_per_byte": 1.564054942435606, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.084120273590088, + "step": 5890, + "unclipped_grad_norm": 0.8752698302268982 + }, + { + "bits_per_byte": 1.5458715607251832, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071516513824463, + "step": 5900, + "unclipped_grad_norm": 0.8957662582397461 + }, + { + "bits_per_byte": 1.5568008862048637, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0790921449661255, + "step": 5910, + "unclipped_grad_norm": 0.8808028101921082 + }, + { + "bits_per_byte": 1.4896764015384112, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0325649976730347, + "step": 5920, + "unclipped_grad_norm": 0.8660398721694946 + }, + { + "bits_per_byte": 1.5908711652887386, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.102707862854004, + "step": 5930, + "unclipped_grad_norm": 0.9604227542877197 + }, + { + "bits_per_byte": 1.4574508083676907, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.010227918624878, + "step": 5940, + "unclipped_grad_norm": 0.8380697965621948 + }, + { + "bits_per_byte": 1.5131755950043528, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0488533973693848, + "step": 5950, + "unclipped_grad_norm": 0.8392192125320435 + }, + { + "bits_per_byte": 1.5452194025131114, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0710644721984863, + "step": 5960, + "unclipped_grad_norm": 0.932554304599762 + }, + { + "bits_per_byte": 1.5213482105733536, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.054518222808838, + "step": 5970, + "unclipped_grad_norm": 0.927780270576477 + }, + { + "bits_per_byte": 1.5601101763727658, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0813859701156616, + "step": 5980, + "unclipped_grad_norm": 0.8920019268989563 + }, + { + "bits_per_byte": 1.5382338112003657, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0662224292755127, + "step": 5990, + "unclipped_grad_norm": 0.9537093639373779 + }, + { + "bits_per_byte": 1.4675097296513546, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.017200231552124, + "step": 6000, + "unclipped_grad_norm": 0.8773758411407471 + }, + { + "bits_per_byte": 1.506333093257125, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0441105365753174, + "step": 6010, + "unclipped_grad_norm": 0.8785988092422485 + }, + { + "bits_per_byte": 1.4794742187065786, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0254933834075928, + "step": 6020, + "unclipped_grad_norm": 0.8875186443328857 + }, + { + "bits_per_byte": 1.5245675538148544, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.056749701499939, + "step": 6030, + "unclipped_grad_norm": 0.88789963722229 + }, + { + "bits_per_byte": 1.5628784091210537, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.083304762840271, + "step": 6040, + "unclipped_grad_norm": 0.9149079322814941 + }, + { + "bits_per_byte": 1.4881958028971334, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.031538724899292, + "step": 6050, + "unclipped_grad_norm": 0.9299342632293701 + }, + { + "bits_per_byte": 1.4785307218839452, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0248394012451172, + "step": 6060, + "unclipped_grad_norm": 0.8783867359161377 + }, + { + "bits_per_byte": 1.5509557118499895, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.075040578842163, + "step": 6070, + "unclipped_grad_norm": 0.9337034821510315 + }, + { + "bits_per_byte": 1.4914187577743023, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0337727069854736, + "step": 6080, + "unclipped_grad_norm": 0.9420101046562195 + }, + { + "bits_per_byte": 1.4968499698885498, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0375373363494873, + "step": 6090, + "unclipped_grad_norm": 0.8664546012878418 + }, + { + "bits_per_byte": 1.4906561867003767, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0332441329956055, + "step": 6100, + "unclipped_grad_norm": 0.8687640428543091 + }, + { + "bits_per_byte": 1.5538661742505406, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.077057957649231, + "step": 6110, + "unclipped_grad_norm": 0.8986173868179321 + }, + { + "bits_per_byte": 1.5476213122150613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0727293491363525, + "step": 6120, + "unclipped_grad_norm": 0.9189077019691467 + }, + { + "bits_per_byte": 1.4789527673091625, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0251319408416748, + "step": 6130, + "unclipped_grad_norm": 0.8851786851882935 + }, + { + "bits_per_byte": 1.4768556108645414, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0236783027648926, + "step": 6140, + "unclipped_grad_norm": 0.9043483138084412 + }, + { + "bits_per_byte": 1.552988202817886, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0764493942260742, + "step": 6150, + "unclipped_grad_norm": 0.9005390405654907 + }, + { + "bits_per_byte": 1.3994871515820966, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.970050573348999, + "step": 6160, + "unclipped_grad_norm": 0.8554620742797852 + }, + { + "bits_per_byte": 1.5302097886417105, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0606606006622314, + "step": 6170, + "unclipped_grad_norm": 0.8954014778137207 + }, + { + "bits_per_byte": 1.4938230752333643, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0354392528533936, + "step": 6180, + "unclipped_grad_norm": 0.892278790473938 + }, + { + "bits_per_byte": 1.516749394489282, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.05133056640625, + "step": 6190, + "unclipped_grad_norm": 0.9185703992843628 + }, + { + "bits_per_byte": 1.5086576107661869, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0457217693328857, + "step": 6200, + "unclipped_grad_norm": 0.8927735090255737 + }, + { + "bits_per_byte": 1.5088584865023946, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.045861005783081, + "step": 6210, + "unclipped_grad_norm": 0.8906724452972412 + }, + { + "bits_per_byte": 1.5118191678369979, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0479131937026978, + "step": 6220, + "unclipped_grad_norm": 0.9077256917953491 + }, + { + "bits_per_byte": 1.4471287536282067, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0030732154846191, + "step": 6230, + "unclipped_grad_norm": 0.8839235305786133 + }, + { + "bits_per_byte": 1.5076786855174757, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0450432300567627, + "step": 6240, + "unclipped_grad_norm": 0.9515005946159363 + }, + { + "bits_per_byte": 1.5408861276419719, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0680608749389648, + "step": 6250, + "unclipped_grad_norm": 0.901561975479126 + }, + { + "bits_per_byte": 1.4200397662908057, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9842965602874756, + "step": 6260, + "unclipped_grad_norm": 0.889621376991272 + }, + { + "bits_per_byte": 1.5216213190229235, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0547075271606445, + "step": 6270, + "unclipped_grad_norm": 0.9411186575889587 + }, + { + "bits_per_byte": 1.5224643779774532, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0552918910980225, + "step": 6280, + "unclipped_grad_norm": 0.9406285881996155 + }, + { + "bits_per_byte": 1.452789390598705, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0069968700408936, + "step": 6290, + "unclipped_grad_norm": 0.8839805126190186 + }, + { + "bits_per_byte": 1.5074908804627336, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0449130535125732, + "step": 6300, + "unclipped_grad_norm": 0.9184122681617737 + }, + { + "bits_per_byte": 1.478450577968643, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0247838497161865, + "step": 6310, + "unclipped_grad_norm": 0.8948619365692139 + }, + { + "bits_per_byte": 1.4214500240278807, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.985274076461792, + "step": 6320, + "unclipped_grad_norm": 0.8749227523803711 + }, + { + "bits_per_byte": 1.5185149683830395, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0525543689727783, + "step": 6330, + "unclipped_grad_norm": 0.9816636443138123 + }, + { + "bits_per_byte": 1.4759477144506365, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.023048996925354, + "step": 6340, + "unclipped_grad_norm": 0.8950188159942627 + }, + { + "bits_per_byte": 1.587707716408768, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1005151271820068, + "step": 6350, + "unclipped_grad_norm": 1.01009202003479 + }, + { + "bits_per_byte": 1.4638088349874379, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0146349668502808, + "step": 6360, + "unclipped_grad_norm": 0.9816907644271851 + }, + { + "bits_per_byte": 1.4382974445064014, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9969518184661865, + "step": 6370, + "unclipped_grad_norm": 0.8970034122467041 + }, + { + "bits_per_byte": 1.4818757844432269, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0271580219268799, + "step": 6380, + "unclipped_grad_norm": 0.9058489799499512 + }, + { + "bits_per_byte": 1.527050123380059, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0584704875946045, + "step": 6390, + "unclipped_grad_norm": 0.929339587688446 + }, + { + "bits_per_byte": 1.5156593684481134, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0505750179290771, + "step": 6400, + "unclipped_grad_norm": 0.9009133577346802 + }, + { + "bits_per_byte": 1.4969940913499729, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0376372337341309, + "step": 6410, + "unclipped_grad_norm": 0.957128643989563 + }, + { + "bits_per_byte": 1.4783625228514012, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0247228145599365, + "step": 6420, + "unclipped_grad_norm": 0.9414587616920471 + }, + { + "bits_per_byte": 1.523227980947284, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.055821180343628, + "step": 6430, + "unclipped_grad_norm": 0.9234302639961243 + }, + { + "bits_per_byte": 1.4623229048839832, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.013604998588562, + "step": 6440, + "unclipped_grad_norm": 0.9177728295326233 + }, + { + "bits_per_byte": 1.5306256426914968, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0609488487243652, + "step": 6450, + "unclipped_grad_norm": 0.9453822374343872 + }, + { + "bits_per_byte": 1.5847898587542297, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0984926223754883, + "step": 6460, + "unclipped_grad_norm": 0.9647909998893738 + }, + { + "bits_per_byte": 1.5690988496201106, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0876164436340332, + "step": 6470, + "unclipped_grad_norm": 0.9592546820640564 + }, + { + "bits_per_byte": 1.4544314809491432, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0081350803375244, + "step": 6480, + "unclipped_grad_norm": 0.9334172010421753 + }, + { + "bits_per_byte": 1.5573938823850386, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0795031785964966, + "step": 6490, + "unclipped_grad_norm": 0.967822790145874 + }, + { + "bits_per_byte": 1.5280063469188563, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0591332912445068, + "step": 6500, + "unclipped_grad_norm": 0.9625112414360046 + }, + { + "bits_per_byte": 1.4687796495453256, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.018080472946167, + "step": 6510, + "unclipped_grad_norm": 0.9510544538497925 + }, + { + "bits_per_byte": 1.4284500618832987, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9901261329650879, + "step": 6520, + "unclipped_grad_norm": 0.8985839486122131 + }, + { + "bits_per_byte": 1.4546447394362132, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0082828998565674, + "step": 6530, + "unclipped_grad_norm": 0.907886803150177 + }, + { + "bits_per_byte": 1.4970680438898438, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0376884937286377, + "step": 6540, + "unclipped_grad_norm": 0.9381445050239563 + }, + { + "bits_per_byte": 1.4280130539674563, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9898232221603394, + "step": 6550, + "unclipped_grad_norm": 0.9031082391738892 + }, + { + "bits_per_byte": 1.5210671909218436, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.054323434829712, + "step": 6560, + "unclipped_grad_norm": 0.9433532357215881 + }, + { + "bits_per_byte": 1.479243074023819, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0253331661224365, + "step": 6570, + "unclipped_grad_norm": 0.9311387538909912 + }, + { + "bits_per_byte": 1.5051876888023787, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0433166027069092, + "step": 6580, + "unclipped_grad_norm": 0.9569925665855408 + }, + { + "bits_per_byte": 1.494823842278735, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0361329317092896, + "step": 6590, + "unclipped_grad_norm": 0.8958868384361267 + }, + { + "bits_per_byte": 1.4302109062541577, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9913466572761536, + "step": 6600, + "unclipped_grad_norm": 0.9136191010475159 + }, + { + "bits_per_byte": 1.436879103584736, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 0.9959686994552612, + "step": 6610, + "unclipped_grad_norm": 0.8710504770278931 + }, + { + "bits_per_byte": 1.4278971376607748, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9897428750991821, + "step": 6620, + "unclipped_grad_norm": 0.9145034551620483 + }, + { + "bits_per_byte": 1.4582168190946336, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.010758876800537, + "step": 6630, + "unclipped_grad_norm": 0.8999127149581909 + }, + { + "bits_per_byte": 1.5240701799885592, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.056404948234558, + "step": 6640, + "unclipped_grad_norm": 0.9801992774009705 + }, + { + "bits_per_byte": 1.5169175935218258, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0514471530914307, + "step": 6650, + "unclipped_grad_norm": 0.8882671594619751 + }, + { + "bits_per_byte": 1.5152221885496202, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.050271987915039, + "step": 6660, + "unclipped_grad_norm": 0.9029646515846252 + }, + { + "bits_per_byte": 1.4896082964086694, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0325177907943726, + "step": 6670, + "unclipped_grad_norm": 0.9797500371932983 + }, + { + "bits_per_byte": 1.4764526555135697, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.023398995399475, + "step": 6680, + "unclipped_grad_norm": 0.9356769919395447 + }, + { + "bits_per_byte": 1.4958203097578342, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0368236303329468, + "step": 6690, + "unclipped_grad_norm": 0.939789891242981 + }, + { + "bits_per_byte": 1.5480882451121538, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.073053002357483, + "step": 6700, + "unclipped_grad_norm": 1.0080333948135376 + }, + { + "bits_per_byte": 1.525887864625528, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0576648712158203, + "step": 6710, + "unclipped_grad_norm": 0.9197862148284912 + }, + { + "bits_per_byte": 1.534906462854123, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0639160871505737, + "step": 6720, + "unclipped_grad_norm": 0.9543269872665405 + }, + { + "bits_per_byte": 1.4820493149379474, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0272783041000366, + "step": 6730, + "unclipped_grad_norm": 0.9817740321159363 + }, + { + "bits_per_byte": 1.4384499931177168, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9970575571060181, + "step": 6740, + "unclipped_grad_norm": 0.9710240364074707 + }, + { + "bits_per_byte": 1.511069151496585, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0473933219909668, + "step": 6750, + "unclipped_grad_norm": 0.9612263441085815 + }, + { + "bits_per_byte": 1.4780942299160553, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0245368480682373, + "step": 6760, + "unclipped_grad_norm": 0.9238441586494446 + }, + { + "bits_per_byte": 1.3767874194686909, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9543163180351257, + "step": 6770, + "unclipped_grad_norm": 0.9215719699859619 + }, + { + "bits_per_byte": 1.4785716537548506, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0248677730560303, + "step": 6780, + "unclipped_grad_norm": 0.9403936266899109 + }, + { + "bits_per_byte": 1.4459957319243224, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0022878646850586, + "step": 6790, + "unclipped_grad_norm": 0.9651554822921753 + }, + { + "bits_per_byte": 1.5175859181030786, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.051910400390625, + "step": 6800, + "unclipped_grad_norm": 0.9926899075508118 + }, + { + "bits_per_byte": 1.5110634760691066, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0473893880844116, + "step": 6810, + "unclipped_grad_norm": 0.9793192148208618 + }, + { + "bits_per_byte": 1.5052492585913877, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0433592796325684, + "step": 6820, + "unclipped_grad_norm": 1.0081095695495605 + }, + { + "bits_per_byte": 1.466343515295854, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0163918733596802, + "step": 6830, + "unclipped_grad_norm": 0.960361659526825 + }, + { + "bits_per_byte": 1.4376153613130795, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9964790344238281, + "step": 6840, + "unclipped_grad_norm": 0.9702624678611755 + }, + { + "bits_per_byte": 1.4922177891702109, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0343265533447266, + "step": 6850, + "unclipped_grad_norm": 0.9893983006477356 + }, + { + "bits_per_byte": 1.4659078832412185, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.016089916229248, + "step": 6860, + "unclipped_grad_norm": 0.9693014621734619 + }, + { + "bits_per_byte": 1.413224953750367, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9795728921890259, + "step": 6870, + "unclipped_grad_norm": 0.9252057075500488 + }, + { + "bits_per_byte": 1.4200107012228098, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9842764139175415, + "step": 6880, + "unclipped_grad_norm": 0.8884434700012207 + }, + { + "bits_per_byte": 1.4366857090938405, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9958346486091614, + "step": 6890, + "unclipped_grad_norm": 0.9679457545280457 + }, + { + "bits_per_byte": 1.5416700245646047, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0686042308807373, + "step": 6900, + "unclipped_grad_norm": 0.9985603094100952 + }, + { + "bits_per_byte": 1.4893800754309745, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0323596000671387, + "step": 6910, + "unclipped_grad_norm": 0.9936915636062622 + }, + { + "bits_per_byte": 1.4499423897963222, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.00502347946167, + "step": 6920, + "unclipped_grad_norm": 0.923648476600647 + }, + { + "bits_per_byte": 1.4406846496917027, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9986065030097961, + "step": 6930, + "unclipped_grad_norm": 0.9748247861862183 + }, + { + "bits_per_byte": 1.408974058568991, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9766263961791992, + "step": 6940, + "unclipped_grad_norm": 0.9357670545578003 + }, + { + "bits_per_byte": 1.434796565665438, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9945251941680908, + "step": 6950, + "unclipped_grad_norm": 0.9624595642089844 + }, + { + "bits_per_byte": 1.49675830313564, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0374737977981567, + "step": 6960, + "unclipped_grad_norm": 0.9387544989585876 + }, + { + "bits_per_byte": 1.4929531869853003, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0348362922668457, + "step": 6970, + "unclipped_grad_norm": 0.9343914985656738 + }, + { + "bits_per_byte": 1.4660633555575984, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.016197681427002, + "step": 6980, + "unclipped_grad_norm": 0.9326976537704468 + }, + { + "bits_per_byte": 1.492015881538098, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034186601638794, + "step": 6990, + "unclipped_grad_norm": 1.0069916248321533 + }, + { + "bits_per_byte": 1.52302985693349, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0556838512420654, + "step": 7000, + "unclipped_grad_norm": 0.9719612002372742 + }, + { + "bits_per_byte": 1.512785538352196, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0485830307006836, + "step": 7010, + "unclipped_grad_norm": 0.9873834848403931 + }, + { + "bits_per_byte": 1.4473323810868282, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0032143592834473, + "step": 7020, + "unclipped_grad_norm": 0.939332127571106 + }, + { + "bits_per_byte": 1.5166639191118032, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0512713193893433, + "step": 7030, + "unclipped_grad_norm": 0.9837853312492371 + }, + { + "bits_per_byte": 1.5072451172546506, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0447427034378052, + "step": 7040, + "unclipped_grad_norm": 0.9734936952590942 + }, + { + "bits_per_byte": 1.4796651194490362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0256257057189941, + "step": 7050, + "unclipped_grad_norm": 1.005554437637329 + }, + { + "bits_per_byte": 1.4927048440374544, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0346641540527344, + "step": 7060, + "unclipped_grad_norm": 0.9903765320777893 + }, + { + "bits_per_byte": 1.4561590466770602, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.009332537651062, + "step": 7070, + "unclipped_grad_norm": 1.009922981262207 + }, + { + "bits_per_byte": 1.4353391709289103, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9949012994766235, + "step": 7080, + "unclipped_grad_norm": 0.9552373290061951 + }, + { + "bits_per_byte": 1.4780041110070032, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0244743824005127, + "step": 7090, + "unclipped_grad_norm": 1.0046122074127197 + }, + { + "bits_per_byte": 1.4728850474040718, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0209261178970337, + "step": 7100, + "unclipped_grad_norm": 0.9868813157081604 + }, + { + "bits_per_byte": 1.4862149067244954, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.030165672302246, + "step": 7110, + "unclipped_grad_norm": 0.9593554735183716 + }, + { + "bits_per_byte": 1.5328834309320238, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.062513828277588, + "step": 7120, + "unclipped_grad_norm": 1.0129024982452393 + }, + { + "bits_per_byte": 1.4811555211014134, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0266587734222412, + "step": 7130, + "unclipped_grad_norm": 1.0546875 + }, + { + "bits_per_byte": 1.4724865636020226, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0206499099731445, + "step": 7140, + "unclipped_grad_norm": 0.9776664972305298 + }, + { + "bits_per_byte": 1.4357171887955067, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.9951633214950562, + "step": 7150, + "unclipped_grad_norm": 0.9291411638259888 + }, + { + "bits_per_byte": 1.5232358921492237, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0558266639709473, + "step": 7160, + "unclipped_grad_norm": 1.0189123153686523 + }, + { + "bits_per_byte": 1.4618840051589814, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0133007764816284, + "step": 7170, + "unclipped_grad_norm": 0.9592419266700745 + }, + { + "bits_per_byte": 1.4762450724539784, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0232551097869873, + "step": 7180, + "unclipped_grad_norm": 0.9680334329605103 + }, + { + "bits_per_byte": 1.4441101141402626, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0009808540344238, + "step": 7190, + "unclipped_grad_norm": 1.0148857831954956 + }, + { + "bits_per_byte": 1.472608843266786, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0207346677780151, + "step": 7200, + "unclipped_grad_norm": 0.9935786128044128 + }, + { + "bits_per_byte": 1.5299951542934338, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.060511827468872, + "step": 7210, + "unclipped_grad_norm": 0.9725626111030579 + }, + { + "bits_per_byte": 1.5238232129019202, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0562337636947632, + "step": 7220, + "unclipped_grad_norm": 1.0141370296478271 + }, + { + "bits_per_byte": 1.4390708504873315, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9974879026412964, + "step": 7230, + "unclipped_grad_norm": 1.0297694206237793 + }, + { + "bits_per_byte": 1.4632025961431465, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0142147541046143, + "step": 7240, + "unclipped_grad_norm": 0.9345177412033081 + }, + { + "bits_per_byte": 1.5031187375124992, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0418825149536133, + "step": 7250, + "unclipped_grad_norm": 1.0408201217651367 + }, + { + "bits_per_byte": 1.3820238612308362, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9579459428787231, + "step": 7260, + "unclipped_grad_norm": 0.9544695615768433 + }, + { + "bits_per_byte": 1.4274823155068936, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9894553422927856, + "step": 7270, + "unclipped_grad_norm": 0.9940797686576843 + }, + { + "bits_per_byte": 1.4990891840027836, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0390894412994385, + "step": 7280, + "unclipped_grad_norm": 1.013985514640808 + }, + { + "bits_per_byte": 1.4370805812602216, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.99610835313797, + "step": 7290, + "unclipped_grad_norm": 0.9820091724395752 + }, + { + "bits_per_byte": 1.4451605841717328, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.001708984375, + "step": 7300, + "unclipped_grad_norm": 0.9996907114982605 + }, + { + "bits_per_byte": 1.4502488628801598, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0052359104156494, + "step": 7310, + "unclipped_grad_norm": 0.9723894000053406 + }, + { + "bits_per_byte": 1.4141343980081293, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9802032709121704, + "step": 7320, + "unclipped_grad_norm": 0.9815637469291687 + }, + { + "bits_per_byte": 1.429772436485783, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9910427331924438, + "step": 7330, + "unclipped_grad_norm": 1.0089058876037598 + }, + { + "bits_per_byte": 1.4033304478809279, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9727145433425903, + "step": 7340, + "unclipped_grad_norm": 0.9830491542816162 + }, + { + "bits_per_byte": 1.4800763299672492, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.02591073513031, + "step": 7350, + "unclipped_grad_norm": 0.9842677712440491 + }, + { + "bits_per_byte": 1.4409101189469837, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9987627863883972, + "step": 7360, + "unclipped_grad_norm": 1.0266367197036743 + }, + { + "bits_per_byte": 1.4700960047550296, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0189929008483887, + "step": 7370, + "unclipped_grad_norm": 0.9935671091079712 + }, + { + "bits_per_byte": 1.4111584102175994, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9781404733657837, + "step": 7380, + "unclipped_grad_norm": 0.9791932702064514 + }, + { + "bits_per_byte": 1.4805810990475317, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0262606143951416, + "step": 7390, + "unclipped_grad_norm": 0.9806758761405945 + }, + { + "bits_per_byte": 1.422500752033327, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9860023856163025, + "step": 7400, + "unclipped_grad_norm": 1.014562726020813 + }, + { + "bits_per_byte": 1.419007526420327, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9835810661315918, + "step": 7410, + "unclipped_grad_norm": 0.9609370231628418 + }, + { + "bits_per_byte": 1.483305992167802, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0281493663787842, + "step": 7420, + "unclipped_grad_norm": 0.9924190640449524 + }, + { + "bits_per_byte": 1.4892452410326982, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0322661399841309, + "step": 7430, + "unclipped_grad_norm": 1.0244429111480713 + }, + { + "bits_per_byte": 1.50888548777858, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0458797216415405, + "step": 7440, + "unclipped_grad_norm": 1.0394790172576904 + }, + { + "bits_per_byte": 1.4553271665948369, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0087559223175049, + "step": 7450, + "unclipped_grad_norm": 0.99248868227005 + }, + { + "bits_per_byte": 1.4425173828106221, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.999876856803894, + "step": 7460, + "unclipped_grad_norm": 1.0004469156265259 + }, + { + "bits_per_byte": 1.437790267669007, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9966002702713013, + "step": 7470, + "unclipped_grad_norm": 1.0109968185424805 + }, + { + "bits_per_byte": 1.4682612938356252, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.017721176147461, + "step": 7480, + "unclipped_grad_norm": 1.0299010276794434 + }, + { + "bits_per_byte": 1.5222762289574092, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.055161476135254, + "step": 7490, + "unclipped_grad_norm": 1.0327476263046265 + }, + { + "bits_per_byte": 1.4747042798848986, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0221871137619019, + "step": 7500, + "unclipped_grad_norm": 0.9886637926101685 + }, + { + "bits_per_byte": 1.4352613487793948, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9948473572731018, + "step": 7510, + "unclipped_grad_norm": 0.9710773825645447 + }, + { + "bits_per_byte": 1.4570311706995855, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.009937047958374, + "step": 7520, + "unclipped_grad_norm": 1.0144232511520386 + }, + { + "bits_per_byte": 1.502727820947088, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0416115522384644, + "step": 7530, + "unclipped_grad_norm": 1.0307852029800415 + }, + { + "bits_per_byte": 1.4572943041554056, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0101194381713867, + "step": 7540, + "unclipped_grad_norm": 0.9732069969177246 + }, + { + "bits_per_byte": 1.4747278415080667, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0222034454345703, + "step": 7550, + "unclipped_grad_norm": 1.0250486135482788 + }, + { + "bits_per_byte": 1.4513507557242378, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0059996843338013, + "step": 7560, + "unclipped_grad_norm": 1.0031380653381348 + }, + { + "bits_per_byte": 1.4127096077370564, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.979215681552887, + "step": 7570, + "unclipped_grad_norm": 0.9720339775085449 + }, + { + "bits_per_byte": 1.587541409185384, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1003998517990112, + "step": 7580, + "unclipped_grad_norm": 1.080464482307434 + }, + { + "bits_per_byte": 1.4421111597892842, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9995952844619751, + "step": 7590, + "unclipped_grad_norm": 0.9815735220909119 + }, + { + "bits_per_byte": 1.4267507013101233, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9889482259750366, + "step": 7600, + "unclipped_grad_norm": 1.029758334159851 + }, + { + "bits_per_byte": 1.4539179407536669, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0077791213989258, + "step": 7610, + "unclipped_grad_norm": 1.031221628189087 + }, + { + "bits_per_byte": 1.4028547438686412, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9723848104476929, + "step": 7620, + "unclipped_grad_norm": 0.9562389850616455 + }, + { + "bits_per_byte": 1.3938100042771153, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9661154747009277, + "step": 7630, + "unclipped_grad_norm": 1.023301124572754 + }, + { + "bits_per_byte": 1.4184527103886437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.983196496963501, + "step": 7640, + "unclipped_grad_norm": 0.9981763958930969 + }, + { + "bits_per_byte": 1.5608753271864544, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.081916332244873, + "step": 7650, + "unclipped_grad_norm": 1.0295648574829102 + }, + { + "bits_per_byte": 1.4665378556913289, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0165265798568726, + "step": 7660, + "unclipped_grad_norm": 1.0587892532348633 + }, + { + "bits_per_byte": 1.4931012640476933, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.034938931465149, + "step": 7670, + "unclipped_grad_norm": 1.0342050790786743 + }, + { + "bits_per_byte": 1.4465245785757257, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0026544332504272, + "step": 7680, + "unclipped_grad_norm": 1.0567405223846436 + }, + { + "bits_per_byte": 1.4541416901824393, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0079342126846313, + "step": 7690, + "unclipped_grad_norm": 1.0190099477767944 + }, + { + "bits_per_byte": 1.3949941048283057, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9669362306594849, + "step": 7700, + "unclipped_grad_norm": 1.002891182899475 + }, + { + "bits_per_byte": 1.4762163513512843, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0232352018356323, + "step": 7710, + "unclipped_grad_norm": 0.9984713792800903 + }, + { + "bits_per_byte": 1.480895139368007, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0264782905578613, + "step": 7720, + "unclipped_grad_norm": 1.0342158079147339 + }, + { + "bits_per_byte": 1.486845911070511, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.030603051185608, + "step": 7730, + "unclipped_grad_norm": 1.027107834815979 + }, + { + "bits_per_byte": 1.4387832095037634, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9972885251045227, + "step": 7740, + "unclipped_grad_norm": 1.0455139875411987 + }, + { + "bits_per_byte": 1.4381068877292456, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9968197345733643, + "step": 7750, + "unclipped_grad_norm": 1.0159157514572144 + }, + { + "bits_per_byte": 1.4538223183997872, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0077128410339355, + "step": 7760, + "unclipped_grad_norm": 0.9867706894874573 + }, + { + "bits_per_byte": 1.3986834766546148, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9694935083389282, + "step": 7770, + "unclipped_grad_norm": 0.9943742156028748 + }, + { + "bits_per_byte": 1.4760859885019304, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0231448411941528, + "step": 7780, + "unclipped_grad_norm": 1.0422483682632446 + }, + { + "bits_per_byte": 1.4162891686407895, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9816968441009521, + "step": 7790, + "unclipped_grad_norm": 0.9937946796417236 + }, + { + "bits_per_byte": 1.45477768202533, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0083750486373901, + "step": 7800, + "unclipped_grad_norm": 1.0428839921951294 + }, + { + "bits_per_byte": 1.4292601001688625, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9906876087188721, + "step": 7810, + "unclipped_grad_norm": 1.028254508972168 + }, + { + "bits_per_byte": 1.4070698666586379, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.975306510925293, + "step": 7820, + "unclipped_grad_norm": 1.0253064632415771 + }, + { + "bits_per_byte": 1.4530960356651936, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0072094202041626, + "step": 7830, + "unclipped_grad_norm": 1.0885143280029297 + }, + { + "bits_per_byte": 1.4330722676078875, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9933300018310547, + "step": 7840, + "unclipped_grad_norm": 1.0074925422668457 + }, + { + "bits_per_byte": 1.3858685333908742, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9606108665466309, + "step": 7850, + "unclipped_grad_norm": 0.9880329966545105 + }, + { + "bits_per_byte": 1.4517502714221922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0062766075134277, + "step": 7860, + "unclipped_grad_norm": 1.0082415342330933 + }, + { + "bits_per_byte": 1.4675547891058807, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0172314643859863, + "step": 7870, + "unclipped_grad_norm": 1.023923397064209 + }, + { + "bits_per_byte": 1.3867994754799946, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9612561464309692, + "step": 7880, + "unclipped_grad_norm": 1.0151392221450806 + }, + { + "bits_per_byte": 1.4568015738606837, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0097779035568237, + "step": 7890, + "unclipped_grad_norm": 1.0505098104476929 + }, + { + "bits_per_byte": 1.446776189193938, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.00282883644104, + "step": 7900, + "unclipped_grad_norm": 1.0279992818832397 + }, + { + "bits_per_byte": 1.461830518554563, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0132637023925781, + "step": 7910, + "unclipped_grad_norm": 1.0125948190689087 + }, + { + "bits_per_byte": 1.4593548282953928, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0115476846694946, + "step": 7920, + "unclipped_grad_norm": 1.0411789417266846 + }, + { + "bits_per_byte": 1.4317973602170415, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9924463033676147, + "step": 7930, + "unclipped_grad_norm": 1.0330348014831543 + }, + { + "bits_per_byte": 1.431196452834927, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9920297861099243, + "step": 7940, + "unclipped_grad_norm": 1.0142537355422974 + }, + { + "bits_per_byte": 1.49434246283897, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.035799264907837, + "step": 7950, + "unclipped_grad_norm": 1.0716009140014648 + }, + { + "bits_per_byte": 1.4186909923514142, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9833616614341736, + "step": 7960, + "unclipped_grad_norm": 1.0115852355957031 + }, + { + "bits_per_byte": 1.4420433126335188, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9995482563972473, + "step": 7970, + "unclipped_grad_norm": 1.0335451364517212 + }, + { + "bits_per_byte": 1.4872165336831205, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0308599472045898, + "step": 7980, + "unclipped_grad_norm": 1.0748283863067627 + }, + { + "bits_per_byte": 1.5234063269562288, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.055944800376892, + "step": 7990, + "unclipped_grad_norm": 1.076164722442627 + }, + { + "bits_per_byte": 1.4309937712808851, + "learning_rate": 3e-05, + "loss_nats": 0.9918892979621887, + "step": 8000, + "unclipped_grad_norm": 1.0322442054748535 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073002.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073002.json new file mode 100644 index 0000000..c08a5f4 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073002.json @@ -0,0 +1,13072 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "b4262e697e2269457fdebf31f75008383c6e8024ee1bbf96e5962fb2ba1dec18", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "population_cv": 0.39438035124793563 + }, + "activation_output_rms_by_block": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005765057983808219, + "first_to_last_ratio": 0.622587017371887, + "imbalance_abs_log_ratio": 0.4738718734697682, + "last_quartile_mean": 0.009259842918254435, + "mean": 0.007398182220640592, + "normalized": [ + 0.49149180917985336, + 0.6965082091758006, + 0.8632718983507628, + 0.9846233376628902, + 0.5069511565390686, + 0.730127144119427, + 0.8901300989627224, + 1.0709228264321597, + 0.6068057253534959, + 0.8292953231982888, + 0.9889841312125875, + 1.119997332175246, + 0.6022405569783624, + 0.8738552664669654, + 1.0668989020126547, + 1.2430080659586231, + 0.6705962885532487, + 0.9336251153230055, + 1.1108121094231989, + 1.3080235563150786, + 0.7274038039370601, + 1.01412651484229, + 1.232101109614035, + 1.4250990487866828, + 0.8294838994154143, + 1.1448566626512724, + 1.427936504016329, + 1.6627087330393202, + 0.7804514393709395, + 1.1025360304896303, + 1.4135242707919937, + 1.6516031296515918 + ], + "population_cv": 0.30590027610070303 + }, + "bits_per_byte": 8.038491115106986, + "branch_output_rms_by_sublayer": [ + 0.0023513585329055786, + 0.0027412420604377985, + 0.0023208872880786657, + 0.002683233004063368, + 0.002778682392090559, + 0.002711885841563344, + 0.002497476991266012, + 0.0027249811682850122, + 0.002628475893288851, + 0.0027012599166482687, + 0.002920303260907531, + 0.002700963756069541, + 0.002932395786046982, + 0.0026683746837079525, + 0.0032228652853518724, + 0.002738022478297353, + 0.003561305580660701, + 0.0027370047755539417, + 0.003354643704369664, + 0.002654933836311102, + 0.0034162327647209167, + 0.002663377905264497, + 0.0036349669098854065, + 0.002684312639757991, + 0.0035210708156228065, + 0.0027356327045708895, + 0.0038089773152023554, + 0.002652956871315837, + 0.003915763925760984, + 0.002679797587916255, + 0.0037244565319269896, + 0.002698024269193411, + 0.004162859171628952, + 0.0026484597474336624, + 0.004167158622294664, + 0.0026680538430809975, + 0.004440344404429197, + 0.002654345240443945, + 0.0041700867004692554, + 0.002591830212622881, + 0.004737259820103645, + 0.002620960585772991, + 0.004552462603896856, + 0.002651234855875373, + 0.004706811159849167, + 0.0026270023081451654, + 0.005186605267226696, + 0.002615687670186162, + 0.005580419674515724, + 0.00258000404573977, + 0.004905648995190859, + 0.002626793459057808, + 0.005227713380008936, + 0.002573782578110695, + 0.005349088925868273, + 0.002592637902125716, + 0.005235649179667234, + 0.002581534907221794, + 0.004925078712403774, + 0.002551236655563116, + 0.005810525268316269, + 0.002563327783718705, + 0.005564783234149218, + 0.002589041367173195 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.010012756197717748, + 0.008904953530422711, + 0.008980942528705073, + 0.007787912846174446, + 0.007907551845183929, + 0.008289745083748344, + 0.007663833460373458, + 0.007350080301903749, + 0.00709564946584838, + 0.006310668475383808, + 0.006285078227083486, + 0.006362421438950662, + 0.005604026016423943, + 0.005959901601525332, + 0.006167718108848569, + 0.005523196426059091, + 0.0054641843609823, + 0.00557989785603791, + 0.005260407000687673, + 0.004763357766419076, + 0.004970214176820314, + 0.004845025235106639, + 0.004921304265254701, + 0.004820106373346748, + 0.0054804096177630225, + 0.0042302744374162005, + 0.004588429583035324, + 0.004469681656036737, + 0.004349111730454971, + 0.003928502929307311, + 0.004468266803351117, + 0.003904642007649288 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.008362221974278682, + "first_to_last_ratio": 1.8887369414995314, + "imbalance_abs_log_ratio": 0.6359083207487641, + "last_quartile_mean": 0.004427414845626747, + "mean": 0.00600782035481319, + "normalized": [ + 1.6666204390908572, + 1.4822269982304765, + 1.4948753455169401, + 1.2962958920592769, + 1.3162097696294732, + 1.3798257261648947, + 1.2756429133626719, + 1.2234187888150154, + 1.1810688480662828, + 1.0504089840715678, + 1.0461494944748422, + 1.0590232502297414, + 0.9327885465041001, + 0.9920239370590588, + 1.0266149359654666, + 0.9193344840336578, + 0.9095119424808776, + 0.9287724210274617, + 0.875593258455752, + 0.7928595538984271, + 0.8272907449435313, + 0.8064530809788657, + 0.8191497039874, + 0.8023053434820334, + 0.9122126318860999, + 0.7041279844573081, + 0.7637428072161453, + 0.7439772483303089, + 0.7239084182952745, + 0.6538982022256998, + 0.7437417465006834, + 0.6499265585598059 + ], + "population_cv": 0.2648157248292805 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.027978340163826942, + 0.014031016267836094, + 0.0140878576785326, + 0.014151837676763535, + 0.014187770895659924, + 0.014265215955674648, + 0.014337530359625816, + 0.014401593245565891, + 0.014477464370429516, + 0.009722241200506687, + 0.009757626801729202, + 0.009786579757928848, + 0.009825476445257664, + 0.009839320555329323, + 0.00987118761986494, + 0.00996283907443285, + 0.009999874979257584, + 0.007562503684312105, + 0.007595349568873644, + 0.007635751739144325, + 0.007658981718122959, + 0.007660485804080963, + 0.007689184043556452, + 0.007726665120571852, + 0.007755559403449297, + 0.006238248664885759, + 0.00625597033649683, + 0.006290202960371971, + 0.006315375678241253, + 0.006327399518340826, + 0.00635211868211627, + 0.006382675375789404, + 0.006408242508769035, + 0.005390914157032967, + 0.005408597644418478, + 0.0054517327807843685, + 0.005475745070725679, + 0.005509276874363422, + 0.005517482291907072, + 0.0055727302096784115, + 0.005585856735706329, + 0.0048178439028561115, + 0.004830927588045597, + 0.004884926602244377, + 0.004899685736745596, + 0.004938251338899136, + 0.004948681220412254, + 0.004975184798240662, + 0.004989070352166891, + 0.004412420094013214, + 0.004419817589223385, + 0.004455919377505779, + 0.004463871940970421, + 0.004530393984168768, + 0.004542885348200798, + 0.0045819394290447235, + 0.004592607729136944, + 0.004127359017729759, + 0.004137600306421518, + 0.004185030702501535, + 0.004193917382508516, + 0.004221425391733646, + 0.004230021964758635, + 0.004281661007553339 + ], + "loss_nats": 5.571857452392578, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0023513585329055786, + 0.0036361459642648697, + 0.004408807028084993, + 0.005152894649654627, + 0.005783648230135441, + 0.006386642809957266, + 0.006751363631337881, + 0.007284422870725393, + 0.002628475893288851, + 0.0037505170330405235, + 0.004723479505628347, + 0.0054016136564314365, + 0.006024772301316261, + 0.006585344672203064, + 0.007445912808179855, + 0.007922882214188576, + 0.003561305580660701, + 0.0044892593286931515, + 0.005543973762542009, + 0.006135277915745974, + 0.006855612155050039, + 0.007316684816032648, + 0.007876631803810596, + 0.0082859443500638, + 0.0035210708156228065, + 0.0044554853811860085, + 0.0058924350887537, + 0.006464940495789051, + 0.007390674669295549, + 0.007893112488090992, + 0.008790239691734314, + 0.009196000173687935, + 0.004162859171628952, + 0.004961193539202213, + 0.006355749908834696, + 0.006907128728926182, + 0.007784162648022175, + 0.008217990398406982, + 0.00937180407345295, + 0.009676996618509293, + 0.004737259820103645, + 0.005381465889513493, + 0.006999304983764887, + 0.007502692751586437, + 0.008728929795324802, + 0.0091153085231781, + 0.010217384435236454, + 0.010543142445385456, + 0.005580419674515724, + 0.006136673036962748, + 0.008008211851119995, + 0.008469858206808567, + 0.010209310799837112, + 0.010564134456217289, + 0.012040933594107628, + 0.012301022186875343, + 0.005235649179667234, + 0.005773921962827444, + 0.007745372597128153, + 0.008156762458384037, + 0.010180563665926456, + 0.010457510128617287, + 0.011906592175364494, + 0.012218860909342766 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.95402777637355e-05, + 6.603818474104628e-05, + 6.243807729333639e-05, + 5.902352859266102e-05, + 5.3786254284204915e-05, + 5.022022014600225e-05, + 4.6831908548483625e-05, + 4.3414886022219434e-05, + 3.877186099998653e-05, + 3.6122120945947245e-05, + 3.3541862649144605e-05, + 3.1072035199031234e-05, + 2.835266786860302e-05, + 2.6703228286351077e-05, + 2.5144798200926743e-05, + 2.3762264390825294e-05, + 2.2584137695957907e-05, + 2.1571571778622456e-05, + 2.0759547624038532e-05, + 1.9997522031189874e-05, + 1.9395642084418796e-05, + 1.8848633771995082e-05, + 1.8380347682978027e-05, + 1.797073673515115e-05, + 1.755586163199041e-05, + 1.7233234757441096e-05, + 1.6949628843576647e-05, + 1.6709716874174774e-05, + 1.630509359529242e-05, + 1.611644620425068e-05, + 1.594743662280962e-05, + 1.580910975462757e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.641166717396118e-05, + "first_to_last_ratio": 3.40273807382454, + "imbalance_abs_log_ratio": 1.2245804233600555, + "last_quartile_mean": 1.6578316035520402e-05, + "mean": 3.084058886315688e-05, + "normalized": [ + 2.254829765809383, + 2.1412750915381364, + 2.0245423189025695, + 1.913826251974467, + 1.7440086673720954, + 1.6283807150646488, + 1.518515381022133, + 1.407719100788124, + 1.2571699318719751, + 1.1712526341901288, + 1.0875882687575642, + 1.0075046017085245, + 0.9193296533476374, + 0.8658469008110146, + 0.8153151132261776, + 0.770486727612793, + 0.7322862023214354, + 0.6994539525278819, + 0.6731242297658762, + 0.648415700488765, + 0.6288998621420432, + 0.6111632256967778, + 0.5959791417905044, + 0.5826975877435189, + 0.5692453444999938, + 0.5587842318415797, + 0.5495883661230995, + 0.5418092679201958, + 0.5286894380532074, + 0.5225725836741036, + 0.517092481391006, + 0.5126072600226392 + ], + "population_cv": 0.5330470009230729 + }, + "activation_output_rms_by_block": [ + 0.005864492617547512, + 0.009892112575471401, + 0.014509912580251694, + 0.019168438389897346, + 0.006793031003326178, + 0.01223041582852602, + 0.017751524224877357, + 0.023574935272336006, + 0.008267601951956749, + 0.014755276031792164, + 0.02151178941130638, + 0.02830074541270733, + 0.009282662533223629, + 0.01676790602505207, + 0.02516039088368416, + 0.0331411249935627, + 0.010437680408358574, + 0.01991143450140953, + 0.029664477333426476, + 0.04036801680922508, + 0.01357664167881012, + 0.025029508396983147, + 0.03699437528848648, + 0.05095698684453964, + 0.015438608825206757, + 0.029965175315737724, + 0.04443887993693352, + 0.060807984322309494, + 0.019359532743692398, + 0.037317242473363876, + 0.05564062297344208, + 0.07935244590044022 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.01372310781152919, + "first_to_last_ratio": 0.3207078305283738, + "imbalance_abs_log_ratio": 1.137224755592113, + "last_quartile_mean": 0.04279006156139076, + "mean": 0.02613224917149637, + "normalized": [ + 0.224415915333617, + 0.37854041994445614, + 0.5552492816453899, + 0.7335165933901026, + 0.2599481949963819, + 0.4680200218611986, + 0.6792956897196492, + 0.9021395409795135, + 0.31637544467372514, + 0.5646385787522038, + 0.8231893577216564, + 1.0829816150527234, + 0.35521865998999624, + 0.6416556766702496, + 0.9628100022529916, + 1.268207905720998, + 0.3994176061868959, + 0.7619487465750876, + 1.1351673994362057, + 1.5447586062839287, + 0.519535903309034, + 0.9578015360531602, + 1.4156598249812316, + 1.9499655965365867, + 0.5907876020884704, + 1.1466741771474498, + 1.700537892674199, + 2.3269326694097012, + 0.7408291807047638, + 1.4280149492094805, + 2.1291938021979306, + 3.0365716085010215 + ], + "population_cv": 0.6534988385002395 + }, + "bits_per_byte": 5.118322357706281, + "branch_output_rms_by_sublayer": [ + 0.003087003482505679, + 0.0049620093777775764, + 0.0026560353580862284, + 0.004914142191410065, + 0.0032712947577238083, + 0.00555405905470252, + 0.002734958194196224, + 0.005823355168104172, + 0.003094236133620143, + 0.006141432095319033, + 0.0030397549271583557, + 0.006136075593531132, + 0.0035086015705019236, + 0.006357426289469004, + 0.0038121570833027363, + 0.0064498125575482845, + 0.0047028325498104095, + 0.006525229662656784, + 0.004038033075630665, + 0.007023691199719906, + 0.004860103130340576, + 0.006682637147605419, + 0.005031581036746502, + 0.006716069765388966, + 0.005196469835937023, + 0.006977807730436325, + 0.005135253071784973, + 0.006669311318546534, + 0.006614624988287687, + 0.006923797074705362, + 0.005866772960871458, + 0.007155857048928738, + 0.006203304044902325, + 0.007240671198815107, + 0.007167549338191748, + 0.007543859537690878, + 0.007609309628605843, + 0.007281317841261625, + 0.008147990331053734, + 0.007711030542850494, + 0.00901782512664795, + 0.00788753479719162, + 0.008936969563364983, + 0.007908415980637074, + 0.009206709451973438, + 0.008078642189502716, + 0.01152731105685234, + 0.008181036449968815, + 0.010577243752777576, + 0.007894126698374748, + 0.011347576975822449, + 0.008452145382761955, + 0.011137762106955051, + 0.008351319469511509, + 0.012936098501086235, + 0.008498509414494038, + 0.013407763093709946, + 0.009082214906811714, + 0.013788831420242786, + 0.009139914996922016, + 0.015265416353940964, + 0.008129997178912163, + 0.0202538650482893, + 0.00888112559914589 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002932670153511829, + 0.0002289733734981176, + 0.00023382191596673618, + 0.00020236837455671756, + 0.0002080571281121399, + 0.00019123637931049836, + 0.00021124835961838567, + 0.00020444780259018514, + 0.00020817260281626697, + 0.00017921191649714578, + 0.0001893830879389061, + 0.0001891854298096901, + 0.00018418310100442068, + 0.00017033524110907422, + 0.00020556753369166975, + 0.00017823357128584385, + 0.0001749817385153215, + 0.00018220334134876856, + 0.00017289789528954973, + 0.00017141475273833558, + 0.00017603112434287998, + 0.00016603952795395127, + 0.0001609386975915838, + 0.00016666055126527177, + 0.00016172141650657888, + 0.00015460675258205218, + 0.00014513294616478987, + 0.00015851211030580876, + 0.00015100080713109572, + 0.00014721653732759288, + 0.00014725738129728224, + 0.00017222312685716471 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002216775436254954, + "first_to_last_ratio": 1.4328688617518024, + "imbalance_abs_log_ratio": 0.35967863157527663, + "last_quartile_mean": 0.00015470888477154567, + "mean": 0.000183954110636719, + "normalized": [ + 1.5942400761586677, + 1.2447309424376207, + 1.2710882902122171, + 1.1001024867357483, + 1.1310273382420937, + 1.0395874201917712, + 1.1483753142954691, + 1.1114065452657267, + 1.1316550747124958, + 0.974220776458003, + 1.02951267184732, + 1.0284381749060347, + 1.0012448233252798, + 0.9259659407419281, + 1.1174935584757544, + 0.9689023565112261, + 0.9512249435995668, + 0.9904825758886794, + 0.9398968834732723, + 0.9318343153355962, + 0.956929550166532, + 0.9026138496130359, + 0.8748850299377811, + 0.9059898182672343, + 0.879139998268121, + 0.8404637006855296, + 0.7889627780669988, + 0.8616937656742324, + 0.8208612822428254, + 0.8002894679441159, + 0.8005115014151157, + 0.936228748904007 + ], + "population_cv": 0.1635293006829388 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.692298173904419, + "mean_weights": [ + 0.48010045289993286, + 0.5198994874954224 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927915811538696, + "mean_weights": [ + 0.5130036473274231, + 0.4869963526725769 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928644776344299, + "mean_weights": [ + 0.5081723928451538, + 0.4918276071548462 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692646861076355, + "mean_weights": [ + 0.5149973034858704, + 0.48500269651412964 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925361156463623, + "mean_weights": [ + 0.5161343812942505, + 0.4838656485080719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928029656410217, + "mean_weights": [ + 0.5107344388961792, + 0.4892655611038208 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924350261688232, + "mean_weights": [ + 0.5171386003494263, + 0.48286139965057373 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929639577865601, + "mean_weights": [ + 0.5072796940803528, + 0.49272027611732483 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0978727340698242, + "mean_weights": [ + 0.34929555654525757, + 0.32294148206710815, + 0.3277629613876343 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098320722579956, + "mean_weights": [ + 0.3432672619819641, + 0.3268624544143677, + 0.329870343208313 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098150610923767, + "mean_weights": [ + 0.3442404866218567, + 0.33018797636032104, + 0.32557153701782227 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984500646591187, + "mean_weights": [ + 0.34007447957992554, + 0.32975536584854126, + 0.3301700949668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982636213302612, + "mean_weights": [ + 0.3434537351131439, + 0.33091628551483154, + 0.3256300091743469 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984963178634644, + "mean_weights": [ + 0.33822712302207947, + 0.33379626274108887, + 0.32797661423683167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980933904647827, + "mean_weights": [ + 0.34460341930389404, + 0.33179372549057007, + 0.3236028254032135 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985530614852905, + "mean_weights": [ + 0.3358767628669739, + 0.3345070779323578, + 0.32961612939834595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860312700271606, + "mean_weights": [ + 0.25749218463897705, + 0.24992284178733826, + 0.24571210145950317, + 0.24687285721302032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862019777297974, + "mean_weights": [ + 0.25314557552337646, + 0.25236955285072327, + 0.24678170680999756, + 0.2477031946182251 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385892391204834, + "mean_weights": [ + 0.25815317034721375, + 0.25234687328338623, + 0.2448991984128952, + 0.24460072815418243 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861613273620605, + "mean_weights": [ + 0.25378289818763733, + 0.25339171290397644, + 0.24767157435417175, + 0.24515381455421448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3855845928192139, + "mean_weights": [ + 0.2592235505580902, + 0.2543467879295349, + 0.2462996244430542, + 0.24013003706932068 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861030340194702, + "mean_weights": [ + 0.25138330459594727, + 0.25586679577827454, + 0.24899962544441223, + 0.24375030398368835 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3856689929962158, + "mean_weights": [ + 0.2574368119239807, + 0.25204822421073914, + 0.2460038810968399, + 0.24451106786727905 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386252999305725, + "mean_weights": [ + 0.25002047419548035, + 0.25257009267807007, + 0.2502796947956085, + 0.24712975323200226 + ], + "sources": 4 + }, + { + "entropy_mean": 1.608945608139038, + "mean_weights": [ + 0.20495271682739258, + 0.2066430151462555, + 0.2000001072883606, + 0.19580814242362976, + 0.19259600341320038 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093038320541382, + "mean_weights": [ + 0.20154938101768494, + 0.20420850813388824, + 0.2010108083486557, + 0.19671520590782166, + 0.19651609659194946 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085759401321411, + "mean_weights": [ + 0.20549985766410828, + 0.209174245595932, + 0.20157773792743683, + 0.19417354464530945, + 0.18957462906837463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093413829803467, + "mean_weights": [ + 0.20104944705963135, + 0.20404505729675293, + 0.2002137303352356, + 0.19850106537342072, + 0.1961907148361206 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6080443859100342, + "mean_weights": [ + 0.20538949966430664, + 0.21283748745918274, + 0.20341980457305908, + 0.19302619993686676, + 0.18532699346542358 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093816757202148, + "mean_weights": [ + 0.2010989934206009, + 0.20253092050552368, + 0.19950580596923828, + 0.19837552309036255, + 0.1984887421131134 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6076874732971191, + "mean_weights": [ + 0.20581099390983582, + 0.21552260220050812, + 0.20295079052448273, + 0.19259145855903625, + 0.18312416970729828 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609355092048645, + "mean_weights": [ + 0.20059309899806976, + 0.203348308801651, + 0.2011984884738922, + 0.1984248012304306, + 0.19643528759479523 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7901467084884644, + "mean_weights": [ + 0.17125868797302246, + 0.18124164640903473, + 0.17160004377365112, + 0.1620369851589203, + 0.15427924692630768, + 0.15958338975906372 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916979789733887, + "mean_weights": [ + 0.16689208149909973, + 0.16927623748779297, + 0.1683787703514099, + 0.16569086909294128, + 0.1651553511619568, + 0.1646067053079605 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789665699005127, + "mean_weights": [ + 0.1720278263092041, + 0.18269017338752747, + 0.17356479167938232, + 0.16241812705993652, + 0.15660038590431213, + 0.15269872546195984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917176485061646, + "mean_weights": [ + 0.16644492745399475, + 0.16854418814182281, + 0.16665583848953247, + 0.16677430272102356, + 0.16461026668548584, + 0.16697049140930176 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893871068954468, + "mean_weights": [ + 0.17303556203842163, + 0.18359911441802979, + 0.17276406288146973, + 0.16385790705680847, + 0.1554735004901886, + 0.15126986801624298 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917413711547852, + "mean_weights": [ + 0.16618916392326355, + 0.16740113496780396, + 0.16675519943237305, + 0.1668717861175537, + 0.1653555929660797, + 0.16742712259292603 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7882882356643677, + "mean_weights": [ + 0.17288538813591003, + 0.18687990307807922, + 0.17607632279396057, + 0.162863090634346, + 0.15389539301395416, + 0.14739990234375 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917335033416748, + "mean_weights": [ + 0.16660982370376587, + 0.16799160838127136, + 0.16742658615112305, + 0.1664734035730362, + 0.16542598605155945, + 0.1660725623369217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941649079322815, + "mean_weights": [ + 0.15037035942077637, + 0.16432975232601166, + 0.15334898233413696, + 0.1418340504169464, + 0.13212506473064423, + 0.12697139382362366, + 0.13102038204669952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458887577056885, + "mean_weights": [ + 0.1431310474872589, + 0.14324922859668732, + 0.14309079945087433, + 0.142366424202919, + 0.14222830533981323, + 0.14392223954200745, + 0.14201194047927856 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9409761428833008, + "mean_weights": [ + 0.1505419909954071, + 0.16567251086235046, + 0.1550290882587433, + 0.14243587851524353, + 0.13249853253364563, + 0.12776729464530945, + 0.12605470418930054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458999633789062, + "mean_weights": [ + 0.14249610900878906, + 0.14256571233272552, + 0.1435922086238861, + 0.14210085570812225, + 0.14355877041816711, + 0.1427185833454132, + 0.14296773076057434 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941575527191162, + "mean_weights": [ + 0.15046924352645874, + 0.1640411913394928, + 0.15395450592041016, + 0.14198558032512665, + 0.13400861620903015, + 0.1288735270500183, + 0.126667320728302 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458911418914795, + "mean_weights": [ + 0.14260169863700867, + 0.14355596899986267, + 0.1428615152835846, + 0.14244115352630615, + 0.14214617013931274, + 0.14205437898635864, + 0.14433911442756653 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9400205612182617, + "mean_weights": [ + 0.1507154405117035, + 0.16689133644104004, + 0.15648439526557922, + 0.1427520215511322, + 0.13353285193443298, + 0.12712189555168152, + 0.12250205874443054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458441734313965, + "mean_weights": [ + 0.1422245353460312, + 0.14142528176307678, + 0.14198407530784607, + 0.14165860414505005, + 0.14288201928138733, + 0.14370128512382507, + 0.14612418413162231 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0741422176361084, + "mean_weights": [ + 0.13300861418247223, + 0.14654949307441711, + 0.1379268318414688, + 0.12711459398269653, + 0.1185426190495491, + 0.1134917140007019, + 0.10956315696239471, + 0.11380299925804138 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794055461883545, + "mean_weights": [ + 0.12428364902734756, + 0.1234029084444046, + 0.12436957657337189, + 0.12428068369626999, + 0.12555459141731262, + 0.12575605511665344, + 0.1263071894645691, + 0.1260453760623932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.071772813796997, + "mean_weights": [ + 0.134201318025589, + 0.1495765745639801, + 0.14145472645759583, + 0.12840476632118225, + 0.11872729659080505, + 0.11272042989730835, + 0.10793697834014893, + 0.1069779098033905 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079404592514038, + "mean_weights": [ + 0.12455718219280243, + 0.12374450266361237, + 0.12391386926174164, + 0.12458803504705429, + 0.1251734495162964, + 0.1251622885465622, + 0.1266230344772339, + 0.1262376308441162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0732176303863525, + "mean_weights": [ + 0.13281872868537903, + 0.1473538726568222, + 0.13968363404273987, + 0.12848341464996338, + 0.12016644328832626, + 0.11505335569381714, + 0.10907343029975891, + 0.1073671206831932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794081687927246, + "mean_weights": [ + 0.12450858950614929, + 0.12424902617931366, + 0.12409761548042297, + 0.12468484044075012, + 0.12504076957702637, + 0.1255234032869339, + 0.12684062123298645, + 0.12505511939525604 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072478771209717, + "mean_weights": [ + 0.13302236795425415, + 0.14877822995185852, + 0.14031866192817688, + 0.12882456183433533, + 0.11896270513534546, + 0.11445862054824829, + 0.11000129580497742, + 0.10563355684280396 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079291820526123, + "mean_weights": [ + 0.12426379323005676, + 0.12249694019556046, + 0.12262053787708282, + 0.12315947562456131, + 0.1255876123905182, + 0.126260906457901, + 0.1273583620786667, + 0.12825237214565277 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1905503273010254, + "mean_weights": [ + 0.11892962455749512, + 0.13242119550704956, + 0.12570281326770782, + 0.11684690415859222, + 0.10851435363292694, + 0.10355254262685776, + 0.09824737906455994, + 0.0950399786233902, + 0.10074522346258163 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971964836120605, + "mean_weights": [ + 0.11044953763484955, + 0.11010265350341797, + 0.1108412891626358, + 0.11044170707464218, + 0.11131919175386429, + 0.11133328080177307, + 0.11097346246242523, + 0.11225177347660065, + 0.11228710412979126 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1902995109558105, + "mean_weights": [ + 0.11926670372486115, + 0.13237422704696655, + 0.12644407153129578, + 0.11611802130937576, + 0.11051423847675323, + 0.10466914623975754, + 0.09992008656263351, + 0.09535142779350281, + 0.09534208476543427 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197113037109375, + "mean_weights": [ + 0.11037027835845947, + 0.10917964577674866, + 0.10961420089006424, + 0.10947252064943314, + 0.11109665036201477, + 0.11174711585044861, + 0.1118166521191597, + 0.11308293044567108, + 0.11361998319625854 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1894307136535645, + "mean_weights": [ + 0.1198490560054779, + 0.13425537943840027, + 0.12682178616523743, + 0.11624304950237274, + 0.10967148095369339, + 0.10377678275108337, + 0.09994889795780182, + 0.0946216955780983, + 0.09481187164783478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.196977138519287, + "mean_weights": [ + 0.10978136211633682, + 0.1075383648276329, + 0.10839901119470596, + 0.11003260314464569, + 0.11053797602653503, + 0.11152469366788864, + 0.11391061544418335, + 0.11424341052770615, + 0.11403194814920425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1905651092529297, + "mean_weights": [ + 0.11943811178207397, + 0.13202990591526031, + 0.12506362795829773, + 0.11625386774539948, + 0.11043960601091385, + 0.1047937273979187, + 0.1010652631521225, + 0.09624719619750977, + 0.0946686640381813 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02856997773051262, + 0.013703026808798313, + 0.015031684190034866, + 0.014947476796805859, + 0.015670204535126686, + 0.015775443986058235, + 0.016530683264136314, + 0.016744175925850868, + 0.01773575134575367, + 0.012103146873414516, + 0.01292925700545311, + 0.013081531040370464, + 0.014021075330674648, + 0.01416850183159113, + 0.015247474424540997, + 0.015494133345782757, + 0.01669454388320446, + 0.012773395515978336, + 0.01373021025210619, + 0.013903867453336716, + 0.01499613095074892, + 0.015176740474998951, + 0.01623532734811306, + 0.016410861164331436, + 0.01758282259106636, + 0.014334342442452908, + 0.015234442427754402, + 0.015468464232981205, + 0.01637510396540165, + 0.01670125499367714, + 0.01772587187588215, + 0.017849057912826538, + 0.01901337131857872, + 0.01606663130223751, + 0.017032917588949203, + 0.0173187293112278, + 0.01834850199520588, + 0.018571525812149048, + 0.019697511568665504, + 0.019770435988903046, + 0.021141810342669487, + 0.018272221088409424, + 0.019469235092401505, + 0.01950262486934662, + 0.02084389142692089, + 0.02077464945614338, + 0.022254284471273422, + 0.0220524650067091, + 0.02407064661383629, + 0.020764723420143127, + 0.022380243986845016, + 0.02203267626464367, + 0.023947052657604218, + 0.023422541096806526, + 0.025450291112065315, + 0.024811791256070137, + 0.027382872998714447, + 0.023658527061343193, + 0.025761734694242477, + 0.025185640901327133, + 0.02756396308541298, + 0.026624467223882675, + 0.02946169301867485, + 0.02854793891310692 + ], + "loss_nats": 3.54775071144104, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1935477256774902, + "mean_weights": [ + 0.10072024166584015, + 0.09480113536119461, + 0.10205776989459991, + 0.11059839278459549, + 0.1157325804233551, + 0.11904929578304291, + 0.11986880004405975, + 0.11937490850687027, + 0.11779685318470001 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.003087003482505679, + 0.005864492617547512, + 0.006683253683149815, + 0.009892112575471401, + 0.010629965923726559, + 0.014509912580251694, + 0.01484723947942257, + 0.019168438389897346, + 0.003094236133620143, + 0.006793031003326178, + 0.007696541491895914, + 0.01223041582852602, + 0.012912816368043423, + 0.017751524224877357, + 0.018475037068128586, + 0.023574935272336006, + 0.0047028325498104095, + 0.008267601951956749, + 0.0095678074285388, + 0.014755276031792164, + 0.01631263643503189, + 0.02151178941130638, + 0.023062339052557945, + 0.02830074541270733, + 0.005196469835937023, + 0.009282662533223629, + 0.011978921480476856, + 0.01676790602505207, + 0.01994149386882782, + 0.02516039088368416, + 0.027569252997636795, + 0.0331411249935627, + 0.006203304044902325, + 0.010437680408358574, + 0.014378104358911514, + 0.01991143450140953, + 0.024205319583415985, + 0.029664477333426476, + 0.03444278985261917, + 0.04036801680922508, + 0.00901782512664795, + 0.01357664167881012, + 0.01929495297372341, + 0.025029508396983147, + 0.030904190614819527, + 0.03699437528848648, + 0.04446368291974068, + 0.05095698684453964, + 0.010577243752777576, + 0.015438608825206757, + 0.023803669959306717, + 0.029965175315737724, + 0.03807948902249336, + 0.04443887993693352, + 0.05412018671631813, + 0.060807984322309494, + 0.013407763093709946, + 0.019359532743692398, + 0.030261604115366936, + 0.037317242473363876, + 0.04958754777908325, + 0.05564062297344208, + 0.07224015146493912, + 0.07935244590044022 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014133188233245164, + 0.0001488620473537594, + 0.00014702342741657048, + 0.0001400294277118519, + 0.00013315585965756327, + 0.00011939816613448784, + 0.0001078194472938776, + 9.923135075950995e-05, + 8.953637006925419e-05, + 8.09328121249564e-05, + 7.468360854545608e-05, + 6.987994856899604e-05, + 7.115858170436695e-05, + 6.467441562563181e-05, + 5.974166560918093e-05, + 5.531677743420005e-05, + 5.705624425900169e-05, + 5.295562368701212e-05, + 4.8012458137236536e-05, + 4.413425631355494e-05, + 5.2346804295666516e-05, + 4.681985228671692e-05, + 4.172014814685099e-05, + 3.8159589166752994e-05, + 3.1379553547594696e-05, + 3.02761654893402e-05, + 2.9319184250198305e-05, + 2.8667578590102494e-05, + 2.2355405235430226e-05, + 2.2064739823690616e-05, + 2.201247843913734e-05, + 2.2052488930057734e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000129606451082509, + "first_to_last_ratio": 4.9818074922726145, + "imbalance_abs_log_ratio": 1.6057927754415957, + "last_quartile_mean": 2.601594928819395e-05, + "mean": 6.850338621688934e-05, + "normalized": [ + 2.0631371693799077, + 2.173061151786652, + 2.146221311616304, + 2.04412417319785, + 1.9437850741564366, + 1.7429527608527258, + 1.5739287245233284, + 1.448561249882341, + 1.3070356812110293, + 1.181442503713818, + 1.0902177639657034, + 1.0200948073975256, + 1.038760061861336, + 0.9441053821903842, + 0.8720979926456799, + 0.8075042780959905, + 0.832896699126015, + 0.7730365841967101, + 0.7008771505867426, + 0.6442638641806842, + 0.7641491492103867, + 0.6834677068149604, + 0.6090231512754765, + 0.5570467574542299, + 0.45807302792658366, + 0.4419659692950432, + 0.4279961308389998, + 0.4184841096663127, + 0.3263401485679923, + 0.32209706763737483, + 0.3213341654300619, + 0.32191823131541414 + ], + "population_cv": 0.5858213540607353 + }, + "activation_output_rms_by_block": [ + 0.021054552868008614, + 0.038058556616306305, + 0.04883526638150215, + 0.05791458487510681, + 0.016990283504128456, + 0.027475843206048012, + 0.03445403650403023, + 0.040619369596242905, + 0.019055280834436417, + 0.029903952032327652, + 0.037492990493774414, + 0.04475866258144379, + 0.0285621490329504, + 0.03507128730416298, + 0.04228900745511055, + 0.05219251662492752, + 0.027010982856154442, + 0.04389137029647827, + 0.054182786494493484, + 0.06662839651107788, + 0.03837457299232483, + 0.041143495589494705, + 0.05716610327363014, + 0.07329611480236053, + 0.028997479006648064, + 0.053451184183359146, + 0.08026835322380066, + 0.10854664444923401, + 0.027061885222792625, + 0.05247335880994797, + 0.08323372155427933, + 0.11460508406162262 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.035675311693921685, + "first_to_last_ratio": 0.5202021080271609, + "imbalance_abs_log_ratio": 0.6535378736360024, + "last_quartile_mean": 0.06857971381396055, + "mean": 0.047658121038693935, + "normalized": [ + 0.4417831087153916, + 0.798574425236075, + 1.0246997849926245, + 1.2152091524566315, + 0.3565034276180115, + 0.5765196488493577, + 0.7229415628043072, + 0.8523074076559539, + 0.3998328179779751, + 0.6274681288431082, + 0.7867072741565622, + 0.9391612931005807, + 0.5993133680146686, + 0.7358932022453922, + 0.8873410561187638, + 1.095144237485823, + 0.5667655851187262, + 0.92096308750491, + 1.136905637771622, + 1.398049168933493, + 0.8052053281993276, + 0.8633050295056751, + 1.1995039256209161, + 1.5379564532737442, + 0.6084478022770731, + 1.121554585418124, + 1.6842534173479116, + 2.277610658655726, + 0.567833658419326, + 1.1010370880409766, + 1.746475096798326, + 2.404733580842896 + ], + "population_cv": 0.48875915937608194 + }, + "bits_per_byte": 3.059021014366605, + "branch_output_rms_by_sublayer": [ + 0.01139580737799406, + 0.01652366667985916, + 0.014748631045222282, + 0.013563322834670544, + 0.012339585460722446, + 0.012220539152622223, + 0.0120800556614995, + 0.0097816726192832, + 0.01439172774553299, + 0.008202088065445423, + 0.0171960536390543, + 0.008006753399968147, + 0.01820177212357521, + 0.008192610926926136, + 0.01519495714455843, + 0.008106541819870472, + 0.016362585127353668, + 0.008609342388808727, + 0.015944689512252808, + 0.008666405454277992, + 0.017555685713887215, + 0.009109876118600368, + 0.01772458106279373, + 0.008975221775472164, + 0.025544000789523125, + 0.009615846909582615, + 0.019797103479504585, + 0.009856111370027065, + 0.02002105489373207, + 0.009463186375796795, + 0.017730148509144783, + 0.009926749393343925, + 0.022795414552092552, + 0.010939708910882473, + 0.02134518511593342, + 0.010758070275187492, + 0.021650495007634163, + 0.011536303907632828, + 0.03255617991089821, + 0.012654256075620651, + 0.033973678946495056, + 0.014695252291858196, + 0.031387828290462494, + 0.016267452389001846, + 0.019825322553515434, + 0.01879013702273369, + 0.013387269340455532, + 0.02227928675711155, + 0.015395586378872395, + 0.021389417350292206, + 0.011897368356585503, + 0.023454682901501656, + 0.014087310060858727, + 0.024781694635748863, + 0.014466723427176476, + 0.026336129754781723, + 0.006911881268024445, + 0.026170482859015465, + 0.005224450025707483, + 0.026235800236463547, + 0.00758614856749773, + 0.030007388442754745, + 0.005234629847109318, + 0.031163375824689865 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00048079443816738296, + 0.0004352800282264352, + 0.00039416580024316875, + 0.0003828745093102654, + 0.0003643213588911513, + 0.0002989310042525185, + 0.0002673721517191021, + 0.00023887606182770654, + 0.0002716851528017292, + 0.00026407625379019375, + 0.00023115090959535868, + 0.00019271062698547964, + 0.00023181255047936123, + 0.00019176941572124543, + 0.00020689480536159778, + 0.0001935784151883457, + 0.00017612030874433946, + 0.00017372524780256995, + 0.0001906158264031438, + 0.00020699034417808569, + 0.0002469606938485226, + 0.00020806313134476826, + 0.00017307229934302834, + 0.00014129096626273445, + 0.00011634997010006574, + 0.00010725656087507846, + 0.00011122939555071074, + 0.00011004876520390145, + 7.494181919924042e-05, + 7.493373834181984e-05, + 7.712439386045853e-05, + 7.424519704760952e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00035782691907971636, + "first_to_last_ratio": 3.8366182378544447, + "imbalance_abs_log_ratio": 1.3445913113590247, + "last_quartile_mean": 9.326623002236059e-05, + "mean": 0.0002159144418958475, + "normalized": [ + 2.226782210331757, + 2.015983851772199, + 1.8255647782620577, + 1.7732695689478515, + 1.6873413176636518, + 1.384488233523149, + 1.2383245389767708, + 1.1063459198478962, + 1.258300048927641, + 1.2230597058328248, + 1.070567154127008, + 0.892532362788586, + 1.0736315201703026, + 0.8881731765481019, + 0.9582258766247762, + 0.896551492751577, + 0.8156948983954307, + 0.8046022594744787, + 0.8828303689620391, + 0.9586683612295531, + 1.1437896033265396, + 0.9636369365469933, + 0.801578152083568, + 0.6543840468572741, + 0.5388707169305075, + 0.49675491798189547, + 0.5151549594091785, + 0.5096869122677149, + 0.34709034996089216, + 0.3470529237593392, + 0.3571988662882573, + 0.34386396943018666 + ], + "population_cv": 0.48652454891086055 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6572662591934204, + "mean_weights": [ + 0.3911391496658325, + 0.6088608503341675 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6751130819320679, + "mean_weights": [ + 0.44147175550460815, + 0.5585282444953918 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6885842084884644, + "mean_weights": [ + 0.4896422326564789, + 0.5103577375411987 + ], + "sources": 2 + }, + { + "entropy_mean": 0.668085515499115, + "mean_weights": [ + 0.4332469701766968, + 0.5667530298233032 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6886032223701477, + "mean_weights": [ + 0.51185142993927, + 0.4881485104560852 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6789880990982056, + "mean_weights": [ + 0.4699174761772156, + 0.5300825834274292 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772009134292603, + "mean_weights": [ + 0.4395230710506439, + 0.5604768991470337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6781555414199829, + "mean_weights": [ + 0.4541339576244354, + 0.5458660125732422 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0917184352874756, + "mean_weights": [ + 0.30861830711364746, + 0.355538010597229, + 0.33584368228912354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0860013961791992, + "mean_weights": [ + 0.30104559659957886, + 0.3248867988586426, + 0.37406760454177856 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0827364921569824, + "mean_weights": [ + 0.2735002934932709, + 0.34776103496551514, + 0.3787387013435364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0724949836730957, + "mean_weights": [ + 0.2886848449707031, + 0.3024986684322357, + 0.40881651639938354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0897927284240723, + "mean_weights": [ + 0.2973601222038269, + 0.3298739790916443, + 0.3727658987045288 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0697213411331177, + "mean_weights": [ + 0.30285951495170593, + 0.28946101665496826, + 0.4076794683933258 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0826778411865234, + "mean_weights": [ + 0.2862996757030487, + 0.3235614597797394, + 0.3901388645172119 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0579445362091064, + "mean_weights": [ + 0.28999701142311096, + 0.2830814719200134, + 0.4269214868545532 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3809051513671875, + "mean_weights": [ + 0.23795948922634125, + 0.2348281741142273, + 0.26517200469970703, + 0.26204031705856323 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3605632781982422, + "mean_weights": [ + 0.24186556041240692, + 0.22099661827087402, + 0.2607640027999878, + 0.27637380361557007 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3829561471939087, + "mean_weights": [ + 0.23731295764446259, + 0.2382979840040207, + 0.2565648853778839, + 0.2678241431713104 + ], + "sources": 4 + }, + { + "entropy_mean": 1.35977041721344, + "mean_weights": [ + 0.23858636617660522, + 0.21978360414505005, + 0.2459542453289032, + 0.29567578434944153 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3790044784545898, + "mean_weights": [ + 0.24452988803386688, + 0.2358427345752716, + 0.24750955402851105, + 0.27211782336235046 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3556556701660156, + "mean_weights": [ + 0.2410905957221985, + 0.22088873386383057, + 0.23317404091358185, + 0.3048466145992279 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804312944412231, + "mean_weights": [ + 0.26568183302879333, + 0.22685569524765015, + 0.2384241223335266, + 0.2690383493900299 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533532619476318, + "mean_weights": [ + 0.24537262320518494, + 0.2171444594860077, + 0.23335666954517365, + 0.30412623286247253 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6053518056869507, + "mean_weights": [ + 0.20177139341831207, + 0.18913303315639496, + 0.18395328521728516, + 0.1997503787279129, + 0.22539189457893372 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585533618927002, + "mean_weights": [ + 0.19244730472564697, + 0.1780712604522705, + 0.17078758776187897, + 0.20516318082809448, + 0.25353068113327026 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6022908687591553, + "mean_weights": [ + 0.1985350400209427, + 0.1897844821214676, + 0.17792725563049316, + 0.1998080015182495, + 0.23394520580768585 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5833771228790283, + "mean_weights": [ + 0.1926397681236267, + 0.1801614761352539, + 0.17439623177051544, + 0.19375719130039215, + 0.2590453326702118 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601109504699707, + "mean_weights": [ + 0.19747966527938843, + 0.19028550386428833, + 0.18151578307151794, + 0.19478440284729004, + 0.23593464493751526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5755603313446045, + "mean_weights": [ + 0.1969345360994339, + 0.17712506651878357, + 0.16844677925109863, + 0.18347150087356567, + 0.27402210235595703 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5980476140975952, + "mean_weights": [ + 0.19043894112110138, + 0.1923028826713562, + 0.1810823678970337, + 0.19125500321388245, + 0.24492081999778748 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5842485427856445, + "mean_weights": [ + 0.1964084804058075, + 0.17743000388145447, + 0.17621047794818878, + 0.18783687055110931, + 0.26211416721343994 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7856926918029785, + "mean_weights": [ + 0.15997162461280823, + 0.16431675851345062, + 0.15234553813934326, + 0.15752777457237244, + 0.18046650290489197, + 0.18537181615829468 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7751184701919556, + "mean_weights": [ + 0.16064736247062683, + 0.15155571699142456, + 0.14907033741474152, + 0.14832919836044312, + 0.18695974349975586, + 0.2034376710653305 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7829411029815674, + "mean_weights": [ + 0.14932186901569366, + 0.16724151372909546, + 0.15592646598815918, + 0.1555134654045105, + 0.17433607578277588, + 0.19766061007976532 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7661566734313965, + "mean_weights": [ + 0.165928453207016, + 0.143815815448761, + 0.1375015527009964, + 0.1444939523935318, + 0.17902101576328278, + 0.22923919558525085 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7807259559631348, + "mean_weights": [ + 0.14958028495311737, + 0.16952179372310638, + 0.15695258975028992, + 0.1499582827091217, + 0.1711740493774414, + 0.20281299948692322 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781292200088501, + "mean_weights": [ + 0.1682473123073578, + 0.15929323434829712, + 0.14874719083309174, + 0.15058940649032593, + 0.1693388819694519, + 0.20378397405147552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7818934917449951, + "mean_weights": [ + 0.14812925457954407, + 0.18386004865169525, + 0.16644306480884552, + 0.15062814950942993, + 0.15809310972690582, + 0.19284634292125702 + ], + "sources": 6 + }, + { + "entropy_mean": 1.77958083152771, + "mean_weights": [ + 0.19125157594680786, + 0.15093471109867096, + 0.14643779397010803, + 0.1490234136581421, + 0.16375312209129333, + 0.19859938323497772 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9289655685424805, + "mean_weights": [ + 0.12320417165756226, + 0.15015369653701782, + 0.13329030573368073, + 0.12274879962205887, + 0.13076716661453247, + 0.1441647708415985, + 0.19567111134529114 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9340521097183228, + "mean_weights": [ + 0.14550700783729553, + 0.14042077958583832, + 0.13418063521385193, + 0.1260203719139099, + 0.12906627357006073, + 0.1355920135974884, + 0.18921294808387756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9230798482894897, + "mean_weights": [ + 0.12067099660634995, + 0.15888366103172302, + 0.14002522826194763, + 0.12021138519048691, + 0.12209220230579376, + 0.13571104407310486, + 0.20240548253059387 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9308109283447266, + "mean_weights": [ + 0.1450628787279129, + 0.14527732133865356, + 0.13644321262836456, + 0.12497064471244812, + 0.12634633481502533, + 0.1283716857433319, + 0.19352790713310242 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273775815963745, + "mean_weights": [ + 0.1260491907596588, + 0.16657930612564087, + 0.14038512110710144, + 0.12195964902639389, + 0.12248437106609344, + 0.13317090272903442, + 0.18937145173549652 + ], + "sources": 7 + }, + { + "entropy_mean": 1.918284296989441, + "mean_weights": [ + 0.13994047045707703, + 0.15779416263103485, + 0.13759920001029968, + 0.11949826776981354, + 0.11786683648824692, + 0.12190048396587372, + 0.20540057122707367 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9060933589935303, + "mean_weights": [ + 0.11515206843614578, + 0.18610316514968872, + 0.15217435359954834, + 0.11993157863616943, + 0.11246243864297867, + 0.12337306141853333, + 0.19080331921577454 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9199097156524658, + "mean_weights": [ + 0.13630829751491547, + 0.1662534922361374, + 0.14684012532234192, + 0.12322510778903961, + 0.12947364151477814, + 0.1381947100162506, + 0.15970461070537567 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0490760803222656, + "mean_weights": [ + 0.10742150247097015, + 0.1722080111503601, + 0.14153704047203064, + 0.1159135103225708, + 0.10868977755308151, + 0.11180940270423889, + 0.1477845162153244, + 0.09463626146316528 + ], + "sources": 8 + }, + { + "entropy_mean": 2.051286220550537, + "mean_weights": [ + 0.1131708174943924, + 0.16089829802513123, + 0.14448881149291992, + 0.11956015229225159, + 0.10795272141695023, + 0.11352235078811646, + 0.13960374891757965, + 0.10080308467149734 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0247201919555664, + "mean_weights": [ + 0.1011081114411354, + 0.1894242763519287, + 0.1531166434288025, + 0.11718504130840302, + 0.10284577310085297, + 0.10429200530052185, + 0.14699342846870422, + 0.08503472805023193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.047537326812744, + "mean_weights": [ + 0.1111423671245575, + 0.16533835232257843, + 0.14553973078727722, + 0.12043843418359756, + 0.10876712203025818, + 0.11067016422748566, + 0.13975468277931213, + 0.09834913164377213 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0282247066497803, + "mean_weights": [ + 0.100978784263134, + 0.18681633472442627, + 0.15267831087112427, + 0.11643697321414948, + 0.10204557329416275, + 0.10493149608373642, + 0.14740025997161865, + 0.08871228992938995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.053830623626709, + "mean_weights": [ + 0.11781682819128036, + 0.1550440490245819, + 0.1398259550333023, + 0.11913557350635529, + 0.10805961489677429, + 0.11164884269237518, + 0.13677509129047394, + 0.11169403791427612 + ], + "sources": 8 + }, + { + "entropy_mean": 2.017721176147461, + "mean_weights": [ + 0.09720280021429062, + 0.19308976829051971, + 0.16317345201969147, + 0.11856384575366974, + 0.10023905336856842, + 0.10130266845226288, + 0.14337101578712463, + 0.08305738866329193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0608584880828857, + "mean_weights": [ + 0.12434182316064835, + 0.14232873916625977, + 0.13082580268383026, + 0.11684279143810272, + 0.10329097509384155, + 0.10883669555187225, + 0.1425117552280426, + 0.13102145493030548 + ], + "sources": 8 + }, + { + "entropy_mean": 2.142296314239502, + "mean_weights": [ + 0.08636107295751572, + 0.16583400964736938, + 0.14278221130371094, + 0.1055997684597969, + 0.08843953907489777, + 0.08854378759860992, + 0.12657995522022247, + 0.07546606659889221, + 0.12039357423782349 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191580057144165, + "mean_weights": [ + 0.1060912162065506, + 0.12397187203168869, + 0.11601399630308151, + 0.1091354638338089, + 0.10782144963741302, + 0.10360677540302277, + 0.12169313430786133, + 0.10359254479408264, + 0.10807354003190994 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1432788372039795, + "mean_weights": [ + 0.08927536010742188, + 0.16873395442962646, + 0.14930182695388794, + 0.112123504281044, + 0.09766747802495956, + 0.0965762585401535, + 0.1252625584602356, + 0.07491002976894379, + 0.08614903688430786 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1724743843078613, + "mean_weights": [ + 0.1105542778968811, + 0.12893787026405334, + 0.11523330211639404, + 0.10665617883205414, + 0.10366716235876083, + 0.10828493535518646, + 0.11819782853126526, + 0.10312190651893616, + 0.10534654557704926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1452555656433105, + "mean_weights": [ + 0.09202822297811508, + 0.1701684445142746, + 0.14628148078918457, + 0.11057832837104797, + 0.09616327285766602, + 0.09830240160226822, + 0.12624149024486542, + 0.08024715632200241, + 0.07998920977115631 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1762189865112305, + "mean_weights": [ + 0.09908370673656464, + 0.13803134858608246, + 0.13144394755363464, + 0.11672185361385345, + 0.1074555367231369, + 0.10124591737985611, + 0.12084682285785675, + 0.08855879306793213, + 0.09661206603050232 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1389729976654053, + "mean_weights": [ + 0.09089216589927673, + 0.17082780599594116, + 0.14892268180847168, + 0.11308471858501434, + 0.10078474879264832, + 0.09951519966125488, + 0.12574949860572815, + 0.07681785523891449, + 0.07340531051158905 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04405451565980911, + 0.018189694732427597, + 0.0240375604480505, + 0.02713710628449917, + 0.030349314212799072, + 0.03182194381952286, + 0.034919846802949905, + 0.03608781099319458, + 0.038470037281513214, + 0.026185201480984688, + 0.02659580484032631, + 0.02762730047106743, + 0.028048591688275337, + 0.028690988197922707, + 0.029997555539011955, + 0.03034786321222782, + 0.031361404806375504, + 0.02331089787185192, + 0.0246393084526062, + 0.024178674444556236, + 0.025546060875058174, + 0.024928271770477295, + 0.02669469080865383, + 0.0256717037409544, + 0.027990806847810745, + 0.020905915647745132, + 0.02195541374385357, + 0.021578654646873474, + 0.023044798523187637, + 0.022400982677936554, + 0.02394026517868042, + 0.023724498227238655, + 0.025360919535160065, + 0.020535636693239212, + 0.02148214913904667, + 0.021988606080412865, + 0.02322523109614849, + 0.023164646700024605, + 0.023834573104977608, + 0.024238524958491325, + 0.02502058818936348, + 0.019457563757896423, + 0.01980534754693508, + 0.019637931138277054, + 0.02057415433228016, + 0.02042427472770214, + 0.02201046608388424, + 0.021937908604741096, + 0.02379545010626316, + 0.021518733352422714, + 0.022529911249876022, + 0.02267260104417801, + 0.023956971243023872, + 0.024102909490466118, + 0.02642984129488468, + 0.025153208523988724, + 0.030454812571406364, + 0.023361310362815857, + 0.02776513062417507, + 0.025513608008623123, + 0.030121317133307457, + 0.027568820863962173, + 0.03041756898164749, + 0.028944427147507668 + ], + "loss_nats": 2.120351791381836, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1635239124298096, + "mean_weights": [ + 0.1242082342505455, + 0.0880277156829834, + 0.0938715785741806, + 0.09339353442192078, + 0.09749531745910645, + 0.10569863021373749, + 0.13438579440116882, + 0.14857196807861328, + 0.11434720456600189 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01139580737799406, + 0.021054552868008614, + 0.02950318343937397, + 0.038058556616306305, + 0.04104945436120033, + 0.04883526638150215, + 0.05214058980345726, + 0.05791458487510681, + 0.01439172774553299, + 0.016990283504128456, + 0.024067245423793793, + 0.027475843206048012, + 0.030465533956885338, + 0.03445403650403023, + 0.036618731915950775, + 0.040619369596242905, + 0.016362585127353668, + 0.019055280834436417, + 0.02618422359228134, + 0.029903952032327652, + 0.033149950206279755, + 0.037492990493774414, + 0.04017872363328934, + 0.04475866258144379, + 0.025544000789523125, + 0.0285621490329504, + 0.030907167121767998, + 0.03507128730416298, + 0.03762345016002655, + 0.04228900745511055, + 0.04704070836305618, + 0.05219251662492752, + 0.022795414552092552, + 0.027010982856154442, + 0.03890290856361389, + 0.04389137029647827, + 0.04891306161880493, + 0.054182786494493484, + 0.060658931732177734, + 0.06662839651107788, + 0.033973678946495056, + 0.03837457299232483, + 0.03364448994398117, + 0.041143495589494705, + 0.04609670862555504, + 0.05716610327363014, + 0.05855504423379898, + 0.07329611480236053, + 0.015395586378872395, + 0.028997479006648064, + 0.03421308845281601, + 0.053451184183359146, + 0.058928847312927246, + 0.08026835322380066, + 0.08527950197458267, + 0.10854664444923401, + 0.006911881268024445, + 0.027061885222792625, + 0.028406694531440735, + 0.05247335880994797, + 0.05527278408408165, + 0.08323372155427933, + 0.0854114517569542, + 0.11460508406162262 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.134936509421095e-05, + 5.603071258519776e-05, + 4.972576425643638e-05, + 4.7571254981448874e-05, + 4.3214979086769745e-05, + 3.9845475839683786e-05, + 3.5890228900825605e-05, + 3.406162795727141e-05, + 3.5466175177134573e-05, + 3.383069270057604e-05, + 3.1626921554561704e-05, + 3.0198520107660443e-05, + 2.883204069803469e-05, + 2.77392227872042e-05, + 2.7800235329777934e-05, + 3.112062404397875e-05, + 6.127587403170764e-05, + 6.331028998829424e-05, + 5.8612553402781487e-05, + 5.708334720111452e-05, + 0.00015888184134382755, + 0.00012872362276539207, + 0.00010434311116114259, + 8.657111175125465e-05, + 5.805831096949987e-05, + 5.078948743175715e-05, + 4.287513002054766e-05, + 3.959577952628024e-05, + 2.2001073375577107e-05, + 1.997212712012697e-05, + 1.8872458895202726e-05, + 1.86532452062238e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.4711176087730564e-05, + "first_to_last_ratio": 1.3207760209544162, + "imbalance_abs_log_ratio": 0.27821945851834845, + "last_quartile_mean": 3.385220156815194e-05, + "mean": 4.981010016535947e-05, + "normalized": [ + 1.0309026668033479, + 1.1248865671658381, + 0.9983068512481783, + 0.9550523854303026, + 0.8675947035501784, + 0.799947715571839, + 0.7205411910772573, + 0.68382974224492, + 0.7120277827066004, + 0.6791934284063869, + 0.6349499689734955, + 0.6062730250974693, + 0.5788392434931497, + 0.5568995584252106, + 0.5581244614543389, + 0.6247854138149606, + 1.2301897371875206, + 1.271033179578376, + 1.1767202476646232, + 1.146019522378179, + 3.189751492495938, + 2.584287570955602, + 2.0948183363362958, + 1.7380232415485222, + 1.165593138274326, + 1.019662423146035, + 0.8607718088943986, + 0.7949347500773989, + 0.4416990390008851, + 0.4009654076948961, + 0.3788881940118565, + 0.3744872052916737 + ], + "population_cv": 0.6175097196118516 + }, + "activation_output_rms_by_block": [ + 0.0955304205417633, + 0.19925470650196075, + 0.2359117865562439, + 0.25103330612182617, + 0.09304249286651611, + 0.11085781455039978, + 0.12910984456539154, + 0.14378803968429565, + 0.04010729491710663, + 0.06472750008106232, + 0.09987020492553711, + 0.14899125695228577, + 0.05702618882060051, + 0.10170809924602509, + 0.15923865139484406, + 0.38537684082984924, + 0.05919196084141731, + 0.08540526777505875, + 0.1145314872264862, + 0.15640178322792053, + 0.06262174993753433, + 0.08427268266677856, + 0.11030438542366028, + 0.13170360028743744, + 0.05311359465122223, + 0.08614693582057953, + 0.12623678147792816, + 0.17503507435321808, + 0.07505694776773453, + 0.13522638380527496, + 0.2187158316373825, + 0.3292153775691986 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15731605142354965, + "first_to_last_ratio": 1.0498699791884785, + "imbalance_abs_log_ratio": 0.04866632715763954, + "last_quartile_mean": 0.14984336588531733, + "mean": 0.13496107165701687, + "normalized": [ + 0.7078368552417796, + 1.4763865169086419, + 1.7479987648273716, + 1.86004232953773, + 0.6894024456398029, + 0.8214058556983611, + 0.9566450753555414, + 1.06540380806779, + 0.2971767667867461, + 0.479601260469815, + 0.7399926786247083, + 1.1039572754054923, + 0.4225380557552472, + 0.7536106374770113, + 1.1798857955094264, + 2.8554666623367226, + 0.4385854388578413, + 0.632814090214868, + 0.8486260950679959, + 1.158865895977708, + 0.4639986121085199, + 0.624422151010662, + 0.8173051982276947, + 0.9758636225277033, + 0.39354751706627217, + 0.6383095122385285, + 0.935356987967166, + 1.2969300866107762, + 0.5561377576971261, + 1.0019658420387498, + 1.6205845800722127, + 2.439335828673987 + ], + "population_cv": 0.5772709429538598 + }, + "bits_per_byte": 2.064953725142933, + "branch_output_rms_by_sublayer": [ + 0.051234737038612366, + 0.07594770938158035, + 0.11323792487382889, + 0.04037529230117798, + 0.05108218640089035, + 0.04642988368868828, + 0.03771067038178444, + 0.02321692928671837, + 0.08815877884626389, + 0.015561277978122234, + 0.03706873208284378, + 0.01583549566566944, + 0.0438041090965271, + 0.020819898694753647, + 0.030363978818058968, + 0.027020027860999107, + 0.03348636254668236, + 0.0216005090624094, + 0.027010800316929817, + 0.024715786799788475, + 0.032826121896505356, + 0.03509491682052612, + 0.031702958047389984, + 0.05490833893418312, + 0.038218315690755844, + 0.037195492535829544, + 0.03341353312134743, + 0.042240556329488754, + 0.04180334508419037, + 0.04918703809380531, + 0.2476748526096344, + 0.04376247525215149, + 0.05078785493969917, + 0.02373618632555008, + 0.0355411171913147, + 0.01774558797478676, + 0.052597854286432266, + 0.022557877004146576, + 0.06600882858037949, + 0.03756217285990715, + 0.050489071756601334, + 0.03271714597940445, + 0.022040290758013725, + 0.03932206705212593, + 0.028325051069259644, + 0.04476166144013405, + 0.02171224355697632, + 0.049177221953868866, + 0.024430278688669205, + 0.0453680194914341, + 0.015834001824259758, + 0.0437605082988739, + 0.018936052918434143, + 0.052028607577085495, + 0.02182060107588768, + 0.060354240238666534, + 0.030554499477148056, + 0.06733348965644836, + 0.016542606055736542, + 0.07210637629032135, + 0.015565099194645882, + 0.09804103523492813, + 0.013472299091517925, + 0.1243291050195694 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00020065673277085515, + 0.0001817740596446822, + 0.0001465045892684325, + 9.755666249346124e-05, + 0.00012624719322665892, + 7.565538032235173e-05, + 8.200655559893429e-05, + 7.97391790009874e-05, + 8.09162329894288e-05, + 7.508564486922528e-05, + 9.13364471873345e-05, + 0.00011039800739152168, + 8.766456722124108e-05, + 8.422239419368561e-05, + 9.398125423258295e-05, + 0.00017161898601766447, + 0.0001497631172902175, + 0.00014237628599353527, + 0.00022365597941099996, + 0.00025566927173363617, + 0.0008281082302532628, + 0.0006115617552883831, + 0.0005788343529396405, + 0.00045039612289279527, + 0.0002758540055807746, + 0.00022394859978287228, + 0.00021964130134669458, + 0.00019854079025805258, + 0.00011545923673689007, + 0.00010570174900205617, + 9.906174415914337e-05, + 0.0001053958635417218 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00012376754404079543, + "first_to_last_ratio": 0.7369290916409895, + "imbalance_abs_log_ratio": 0.30526360357753907, + "last_quartile_mean": 0.00016795041130102567, + "mean": 0.00019904163414499135, + "normalized": [ + 1.0081143758329842, + 0.9132464191500225, + 0.7360499721465892, + 0.49013194105106844, + 0.6342753051087516, + 0.3800982676179234, + 0.4120070454164219, + 0.40061557645221907, + 0.4065291833892681, + 0.3772358742519635, + 0.4588811159016144, + 0.5546478145929135, + 0.4404333173700838, + 0.42313958361261256, + 0.4721688235543854, + 0.8622265726207266, + 0.7524210597121753, + 0.7153090689047581, + 1.123664304564935, + 1.2845014704179658, + 4.16047744890413, + 3.072531824386513, + 2.908106916555911, + 2.2628236792142964, + 1.385911077176092, + 1.1251344511155768, + 1.1034942628470255, + 0.9974837229954918, + 0.5800758079225982, + 0.5310534625386868, + 0.49769358347903286, + 0.5295166711952659 + ], + "population_cv": 0.8756839263617585 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4580664336681366, + "mean_weights": [ + 0.81428062915802, + 0.18571940064430237 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6350213885307312, + "mean_weights": [ + 0.5189062356948853, + 0.48109376430511475 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4277062714099884, + "mean_weights": [ + 0.8402144908905029, + 0.15978553891181946 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868079900741577, + "mean_weights": [ + 0.493515282869339, + 0.5064846873283386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.38686758279800415, + "mean_weights": [ + 0.8638634085655212, + 0.13613657653331757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772255897521973, + "mean_weights": [ + 0.5679300427436829, + 0.43206995725631714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5690979361534119, + "mean_weights": [ + 0.7181376218795776, + 0.28186243772506714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6592018008232117, + "mean_weights": [ + 0.5172787308692932, + 0.4827212691307068 + ], + "sources": 2 + }, + { + "entropy_mean": 1.038456916809082, + "mean_weights": [ + 0.4540913999080658, + 0.23118522763252258, + 0.3147233724594116 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0903832912445068, + "mean_weights": [ + 0.3152569830417633, + 0.30281925201416016, + 0.38192376494407654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0411981344223022, + "mean_weights": [ + 0.4020236134529114, + 0.2803056836128235, + 0.31767070293426514 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0905810594558716, + "mean_weights": [ + 0.3278552293777466, + 0.2872077226638794, + 0.384937047958374 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0150185823440552, + "mean_weights": [ + 0.48287349939346313, + 0.22136211395263672, + 0.29576438665390015 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0816482305526733, + "mean_weights": [ + 0.3306047320365906, + 0.2639211118221283, + 0.40547415614128113 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0114538669586182, + "mean_weights": [ + 0.4296638071537018, + 0.22373110055923462, + 0.346605122089386 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0918333530426025, + "mean_weights": [ + 0.36806726455688477, + 0.3041593134403229, + 0.32777342200279236 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3290088176727295, + "mean_weights": [ + 0.35130923986434937, + 0.17179939150810242, + 0.21658103168010712, + 0.2603103518486023 + ], + "sources": 4 + }, + { + "entropy_mean": 1.379033088684082, + "mean_weights": [ + 0.29040855169296265, + 0.23445811867713928, + 0.2306380271911621, + 0.24449528753757477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3394957780838013, + "mean_weights": [ + 0.3262045383453369, + 0.18669262528419495, + 0.22525924444198608, + 0.26184359192848206 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3677098751068115, + "mean_weights": [ + 0.29889944195747375, + 0.22409284114837646, + 0.1924189329147339, + 0.2845887541770935 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2910308837890625, + "mean_weights": [ + 0.3990757167339325, + 0.18164023756980896, + 0.16051796078681946, + 0.2587660551071167 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3623619079589844, + "mean_weights": [ + 0.3024129569530487, + 0.21113446354866028, + 0.18768057227134705, + 0.29877203702926636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2281851768493652, + "mean_weights": [ + 0.46696996688842773, + 0.16045212745666504, + 0.13728894293308258, + 0.23528897762298584 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772211074829102, + "mean_weights": [ + 0.28748467564582825, + 0.24638661742210388, + 0.22340166568756104, + 0.24272704124450684 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5519046783447266, + "mean_weights": [ + 0.30400484800338745, + 0.1470816731452942, + 0.13316048681735992, + 0.1807912439107895, + 0.23496176302433014 + ], + "sources": 5 + }, + { + "entropy_mean": 1.572178602218628, + "mean_weights": [ + 0.21885336935520172, + 0.2029750645160675, + 0.14561443030834198, + 0.14960341155529022, + 0.28295373916625977 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5391805171966553, + "mean_weights": [ + 0.2898171544075012, + 0.15984714031219482, + 0.1228363886475563, + 0.16408921778202057, + 0.26341012120246887 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5521924495697021, + "mean_weights": [ + 0.210292249917984, + 0.185291588306427, + 0.14173434674739838, + 0.14211969077587128, + 0.32056212425231934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5468778610229492, + "mean_weights": [ + 0.2756267488002777, + 0.16966477036476135, + 0.1333748996257782, + 0.1590600460767746, + 0.26227355003356934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5309922695159912, + "mean_weights": [ + 0.21606209874153137, + 0.27386677265167236, + 0.12764990329742432, + 0.10704417526721954, + 0.2753770351409912 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5832164287567139, + "mean_weights": [ + 0.22763225436210632, + 0.24636119604110718, + 0.17577087879180908, + 0.1817018985748291, + 0.1685338020324707 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5793015956878662, + "mean_weights": [ + 0.21581211686134338, + 0.21387425065040588, + 0.1464191973209381, + 0.17280855774879456, + 0.25108590722084045 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7719972133636475, + "mean_weights": [ + 0.16962555050849915, + 0.20328447222709656, + 0.1659279465675354, + 0.14008209109306335, + 0.12920910120010376, + 0.1918708235025406 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689546823501587, + "mean_weights": [ + 0.16601097583770752, + 0.1523149460554123, + 0.0875597670674324, + 0.101267509162426, + 0.1655988097190857, + 0.3272479772567749 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5251271724700928, + "mean_weights": [ + 0.13745036721229553, + 0.11702707409858704, + 0.07592719793319702, + 0.08887732028961182, + 0.11522199213504791, + 0.4654960632324219 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7189464569091797, + "mean_weights": [ + 0.1744517982006073, + 0.15982553362846375, + 0.09849169105291367, + 0.10780535638332367, + 0.16236206889152527, + 0.29706352949142456 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5877950191497803, + "mean_weights": [ + 0.14238224923610687, + 0.1503014713525772, + 0.10275131464004517, + 0.10159259289503098, + 0.08505517989397049, + 0.4179171919822693 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7369492053985596, + "mean_weights": [ + 0.17980654537677765, + 0.1706826537847519, + 0.10907262563705444, + 0.1106589287519455, + 0.15735921263694763, + 0.2724200189113617 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5987625122070312, + "mean_weights": [ + 0.15493065118789673, + 0.15333226323127747, + 0.10736657679080963, + 0.1186048835515976, + 0.07128532975912094, + 0.39448028802871704 + ], + "sources": 6 + }, + { + "entropy_mean": 1.75795578956604, + "mean_weights": [ + 0.1946164071559906, + 0.17554126679897308, + 0.11572675406932831, + 0.1155121773481369, + 0.1708213835954666, + 0.2277819961309433 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5344648361206055, + "mean_weights": [ + 0.0905577763915062, + 0.0786280632019043, + 0.06272733211517334, + 0.07359343767166138, + 0.050107162445783615, + 0.13821640610694885, + 0.5061697959899902 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7138500213623047, + "mean_weights": [ + 0.13302072882652283, + 0.10984362661838531, + 0.07243296504020691, + 0.07083481550216675, + 0.08885611593723297, + 0.11790697276592255, + 0.4071047604084015 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4565383195877075, + "mean_weights": [ + 0.09418222308158875, + 0.07457256317138672, + 0.057202234864234924, + 0.06210128590464592, + 0.04410415142774582, + 0.11846546828746796, + 0.5493720769882202 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7785513401031494, + "mean_weights": [ + 0.1471671462059021, + 0.12308044731616974, + 0.07605892419815063, + 0.07563325762748718, + 0.10904586315155029, + 0.11226382851600647, + 0.3567505478858948 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5135828256607056, + "mean_weights": [ + 0.10121430456638336, + 0.07933001220226288, + 0.061728835105895996, + 0.06817515194416046, + 0.048567138612270355, + 0.12093988060951233, + 0.5200446844100952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7546210289001465, + "mean_weights": [ + 0.14427465200424194, + 0.12224892526865005, + 0.08193168044090271, + 0.07876674830913544, + 0.09007281810045242, + 0.10526382178068161, + 0.3774413466453552 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5284571647644043, + "mean_weights": [ + 0.10831618309020996, + 0.08754566311836243, + 0.06345342099666595, + 0.06688661873340607, + 0.05242554843425751, + 0.11085730791091919, + 0.5105152726173401 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8309729099273682, + "mean_weights": [ + 0.1468513160943985, + 0.13658970594406128, + 0.09501184523105621, + 0.0928751677274704, + 0.10057065635919571, + 0.11721575260162354, + 0.31088554859161377 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6937754154205322, + "mean_weights": [ + 0.09603364765644073, + 0.06767555326223373, + 0.045112669467926025, + 0.047895483672618866, + 0.03816477954387665, + 0.07518067210912704, + 0.3669250011444092, + 0.26301223039627075 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9070472717285156, + "mean_weights": [ + 0.10919073224067688, + 0.11743452399969101, + 0.07601121068000793, + 0.0681401938199997, + 0.0658918172121048, + 0.07979628443717957, + 0.24526360630989075, + 0.2382715940475464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.776512861251831, + "mean_weights": [ + 0.08731270581483841, + 0.08430957794189453, + 0.056895699352025986, + 0.06295455247163773, + 0.04594648256897926, + 0.09297705441713333, + 0.33455103635787964, + 0.23505286872386932 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9573358297348022, + "mean_weights": [ + 0.11900999397039413, + 0.11752337962388992, + 0.07979564368724823, + 0.07266241312026978, + 0.07412992417812347, + 0.08811671286821365, + 0.22605092823505402, + 0.2227109968662262 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7231189012527466, + "mean_weights": [ + 0.09091638028621674, + 0.07795695960521698, + 0.05235600098967552, + 0.053361356258392334, + 0.04162060469388962, + 0.07854442298412323, + 0.3417898118495941, + 0.2634544372558594 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8967883586883545, + "mean_weights": [ + 0.1089041531085968, + 0.1000017300248146, + 0.07706406712532043, + 0.07702124863862991, + 0.06017591059207916, + 0.07969298958778381, + 0.261149525642395, + 0.23599036037921906 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801592230796814, + "mean_weights": [ + 0.09432460367679596, + 0.09541776776313782, + 0.06706254929304123, + 0.07247316837310791, + 0.048089414834976196, + 0.08817503601312637, + 0.3473600149154663, + 0.1870974600315094 + ], + "sources": 8 + }, + { + "entropy_mean": 1.919801950454712, + "mean_weights": [ + 0.10887958109378815, + 0.10445678234100342, + 0.0838727131485939, + 0.0852852314710617, + 0.05865198373794556, + 0.08557502925395966, + 0.27338963747024536, + 0.19988907873630524 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9332969188690186, + "mean_weights": [ + 0.07704077661037445, + 0.08837630599737167, + 0.05746292322874069, + 0.06001049652695656, + 0.04015304520726204, + 0.06584024429321289, + 0.2511133551597595, + 0.10944834351539612, + 0.25055450201034546 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0115504264831543, + "mean_weights": [ + 0.08485299348831177, + 0.08635596185922623, + 0.06204982101917267, + 0.06870189309120178, + 0.0476498119533062, + 0.08640170097351074, + 0.2653427720069885, + 0.13704031705856323, + 0.16160473227500916 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781036376953125, + "mean_weights": [ + 0.08389372378587723, + 0.08841055631637573, + 0.060679104179143906, + 0.0658775195479393, + 0.045975662767887115, + 0.08423998951911926, + 0.29481440782546997, + 0.12572166323661804, + 0.15038736164569855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0586161613464355, + "mean_weights": [ + 0.1038442850112915, + 0.09918913245201111, + 0.08148118108510971, + 0.0698266476392746, + 0.059573929756879807, + 0.08160553872585297, + 0.22791330516338348, + 0.13585451245307922, + 0.1407114714384079 + ], + "sources": 9 + }, + { + "entropy_mean": 2.022735118865967, + "mean_weights": [ + 0.09707076847553253, + 0.09551473706960678, + 0.06701318919658661, + 0.0673459842801094, + 0.04480960965156555, + 0.08329527080059052, + 0.24251294136047363, + 0.15929152071475983, + 0.14314597845077515 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9317231178283691, + "mean_weights": [ + 0.07746735960245132, + 0.08971680700778961, + 0.06909166276454926, + 0.08098623156547546, + 0.04421798139810562, + 0.0916394367814064, + 0.3483121693134308, + 0.10497690737247467, + 0.09359145164489746 + ], + "sources": 9 + }, + { + "entropy_mean": 2.053220272064209, + "mean_weights": [ + 0.09178322553634644, + 0.10647953301668167, + 0.07523074746131897, + 0.08394691348075867, + 0.04770101606845856, + 0.09626083076000214, + 0.26201331615448, + 0.13223087787628174, + 0.10435353964567184 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0558863990008831, + 0.04748110845685005, + 0.061753273010253906, + 0.06085919216275215, + 0.11003358662128448, + 0.06251973658800125, + 0.1138310506939888, + 0.08963073790073395, + 0.13082429766654968, + 0.08407715708017349, + 0.10403942316770554, + 0.1004372090101242, + 0.1062953919172287, + 0.09022688120603561, + 0.10766621679067612, + 0.09703134745359421, + 0.11238158494234085, + 0.0712415874004364, + 0.08696430176496506, + 0.07720176130533218, + 0.08355452120304108, + 0.07461326569318771, + 0.08609789609909058, + 0.0729745551943779, + 0.100697360932827, + 0.06531906127929688, + 0.07805952429771423, + 0.06964895129203796, + 0.08197034895420074, + 0.08001933246850967, + 0.10499614477157593, + 0.11749708652496338, + 0.13756641745567322, + 0.09965827316045761, + 0.09882861375808716, + 0.0854005292057991, + 0.10110001266002655, + 0.08426972478628159, + 0.10269922018051147, + 0.0886540412902832, + 0.10988812148571014, + 0.04304825887084007, + 0.0575832724571228, + 0.0459144227206707, + 0.06622003018856049, + 0.05584317073225975, + 0.06460635364055634, + 0.06311307847499847, + 0.07165247201919556, + 0.05157631263136864, + 0.055589381605386734, + 0.05562245845794678, + 0.05917651578783989, + 0.059894196689128876, + 0.06219335272908211, + 0.06387592852115631, + 0.06750614196062088, + 0.050557736307382584, + 0.06024058535695076, + 0.06106090545654297, + 0.06803743541240692, + 0.06874485313892365, + 0.06952574104070663, + 0.07015988230705261 + ], + "loss_nats": 1.43131685256958, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.843801498413086, + "mean_weights": [ + 0.11151862144470215, + 0.061874739825725555, + 0.05308101326227188, + 0.040848225355148315, + 0.042743273079395294, + 0.053944505751132965, + 0.1199670359492302, + 0.3250349462032318, + 0.19098761677742004 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.051234737038612366, + 0.0955304205417633, + 0.1796528398990631, + 0.19925470650196075, + 0.21105031669139862, + 0.2359117865562439, + 0.23972436785697937, + 0.25103330612182617, + 0.08815877884626389, + 0.09304249286651611, + 0.1068187803030014, + 0.11085781455039978, + 0.12298524379730225, + 0.12910984456539154, + 0.13432784378528595, + 0.14378803968429565, + 0.03348636254668236, + 0.04010729491710663, + 0.051035381853580475, + 0.06472750008106232, + 0.07822930067777634, + 0.09987020492553711, + 0.11166217923164368, + 0.14899125695228577, + 0.038218315690755844, + 0.05702618882060051, + 0.07459890842437744, + 0.10170809924602509, + 0.12532564997673035, + 0.15923865139484406, + 0.36271119117736816, + 0.38537684082984924, + 0.05078785493969917, + 0.05919196084141731, + 0.0794382095336914, + 0.08540526777505875, + 0.10824274271726608, + 0.1145314872264862, + 0.14269457757472992, + 0.15640178322792053, + 0.050489071756601334, + 0.06262174993753433, + 0.06721649318933487, + 0.08427268266677856, + 0.09199884533882141, + 0.11030438542366028, + 0.11444488167762756, + 0.13170360028743744, + 0.024430278688669205, + 0.05311359465122223, + 0.05695623904466629, + 0.08614693582057953, + 0.09002252668142319, + 0.12623678147792816, + 0.130380317568779, + 0.17503507435321808, + 0.030554499477148056, + 0.07505694776773453, + 0.07779927551746368, + 0.13522638380527496, + 0.13734298944473267, + 0.2187158316373825, + 0.22044414281845093, + 0.3292153775691986 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.57800890924409e-05, + 3.942943658330478e-05, + 3.388081677258015e-05, + 3.245583502575755e-05, + 3.3783475373638794e-05, + 2.915231198130641e-05, + 2.5950168492272496e-05, + 2.450849387969356e-05, + 2.7269536076346412e-05, + 2.568827403592877e-05, + 2.400098310317844e-05, + 2.2796592020313255e-05, + 2.010442767641507e-05, + 1.8936527339974418e-05, + 1.9039980543311685e-05, + 2.3072669137036428e-05, + 4.836148946196772e-05, + 4.9383146688342094e-05, + 4.5993543608346954e-05, + 4.280430221115239e-05, + 0.00014426946290768683, + 0.00011744296352844685, + 9.347966988570988e-05, + 7.698025729041547e-05, + 6.582303467439488e-05, + 5.6237244280055165e-05, + 4.4205091398907825e-05, + 3.852568261208944e-05, + 2.7434407456894405e-05, + 2.285257687617559e-05, + 2.028334711212665e-05, + 1.9826158677460626e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.186757840012433e-05, + "first_to_last_ratio": 0.8636564556008468, + "imbalance_abs_log_ratio": 0.146580210082559, + "last_quartile_mean": 3.689844288601307e-05, + "mean": 4.2179749868864747e-05, + "normalized": [ + 0.8482764645044092, + 0.9347954102594115, + 0.8032484042203742, + 0.7694648528419803, + 0.8009406285876599, + 0.6911447334785021, + 0.6152281265998635, + 0.5810488197746441, + 0.646507771172811, + 0.6090191173677569, + 0.569016724323795, + 0.5404629494292167, + 0.4766369582304063, + 0.4489483081063157, + 0.4514009827584627, + 0.5470082020108817, + 1.146557047216301, + 1.170778557053383, + 1.090417647125437, + 1.014806923801809, + 3.420348943656972, + 2.7843447126541196, + 2.216221532283493, + 1.825052484420704, + 1.5605363919661974, + 1.3332759073938238, + 1.048016916561615, + 0.913369157756135, + 0.6504165515961319, + 0.5417902416971032, + 0.48087879077488155, + 0.470039740375403 + ], + "population_cv": 0.6827037752697015 + }, + "activation_output_rms_by_block": [ + 0.15143299102783203, + 0.30262914299964905, + 0.3626292943954468, + 0.38562148809432983, + 0.13662196695804596, + 0.15442071855068207, + 0.18618692457675934, + 0.2074802666902542, + 0.04909220710396767, + 0.0881529152393341, + 0.14954733848571777, + 0.23951733112335205, + 0.08217982947826385, + 0.14409078657627106, + 0.22605396807193756, + 0.6592875719070435, + 0.06389845162630081, + 0.09458807110786438, + 0.14630787074565887, + 0.21908092498779297, + 0.07773751765489578, + 0.10736925154924393, + 0.1424422264099121, + 0.17246152460575104, + 0.06572399288415909, + 0.10123347491025925, + 0.14551502466201782, + 0.19983991980552673, + 0.09023923426866531, + 0.15600618720054626, + 0.24885882437229156, + 0.3860291540622711 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2358778491616249, + "first_to_last_ratio": 1.3542132581102162, + "imbalance_abs_log_ratio": 0.30322066438495643, + "last_quartile_mean": 0.17418072652071714, + "mean": 0.18569613725412637, + "normalized": [ + 0.8154881047449846, + 1.6297007976288655, + 1.9528101109566227, + 2.076626330501449, + 0.7357286423846173, + 0.8315774406193291, + 1.0026429592445498, + 1.1173106223869165, + 0.26436848838048077, + 0.4747159340137284, + 0.8053335987331889, + 1.2898347518967022, + 0.442550021198342, + 0.7759492938675503, + 1.2173326349948854, + 3.5503569522547695, + 0.3441022122008675, + 0.5093701598026241, + 0.7878886061342009, + 1.1797818103667892, + 0.41862754284711634, + 0.5781986267291517, + 0.7670715638795383, + 0.9287297364173834, + 0.3539330104331431, + 0.5451566004936365, + 0.7836190176798323, + 1.0761662722797756, + 0.4859510574803844, + 0.8401154138551131, + 1.3401400161152877, + 2.078821669478174 + ], + "population_cv": 0.6621635595354842 + }, + "bits_per_byte": 1.9288425276664853, + "branch_output_rms_by_sublayer": [ + 0.08832799643278122, + 0.11756844818592072, + 0.17041416466236115, + 0.05185626819729805, + 0.08745641261339188, + 0.0881219431757927, + 0.05030680447816849, + 0.03964918479323387, + 0.12814420461654663, + 0.022397184744477272, + 0.05527547001838684, + 0.022468235343694687, + 0.06746833771467209, + 0.031668391078710556, + 0.04974694177508354, + 0.045926205813884735, + 0.03680809214711189, + 0.03180550038814545, + 0.03558321297168732, + 0.03912416473031044, + 0.051462169736623764, + 0.06056958809494972, + 0.0526844747364521, + 0.10351218283176422, + 0.05210389941930771, + 0.05620818957686424, + 0.03620988130569458, + 0.0675361230969429, + 0.04957791045308113, + 0.0823286920785904, + 0.4858623445034027, + 0.07062793523073196, + 0.04707946628332138, + 0.0371064618229866, + 0.04491531476378441, + 0.023919809609651566, + 0.07858023792505264, + 0.035764746367931366, + 0.09729185700416565, + 0.07103991508483887, + 0.05887215584516525, + 0.045838613063097, + 0.026784580200910568, + 0.05455324053764343, + 0.041033755987882614, + 0.06383209675550461, + 0.032232560217380524, + 0.07182251662015915, + 0.02874751202762127, + 0.05687069892883301, + 0.022572210058569908, + 0.052836012095212936, + 0.02659173309803009, + 0.06559387594461441, + 0.025225715711712837, + 0.07689779996871948, + 0.041177187114953995, + 0.07740812748670578, + 0.027006076648831367, + 0.08842137455940247, + 0.023083338513970375, + 0.11899886280298233, + 0.016772178933024406, + 0.16321057081222534 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00017320581296885245, + 0.0001375362640366496, + 0.00014375813569971144, + 8.981648567965897e-05, + 0.00010459593200616139, + 6.156395218407552e-05, + 7.281931709149581e-05, + 7.667720183186952e-05, + 7.236475672264944e-05, + 7.890209128468536e-05, + 9.134838485444255e-05, + 0.00011250740948005864, + 7.70610699389613e-05, + 7.611216582142322e-05, + 8.146874247135892e-05, + 0.00017928549432252573, + 0.00013712162717798398, + 0.00013184310037104538, + 0.00022367504753246386, + 0.0002691717754589954, + 0.0008753648193242955, + 0.0006614972326966637, + 0.0006052862860379005, + 0.0005306195765578692, + 0.00037252876038501256, + 0.00029795485316373564, + 0.0002868510641817651, + 0.00023542884481748448, + 0.00018331496216173493, + 0.00013991664250432274, + 0.00012258431160994327, + 0.00012212563876750547 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00010749663768730933, + "first_to_last_ratio": 0.4884254111851857, + "imbalance_abs_log_ratio": 0.7165685086453754, + "last_quartile_mean": 0.00022008813469893802, + "mean": 0.0002132596174732282, + "normalized": [ + 0.812182892481255, + 0.6449240867362775, + 0.6740991914128248, + 0.42116030565859097, + 0.4904629073494984, + 0.2886807775119639, + 0.3414585375059921, + 0.3595486231306557, + 0.33932704925597945, + 0.3699813974138373, + 0.428343565166106, + 0.5275607769210333, + 0.36134862686150726, + 0.35689910130771973, + 0.3820167335786683, + 0.840691249692561, + 0.6429798046280267, + 0.6182281574597519, + 1.0488392044524941, + 1.262178834644052, + 4.104690938190328, + 3.101840097690814, + 2.838260207016858, + 2.4881390243724, + 1.746832170097925, + 1.3971461484082413, + 1.3450791461621905, + 1.1039541738230836, + 0.8595859091079332, + 0.6560859677143862, + 0.5748125831902143, + 0.5726618110568292 + ], + "population_cv": 0.9042592730039112 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2508508265018463, + "mean_weights": [ + 0.9251874089241028, + 0.07481260597705841 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5796181559562683, + "mean_weights": [ + 0.6163991689682007, + 0.38360080122947693 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3817020058631897, + "mean_weights": [ + 0.8651303648948669, + 0.13486966490745544 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6884468793869019, + "mean_weights": [ + 0.4932539463043213, + 0.5067461133003235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2797870337963104, + "mean_weights": [ + 0.9142746329307556, + 0.08572537451982498 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6642253994941711, + "mean_weights": [ + 0.605044960975647, + 0.39495497941970825 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5038842558860779, + "mean_weights": [ + 0.7756115198135376, + 0.2243884801864624 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6434851884841919, + "mean_weights": [ + 0.5374104976654053, + 0.4625895023345947 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0016276836395264, + "mean_weights": [ + 0.4353652000427246, + 0.18586981296539307, + 0.3787649869918823 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0655895471572876, + "mean_weights": [ + 0.2726097106933594, + 0.2790643572807312, + 0.4483259320259094 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0300546884536743, + "mean_weights": [ + 0.40803641080856323, + 0.2817816734313965, + 0.3101818859577179 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0874333381652832, + "mean_weights": [ + 0.36608952283859253, + 0.27405625581741333, + 0.35985422134399414 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9672750234603882, + "mean_weights": [ + 0.5343947410583496, + 0.1846013069152832, + 0.2810039520263672 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0731852054595947, + "mean_weights": [ + 0.3801807165145874, + 0.24373164772987366, + 0.37608763575553894 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9721636772155762, + "mean_weights": [ + 0.49137914180755615, + 0.2109004259109497, + 0.29772043228149414 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0864832401275635, + "mean_weights": [ + 0.39626431465148926, + 0.29088544845581055, + 0.3128502070903778 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3038380146026611, + "mean_weights": [ + 0.35696184635162354, + 0.1472446322441101, + 0.19980239868164062, + 0.29599112272262573 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3611116409301758, + "mean_weights": [ + 0.3264957070350647, + 0.22676822543144226, + 0.20138126611709595, + 0.2453548014163971 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3161137104034424, + "mean_weights": [ + 0.3259783387184143, + 0.17862647771835327, + 0.20532535016536713, + 0.2900698482990265 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568220138549805, + "mean_weights": [ + 0.3223523497581482, + 0.21851874887943268, + 0.1804703176021576, + 0.27865859866142273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2413711547851562, + "mean_weights": [ + 0.4272037148475647, + 0.16667884588241577, + 0.1365978717803955, + 0.269519567489624 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3540384769439697, + "mean_weights": [ + 0.32447606325149536, + 0.2030642032623291, + 0.18181107938289642, + 0.2906486392021179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1544246673583984, + "mean_weights": [ + 0.5128031969070435, + 0.13654224574565887, + 0.12138728052377701, + 0.22926729917526245 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3712997436523438, + "mean_weights": [ + 0.3103735148906708, + 0.24219870567321777, + 0.2247106432914734, + 0.22271710634231567 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5205519199371338, + "mean_weights": [ + 0.3307538628578186, + 0.1283297836780548, + 0.12636876106262207, + 0.17177441716194153, + 0.2427731454372406 + ], + "sources": 5 + }, + { + "entropy_mean": 1.518624186515808, + "mean_weights": [ + 0.2150207757949829, + 0.17106646299362183, + 0.13557285070419312, + 0.11898039281368256, + 0.3593595027923584 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4772696495056152, + "mean_weights": [ + 0.30324018001556396, + 0.13238273561000824, + 0.09796822816133499, + 0.15785877406597137, + 0.30855005979537964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4979676008224487, + "mean_weights": [ + 0.20932656526565552, + 0.16542977094650269, + 0.1272144764661789, + 0.1160244345664978, + 0.3820047676563263 + ], + "sources": 5 + }, + { + "entropy_mean": 1.49334716796875, + "mean_weights": [ + 0.28671571612358093, + 0.13947691023349762, + 0.11353637278079987, + 0.14176231622695923, + 0.31850868463516235 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5120060443878174, + "mean_weights": [ + 0.21571135520935059, + 0.28788846731185913, + 0.131511390209198, + 0.08343321830034256, + 0.2814555764198303 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5766459703445435, + "mean_weights": [ + 0.2565065026283264, + 0.22934341430664062, + 0.1593727320432663, + 0.1524350941181183, + 0.20234227180480957 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5796699523925781, + "mean_weights": [ + 0.2356114238500595, + 0.21223093569278717, + 0.1439642310142517, + 0.1658918261528015, + 0.2423015832901001 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7626371383666992, + "mean_weights": [ + 0.16389364004135132, + 0.20886825025081635, + 0.18027648329734802, + 0.1462823748588562, + 0.11240866780281067, + 0.18827059864997864 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6546597480773926, + "mean_weights": [ + 0.17577320337295532, + 0.14207448065280914, + 0.08111412078142166, + 0.0931762307882309, + 0.15011057257652283, + 0.35775139927864075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.430190086364746, + "mean_weights": [ + 0.1354856789112091, + 0.09479984641075134, + 0.07003438472747803, + 0.0834486186504364, + 0.09165525436401367, + 0.5245761871337891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7008471488952637, + "mean_weights": [ + 0.18653985857963562, + 0.149653822183609, + 0.09563066065311432, + 0.09960515052080154, + 0.15559062361717224, + 0.31297987699508667 + ], + "sources": 6 + }, + { + "entropy_mean": 1.514115333557129, + "mean_weights": [ + 0.14724519848823547, + 0.13274209201335907, + 0.096603162586689, + 0.11347334086894989, + 0.0527515783905983, + 0.4571846127510071 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7213332653045654, + "mean_weights": [ + 0.19527462124824524, + 0.16114139556884766, + 0.10406571626663208, + 0.10423287749290466, + 0.14977452158927917, + 0.2855108380317688 + ], + "sources": 6 + }, + { + "entropy_mean": 1.503009557723999, + "mean_weights": [ + 0.1677132397890091, + 0.12867671251296997, + 0.09363879263401031, + 0.12237933278083801, + 0.04204780235886574, + 0.44554415345191956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7531253099441528, + "mean_weights": [ + 0.21246549487113953, + 0.17559614777565002, + 0.11484666168689728, + 0.11058571934700012, + 0.1656094789505005, + 0.22089649736881256 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3981997966766357, + "mean_weights": [ + 0.08501995354890823, + 0.06276540458202362, + 0.0543542318046093, + 0.06455785036087036, + 0.03336101397871971, + 0.12979380786418915, + 0.5701477527618408 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6064693927764893, + "mean_weights": [ + 0.12975823879241943, + 0.09198254346847534, + 0.06163205951452255, + 0.06021754443645477, + 0.078399658203125, + 0.10503693670034409, + 0.4729730188846588 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3297276496887207, + "mean_weights": [ + 0.09028719365596771, + 0.057085148990154266, + 0.04790058359503746, + 0.05685827136039734, + 0.031231366097927094, + 0.11244906485080719, + 0.6041883826255798 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6772459745407104, + "mean_weights": [ + 0.1455874890089035, + 0.10527922213077545, + 0.06669703871011734, + 0.06069611385464668, + 0.09951996803283691, + 0.09685218334197998, + 0.4253680109977722 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3742375373840332, + "mean_weights": [ + 0.09774194657802582, + 0.06367622315883636, + 0.05300067365169525, + 0.05981069803237915, + 0.03241024166345596, + 0.10956928879022598, + 0.5837908983230591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.672610878944397, + "mean_weights": [ + 0.14811497926712036, + 0.10026681423187256, + 0.0711982250213623, + 0.06714795529842377, + 0.08556141704320908, + 0.09809904545545578, + 0.42961156368255615 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4325350522994995, + "mean_weights": [ + 0.11123399436473846, + 0.07182648032903671, + 0.05620439350605011, + 0.06410455703735352, + 0.03741488978266716, + 0.10516737401485443, + 0.5540482997894287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7746202945709229, + "mean_weights": [ + 0.15383490920066833, + 0.11435382068157196, + 0.09083531051874161, + 0.08329388499259949, + 0.08837151527404785, + 0.11093884706497192, + 0.3583717346191406 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5557539463043213, + "mean_weights": [ + 0.09889799356460571, + 0.05227670818567276, + 0.03886467218399048, + 0.039664674550294876, + 0.02741250954568386, + 0.06309205293655396, + 0.40426212549209595, + 0.27552926540374756 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7942585945129395, + "mean_weights": [ + 0.10892648994922638, + 0.08669497072696686, + 0.060229551047086716, + 0.05406671017408371, + 0.05429235100746155, + 0.07220236957073212, + 0.24074554443359375, + 0.322842001914978 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6406831741333008, + "mean_weights": [ + 0.08106359094381332, + 0.06122073531150818, + 0.04382575303316116, + 0.0495278537273407, + 0.031025834381580353, + 0.08371539413928986, + 0.3212447166442871, + 0.3283761143684387 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8771746158599854, + "mean_weights": [ + 0.11823929846286774, + 0.09187044948339462, + 0.06966835260391235, + 0.05918115749955177, + 0.06588791310787201, + 0.07802121341228485, + 0.22871823608875275, + 0.2884133756160736 + ], + "sources": 8 + }, + { + "entropy_mean": 1.612527847290039, + "mean_weights": [ + 0.09160974621772766, + 0.05898914486169815, + 0.039636120200157166, + 0.04581688344478607, + 0.032106831669807434, + 0.06671778112649918, + 0.3060194253921509, + 0.35910406708717346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8253357410430908, + "mean_weights": [ + 0.10355446487665176, + 0.08380813896656036, + 0.06609268486499786, + 0.07030215114355087, + 0.04850628599524498, + 0.0775601714849472, + 0.25825831294059753, + 0.2919177711009979 + ], + "sources": 8 + }, + { + "entropy_mean": 1.714170217514038, + "mean_weights": [ + 0.09696917235851288, + 0.07828262448310852, + 0.055068351328372955, + 0.0616651251912117, + 0.03598042577505112, + 0.07802210748195648, + 0.3351396918296814, + 0.25887250900268555 + ], + "sources": 8 + }, + { + "entropy_mean": 1.875918984413147, + "mean_weights": [ + 0.11156761646270752, + 0.0880264937877655, + 0.07785218954086304, + 0.07930342108011246, + 0.04990004748106003, + 0.08684437721967697, + 0.2776731252670288, + 0.22883272171020508 + ], + "sources": 8 + }, + { + "entropy_mean": 1.731968641281128, + "mean_weights": [ + 0.06617744266986847, + 0.05556279793381691, + 0.04319432005286217, + 0.045391663908958435, + 0.026374589651823044, + 0.054588086903095245, + 0.21148523688316345, + 0.10446193069219589, + 0.3927639424800873 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9649744033813477, + "mean_weights": [ + 0.07813189923763275, + 0.08751581609249115, + 0.05837181955575943, + 0.07608447968959808, + 0.03110998496413231, + 0.09630586206912994, + 0.27242881059646606, + 0.1154000535607338, + 0.18465125560760498 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8779311180114746, + "mean_weights": [ + 0.07922039180994034, + 0.064256951212883, + 0.04481552541255951, + 0.05368471518158913, + 0.03291679173707962, + 0.07845190167427063, + 0.26414477825164795, + 0.13897250592708588, + 0.24353644251823425 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9959204196929932, + "mean_weights": [ + 0.10073552280664444, + 0.0823616087436676, + 0.0686727911233902, + 0.0658436119556427, + 0.048376426100730896, + 0.07448098808526993, + 0.23861894011497498, + 0.14355987310409546, + 0.177350252866745 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9298079013824463, + "mean_weights": [ + 0.09570661187171936, + 0.07176163792610168, + 0.05499213933944702, + 0.057467732578516006, + 0.03367581591010094, + 0.06890923529863358, + 0.21427617967128754, + 0.18804281949996948, + 0.2151678204536438 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8635971546173096, + "mean_weights": [ + 0.07685244828462601, + 0.07133444398641586, + 0.05962139368057251, + 0.08040933310985565, + 0.033340416848659515, + 0.10067727416753769, + 0.36328762769699097, + 0.11175162345170975, + 0.10272547602653503 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0009379386901855, + "mean_weights": [ + 0.09139549732208252, + 0.08807491511106491, + 0.06457877159118652, + 0.07007958739995956, + 0.03462113067507744, + 0.09155915677547455, + 0.2692604660987854, + 0.15421991050243378, + 0.13621056079864502 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059407927095890045, + 0.05594591051340103, + 0.07886742800474167, + 0.07073435187339783, + 0.16177938878536224, + 0.06750936061143875, + 0.15622827410697937, + 0.10916222631931305, + 0.1895449161529541, + 0.11270010471343994, + 0.15217198431491852, + 0.14727257192134857, + 0.14916078746318817, + 0.11801524460315704, + 0.14915435016155243, + 0.13372685015201569, + 0.16036193072795868, + 0.09399056434631348, + 0.12247956544160843, + 0.10827244818210602, + 0.12010690569877625, + 0.10214843600988388, + 0.12369577586650848, + 0.09725081920623779, + 0.14881008863449097, + 0.0902416780591011, + 0.10321364551782608, + 0.09460548311471939, + 0.1168464794754982, + 0.11164536327123642, + 0.16267083585262299, + 0.19675607979297638, + 0.21974295377731323, + 0.15372900664806366, + 0.1438056081533432, + 0.10524243861436844, + 0.14797262847423553, + 0.10666082799434662, + 0.15148812532424927, + 0.11902136355638504, + 0.1692686378955841, + 0.05537491664290428, + 0.07994630187749863, + 0.06347332894802094, + 0.09600856900215149, + 0.07961282879114151, + 0.09550122171640396, + 0.09120825678110123, + 0.10489360988140106, + 0.07538540661334991, + 0.07298607379198074, + 0.07336290925741196, + 0.08083610236644745, + 0.07823792845010757, + 0.0843365490436554, + 0.08682125806808472, + 0.09205732494592667, + 0.057621169835329056, + 0.07988084107637405, + 0.0771951824426651, + 0.08835897594690323, + 0.09140052646398544, + 0.09466564655303955, + 0.09293081611394882 + ], + "loss_nats": 1.3369717597961426, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7290174961090088, + "mean_weights": [ + 0.10267993807792664, + 0.05008202791213989, + 0.044380687177181244, + 0.0326685830950737, + 0.033161722123622894, + 0.043591417372226715, + 0.1060870811343193, + 0.3520195186138153, + 0.2353290319442749 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08832799643278122, + 0.15143299102783203, + 0.2790028154850006, + 0.30262914299964905, + 0.315830796957016, + 0.3626292943954468, + 0.3664044439792633, + 0.38562148809432983, + 0.12814420461654663, + 0.13662196695804596, + 0.14859555661678314, + 0.15442071855068207, + 0.17815031111240387, + 0.18618692457675934, + 0.19290222227573395, + 0.2074802666902542, + 0.03680809214711189, + 0.04909220710396767, + 0.06554166972637177, + 0.0881529152393341, + 0.11251214146614075, + 0.14954733848571777, + 0.16864995658397675, + 0.23951733112335205, + 0.05210389941930771, + 0.08217982947826385, + 0.09766767174005508, + 0.14409078657627106, + 0.16688434779644012, + 0.22605396807193756, + 0.6339711546897888, + 0.6592875719070435, + 0.04707946628332138, + 0.06389845162630081, + 0.084575355052948, + 0.09458807110786438, + 0.13482175767421722, + 0.14630787074565887, + 0.19051826000213623, + 0.21908092498779297, + 0.05887215584516525, + 0.07773751765489578, + 0.08281512558460236, + 0.10736925154924393, + 0.11754488199949265, + 0.1424422264099121, + 0.14818532764911652, + 0.17246152460575104, + 0.02874751202762127, + 0.06572399288415909, + 0.07124026119709015, + 0.10123347491025925, + 0.10666517913341522, + 0.14551502466201782, + 0.14956052601337433, + 0.19983991980552673, + 0.041177187114953995, + 0.09023923426866531, + 0.09537366032600403, + 0.15600618720054626, + 0.15850447118282318, + 0.24885882437229156, + 0.25068944692611694, + 0.3860291540622711 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "population_cv": 0.7299683279122143 + }, + "activation_output_rms_by_block": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24231436662375927, + "first_to_last_ratio": 1.7382073612286646, + "imbalance_abs_log_ratio": 0.5528543299844475, + "last_quartile_mean": 0.139404752291739, + "mean": 0.17594548198394477, + "normalized": [ + 0.872176901111452, + 1.8466899321818973, + 2.1703463827261262, + 2.3075366266331874, + 0.767871429869389, + 0.870980627700565, + 1.0231460974138593, + 1.1589542538312023, + 0.24864957835709345, + 0.4666800200245053, + 0.7845885991457198, + 1.3049357734104974, + 0.46603991847126747, + 0.789568230136157, + 1.2326042132027004, + 4.183179000808161, + 0.32473565854469494, + 0.4690011715552792, + 0.7196215097851506, + 1.1150703717560293, + 0.38383209436703286, + 0.5425045292082467, + 0.7244651255474189, + 0.8882788682598759, + 0.3321309360788757, + 0.49982187502145003, + 0.6961071361611404, + 0.9322692927740504, + 0.42842289992260757, + 0.6985518108614562, + 1.0888090156129309, + 1.6624301195199809 + ], + "population_cv": 0.7619463065887785 + }, + "bits_per_byte": 1.877417995231997, + "branch_output_rms_by_sublayer": [ + 0.09253507107496262, + 0.116500623524189, + 0.19514523446559906, + 0.0468219518661499, + 0.07917427271604538, + 0.09375449270009995, + 0.04215213656425476, + 0.041188232600688934, + 0.12552575767040253, + 0.024237047880887985, + 0.05027730390429497, + 0.022429833188652992, + 0.054851483553647995, + 0.03027583472430706, + 0.036243390291929245, + 0.04546776041388512, + 0.031880609691143036, + 0.030174262821674347, + 0.035054463893175125, + 0.03663193807005882, + 0.050445348024368286, + 0.05748353153467178, + 0.05600574240088463, + 0.10596779733896255, + 0.053015924990177155, + 0.05394246056675911, + 0.032114963978528976, + 0.06537146866321564, + 0.048427484929561615, + 0.08060523867607117, + 0.5742782950401306, + 0.06690707802772522, + 0.04075472429394722, + 0.033449217677116394, + 0.03505101054906845, + 0.022565271705389023, + 0.06577442586421967, + 0.036697570234537125, + 0.08553697913885117, + 0.07367803901433945, + 0.0493597537279129, + 0.042231619358062744, + 0.023477228358387947, + 0.052135858684778214, + 0.03512806445360184, + 0.06269875913858414, + 0.030729109421372414, + 0.0728035643696785, + 0.023035654798150063, + 0.051585447043180466, + 0.019568851217627525, + 0.04862172156572342, + 0.02376176416873932, + 0.05972515419125557, + 0.02176014520227909, + 0.06787826865911484, + 0.03424545377492905, + 0.06439156830310822, + 0.021422436460852623, + 0.07232685387134552, + 0.019477492198348045, + 0.09627991914749146, + 0.017491821199655533, + 0.1271906942129135 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00028365814152869, + 0.0002107335686261075, + 0.00020559059302825965, + 0.00013034830149957224, + 0.00015811506316286034, + 8.078727112532468e-05, + 9.745502139202278e-05, + 0.00010158534732968689, + 0.00010687310625858127, + 0.00011249947044062199, + 0.00014466041390513986, + 0.00018570148639523412, + 0.0001198924920321974, + 0.00012288623237884872, + 0.00013187077826681118, + 0.000321118955435273, + 0.0002173403860671073, + 0.00022706794561596516, + 0.0003846029635623724, + 0.00045310883243459936, + 0.001499135691470673, + 0.0011394398675334944, + 0.0010162826769661553, + 0.0008650696174683238, + 0.0006964490714203023, + 0.0005559793522399823, + 0.0005128319422021078, + 0.0003792784055626199, + 0.00037680605737877213, + 0.0002696585960614849, + 0.00021598580145167561, + 0.00019464643295656736 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015853416346156552, + "first_to_last_ratio": 0.39613292787359505, + "imbalance_abs_log_ratio": 0.9260054476059185, + "last_quartile_mean": 0.000400204457409189, + "mean": 0.0003599206213499198, + "normalized": [ + 0.7881130579981791, + 0.5855001245433764, + 0.5712109305023165, + 0.36215846986118033, + 0.439305373973387, + 0.2244585787341764, + 0.27076809610549, + 0.28224375404965807, + 0.29693521270813145, + 0.3125674489521634, + 0.4019231056074973, + 0.5159512275199496, + 0.3331081491872516, + 0.34142592863379456, + 0.36638850469921974, + 0.892193823823939, + 0.6038564427121444, + 0.6308834008018855, + 1.0685771827128963, + 1.258913231298522, + 4.165184217142125, + 3.165808792116179, + 2.8236300358520197, + 2.4035011226191765, + 1.9350074158246267, + 1.544727696220138, + 1.4248473462806275, + 1.0537834818691318, + 1.0469143334036313, + 0.7492168552335295, + 0.6000928778173319, + 0.540803781196325 + ], + "population_cv": 0.9317377718256953 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2200722098350525, + "mean_weights": [ + 0.9384047985076904, + 0.06159520894289017 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5651201605796814, + "mean_weights": [ + 0.6557347774505615, + 0.3442652225494385 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4018177092075348, + "mean_weights": [ + 0.856421947479248, + 0.14357808232307434 + ], + "sources": 2 + }, + { + "entropy_mean": 0.687358021736145, + "mean_weights": [ + 0.46872246265411377, + 0.5312775373458862 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2678314447402954, + "mean_weights": [ + 0.920531153678894, + 0.07946885377168655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.657238245010376, + "mean_weights": [ + 0.6222448348999023, + 0.37775513529777527 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044373869895935, + "mean_weights": [ + 0.7792346477508545, + 0.2207653522491455 + ], + "sources": 2 + }, + { + "entropy_mean": 0.650418758392334, + "mean_weights": [ + 0.5397791862487793, + 0.4602207541465759 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0032464265823364, + "mean_weights": [ + 0.4337552487850189, + 0.18956376612186432, + 0.37668097019195557 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0385558605194092, + "mean_weights": [ + 0.24478834867477417, + 0.2602521479129791, + 0.4949595034122467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.031773328781128, + "mean_weights": [ + 0.40370821952819824, + 0.2651250958442688, + 0.33116668462753296 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0838522911071777, + "mean_weights": [ + 0.35612422227859497, + 0.2625962495803833, + 0.38127952814102173 + ], + "sources": 3 + }, + { + "entropy_mean": 0.96053147315979, + "mean_weights": [ + 0.5271762013435364, + 0.1581989824771881, + 0.3146248161792755 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0704245567321777, + "mean_weights": [ + 0.3788670599460602, + 0.23852786421775818, + 0.38260507583618164 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9674104452133179, + "mean_weights": [ + 0.48829180002212524, + 0.18356263637542725, + 0.3281455934047699 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832805633544922, + "mean_weights": [ + 0.40388280153274536, + 0.2811351418495178, + 0.3149820566177368 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2917464971542358, + "mean_weights": [ + 0.3349481225013733, + 0.1244347095489502, + 0.20291922986507416, + 0.33769792318344116 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533791303634644, + "mean_weights": [ + 0.32115638256073, + 0.20602071285247803, + 0.18808895349502563, + 0.28473395109176636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308367371559143, + "mean_weights": [ + 0.31139707565307617, + 0.16179001331329346, + 0.2296154499053955, + 0.29719749093055725 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3437637090682983, + "mean_weights": [ + 0.3217787742614746, + 0.1920534074306488, + 0.1763250231742859, + 0.3098427951335907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2280659675598145, + "mean_weights": [ + 0.425346314907074, + 0.14848899841308594, + 0.13232898712158203, + 0.29383569955825806 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3462648391723633, + "mean_weights": [ + 0.3221249580383301, + 0.18557843565940857, + 0.18214309215545654, + 0.3101535141468048 + ], + "sources": 4 + }, + { + "entropy_mean": 1.147221326828003, + "mean_weights": [ + 0.505380392074585, + 0.12089012563228607, + 0.11514648050069809, + 0.2585829496383667 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3729398250579834, + "mean_weights": [ + 0.30785465240478516, + 0.23079943656921387, + 0.23862706124782562, + 0.22271886467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5232064723968506, + "mean_weights": [ + 0.3136816620826721, + 0.11285122483968735, + 0.13196846842765808, + 0.19688263535499573, + 0.24461600184440613 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4876129627227783, + "mean_weights": [ + 0.2035508155822754, + 0.15372204780578613, + 0.13167491555213928, + 0.11437506973743439, + 0.396677166223526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4587602615356445, + "mean_weights": [ + 0.28519001603126526, + 0.10532154142856598, + 0.09675675630569458, + 0.17811742424964905, + 0.33461427688598633 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4565696716308594, + "mean_weights": [ + 0.20232918858528137, + 0.14928825199604034, + 0.12029382586479187, + 0.1059027910232544, + 0.4221859574317932 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4788098335266113, + "mean_weights": [ + 0.2697942852973938, + 0.11948063224554062, + 0.11090938746929169, + 0.15228068828582764, + 0.34753501415252686 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5018762350082397, + "mean_weights": [ + 0.208971306681633, + 0.27331358194351196, + 0.12766750156879425, + 0.07916607707738876, + 0.31088152527809143 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5627843141555786, + "mean_weights": [ + 0.27452588081359863, + 0.21448183059692383, + 0.143874391913414, + 0.14059773087501526, + 0.22652018070220947 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580754280090332, + "mean_weights": [ + 0.2518634498119354, + 0.18758198618888855, + 0.13903988897800446, + 0.18044015765190125, + 0.24107450246810913 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7564777135849, + "mean_weights": [ + 0.15440437197685242, + 0.19411900639533997, + 0.17806550860404968, + 0.14696577191352844, + 0.10594475269317627, + 0.2205006331205368 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5899518728256226, + "mean_weights": [ + 0.1636897623538971, + 0.11789832264184952, + 0.07030131667852402, + 0.09238424897193909, + 0.1399868130683899, + 0.4157395362854004 + ], + "sources": 6 + }, + { + "entropy_mean": 1.2894535064697266, + "mean_weights": [ + 0.1212274506688118, + 0.06944876164197922, + 0.05726535990834236, + 0.09096578508615494, + 0.06335186958312988, + 0.5977407693862915 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6581614017486572, + "mean_weights": [ + 0.18132120370864868, + 0.12711825966835022, + 0.08560528606176376, + 0.09748884290456772, + 0.15072739124298096, + 0.35773903131484985 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3699336051940918, + "mean_weights": [ + 0.13604651391506195, + 0.10151475667953491, + 0.07715043425559998, + 0.11300846189260483, + 0.033717285841703415, + 0.538562536239624 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6926933526992798, + "mean_weights": [ + 0.19425281882286072, + 0.14001017808914185, + 0.09434761106967926, + 0.10331346094608307, + 0.14820793271064758, + 0.3198680281639099 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4148362874984741, + "mean_weights": [ + 0.16652247309684753, + 0.10329679399728775, + 0.08063912391662598, + 0.1165856122970581, + 0.033275946974754333, + 0.4996800422668457 + ], + "sources": 6 + }, + { + "entropy_mean": 1.740808129310608, + "mean_weights": [ + 0.21459883451461792, + 0.15640267729759216, + 0.10570268332958221, + 0.11097012460231781, + 0.17001523077487946, + 0.24231043457984924 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3179612159729004, + "mean_weights": [ + 0.08068332821130753, + 0.049232255667448044, + 0.045073021203279495, + 0.06111945956945419, + 0.028317857533693314, + 0.13192236423492432, + 0.6036517024040222 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5319744348526, + "mean_weights": [ + 0.12406684458255768, + 0.07462044060230255, + 0.05378449708223343, + 0.05575074627995491, + 0.07357081025838852, + 0.10496053099632263, + 0.5132461786270142 + ], + "sources": 7 + }, + { + "entropy_mean": 1.2461028099060059, + "mean_weights": [ + 0.08556011319160461, + 0.0446893684566021, + 0.04084867238998413, + 0.05386578291654587, + 0.025119207799434662, + 0.11240880191326141, + 0.6375080347061157 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6323832273483276, + "mean_weights": [ + 0.1418324112892151, + 0.08991365134716034, + 0.06188817322254181, + 0.05739111453294754, + 0.09848876297473907, + 0.0972195416688919, + 0.45326635241508484 + ], + "sources": 7 + }, + { + "entropy_mean": 1.301480770111084, + "mean_weights": [ + 0.09513165056705475, + 0.05029121786355972, + 0.045042067766189575, + 0.05902485549449921, + 0.02615261822938919, + 0.1109299436211586, + 0.6134276390075684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.629331111907959, + "mean_weights": [ + 0.14790503680706024, + 0.08658000826835632, + 0.06608416140079498, + 0.06427294760942459, + 0.0819322019815445, + 0.097825787961483, + 0.45539984107017517 + ], + "sources": 7 + }, + { + "entropy_mean": 1.338958501815796, + "mean_weights": [ + 0.10803166776895523, + 0.0559723861515522, + 0.047874726355075836, + 0.060293328016996384, + 0.029215168207883835, + 0.10392352938652039, + 0.5946891903877258 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7611865997314453, + "mean_weights": [ + 0.1573912799358368, + 0.10317619144916534, + 0.0853322446346283, + 0.08218386769294739, + 0.09052872657775879, + 0.11341015249490738, + 0.3679775595664978 + ], + "sources": 7 + }, + { + "entropy_mean": 1.476489782333374, + "mean_weights": [ + 0.09070298075675964, + 0.037138283252716064, + 0.030013620853424072, + 0.03533655032515526, + 0.020730752497911453, + 0.06278065592050552, + 0.3836256265640259, + 0.3396714925765991 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7213938236236572, + "mean_weights": [ + 0.10584281384944916, + 0.06887699663639069, + 0.05207539349794388, + 0.050678279250860214, + 0.04768219590187073, + 0.06809806823730469, + 0.2275032103061676, + 0.37924304604530334 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5405172109603882, + "mean_weights": [ + 0.07428755611181259, + 0.04491765797138214, + 0.03515764698386192, + 0.046252522617578506, + 0.023427214473485947, + 0.07749318331480026, + 0.29666566848754883, + 0.4017985761165619 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8063910007476807, + "mean_weights": [ + 0.11433833837509155, + 0.07374332845211029, + 0.05932536721229553, + 0.05389238893985748, + 0.061260174959897995, + 0.07349519431591034, + 0.22022446990013123, + 0.3437207341194153 + ], + "sources": 8 + }, + { + "entropy_mean": 1.4859801530838013, + "mean_weights": [ + 0.08387120068073273, + 0.041406162083148956, + 0.032444946467876434, + 0.03686721250414848, + 0.025052767246961594, + 0.060124777257442474, + 0.27774184942245483, + 0.4424910843372345 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7777256965637207, + "mean_weights": [ + 0.10586117208003998, + 0.07087038457393646, + 0.05850498005747795, + 0.06360965967178345, + 0.04672512412071228, + 0.07485505938529968, + 0.2385886013507843, + 0.3409850001335144 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6318418979644775, + "mean_weights": [ + 0.09244748204946518, + 0.05762210488319397, + 0.045099519193172455, + 0.056236449629068375, + 0.029373466968536377, + 0.0723743885755539, + 0.31761035323143005, + 0.3292362093925476 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8334553241729736, + "mean_weights": [ + 0.1157165914773941, + 0.07161639630794525, + 0.06577154994010925, + 0.07425849884748459, + 0.04881399869918823, + 0.08197253942489624, + 0.27155569195747375, + 0.270294725894928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6227548122406006, + "mean_weights": [ + 0.0605507455766201, + 0.040063511580228806, + 0.032619886100292206, + 0.03868011012673378, + 0.02029048651456833, + 0.04760073497891426, + 0.19551873207092285, + 0.11294309794902802, + 0.45173269510269165 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9787883758544922, + "mean_weights": [ + 0.0788881778717041, + 0.08108963817358017, + 0.057054273784160614, + 0.08211588114500046, + 0.026040803641080856, + 0.10537256300449371, + 0.24502354860305786, + 0.12593510746955872, + 0.1984800100326538 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7590525150299072, + "mean_weights": [ + 0.07131639868021011, + 0.044687941670417786, + 0.033290669322013855, + 0.043597716838121414, + 0.02658817172050476, + 0.06922741234302521, + 0.23554196953773499, + 0.14986532926559448, + 0.3258844017982483 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9736928939819336, + "mean_weights": [ + 0.09849467873573303, + 0.068942591547966, + 0.0599900484085083, + 0.06335455179214478, + 0.04671281576156616, + 0.0713595598936081, + 0.22560220956802368, + 0.1557777225971222, + 0.20976582169532776 + ], + "sources": 9 + }, + { + "entropy_mean": 1.809161901473999, + "mean_weights": [ + 0.08854075521230698, + 0.05356752127408981, + 0.04243364930152893, + 0.04702243581414223, + 0.02648795023560524, + 0.05820769816637039, + 0.18489578366279602, + 0.21069815754890442, + 0.2881460189819336 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8544633388519287, + "mean_weights": [ + 0.07977832853794098, + 0.05858254432678223, + 0.053744811564683914, + 0.07626348733901978, + 0.031090255826711655, + 0.0994817242026329, + 0.3525030016899109, + 0.1274903416633606, + 0.12106549739837646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9456019401550293, + "mean_weights": [ + 0.08823960274457932, + 0.06737187504768372, + 0.05197303742170334, + 0.0631948709487915, + 0.02739548310637474, + 0.08291902393102646, + 0.2524128556251526, + 0.186361163854599, + 0.18013209104537964 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.06279198825359344, + 0.0596221424639225, + 0.07690861076116562, + 0.07634982466697693, + 0.18058517575263977, + 0.06968431174755096, + 0.15882407128810883, + 0.11288930475711823, + 0.19892723858356476, + 0.11885104328393936, + 0.15647411346435547, + 0.1495160460472107, + 0.1533493995666504, + 0.11554427444934845, + 0.1527898609638214, + 0.13194940984249115, + 0.16355258226394653, + 0.08884616196155548, + 0.11789151281118393, + 0.10897023230791092, + 0.11456070095300674, + 0.09843935817480087, + 0.12148918211460114, + 0.09431816637516022, + 0.15094918012619019, + 0.09025302529335022, + 0.09854138642549515, + 0.0895693451166153, + 0.11213528364896774, + 0.10807885229587555, + 0.16254977881908417, + 0.2204425185918808, + 0.2323070764541626, + 0.15380074083805084, + 0.1390199512243271, + 0.08548793941736221, + 0.14812250435352325, + 0.09071414917707443, + 0.15333011746406555, + 0.10860678553581238, + 0.1781008094549179, + 0.04808543995022774, + 0.07593096792697906, + 0.05631518363952637, + 0.09691466391086578, + 0.07238366454839706, + 0.09368696808815002, + 0.08572373539209366, + 0.10724716633558273, + 0.06546958535909653, + 0.06538715958595276, + 0.06323372572660446, + 0.07412076741456985, + 0.06869969516992569, + 0.07655765861272812, + 0.07729499042034149, + 0.08420819044113159, + 0.04753003641963005, + 0.07230556756258011, + 0.06484207510948181, + 0.07723049819469452, + 0.07835857570171356, + 0.08422473073005676, + 0.08203105628490448 + ], + "loss_nats": 1.3013269901275635, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.596132755279541, + "mean_weights": [ + 0.09668220579624176, + 0.03708431497216225, + 0.034579962491989136, + 0.026012204587459564, + 0.0258483849465847, + 0.037477150559425354, + 0.0845583826303482, + 0.3670029640197754, + 0.29075440764427185 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.09253507107496262, + 0.15345558524131775, + 0.30484533309936523, + 0.3249167501926422, + 0.33420246839523315, + 0.3818626403808594, + 0.38682517409324646, + 0.40600064396858215, + 0.12552575767040253, + 0.1351035088300705, + 0.14749252796173096, + 0.15324510633945465, + 0.1726136952638626, + 0.18001793324947357, + 0.1892337054014206, + 0.20391276478767395, + 0.031880609691143036, + 0.04374876990914345, + 0.06123264133930206, + 0.08211024105548859, + 0.10403046011924744, + 0.1380448192358017, + 0.15778414905071259, + 0.2295975536108017, + 0.053015924990177155, + 0.08199761807918549, + 0.09510950744152069, + 0.13892096281051636, + 0.1597633957862854, + 0.21687114238739014, + 0.7120755910873413, + 0.7360114455223083, + 0.04075472429394722, + 0.057135771960020065, + 0.07218439877033234, + 0.08251863718032837, + 0.11449843645095825, + 0.12661415338516235, + 0.16539782285690308, + 0.19619159400463104, + 0.0493597537279129, + 0.06753352284431458, + 0.07179203629493713, + 0.095451220870018, + 0.10295005142688751, + 0.1274663656949997, + 0.1323672980070114, + 0.15628865361213684, + 0.023035654798150063, + 0.058436937630176544, + 0.06294231116771698, + 0.08794140070676804, + 0.09175154566764832, + 0.12247690558433533, + 0.1252463310956955, + 0.1640285700559616, + 0.03424545377492905, + 0.07537907361984253, + 0.07851024717092514, + 0.12290703505277634, + 0.12469635903835297, + 0.19157102704048157, + 0.19299884140491486, + 0.29249706864356995 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.034013546791773, + "cross_entropy_nats": 5.5687538385391235, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "cross_entropy_nats": 3.5575302243232727, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "cross_entropy_nats": 2.1426188945770264, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "cross_entropy_nats": 1.3501944988965988, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "cross_entropy_nats": 1.241536632180214, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "cross_entropy_nats": 1.1848131567239761, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d", + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.03141246394478, + "measured_steps": 7980, + "median_ms": 108.93342149211094, + "p95_ms": 109.53918776649516, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.018870646365954, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.558257579803467, + "step": 1, + "unclipped_grad_norm": 26.834381103515625 + }, + { + "bits_per_byte": 7.214299345307752, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.000571250915527, + "step": 10, + "unclipped_grad_norm": 12.740906715393066 + }, + { + "bits_per_byte": 6.684460697974439, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.633315086364746, + "step": 20, + "unclipped_grad_norm": 3.454355239868164 + }, + { + "bits_per_byte": 6.541223915307444, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.534030914306641, + "step": 30, + "unclipped_grad_norm": 3.060823440551758 + }, + { + "bits_per_byte": 6.411011034824987, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.443774223327637, + "step": 40, + "unclipped_grad_norm": 3.0002496242523193 + }, + { + "bits_per_byte": 6.085588902515397, + "learning_rate": 3.75e-05, + "loss_nats": 4.2182087898254395, + "step": 50, + "unclipped_grad_norm": 2.976402997970581 + }, + { + "bits_per_byte": 5.879418164452873, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.0753021240234375, + "step": 60, + "unclipped_grad_norm": 2.708517551422119 + }, + { + "bits_per_byte": 5.710178980836628, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9579944610595703, + "step": 70, + "unclipped_grad_norm": 2.559852361679077 + }, + { + "bits_per_byte": 5.512089019607543, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8206889629364014, + "step": 80, + "unclipped_grad_norm": 2.5563650131225586 + }, + { + "bits_per_byte": 5.341444737733749, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7024073600769043, + "step": 90, + "unclipped_grad_norm": 2.3861565589904785 + }, + { + "bits_per_byte": 5.169528816139475, + "learning_rate": 7.5e-05, + "loss_nats": 3.5832443237304688, + "step": 100, + "unclipped_grad_norm": 2.241894006729126 + }, + { + "bits_per_byte": 5.000991665704052, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.466423273086548, + "step": 110, + "unclipped_grad_norm": 2.1930899620056152 + }, + { + "bits_per_byte": 4.733823840860015, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2812366485595703, + "step": 120, + "unclipped_grad_norm": 1.8853988647460938 + }, + { + "bits_per_byte": 4.585053344418857, + "learning_rate": 9.75e-05, + "loss_nats": 3.178116798400879, + "step": 130, + "unclipped_grad_norm": 1.6627076864242554 + }, + { + "bits_per_byte": 4.391709416422983, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0441009998321533, + "step": 140, + "unclipped_grad_norm": 1.4586857557296753 + }, + { + "bits_per_byte": 4.260228335873064, + "learning_rate": 0.0001125, + "loss_nats": 2.952965259552002, + "step": 150, + "unclipped_grad_norm": 1.235063076019287 + }, + { + "bits_per_byte": 4.153197684831009, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.878777265548706, + "step": 160, + "unclipped_grad_norm": 1.0407519340515137 + }, + { + "bits_per_byte": 3.997309514401244, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.770723819732666, + "step": 170, + "unclipped_grad_norm": 0.9049285054206848 + }, + { + "bits_per_byte": 3.860879805055398, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.6761579513549805, + "step": 180, + "unclipped_grad_norm": 1.0858614444732666 + }, + { + "bits_per_byte": 3.870312021559318, + "learning_rate": 0.0001425, + "loss_nats": 2.6826958656311035, + "step": 190, + "unclipped_grad_norm": 0.6896197199821472 + }, + { + "bits_per_byte": 3.747811875037455, + "learning_rate": 0.00015, + "loss_nats": 2.597785234451294, + "step": 200, + "unclipped_grad_norm": 0.712465226650238 + }, + { + "bits_per_byte": 3.69472358243852, + "learning_rate": 0.0001575, + "loss_nats": 2.5609872341156006, + "step": 210, + "unclipped_grad_norm": 0.8807040452957153 + }, + { + "bits_per_byte": 3.662194783854328, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.538439989089966, + "step": 220, + "unclipped_grad_norm": 0.7631296515464783 + }, + { + "bits_per_byte": 3.6752317567203328, + "learning_rate": 0.0001725, + "loss_nats": 2.5474765300750732, + "step": 230, + "unclipped_grad_norm": 0.6869579553604126 + }, + { + "bits_per_byte": 3.6565805821995627, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.53454852104187, + "step": 240, + "unclipped_grad_norm": 0.781292736530304 + }, + { + "bits_per_byte": 3.5245367744160205, + "learning_rate": 0.0001875, + "loss_nats": 2.4430227279663086, + "step": 250, + "unclipped_grad_norm": 1.0127525329589844 + }, + { + "bits_per_byte": 3.524651658826797, + "learning_rate": 0.000195, + "loss_nats": 2.4431023597717285, + "step": 260, + "unclipped_grad_norm": 0.8615073561668396 + }, + { + "bits_per_byte": 3.485441678221887, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.415924072265625, + "step": 270, + "unclipped_grad_norm": 0.7731815576553345 + }, + { + "bits_per_byte": 3.4956767097400343, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.423018455505371, + "step": 280, + "unclipped_grad_norm": 0.9061474800109863 + }, + { + "bits_per_byte": 3.3938416545805445, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3524317741394043, + "step": 290, + "unclipped_grad_norm": 1.0149996280670166 + }, + { + "bits_per_byte": 3.483576010425327, + "learning_rate": 0.000225, + "loss_nats": 2.414630889892578, + "step": 300, + "unclipped_grad_norm": 1.1414116621017456 + }, + { + "bits_per_byte": 3.381058528107214, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.343571186065674, + "step": 310, + "unclipped_grad_norm": 1.1472890377044678 + }, + { + "bits_per_byte": 3.3707543595634197, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3364288806915283, + "step": 320, + "unclipped_grad_norm": 1.2828246355056763 + }, + { + "bits_per_byte": 3.3462052120139605, + "learning_rate": 0.0002475, + "loss_nats": 2.3194127082824707, + "step": 330, + "unclipped_grad_norm": 0.9300209879875183 + }, + { + "bits_per_byte": 3.3510090314178607, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.322742462158203, + "step": 340, + "unclipped_grad_norm": 0.7300398945808411 + }, + { + "bits_per_byte": 3.324897249502662, + "learning_rate": 0.0002625, + "loss_nats": 2.304643154144287, + "step": 350, + "unclipped_grad_norm": 0.96440589427948 + }, + { + "bits_per_byte": 3.301325651340707, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.288304567337036, + "step": 360, + "unclipped_grad_norm": 1.186806321144104 + }, + { + "bits_per_byte": 3.330423052074884, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3084733486175537, + "step": 370, + "unclipped_grad_norm": 1.40411376953125 + }, + { + "bits_per_byte": 3.351273196769586, + "learning_rate": 0.000285, + "loss_nats": 2.322925567626953, + "step": 380, + "unclipped_grad_norm": 0.8571750521659851 + }, + { + "bits_per_byte": 3.2969108566930587, + "learning_rate": 0.0002925, + "loss_nats": 2.2852444648742676, + "step": 390, + "unclipped_grad_norm": 1.172652006149292 + }, + { + "bits_per_byte": 3.2316042286462263, + "learning_rate": 0.0003, + "loss_nats": 2.2399773597717285, + "step": 400, + "unclipped_grad_norm": 1.0324530601501465 + }, + { + "bits_per_byte": 3.191360632309631, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.212082624435425, + "step": 410, + "unclipped_grad_norm": 0.9553669691085815 + }, + { + "bits_per_byte": 3.2040835648551598, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2209014892578125, + "step": 420, + "unclipped_grad_norm": 1.0212959051132202 + }, + { + "bits_per_byte": 3.1886670400318176, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2102155685424805, + "step": 430, + "unclipped_grad_norm": 0.8366522789001465 + }, + { + "bits_per_byte": 3.2100664973133752, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.225048542022705, + "step": 440, + "unclipped_grad_norm": 1.1677532196044922 + }, + { + "bits_per_byte": 3.1886219805772913, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.210184335708618, + "step": 450, + "unclipped_grad_norm": 1.3463479280471802 + }, + { + "bits_per_byte": 3.136037597145384, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.1737356185913086, + "step": 460, + "unclipped_grad_norm": 1.704139232635498 + }, + { + "bits_per_byte": 3.1589835224235, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.189640522003174, + "step": 470, + "unclipped_grad_norm": 1.1849488019943237 + }, + { + "bits_per_byte": 3.1210654754918714, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.163357734680176, + "step": 480, + "unclipped_grad_norm": 0.8924318552017212 + }, + { + "bits_per_byte": 3.0823876092087312, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1365482807159424, + "step": 490, + "unclipped_grad_norm": 1.3279240131378174 + }, + { + "bits_per_byte": 3.12389562199447, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1653194427490234, + "step": 500, + "unclipped_grad_norm": 1.3776742219924927 + }, + { + "bits_per_byte": 3.102860080002138, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1507387161254883, + "step": 510, + "unclipped_grad_norm": 1.061196208000183 + }, + { + "bits_per_byte": 2.961672642602824, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.05287504196167, + "step": 520, + "unclipped_grad_norm": 1.0864008665084839 + }, + { + "bits_per_byte": 3.0521231342057993, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.115570545196533, + "step": 530, + "unclipped_grad_norm": 0.9000556468963623 + }, + { + "bits_per_byte": 2.992112883874938, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.073974609375, + "step": 540, + "unclipped_grad_norm": 0.8655318021774292 + }, + { + "bits_per_byte": 2.9456885750316344, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0417957305908203, + "step": 550, + "unclipped_grad_norm": 0.8011268973350525 + }, + { + "bits_per_byte": 2.9526635034200264, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.046630382537842, + "step": 560, + "unclipped_grad_norm": 1.2049078941345215 + }, + { + "bits_per_byte": 2.943776471919342, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0404703617095947, + "step": 570, + "unclipped_grad_norm": 0.9911091923713684 + }, + { + "bits_per_byte": 2.836173462615215, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.9658856391906738, + "step": 580, + "unclipped_grad_norm": 0.9621384143829346 + }, + { + "bits_per_byte": 2.898327304706431, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.008967399597168, + "step": 590, + "unclipped_grad_norm": 1.0092018842697144 + }, + { + "bits_per_byte": 2.807946982025008, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9463205337524414, + "step": 600, + "unclipped_grad_norm": 1.041255235671997 + }, + { + "bits_per_byte": 2.8662821213364653, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.98675537109375, + "step": 610, + "unclipped_grad_norm": 1.0585267543792725 + }, + { + "bits_per_byte": 2.7525066546928727, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9078922271728516, + "step": 620, + "unclipped_grad_norm": 0.9587937593460083 + }, + { + "bits_per_byte": 2.7676626258001575, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9183975458145142, + "step": 630, + "unclipped_grad_norm": 0.9836012721061707 + }, + { + "bits_per_byte": 2.7750938241612886, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9235484600067139, + "step": 640, + "unclipped_grad_norm": 0.9689151644706726 + }, + { + "bits_per_byte": 2.6921524390602856, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8660578727722168, + "step": 650, + "unclipped_grad_norm": 1.0390667915344238 + }, + { + "bits_per_byte": 2.6851900654054064, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8612319231033325, + "step": 660, + "unclipped_grad_norm": 1.6513670682907104 + }, + { + "bits_per_byte": 2.6846809967588525, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8608790636062622, + "step": 670, + "unclipped_grad_norm": 0.9175130724906921 + }, + { + "bits_per_byte": 2.7078205745018398, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8769181966781616, + "step": 680, + "unclipped_grad_norm": 1.0189234018325806 + }, + { + "bits_per_byte": 2.5875252455640383, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.793535828590393, + "step": 690, + "unclipped_grad_norm": 0.9444974064826965 + }, + { + "bits_per_byte": 2.5413828162854992, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.761552333831787, + "step": 700, + "unclipped_grad_norm": 1.1904406547546387 + }, + { + "bits_per_byte": 2.623998466245768, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.818817138671875, + "step": 710, + "unclipped_grad_norm": 1.1858620643615723 + }, + { + "bits_per_byte": 2.683836561943116, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8602937459945679, + "step": 720, + "unclipped_grad_norm": 0.9852609634399414 + }, + { + "bits_per_byte": 2.4849188482154285, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.722414493560791, + "step": 730, + "unclipped_grad_norm": 1.1975579261779785 + }, + { + "bits_per_byte": 2.5244201674309004, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7497947216033936, + "step": 740, + "unclipped_grad_norm": 0.9646193385124207 + }, + { + "bits_per_byte": 2.5296962511940695, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7534518241882324, + "step": 750, + "unclipped_grad_norm": 1.191988468170166 + }, + { + "bits_per_byte": 2.4988650933565446, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7320812940597534, + "step": 760, + "unclipped_grad_norm": 1.092863917350769 + }, + { + "bits_per_byte": 2.4838112799438723, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.721646785736084, + "step": 770, + "unclipped_grad_norm": 1.0348618030548096 + }, + { + "bits_per_byte": 2.4223032327066463, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.679012656211853, + "step": 780, + "unclipped_grad_norm": 1.034103274345398 + }, + { + "bits_per_byte": 2.485370818621896, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7227277755737305, + "step": 790, + "unclipped_grad_norm": 1.060265302658081 + }, + { + "bits_per_byte": 2.4233009040643014, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.679704189300537, + "step": 800, + "unclipped_grad_norm": 1.3169445991516113 + }, + { + "bits_per_byte": 2.4599308009409677, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7050940990447998, + "step": 810, + "unclipped_grad_norm": 0.9821087121963501 + }, + { + "bits_per_byte": 2.425854846429614, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6814744472503662, + "step": 820, + "unclipped_grad_norm": 1.0924170017242432 + }, + { + "bits_per_byte": 2.4277851797028984, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6828124523162842, + "step": 830, + "unclipped_grad_norm": 1.055789828300476 + }, + { + "bits_per_byte": 2.3982261775338056, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6623237133026123, + "step": 840, + "unclipped_grad_norm": 1.230089545249939 + }, + { + "bits_per_byte": 2.339956047664374, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.621933937072754, + "step": 850, + "unclipped_grad_norm": 0.94926518201828 + }, + { + "bits_per_byte": 2.3803994878412724, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6499671936035156, + "step": 860, + "unclipped_grad_norm": 1.1705771684646606 + }, + { + "bits_per_byte": 2.3436123988217177, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6244683265686035, + "step": 870, + "unclipped_grad_norm": 1.1496235132217407 + }, + { + "bits_per_byte": 2.3796778486382517, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6494669914245605, + "step": 880, + "unclipped_grad_norm": 1.1764546632766724 + }, + { + "bits_per_byte": 2.357840007579645, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.634330153465271, + "step": 890, + "unclipped_grad_norm": 0.9926145672798157 + }, + { + "bits_per_byte": 2.2929997964819524, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.5893863439559937, + "step": 900, + "unclipped_grad_norm": 1.009945034980774 + }, + { + "bits_per_byte": 2.3411444477818364, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6227576732635498, + "step": 910, + "unclipped_grad_norm": 1.2830723524093628 + }, + { + "bits_per_byte": 2.2736926801307917, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5760036706924438, + "step": 920, + "unclipped_grad_norm": 1.0900828838348389 + }, + { + "bits_per_byte": 2.3038844064724087, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.596930980682373, + "step": 930, + "unclipped_grad_norm": 1.1090145111083984 + }, + { + "bits_per_byte": 2.2389674911124313, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.551934003829956, + "step": 940, + "unclipped_grad_norm": 0.9897664189338684 + }, + { + "bits_per_byte": 2.2434025797128814, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5550081729888916, + "step": 950, + "unclipped_grad_norm": 1.0138952732086182 + }, + { + "bits_per_byte": 2.2236732459538526, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5413328409194946, + "step": 960, + "unclipped_grad_norm": 1.0875126123428345 + }, + { + "bits_per_byte": 2.2667958318658914, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5712231397628784, + "step": 970, + "unclipped_grad_norm": 1.245374321937561 + }, + { + "bits_per_byte": 2.2344890628839638, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5488297939300537, + "step": 980, + "unclipped_grad_norm": 1.1757959127426147 + }, + { + "bits_per_byte": 2.1613678871472266, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.4981460571289062, + "step": 990, + "unclipped_grad_norm": 1.066755771636963 + }, + { + "bits_per_byte": 2.207448574654106, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5300867557525635, + "step": 1000, + "unclipped_grad_norm": 0.9658553004264832 + }, + { + "bits_per_byte": 2.213320750285166, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5341570377349854, + "step": 1010, + "unclipped_grad_norm": 1.1267611980438232 + }, + { + "bits_per_byte": 2.2186436133293697, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.537846565246582, + "step": 1020, + "unclipped_grad_norm": 0.9853240251541138 + }, + { + "bits_per_byte": 2.1280228589020402, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4750330448150635, + "step": 1030, + "unclipped_grad_norm": 0.972481369972229 + }, + { + "bits_per_byte": 2.2322469250646653, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5472756624221802, + "step": 1040, + "unclipped_grad_norm": 1.0531573295593262 + }, + { + "bits_per_byte": 2.179971250491057, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5110409259796143, + "step": 1050, + "unclipped_grad_norm": 0.9985654950141907 + }, + { + "bits_per_byte": 2.1476089311129005, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.4886090755462646, + "step": 1060, + "unclipped_grad_norm": 1.100019931793213 + }, + { + "bits_per_byte": 2.2640807417567204, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5693411827087402, + "step": 1070, + "unclipped_grad_norm": 1.0959177017211914 + }, + { + "bits_per_byte": 2.1797610876917024, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5108952522277832, + "step": 1080, + "unclipped_grad_norm": 1.088409423828125 + }, + { + "bits_per_byte": 2.1080023585151006, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.461155891418457, + "step": 1090, + "unclipped_grad_norm": 1.0188393592834473 + }, + { + "bits_per_byte": 2.1918825969071642, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5192972421646118, + "step": 1100, + "unclipped_grad_norm": 0.9501848816871643 + }, + { + "bits_per_byte": 2.205643788715952, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5288357734680176, + "step": 1110, + "unclipped_grad_norm": 0.9797388911247253 + }, + { + "bits_per_byte": 2.0930500148664373, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4507917165756226, + "step": 1120, + "unclipped_grad_norm": 0.8856902122497559 + }, + { + "bits_per_byte": 2.095190166973774, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4522751569747925, + "step": 1130, + "unclipped_grad_norm": 0.9447507262229919 + }, + { + "bits_per_byte": 2.116739593126884, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4672120809555054, + "step": 1140, + "unclipped_grad_norm": 1.0488460063934326 + }, + { + "bits_per_byte": 2.1023439573190634, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4572337865829468, + "step": 1150, + "unclipped_grad_norm": 0.9838997721672058 + }, + { + "bits_per_byte": 2.0562952585852448, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.425315260887146, + "step": 1160, + "unclipped_grad_norm": 0.9282211065292358 + }, + { + "bits_per_byte": 2.160992449020393, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.497885823249817, + "step": 1170, + "unclipped_grad_norm": 1.0253955125808716 + }, + { + "bits_per_byte": 2.0716469459492117, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4359562397003174, + "step": 1180, + "unclipped_grad_norm": 0.9352066516876221 + }, + { + "bits_per_byte": 2.0893013090255805, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4481933116912842, + "step": 1190, + "unclipped_grad_norm": 1.0826658010482788 + }, + { + "bits_per_byte": 2.0752424152481495, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.438448429107666, + "step": 1200, + "unclipped_grad_norm": 0.9942638874053955 + }, + { + "bits_per_byte": 2.2049758081000004, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5283727645874023, + "step": 1210, + "unclipped_grad_norm": 1.1482313871383667 + }, + { + "bits_per_byte": 2.0506430487646385, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4213974475860596, + "step": 1220, + "unclipped_grad_norm": 0.9142818450927734 + }, + { + "bits_per_byte": 2.00269480165204, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.3881622552871704, + "step": 1230, + "unclipped_grad_norm": 0.9387761354446411 + }, + { + "bits_per_byte": 2.0166881700394925, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.3978617191314697, + "step": 1240, + "unclipped_grad_norm": 1.0545530319213867 + }, + { + "bits_per_byte": 2.1787518934964396, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5101957321166992, + "step": 1250, + "unclipped_grad_norm": 1.1504697799682617 + }, + { + "bits_per_byte": 2.1047604855463367, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4589087963104248, + "step": 1260, + "unclipped_grad_norm": 0.9371413588523865 + }, + { + "bits_per_byte": 1.9885089846782722, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3783293962478638, + "step": 1270, + "unclipped_grad_norm": 0.9821643233299255 + }, + { + "bits_per_byte": 2.1346452228561636, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4796233177185059, + "step": 1280, + "unclipped_grad_norm": 0.9382128715515137 + }, + { + "bits_per_byte": 2.022505311222276, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4018938541412354, + "step": 1290, + "unclipped_grad_norm": 0.9164487719535828 + }, + { + "bits_per_byte": 2.050535903573151, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4213231801986694, + "step": 1300, + "unclipped_grad_norm": 0.9040083885192871 + }, + { + "bits_per_byte": 2.128253659619498, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4751930236816406, + "step": 1310, + "unclipped_grad_norm": 1.045804738998413 + }, + { + "bits_per_byte": 2.0627867437420613, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4298148155212402, + "step": 1320, + "unclipped_grad_norm": 1.0330690145492554 + }, + { + "bits_per_byte": 1.954123977343318, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3544955253601074, + "step": 1330, + "unclipped_grad_norm": 0.9727330803871155 + }, + { + "bits_per_byte": 1.9791288789175636, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3718276023864746, + "step": 1340, + "unclipped_grad_norm": 0.966886043548584 + }, + { + "bits_per_byte": 2.1481475807754027, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4889824390411377, + "step": 1350, + "unclipped_grad_norm": 0.9399362802505493 + }, + { + "bits_per_byte": 1.9898208683390537, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3792387247085571, + "step": 1360, + "unclipped_grad_norm": 0.9223746061325073 + }, + { + "bits_per_byte": 2.0666878262115826, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4325188398361206, + "step": 1370, + "unclipped_grad_norm": 0.9086402654647827 + }, + { + "bits_per_byte": 2.05495087020345, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4243834018707275, + "step": 1380, + "unclipped_grad_norm": 0.954355776309967 + }, + { + "bits_per_byte": 2.067550663170961, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4331169128417969, + "step": 1390, + "unclipped_grad_norm": 0.8977138996124268 + }, + { + "bits_per_byte": 2.0721085473841274, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4362761974334717, + "step": 1400, + "unclipped_grad_norm": 0.8833977580070496 + }, + { + "bits_per_byte": 1.9799110560136877, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.3723697662353516, + "step": 1410, + "unclipped_grad_norm": 0.9529625177383423 + }, + { + "bits_per_byte": 2.038817521691312, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4132006168365479, + "step": 1420, + "unclipped_grad_norm": 1.083083152770996 + }, + { + "bits_per_byte": 2.0292872750764, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4065947532653809, + "step": 1430, + "unclipped_grad_norm": 0.9355389475822449 + }, + { + "bits_per_byte": 1.987968787171912, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3779549598693848, + "step": 1440, + "unclipped_grad_norm": 0.8836045265197754 + }, + { + "bits_per_byte": 2.0423230440638482, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4156304597854614, + "step": 1450, + "unclipped_grad_norm": 0.924534022808075 + }, + { + "bits_per_byte": 1.9816687187055055, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3735880851745605, + "step": 1460, + "unclipped_grad_norm": 0.8437807559967041 + }, + { + "bits_per_byte": 1.9797782854072217, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3722777366638184, + "step": 1470, + "unclipped_grad_norm": 0.9143778085708618 + }, + { + "bits_per_byte": 1.8767507025466492, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3008644580841064, + "step": 1480, + "unclipped_grad_norm": 0.8930306434631348 + }, + { + "bits_per_byte": 2.001770738868954, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.387521743774414, + "step": 1490, + "unclipped_grad_norm": 0.8246955871582031 + }, + { + "bits_per_byte": 1.9125255016832254, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3256616592407227, + "step": 1500, + "unclipped_grad_norm": 0.9061107635498047 + }, + { + "bits_per_byte": 1.973825621895558, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3681516647338867, + "step": 1510, + "unclipped_grad_norm": 0.9889123439788818 + }, + { + "bits_per_byte": 1.980869687309597, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3730342388153076, + "step": 1520, + "unclipped_grad_norm": 0.9045401811599731 + }, + { + "bits_per_byte": 1.9378121108395843, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.343189001083374, + "step": 1530, + "unclipped_grad_norm": 0.8896906971931458 + }, + { + "bits_per_byte": 2.017761685746178, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3986058235168457, + "step": 1540, + "unclipped_grad_norm": 0.9600386619567871 + }, + { + "bits_per_byte": 1.9800400430018348, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3724591732025146, + "step": 1550, + "unclipped_grad_norm": 0.8903042078018188 + }, + { + "bits_per_byte": 2.037820882229562, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.412509799003601, + "step": 1560, + "unclipped_grad_norm": 0.9084417819976807 + }, + { + "bits_per_byte": 2.015308525214271, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3969054222106934, + "step": 1570, + "unclipped_grad_norm": 0.9122006893157959 + }, + { + "bits_per_byte": 1.985575476602505, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.376296043395996, + "step": 1580, + "unclipped_grad_norm": 0.8903481960296631 + }, + { + "bits_per_byte": 1.996281568601366, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3837169408798218, + "step": 1590, + "unclipped_grad_norm": 0.933029294013977 + }, + { + "bits_per_byte": 2.002402775110875, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3879598379135132, + "step": 1600, + "unclipped_grad_norm": 0.9777830243110657 + }, + { + "bits_per_byte": 2.0119596790366705, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3945841789245605, + "step": 1610, + "unclipped_grad_norm": 0.9627588391304016 + }, + { + "bits_per_byte": 2.0159194075901357, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3973288536071777, + "step": 1620, + "unclipped_grad_norm": 0.8731178641319275 + }, + { + "bits_per_byte": 1.8905791395719242, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3104496002197266, + "step": 1630, + "unclipped_grad_norm": 0.8998029828071594 + }, + { + "bits_per_byte": 2.0235616286638756, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4026260375976562, + "step": 1640, + "unclipped_grad_norm": 0.9714035391807556 + }, + { + "bits_per_byte": 1.9588359580116574, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3577616214752197, + "step": 1650, + "unclipped_grad_norm": 0.9218164682388306 + }, + { + "bits_per_byte": 1.936817019221692, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3424992561340332, + "step": 1660, + "unclipped_grad_norm": 0.8974215984344482 + }, + { + "bits_per_byte": 2.096671109580354, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4533016681671143, + "step": 1670, + "unclipped_grad_norm": 0.9378093481063843 + }, + { + "bits_per_byte": 2.0150522710644854, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3967278003692627, + "step": 1680, + "unclipped_grad_norm": 0.8720370531082153 + }, + { + "bits_per_byte": 1.8838673446393523, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3057973384857178, + "step": 1690, + "unclipped_grad_norm": 0.8577803373336792 + }, + { + "bits_per_byte": 1.927341291107104, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3359311819076538, + "step": 1700, + "unclipped_grad_norm": 0.9562329053878784 + }, + { + "bits_per_byte": 1.9962937793695772, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3837254047393799, + "step": 1710, + "unclipped_grad_norm": 0.9462636113166809 + }, + { + "bits_per_byte": 1.995081645646296, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.382885217666626, + "step": 1720, + "unclipped_grad_norm": 0.8408851027488708 + }, + { + "bits_per_byte": 1.894888164889292, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3134363889694214, + "step": 1730, + "unclipped_grad_norm": 0.8187386989593506 + }, + { + "bits_per_byte": 1.9475287866480313, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.349924087524414, + "step": 1740, + "unclipped_grad_norm": 0.9181671142578125 + }, + { + "bits_per_byte": 2.0484275682562236, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4198617935180664, + "step": 1750, + "unclipped_grad_norm": 0.9259082674980164 + }, + { + "bits_per_byte": 1.9089950418263137, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3232145309448242, + "step": 1760, + "unclipped_grad_norm": 0.9000674486160278 + }, + { + "bits_per_byte": 1.9653874651136254, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3623027801513672, + "step": 1770, + "unclipped_grad_norm": 0.8686579465866089 + }, + { + "bits_per_byte": 1.9032847018697159, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3192564249038696, + "step": 1780, + "unclipped_grad_norm": 0.8219738006591797 + }, + { + "bits_per_byte": 1.8754542973244455, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2999658584594727, + "step": 1790, + "unclipped_grad_norm": 0.8401536345481873 + }, + { + "bits_per_byte": 1.9695277754504965, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3651726245880127, + "step": 1800, + "unclipped_grad_norm": 0.9855192303657532 + }, + { + "bits_per_byte": 1.952977025044714, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3537005186080933, + "step": 1810, + "unclipped_grad_norm": 0.9152796268463135 + }, + { + "bits_per_byte": 2.0236675699768067, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4026994705200195, + "step": 1820, + "unclipped_grad_norm": 0.9160932898521423 + }, + { + "bits_per_byte": 1.863515605666851, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.2916905879974365, + "step": 1830, + "unclipped_grad_norm": 0.8749093413352966 + }, + { + "bits_per_byte": 1.9532654399502112, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.35390043258667, + "step": 1840, + "unclipped_grad_norm": 0.8691527843475342 + }, + { + "bits_per_byte": 1.8425976998077092, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.277191400527954, + "step": 1850, + "unclipped_grad_norm": 0.918012797832489 + }, + { + "bits_per_byte": 1.9936586611930571, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3818988800048828, + "step": 1860, + "unclipped_grad_norm": 0.9178117513656616 + }, + { + "bits_per_byte": 1.995530520365048, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3831963539123535, + "step": 1870, + "unclipped_grad_norm": 0.8769021034240723 + }, + { + "bits_per_byte": 1.8636951555543517, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.2918150424957275, + "step": 1880, + "unclipped_grad_norm": 0.8614026308059692 + }, + { + "bits_per_byte": 1.9426209177403593, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3465222120285034, + "step": 1890, + "unclipped_grad_norm": 0.8975904583930969 + }, + { + "bits_per_byte": 1.861964838104021, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2906156778335571, + "step": 1900, + "unclipped_grad_norm": 0.8269307613372803 + }, + { + "bits_per_byte": 1.8502344174366214, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.282484769821167, + "step": 1910, + "unclipped_grad_norm": 0.8237605690956116 + }, + { + "bits_per_byte": 1.9149965484108225, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3273744583129883, + "step": 1920, + "unclipped_grad_norm": 0.8641052842140198 + }, + { + "bits_per_byte": 1.8801597426520518, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.303227424621582, + "step": 1930, + "unclipped_grad_norm": 0.8907292485237122 + }, + { + "bits_per_byte": 1.8629336163763313, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2912871837615967, + "step": 1940, + "unclipped_grad_norm": 0.8837904930114746 + }, + { + "bits_per_byte": 1.923144226495448, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3330219984054565, + "step": 1950, + "unclipped_grad_norm": 1.0997384786605835 + }, + { + "bits_per_byte": 1.88083511852199, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.303695559501648, + "step": 1960, + "unclipped_grad_norm": 0.8385953307151794 + }, + { + "bits_per_byte": 1.8359603733629608, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2725907564163208, + "step": 1970, + "unclipped_grad_norm": 0.839948832988739 + }, + { + "bits_per_byte": 1.7988822896981462, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2468901872634888, + "step": 1980, + "unclipped_grad_norm": 0.7658324837684631 + }, + { + "bits_per_byte": 1.9198664091526536, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3307499885559082, + "step": 1990, + "unclipped_grad_norm": 0.8840731382369995 + }, + { + "bits_per_byte": 1.863330208369221, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.2915620803833008, + "step": 2000, + "unclipped_grad_norm": 0.8192090392112732 + }, + { + "bits_per_byte": 1.9331472534175815, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3399555683135986, + "step": 2010, + "unclipped_grad_norm": 0.8343153595924377 + }, + { + "bits_per_byte": 1.9117808168049895, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3251454830169678, + "step": 2020, + "unclipped_grad_norm": 0.913158118724823 + }, + { + "bits_per_byte": 1.8970498148279866, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3149347305297852, + "step": 2030, + "unclipped_grad_norm": 0.8877862691879272 + }, + { + "bits_per_byte": 1.9115138997308505, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.324960470199585, + "step": 2040, + "unclipped_grad_norm": 0.8749900460243225 + }, + { + "bits_per_byte": 1.8657040848990802, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2932075262069702, + "step": 2050, + "unclipped_grad_norm": 0.9185486435890198 + }, + { + "bits_per_byte": 1.7675556498434857, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.225176215171814, + "step": 2060, + "unclipped_grad_norm": 0.7904725670814514 + }, + { + "bits_per_byte": 1.8617049723185672, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.290435552597046, + "step": 2070, + "unclipped_grad_norm": 0.8348630666732788 + }, + { + "bits_per_byte": 1.973646415973359, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3680274486541748, + "step": 2080, + "unclipped_grad_norm": 0.8117388486862183 + }, + { + "bits_per_byte": 1.8279640400110946, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2670481204986572, + "step": 2090, + "unclipped_grad_norm": 0.7758682370185852 + }, + { + "bits_per_byte": 1.8962239541385433, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3143622875213623, + "step": 2100, + "unclipped_grad_norm": 0.8137099146842957 + }, + { + "bits_per_byte": 1.8350517890184526, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.271960973739624, + "step": 2110, + "unclipped_grad_norm": 0.9251447319984436 + }, + { + "bits_per_byte": 1.870564486595114, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2965764999389648, + "step": 2120, + "unclipped_grad_norm": 0.9138389825820923 + }, + { + "bits_per_byte": 1.8882444750864615, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3088313341140747, + "step": 2130, + "unclipped_grad_norm": 0.8262670040130615 + }, + { + "bits_per_byte": 1.7726164113277747, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2286840677261353, + "step": 2140, + "unclipped_grad_norm": 0.7359873056411743 + }, + { + "bits_per_byte": 1.8268258588276844, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2662591934204102, + "step": 2150, + "unclipped_grad_norm": 0.8173519372940063 + }, + { + "bits_per_byte": 1.8637190611428216, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.291831612586975, + "step": 2160, + "unclipped_grad_norm": 0.8174240589141846 + }, + { + "bits_per_byte": 1.8528119214251022, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2842713594436646, + "step": 2170, + "unclipped_grad_norm": 0.8957990407943726 + }, + { + "bits_per_byte": 1.782856774308099, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2357821464538574, + "step": 2180, + "unclipped_grad_norm": 0.8048707246780396 + }, + { + "bits_per_byte": 1.8923490130319534, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3116763830184937, + "step": 2190, + "unclipped_grad_norm": 0.8952655792236328 + }, + { + "bits_per_byte": 1.9442205283760343, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3476309776306152, + "step": 2200, + "unclipped_grad_norm": 0.8895230889320374 + }, + { + "bits_per_byte": 1.9876000563684622, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.377699375152588, + "step": 2210, + "unclipped_grad_norm": 0.8686981797218323 + }, + { + "bits_per_byte": 1.852814845130167, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.284273386001587, + "step": 2220, + "unclipped_grad_norm": 0.8053512573242188 + }, + { + "bits_per_byte": 1.859576858996791, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2889604568481445, + "step": 2230, + "unclipped_grad_norm": 0.8349927663803101 + }, + { + "bits_per_byte": 1.8206912376714082, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2620069980621338, + "step": 2240, + "unclipped_grad_norm": 0.8471757769584656 + }, + { + "bits_per_byte": 1.8270817690121683, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2664365768432617, + "step": 2250, + "unclipped_grad_norm": 0.8009039163589478 + }, + { + "bits_per_byte": 1.8453999851208678, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2791337966918945, + "step": 2260, + "unclipped_grad_norm": 0.8003636002540588 + }, + { + "bits_per_byte": 1.9009572606555896, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.317643165588379, + "step": 2270, + "unclipped_grad_norm": 0.8468067646026611 + }, + { + "bits_per_byte": 1.7920475271702074, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2421526908874512, + "step": 2280, + "unclipped_grad_norm": 0.9071624875068665 + }, + { + "bits_per_byte": 1.8281408381761814, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2671706676483154, + "step": 2290, + "unclipped_grad_norm": 0.822325587272644 + }, + { + "bits_per_byte": 1.8827026781277094, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3049900531768799, + "step": 2300, + "unclipped_grad_norm": 0.8915430307388306 + }, + { + "bits_per_byte": 1.831042185496237, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.269181728363037, + "step": 2310, + "unclipped_grad_norm": 0.8500391840934753 + }, + { + "bits_per_byte": 1.8431096921593277, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2775462865829468, + "step": 2320, + "unclipped_grad_norm": 0.8417274355888367 + }, + { + "bits_per_byte": 1.7906957435344257, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.241215705871582, + "step": 2330, + "unclipped_grad_norm": 0.777140736579895 + }, + { + "bits_per_byte": 1.825728265549878, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.265498399734497, + "step": 2340, + "unclipped_grad_norm": 0.8567083477973938 + }, + { + "bits_per_byte": 1.7348746466132805, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2025234699249268, + "step": 2350, + "unclipped_grad_norm": 0.7593752145767212 + }, + { + "bits_per_byte": 1.8440781264663364, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2782175540924072, + "step": 2360, + "unclipped_grad_norm": 0.8551052212715149 + }, + { + "bits_per_byte": 1.871989878805465, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2975645065307617, + "step": 2370, + "unclipped_grad_norm": 0.8494746685028076 + }, + { + "bits_per_byte": 1.7757177744707837, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2308337688446045, + "step": 2380, + "unclipped_grad_norm": 0.7841326594352722 + }, + { + "bits_per_byte": 1.7735628318554726, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2293400764465332, + "step": 2390, + "unclipped_grad_norm": 0.8951802849769592 + }, + { + "bits_per_byte": 1.8680005692360513, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2947993278503418, + "step": 2400, + "unclipped_grad_norm": 0.8339796662330627 + }, + { + "bits_per_byte": 1.8459044102358484, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.279483437538147, + "step": 2410, + "unclipped_grad_norm": 0.9356110692024231 + }, + { + "bits_per_byte": 1.931278833898608, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.338660478591919, + "step": 2420, + "unclipped_grad_norm": 0.8573734760284424 + }, + { + "bits_per_byte": 1.825236739133712, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265157699584961, + "step": 2430, + "unclipped_grad_norm": 0.8783476948738098 + }, + { + "bits_per_byte": 1.7732133631089195, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.229097843170166, + "step": 2440, + "unclipped_grad_norm": 0.8393950462341309 + }, + { + "bits_per_byte": 1.8696945983470499, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.295973539352417, + "step": 2450, + "unclipped_grad_norm": 0.8328628540039062 + }, + { + "bits_per_byte": 1.8721511985319743, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2976763248443604, + "step": 2460, + "unclipped_grad_norm": 0.8645251989364624 + }, + { + "bits_per_byte": 1.8203225068679583, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.261751413345337, + "step": 2470, + "unclipped_grad_norm": 0.8000954985618591 + }, + { + "bits_per_byte": 1.8206716316492098, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.261993408203125, + "step": 2480, + "unclipped_grad_norm": 0.7867690920829773 + }, + { + "bits_per_byte": 1.7380601092725614, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2047314643859863, + "step": 2490, + "unclipped_grad_norm": 0.7984139919281006 + }, + { + "bits_per_byte": 1.822108202731867, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2629891633987427, + "step": 2500, + "unclipped_grad_norm": 0.7798662781715393 + }, + { + "bits_per_byte": 1.7878630172920644, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2392522096633911, + "step": 2510, + "unclipped_grad_norm": 0.8921790719032288 + }, + { + "bits_per_byte": 1.7912486677569497, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2415989637374878, + "step": 2520, + "unclipped_grad_norm": 0.8250295519828796 + }, + { + "bits_per_byte": 1.7871405181757898, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2387514114379883, + "step": 2530, + "unclipped_grad_norm": 0.7922425270080566 + }, + { + "bits_per_byte": 1.7956409326773348, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2446434497833252, + "step": 2540, + "unclipped_grad_norm": 0.8244069218635559 + }, + { + "bits_per_byte": 1.833951787983534, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2711985111236572, + "step": 2550, + "unclipped_grad_norm": 0.8060020804405212 + }, + { + "bits_per_byte": 1.7851159384098327, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2373480796813965, + "step": 2560, + "unclipped_grad_norm": 0.7821792960166931 + }, + { + "bits_per_byte": 1.768823849910948, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2260552644729614, + "step": 2570, + "unclipped_grad_norm": 0.7924907207489014 + }, + { + "bits_per_byte": 1.8169237857216074, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2593955993652344, + "step": 2580, + "unclipped_grad_norm": 0.813592255115509 + }, + { + "bits_per_byte": 1.7917042497990852, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2419147491455078, + "step": 2590, + "unclipped_grad_norm": 0.8239418268203735 + }, + { + "bits_per_byte": 1.8340940176357976, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2712970972061157, + "step": 2600, + "unclipped_grad_norm": 0.7961696982383728 + }, + { + "bits_per_byte": 1.7288738279593754, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.198364019393921, + "step": 2610, + "unclipped_grad_norm": 0.7893867492675781 + }, + { + "bits_per_byte": 1.7710386424887592, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2275904417037964, + "step": 2620, + "unclipped_grad_norm": 0.7791327238082886 + }, + { + "bits_per_byte": 1.783225161146247, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2360374927520752, + "step": 2630, + "unclipped_grad_norm": 0.7614359259605408 + }, + { + "bits_per_byte": 1.8200944578729141, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2615933418273926, + "step": 2640, + "unclipped_grad_norm": 0.831463098526001 + }, + { + "bits_per_byte": 1.7566091260987187, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2175886631011963, + "step": 2650, + "unclipped_grad_norm": 0.7739477157592773 + }, + { + "bits_per_byte": 1.7470520501902724, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2109642028808594, + "step": 2660, + "unclipped_grad_norm": 0.7929992079734802 + }, + { + "bits_per_byte": 1.775358846678433, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.230584979057312, + "step": 2670, + "unclipped_grad_norm": 0.7438913583755493 + }, + { + "bits_per_byte": 1.7678870604116983, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2254059314727783, + "step": 2680, + "unclipped_grad_norm": 0.8285056352615356 + }, + { + "bits_per_byte": 1.8436208245976922, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2779005765914917, + "step": 2690, + "unclipped_grad_norm": 0.7684076428413391 + }, + { + "bits_per_byte": 1.825501592416041, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2653412818908691, + "step": 2700, + "unclipped_grad_norm": 0.8987715840339661 + }, + { + "bits_per_byte": 1.7152053347994027, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.188889741897583, + "step": 2710, + "unclipped_grad_norm": 0.8146651983261108 + }, + { + "bits_per_byte": 1.8198911743795942, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2614524364471436, + "step": 2720, + "unclipped_grad_norm": 0.8013731241226196 + }, + { + "bits_per_byte": 1.7448410412307802, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2094316482543945, + "step": 2730, + "unclipped_grad_norm": 0.7481213212013245 + }, + { + "bits_per_byte": 1.7681415947349752, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2255823612213135, + "step": 2740, + "unclipped_grad_norm": 0.8155582547187805 + }, + { + "bits_per_byte": 1.781660806953999, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2349531650543213, + "step": 2750, + "unclipped_grad_norm": 0.816902756690979 + }, + { + "bits_per_byte": 1.869134278870539, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2955851554870605, + "step": 2760, + "unclipped_grad_norm": 0.8422117233276367 + }, + { + "bits_per_byte": 1.7766440730283308, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.231475830078125, + "step": 2770, + "unclipped_grad_norm": 0.8072130084037781 + }, + { + "bits_per_byte": 1.778421169759696, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2327076196670532, + "step": 2780, + "unclipped_grad_norm": 0.8882554173469543 + }, + { + "bits_per_byte": 1.7671284449387423, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2248800992965698, + "step": 2790, + "unclipped_grad_norm": 0.8406802415847778 + }, + { + "bits_per_byte": 1.719355792112675, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.191766619682312, + "step": 2800, + "unclipped_grad_norm": 0.8179776072502136 + }, + { + "bits_per_byte": 1.7902867687906738, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2409322261810303, + "step": 2810, + "unclipped_grad_norm": 0.8046079874038696 + }, + { + "bits_per_byte": 1.7916516231079211, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2418782711029053, + "step": 2820, + "unclipped_grad_norm": 0.7889978885650635 + }, + { + "bits_per_byte": 1.717342391219024, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.190371036529541, + "step": 2830, + "unclipped_grad_norm": 0.7270697355270386 + }, + { + "bits_per_byte": 1.8021411889493455, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2491490840911865, + "step": 2840, + "unclipped_grad_norm": 0.8529788255691528 + }, + { + "bits_per_byte": 1.7264863648000979, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.196709156036377, + "step": 2850, + "unclipped_grad_norm": 0.7487845420837402 + }, + { + "bits_per_byte": 1.698717702026487, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1774613857269287, + "step": 2860, + "unclipped_grad_norm": 0.8107967972755432 + }, + { + "bits_per_byte": 1.8323354950307253, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.270078182220459, + "step": 2870, + "unclipped_grad_norm": 0.822874128818512 + }, + { + "bits_per_byte": 1.7418315168233318, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2073456048965454, + "step": 2880, + "unclipped_grad_norm": 0.7772940993309021 + }, + { + "bits_per_byte": 1.7973201712803593, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.245807409286499, + "step": 2890, + "unclipped_grad_norm": 0.7877682447433472 + }, + { + "bits_per_byte": 1.7076477291898875, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1836512088775635, + "step": 2900, + "unclipped_grad_norm": 0.8104777932167053 + }, + { + "bits_per_byte": 1.7337485042154308, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2017428874969482, + "step": 2910, + "unclipped_grad_norm": 0.8031578063964844 + }, + { + "bits_per_byte": 1.8229534974608574, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2635750770568848, + "step": 2920, + "unclipped_grad_norm": 0.7966766953468323 + }, + { + "bits_per_byte": 1.766033431400699, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.22412109375, + "step": 2930, + "unclipped_grad_norm": 0.8636581301689148 + }, + { + "bits_per_byte": 1.8135838826418518, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2570805549621582, + "step": 2940, + "unclipped_grad_norm": 0.8673250675201416 + }, + { + "bits_per_byte": 1.7613176671140407, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208523750305176, + "step": 2950, + "unclipped_grad_norm": 0.7721947431564331 + }, + { + "bits_per_byte": 1.7305283010606756, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.1995108127593994, + "step": 2960, + "unclipped_grad_norm": 0.7505601644515991 + }, + { + "bits_per_byte": 1.8081494028572378, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.253313660621643, + "step": 2970, + "unclipped_grad_norm": 0.7939547896385193 + }, + { + "bits_per_byte": 1.72656255311443, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.196761965751648, + "step": 2980, + "unclipped_grad_norm": 0.7610272765159607 + }, + { + "bits_per_byte": 1.8584056571444152, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2881486415863037, + "step": 2990, + "unclipped_grad_norm": 0.833854615688324 + }, + { + "bits_per_byte": 1.7959412143857414, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2448515892028809, + "step": 3000, + "unclipped_grad_norm": 0.7871666550636292 + }, + { + "bits_per_byte": 1.7543190051198296, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.216001272201538, + "step": 3010, + "unclipped_grad_norm": 0.7730440497398376 + }, + { + "bits_per_byte": 1.7304921847039942, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1994857788085938, + "step": 3020, + "unclipped_grad_norm": 0.8199000954627991 + }, + { + "bits_per_byte": 1.7578830015936595, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.218471646308899, + "step": 3030, + "unclipped_grad_norm": 0.9139093160629272 + }, + { + "bits_per_byte": 1.7370483353375354, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.204030156135559, + "step": 3040, + "unclipped_grad_norm": 0.7832452058792114 + }, + { + "bits_per_byte": 1.745498014957076, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2098870277404785, + "step": 3050, + "unclipped_grad_norm": 0.7990915179252625 + }, + { + "bits_per_byte": 1.7370017280391516, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2039978504180908, + "step": 3060, + "unclipped_grad_norm": 0.8140543103218079 + }, + { + "bits_per_byte": 1.7324813360438736, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.200864553451538, + "step": 3070, + "unclipped_grad_norm": 0.8290132284164429 + }, + { + "bits_per_byte": 1.73554830265671, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2029904127120972, + "step": 3080, + "unclipped_grad_norm": 0.8363073468208313 + }, + { + "bits_per_byte": 1.7064868462965634, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.1828465461730957, + "step": 3090, + "unclipped_grad_norm": 0.7568578124046326 + }, + { + "bits_per_byte": 1.8015361539836103, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2487297058105469, + "step": 3100, + "unclipped_grad_norm": 0.8538148999214172 + }, + { + "bits_per_byte": 1.7730212584879057, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2289646863937378, + "step": 3110, + "unclipped_grad_norm": 0.8123647570610046 + }, + { + "bits_per_byte": 1.664611306585931, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1538206338882446, + "step": 3120, + "unclipped_grad_norm": 0.814990758895874 + }, + { + "bits_per_byte": 1.7603762340832176, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2201998233795166, + "step": 3130, + "unclipped_grad_norm": 0.8308927416801453 + }, + { + "bits_per_byte": 1.713619138810495, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1877902746200562, + "step": 3140, + "unclipped_grad_norm": 0.7710813879966736 + }, + { + "bits_per_byte": 1.7093383186478688, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1848230361938477, + "step": 3150, + "unclipped_grad_norm": 0.7859835624694824 + }, + { + "bits_per_byte": 1.7563532159142348, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2174112796783447, + "step": 3160, + "unclipped_grad_norm": 0.811743974685669 + }, + { + "bits_per_byte": 1.8161149513145995, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.258834958076477, + "step": 3170, + "unclipped_grad_norm": 0.8695415258407593 + }, + { + "bits_per_byte": 1.6889931150161004, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1707208156585693, + "step": 3180, + "unclipped_grad_norm": 0.8201394081115723 + }, + { + "bits_per_byte": 1.7704115937437135, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2271558046340942, + "step": 3190, + "unclipped_grad_norm": 0.8031932711601257 + }, + { + "bits_per_byte": 1.6638831320421776, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1533159017562866, + "step": 3200, + "unclipped_grad_norm": 0.7531365156173706 + }, + { + "bits_per_byte": 1.7080410535124106, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1839238405227661, + "step": 3210, + "unclipped_grad_norm": 0.7989531755447388 + }, + { + "bits_per_byte": 1.7142219380017691, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1882081031799316, + "step": 3220, + "unclipped_grad_norm": 0.8516196012496948 + }, + { + "bits_per_byte": 1.687988048404458, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1700241565704346, + "step": 3230, + "unclipped_grad_norm": 0.8453141450881958 + }, + { + "bits_per_byte": 1.725164850110868, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1957931518554688, + "step": 3240, + "unclipped_grad_norm": 0.8131914734840393 + }, + { + "bits_per_byte": 1.7438406181507111, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2087382078170776, + "step": 3250, + "unclipped_grad_norm": 0.8390238285064697 + }, + { + "bits_per_byte": 1.8351064795014271, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2719988822937012, + "step": 3260, + "unclipped_grad_norm": 0.8244672417640686 + }, + { + "bits_per_byte": 1.6914478233918653, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1724222898483276, + "step": 3270, + "unclipped_grad_norm": 0.8076629042625427 + }, + { + "bits_per_byte": 1.6199020087370317, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1228305101394653, + "step": 3280, + "unclipped_grad_norm": 0.7831076979637146 + }, + { + "bits_per_byte": 1.6792207168287738, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1639471054077148, + "step": 3290, + "unclipped_grad_norm": 0.8022177219390869 + }, + { + "bits_per_byte": 1.7567814527148833, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2177081108093262, + "step": 3300, + "unclipped_grad_norm": 0.8066262602806091 + }, + { + "bits_per_byte": 1.745768887632185, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.210074782371521, + "step": 3310, + "unclipped_grad_norm": 0.8170367479324341 + }, + { + "bits_per_byte": 1.7488066171943748, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2121803760528564, + "step": 3320, + "unclipped_grad_norm": 0.8205949664115906 + }, + { + "bits_per_byte": 1.6972405430382262, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.176437497138977, + "step": 3330, + "unclipped_grad_norm": 0.7750418782234192 + }, + { + "bits_per_byte": 1.7083963696690934, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.184170126914978, + "step": 3340, + "unclipped_grad_norm": 0.7907351851463318 + }, + { + "bits_per_byte": 1.7221589373390878, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1937096118927002, + "step": 3350, + "unclipped_grad_norm": 0.7935147285461426 + }, + { + "bits_per_byte": 1.6939794080125659, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1741770505905151, + "step": 3360, + "unclipped_grad_norm": 0.7715696096420288 + }, + { + "bits_per_byte": 1.6536716621471985, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.146237850189209, + "step": 3370, + "unclipped_grad_norm": 0.7847044467926025 + }, + { + "bits_per_byte": 1.7430156173745224, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2081663608551025, + "step": 3380, + "unclipped_grad_norm": 0.9182605743408203 + }, + { + "bits_per_byte": 1.7282608817917002, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.197939157485962, + "step": 3390, + "unclipped_grad_norm": 0.8387140035629272 + }, + { + "bits_per_byte": 1.7168453613580306, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1900265216827393, + "step": 3400, + "unclipped_grad_norm": 0.827458918094635 + }, + { + "bits_per_byte": 1.744452876387783, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2091625928878784, + "step": 3410, + "unclipped_grad_norm": 0.8461803197860718 + }, + { + "bits_per_byte": 1.7169175940713928, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1900765895843506, + "step": 3420, + "unclipped_grad_norm": 0.7754979133605957 + }, + { + "bits_per_byte": 1.665198971303929, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1542279720306396, + "step": 3430, + "unclipped_grad_norm": 0.784263551235199 + }, + { + "bits_per_byte": 1.6807204055442975, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1649866104125977, + "step": 3440, + "unclipped_grad_norm": 0.768701434135437 + }, + { + "bits_per_byte": 1.674807642007634, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1608881950378418, + "step": 3450, + "unclipped_grad_norm": 0.8573304414749146 + }, + { + "bits_per_byte": 1.6607244986764313, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.151126503944397, + "step": 3460, + "unclipped_grad_norm": 0.7546505331993103 + }, + { + "bits_per_byte": 1.7168016777647113, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1899962425231934, + "step": 3470, + "unclipped_grad_norm": 0.8346678614616394 + }, + { + "bits_per_byte": 1.7794081781929978, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2333917617797852, + "step": 3480, + "unclipped_grad_norm": 0.8080501556396484 + }, + { + "bits_per_byte": 1.7098234817059528, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1851593255996704, + "step": 3490, + "unclipped_grad_norm": 0.7828541398048401 + }, + { + "bits_per_byte": 1.6618220919542377, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.15188729763031, + "step": 3500, + "unclipped_grad_norm": 0.8354499936103821 + }, + { + "bits_per_byte": 1.670962969847596, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.158223271369934, + "step": 3510, + "unclipped_grad_norm": 0.7484581470489502 + }, + { + "bits_per_byte": 1.7077951183216769, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1837533712387085, + "step": 3520, + "unclipped_grad_norm": 0.8215554356575012 + }, + { + "bits_per_byte": 1.6640239858332344, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1534135341644287, + "step": 3530, + "unclipped_grad_norm": 0.7679988741874695 + }, + { + "bits_per_byte": 1.6934656958344385, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.173820972442627, + "step": 3540, + "unclipped_grad_norm": 0.849611759185791 + }, + { + "bits_per_byte": 1.7380399873024104, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2047175168991089, + "step": 3550, + "unclipped_grad_norm": 0.8348926305770874 + }, + { + "bits_per_byte": 1.6724150193688307, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1592297554016113, + "step": 3560, + "unclipped_grad_norm": 0.82209312915802 + }, + { + "bits_per_byte": 1.707797526078789, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1837550401687622, + "step": 3570, + "unclipped_grad_norm": 0.814924955368042 + }, + { + "bits_per_byte": 1.725196322935976, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1958149671554565, + "step": 3580, + "unclipped_grad_norm": 0.8165888786315918 + }, + { + "bits_per_byte": 1.7086591591596116, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1843522787094116, + "step": 3590, + "unclipped_grad_norm": 0.8172020316123962 + }, + { + "bits_per_byte": 1.7535423314685332, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2154629230499268, + "step": 3600, + "unclipped_grad_norm": 0.8490144610404968 + }, + { + "bits_per_byte": 1.7320164669385913, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.2005423307418823, + "step": 3610, + "unclipped_grad_norm": 0.8558303117752075 + }, + { + "bits_per_byte": 1.671523289324107, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1586116552352905, + "step": 3620, + "unclipped_grad_norm": 0.7700185179710388 + }, + { + "bits_per_byte": 1.6783824733884685, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1633660793304443, + "step": 3630, + "unclipped_grad_norm": 0.8705776333808899 + }, + { + "bits_per_byte": 1.6085100499265301, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1149342060089111, + "step": 3640, + "unclipped_grad_norm": 0.7363900542259216 + }, + { + "bits_per_byte": 1.69346002040696, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1738170385360718, + "step": 3650, + "unclipped_grad_norm": 0.7923496961593628 + }, + { + "bits_per_byte": 1.6919579239343243, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1727758646011353, + "step": 3660, + "unclipped_grad_norm": 0.8016297221183777 + }, + { + "bits_per_byte": 1.7219477426438283, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1935632228851318, + "step": 3670, + "unclipped_grad_norm": 0.8946071863174438 + }, + { + "bits_per_byte": 1.6611216066172736, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1514017581939697, + "step": 3680, + "unclipped_grad_norm": 0.7832621932029724 + }, + { + "bits_per_byte": 1.7299588665036687, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1991161108016968, + "step": 3690, + "unclipped_grad_norm": 0.8317174911499023 + }, + { + "bits_per_byte": 1.6188276331170917, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1220858097076416, + "step": 3700, + "unclipped_grad_norm": 0.7848891615867615 + }, + { + "bits_per_byte": 1.69280218676741, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.17336106300354, + "step": 3710, + "unclipped_grad_norm": 0.7887690663337708 + }, + { + "bits_per_byte": 1.7133850704226707, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1876280307769775, + "step": 3720, + "unclipped_grad_norm": 0.8315525650978088 + }, + { + "bits_per_byte": 1.6847540866376338, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1677825450897217, + "step": 3730, + "unclipped_grad_norm": 0.8175208568572998 + }, + { + "bits_per_byte": 1.6646517225088835, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.153848648071289, + "step": 3740, + "unclipped_grad_norm": 0.795343816280365 + }, + { + "bits_per_byte": 1.6976003307438312, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1766868829727173, + "step": 3750, + "unclipped_grad_norm": 0.8138338327407837 + }, + { + "bits_per_byte": 1.7200370153927425, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1922388076782227, + "step": 3760, + "unclipped_grad_norm": 0.8444334268569946 + }, + { + "bits_per_byte": 1.7215936303657018, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1933177709579468, + "step": 3770, + "unclipped_grad_norm": 0.7583279609680176 + }, + { + "bits_per_byte": 1.7717356881727062, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2280735969543457, + "step": 3780, + "unclipped_grad_norm": 0.8467698693275452 + }, + { + "bits_per_byte": 1.6571842358084203, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1486725807189941, + "step": 3790, + "unclipped_grad_norm": 0.769040584564209 + }, + { + "bits_per_byte": 1.6766210270783317, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1621451377868652, + "step": 3800, + "unclipped_grad_norm": 0.7948989272117615 + }, + { + "bits_per_byte": 1.6280214816669156, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1284584999084473, + "step": 3810, + "unclipped_grad_norm": 0.8209450840950012 + }, + { + "bits_per_byte": 1.6241315780697008, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1257622241973877, + "step": 3820, + "unclipped_grad_norm": 0.81658935546875 + }, + { + "bits_per_byte": 1.6981439679032087, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1770637035369873, + "step": 3830, + "unclipped_grad_norm": 0.8478888273239136 + }, + { + "bits_per_byte": 1.649875145129402, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1436063051223755, + "step": 3840, + "unclipped_grad_norm": 0.8229350447654724 + }, + { + "bits_per_byte": 1.621423539249215, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.123885154724121, + "step": 3850, + "unclipped_grad_norm": 0.8153557181358337 + }, + { + "bits_per_byte": 1.6024271955481633, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1107178926467896, + "step": 3860, + "unclipped_grad_norm": 0.794911801815033 + }, + { + "bits_per_byte": 1.5664059452729007, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.085749864578247, + "step": 3870, + "unclipped_grad_norm": 0.75998854637146 + }, + { + "bits_per_byte": 1.6739675067581694, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1603058576583862, + "step": 3880, + "unclipped_grad_norm": 0.8571500182151794 + }, + { + "bits_per_byte": 1.6411153807903556, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.137534499168396, + "step": 3890, + "unclipped_grad_norm": 0.8254691958427429 + }, + { + "bits_per_byte": 1.6427378651185953, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.138659119606018, + "step": 3900, + "unclipped_grad_norm": 0.8312620520591736 + }, + { + "bits_per_byte": 1.6573687731927962, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1488004922866821, + "step": 3910, + "unclipped_grad_norm": 0.8051927089691162 + }, + { + "bits_per_byte": 1.6875147521492837, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1696960926055908, + "step": 3920, + "unclipped_grad_norm": 0.8500935435295105 + }, + { + "bits_per_byte": 1.685968628118027, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1686244010925293, + "step": 3930, + "unclipped_grad_norm": 0.8186569213867188 + }, + { + "bits_per_byte": 1.6142946863883005, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1189438104629517, + "step": 3940, + "unclipped_grad_norm": 0.7923241853713989 + }, + { + "bits_per_byte": 1.7195771337843353, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1919200420379639, + "step": 3950, + "unclipped_grad_norm": 0.8632136583328247 + }, + { + "bits_per_byte": 1.7081789835984027, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1840194463729858, + "step": 3960, + "unclipped_grad_norm": 0.8128528594970703 + }, + { + "bits_per_byte": 1.6418629893736563, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1380527019500732, + "step": 3970, + "unclipped_grad_norm": 0.8211596608161926 + }, + { + "bits_per_byte": 1.6416170541829225, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1378822326660156, + "step": 3980, + "unclipped_grad_norm": 0.8518010973930359 + }, + { + "bits_per_byte": 1.672920992327669, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1595804691314697, + "step": 3990, + "unclipped_grad_norm": 0.8055341243743896 + }, + { + "bits_per_byte": 1.6017095119461129, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1102204322814941, + "step": 4000, + "unclipped_grad_norm": 0.7798590064048767 + }, + { + "bits_per_byte": 1.6685314791296977, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1565378904342651, + "step": 4010, + "unclipped_grad_norm": 0.8833235502243042 + }, + { + "bits_per_byte": 1.6832612772281446, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.166747808456421, + "step": 4020, + "unclipped_grad_norm": 0.8783014416694641 + }, + { + "bits_per_byte": 1.6355704881438877, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1336910724639893, + "step": 4030, + "unclipped_grad_norm": 0.831271231174469 + }, + { + "bits_per_byte": 1.6221185211413516, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1243668794631958, + "step": 4040, + "unclipped_grad_norm": 0.828468382358551 + }, + { + "bits_per_byte": 1.6444031731269, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1398134231567383, + "step": 4050, + "unclipped_grad_norm": 0.8378818035125732 + }, + { + "bits_per_byte": 1.6213299806871455, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1238203048706055, + "step": 4060, + "unclipped_grad_norm": 0.808568000793457 + }, + { + "bits_per_byte": 1.749336667724334, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.212547779083252, + "step": 4070, + "unclipped_grad_norm": 0.8772934675216675 + }, + { + "bits_per_byte": 1.5885019322904526, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1010656356811523, + "step": 4080, + "unclipped_grad_norm": 0.8058938980102539 + }, + { + "bits_per_byte": 1.6013793052564562, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1099915504455566, + "step": 4090, + "unclipped_grad_norm": 0.8040578365325928 + }, + { + "bits_per_byte": 1.737544161319973, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.204373836517334, + "step": 4100, + "unclipped_grad_norm": 0.8918697834014893 + }, + { + "bits_per_byte": 1.6258001537483717, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1269187927246094, + "step": 4110, + "unclipped_grad_norm": 0.8254836797714233 + }, + { + "bits_per_byte": 1.6466504704257243, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1413711309432983, + "step": 4120, + "unclipped_grad_norm": 0.8797640800476074 + }, + { + "bits_per_byte": 1.7012891866221878, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.179243803024292, + "step": 4130, + "unclipped_grad_norm": 0.8626418709754944 + }, + { + "bits_per_byte": 1.6871897049391529, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1694707870483398, + "step": 4140, + "unclipped_grad_norm": 0.8501971364021301 + }, + { + "bits_per_byte": 1.629858600343432, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1297318935394287, + "step": 4150, + "unclipped_grad_norm": 0.8614740371704102 + }, + { + "bits_per_byte": 1.6422026551091102, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138288140296936, + "step": 4160, + "unclipped_grad_norm": 0.8621925115585327 + }, + { + "bits_per_byte": 1.6599294228814925, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1505753993988037, + "step": 4170, + "unclipped_grad_norm": 0.8789071440696716 + }, + { + "bits_per_byte": 1.6412854716320588, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1376523971557617, + "step": 4180, + "unclipped_grad_norm": 0.8332215547561646 + }, + { + "bits_per_byte": 1.6753517951149641, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1612653732299805, + "step": 4190, + "unclipped_grad_norm": 0.8483413457870483 + }, + { + "bits_per_byte": 1.6524258198243482, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1453742980957031, + "step": 4200, + "unclipped_grad_norm": 0.8514559268951416 + }, + { + "bits_per_byte": 1.6764456047744516, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1620235443115234, + "step": 4210, + "unclipped_grad_norm": 0.7994921803474426 + }, + { + "bits_per_byte": 1.6553684429806108, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.147413969039917, + "step": 4220, + "unclipped_grad_norm": 0.8137004375457764 + }, + { + "bits_per_byte": 1.654246428166382, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1466362476348877, + "step": 4230, + "unclipped_grad_norm": 0.8624707460403442 + }, + { + "bits_per_byte": 1.670982919835096, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.158237099647522, + "step": 4240, + "unclipped_grad_norm": 0.83209228515625 + }, + { + "bits_per_byte": 1.605450134602379, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1128132343292236, + "step": 4250, + "unclipped_grad_norm": 0.8165753483772278 + }, + { + "bits_per_byte": 1.64798711958823, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.142297625541687, + "step": 4260, + "unclipped_grad_norm": 0.8390073180198669 + }, + { + "bits_per_byte": 1.5630770490828, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.0834424495697021, + "step": 4270, + "unclipped_grad_norm": 0.7599806189537048 + }, + { + "bits_per_byte": 1.634252757072977, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1327776908874512, + "step": 4280, + "unclipped_grad_norm": 0.8349722027778625 + }, + { + "bits_per_byte": 1.622665597953746, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1247460842132568, + "step": 4290, + "unclipped_grad_norm": 0.8544122576713562 + }, + { + "bits_per_byte": 1.5775992641216556, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0935084819793701, + "step": 4300, + "unclipped_grad_norm": 0.8515903353691101 + }, + { + "bits_per_byte": 1.5682213941354084, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.0870082378387451, + "step": 4310, + "unclipped_grad_norm": 0.8175749182701111 + }, + { + "bits_per_byte": 1.709660270170284, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1850461959838867, + "step": 4320, + "unclipped_grad_norm": 0.9185000061988831 + }, + { + "bits_per_byte": 1.6398645509706304, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1366674900054932, + "step": 4330, + "unclipped_grad_norm": 0.8850803971290588 + }, + { + "bits_per_byte": 1.5869262272432476, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.099973440170288, + "step": 4340, + "unclipped_grad_norm": 0.8476384878158569 + }, + { + "bits_per_byte": 1.5594358323987327, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.080918550491333, + "step": 4350, + "unclipped_grad_norm": 0.8173791170120239 + }, + { + "bits_per_byte": 1.6383724294917448, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1356332302093506, + "step": 4360, + "unclipped_grad_norm": 0.8429195880889893 + }, + { + "bits_per_byte": 1.6201880158854165, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1230287551879883, + "step": 4370, + "unclipped_grad_norm": 0.8339970707893372 + }, + { + "bits_per_byte": 1.5560377991829855, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.0785632133483887, + "step": 4380, + "unclipped_grad_norm": 0.7866214513778687 + }, + { + "bits_per_byte": 1.7055524645544258, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1821988821029663, + "step": 4390, + "unclipped_grad_norm": 0.8178918361663818 + }, + { + "bits_per_byte": 1.6491636529027824, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.143113136291504, + "step": 4400, + "unclipped_grad_norm": 0.8444706797599792 + }, + { + "bits_per_byte": 1.6592194784987309, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.150083303451538, + "step": 4410, + "unclipped_grad_norm": 0.8914594054222107 + }, + { + "bits_per_byte": 1.676016508060549, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1617261171340942, + "step": 4420, + "unclipped_grad_norm": 0.894481897354126 + }, + { + "bits_per_byte": 1.6150391992838857, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.119459867477417, + "step": 4430, + "unclipped_grad_norm": 0.8697190880775452 + }, + { + "bits_per_byte": 1.63267068866769, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1316810846328735, + "step": 4440, + "unclipped_grad_norm": 0.8085072636604309 + }, + { + "bits_per_byte": 1.6597337066248108, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.150439739227295, + "step": 4450, + "unclipped_grad_norm": 0.8258711099624634 + }, + { + "bits_per_byte": 1.6374223973283786, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.134974718093872, + "step": 4460, + "unclipped_grad_norm": 0.8935436010360718 + }, + { + "bits_per_byte": 1.6156965169754833, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.11991548538208, + "step": 4470, + "unclipped_grad_norm": 0.8515243530273438 + }, + { + "bits_per_byte": 1.663532975365021, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1530731916427612, + "step": 4480, + "unclipped_grad_norm": 0.8462327718734741 + }, + { + "bits_per_byte": 1.6896239473794652, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1711580753326416, + "step": 4490, + "unclipped_grad_norm": 0.8624916672706604 + }, + { + "bits_per_byte": 1.6220770733224936, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.124338150024414, + "step": 4500, + "unclipped_grad_norm": 0.830740213394165 + }, + { + "bits_per_byte": 1.6091536090060587, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1153802871704102, + "step": 4510, + "unclipped_grad_norm": 0.8425999879837036 + }, + { + "bits_per_byte": 1.6507618876772505, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1442209482192993, + "step": 4520, + "unclipped_grad_norm": 0.8468135595321655 + }, + { + "bits_per_byte": 1.6238777316770272, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1255862712860107, + "step": 4530, + "unclipped_grad_norm": 0.8532826900482178 + }, + { + "bits_per_byte": 1.6469891042652731, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1416058540344238, + "step": 4540, + "unclipped_grad_norm": 0.8736191987991333 + }, + { + "bits_per_byte": 1.6559469926181134, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1478149890899658, + "step": 4550, + "unclipped_grad_norm": 0.8442186713218689 + }, + { + "bits_per_byte": 1.6446332859137545, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1399729251861572, + "step": 4560, + "unclipped_grad_norm": 0.9025750756263733 + }, + { + "bits_per_byte": 1.637351540476223, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1349256038665771, + "step": 4570, + "unclipped_grad_norm": 0.8963979482650757 + }, + { + "bits_per_byte": 1.6074537324849307, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1142020225524902, + "step": 4580, + "unclipped_grad_norm": 0.8524222373962402 + }, + { + "bits_per_byte": 1.592079171428399, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1035451889038086, + "step": 4590, + "unclipped_grad_norm": 0.8329559564590454 + }, + { + "bits_per_byte": 1.593515914493707, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1045410633087158, + "step": 4600, + "unclipped_grad_norm": 0.856568455696106 + }, + { + "bits_per_byte": 1.6099590037600493, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1159385442733765, + "step": 4610, + "unclipped_grad_norm": 0.8378997445106506 + }, + { + "bits_per_byte": 1.6641679353120065, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1535133123397827, + "step": 4620, + "unclipped_grad_norm": 0.8407614827156067 + }, + { + "bits_per_byte": 1.64286238055782, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1387454271316528, + "step": 4630, + "unclipped_grad_norm": 0.9177886247634888 + }, + { + "bits_per_byte": 1.6574955244064822, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.148888349533081, + "step": 4640, + "unclipped_grad_norm": 0.8678322434425354 + }, + { + "bits_per_byte": 1.6864685816840852, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1689709424972534, + "step": 4650, + "unclipped_grad_norm": 0.8395454287528992 + }, + { + "bits_per_byte": 1.6055636431519484, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1128919124603271, + "step": 4660, + "unclipped_grad_norm": 0.8130927681922913 + }, + { + "bits_per_byte": 1.6111651180905502, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.116774559020996, + "step": 4670, + "unclipped_grad_norm": 0.8343866467475891 + }, + { + "bits_per_byte": 1.6120890088909852, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.117414951324463, + "step": 4680, + "unclipped_grad_norm": 0.9091700911521912 + }, + { + "bits_per_byte": 1.598018764258597, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1076622009277344, + "step": 4690, + "unclipped_grad_norm": 0.8087259531021118 + }, + { + "bits_per_byte": 1.6257145063882419, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.126859426498413, + "step": 4700, + "unclipped_grad_norm": 0.8473510146141052 + }, + { + "bits_per_byte": 1.6037934257266175, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.111664891242981, + "step": 4710, + "unclipped_grad_norm": 0.8299785256385803 + }, + { + "bits_per_byte": 1.6360324335441052, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1340112686157227, + "step": 4720, + "unclipped_grad_norm": 0.8852627873420715 + }, + { + "bits_per_byte": 1.6613310814860245, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1515469551086426, + "step": 4730, + "unclipped_grad_norm": 0.8318192958831787 + }, + { + "bits_per_byte": 1.6077376758415052, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1143988370895386, + "step": 4740, + "unclipped_grad_norm": 0.9029622673988342 + }, + { + "bits_per_byte": 1.5803602735986073, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.0954222679138184, + "step": 4750, + "unclipped_grad_norm": 0.8787683248519897 + }, + { + "bits_per_byte": 1.5545300272828713, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.077518105506897, + "step": 4760, + "unclipped_grad_norm": 0.8357442021369934 + }, + { + "bits_per_byte": 1.5563697256991507, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.0787932872772217, + "step": 4770, + "unclipped_grad_norm": 0.8196295499801636 + }, + { + "bits_per_byte": 1.593790398804484, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1047313213348389, + "step": 4780, + "unclipped_grad_norm": 0.9009830951690674 + }, + { + "bits_per_byte": 1.5906680537780697, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1025670766830444, + "step": 4790, + "unclipped_grad_norm": 0.8538159728050232 + }, + { + "bits_per_byte": 1.62577985979557, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1269047260284424, + "step": 4800, + "unclipped_grad_norm": 0.9271508455276489 + }, + { + "bits_per_byte": 1.5952574108163438, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.105748176574707, + "step": 4810, + "unclipped_grad_norm": 0.8505629301071167 + }, + { + "bits_per_byte": 1.5898409892100704, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1019937992095947, + "step": 4820, + "unclipped_grad_norm": 0.8469997048377991 + }, + { + "bits_per_byte": 1.5923758415011373, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1037508249282837, + "step": 4830, + "unclipped_grad_norm": 0.8425012826919556 + }, + { + "bits_per_byte": 1.5390662072305417, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.0667994022369385, + "step": 4840, + "unclipped_grad_norm": 0.8439065217971802 + }, + { + "bits_per_byte": 1.6086853002477592, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1150556802749634, + "step": 4850, + "unclipped_grad_norm": 0.8472049236297607 + }, + { + "bits_per_byte": 1.5891265732783864, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1014986038208008, + "step": 4860, + "unclipped_grad_norm": 0.8380921483039856 + }, + { + "bits_per_byte": 1.603959560967351, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.111780047416687, + "step": 4870, + "unclipped_grad_norm": 0.8809474110603333 + }, + { + "bits_per_byte": 1.5692639529649388, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.087730884552002, + "step": 4880, + "unclipped_grad_norm": 0.7983198761940002 + }, + { + "bits_per_byte": 1.525049621185223, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0570838451385498, + "step": 4890, + "unclipped_grad_norm": 0.874304473400116 + }, + { + "bits_per_byte": 1.5072996357549742, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0447804927825928, + "step": 4900, + "unclipped_grad_norm": 0.8955963253974915 + }, + { + "bits_per_byte": 1.5786693401753242, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.094250202178955, + "step": 4910, + "unclipped_grad_norm": 0.8447177410125732 + }, + { + "bits_per_byte": 1.585648052182035, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.0990874767303467, + "step": 4920, + "unclipped_grad_norm": 0.8577683568000793 + }, + { + "bits_per_byte": 1.5737733380705616, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0908565521240234, + "step": 4930, + "unclipped_grad_norm": 0.8645251393318176 + }, + { + "bits_per_byte": 1.6512401714293001, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.14455246925354, + "step": 4940, + "unclipped_grad_norm": 0.9303934574127197 + }, + { + "bits_per_byte": 1.5141001737353912, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0494942665100098, + "step": 4950, + "unclipped_grad_norm": 0.8813614249229431 + }, + { + "bits_per_byte": 1.592528046147151, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1038563251495361, + "step": 4960, + "unclipped_grad_norm": 0.915719747543335 + }, + { + "bits_per_byte": 1.6794828183886887, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1641287803649902, + "step": 4970, + "unclipped_grad_norm": 0.9402682781219482 + }, + { + "bits_per_byte": 1.543928500735735, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.0701696872711182, + "step": 4980, + "unclipped_grad_norm": 0.9095892310142517 + }, + { + "bits_per_byte": 1.6067152389821258, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1136901378631592, + "step": 4990, + "unclipped_grad_norm": 0.934682309627533 + }, + { + "bits_per_byte": 1.6283919322968743, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1287152767181396, + "step": 5000, + "unclipped_grad_norm": 0.8894299268722534 + }, + { + "bits_per_byte": 1.5848010376265358, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.098500370979309, + "step": 5010, + "unclipped_grad_norm": 0.8762696385383606 + }, + { + "bits_per_byte": 1.5972207647585936, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1071090698242188, + "step": 5020, + "unclipped_grad_norm": 0.87384033203125 + }, + { + "bits_per_byte": 1.5759778116893213, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0923845767974854, + "step": 5030, + "unclipped_grad_norm": 0.875623881816864 + }, + { + "bits_per_byte": 1.5723771829108575, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0898888111114502, + "step": 5040, + "unclipped_grad_norm": 0.8759424686431885 + }, + { + "bits_per_byte": 1.6142312247901323, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1188998222351074, + "step": 5050, + "unclipped_grad_norm": 0.886820912361145 + }, + { + "bits_per_byte": 1.5584469321562717, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.080233097076416, + "step": 5060, + "unclipped_grad_norm": 0.880901038646698 + }, + { + "bits_per_byte": 1.6299523308881525, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1297968626022339, + "step": 5070, + "unclipped_grad_norm": 0.9343949556350708 + }, + { + "bits_per_byte": 1.651922254622622, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1450252532958984, + "step": 5080, + "unclipped_grad_norm": 0.9880313277244568 + }, + { + "bits_per_byte": 1.6199948793684975, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1228948831558228, + "step": 5090, + "unclipped_grad_norm": 1.0230180025100708 + }, + { + "bits_per_byte": 1.5480533326340287, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.073028802871704, + "step": 5100, + "unclipped_grad_norm": 0.9155668020248413 + }, + { + "bits_per_byte": 1.5785750936826513, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0941848754882812, + "step": 5110, + "unclipped_grad_norm": 0.8749560713768005 + }, + { + "bits_per_byte": 1.553970911684916, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0771305561065674, + "step": 5120, + "unclipped_grad_norm": 0.9114576578140259 + }, + { + "bits_per_byte": 1.6072428817549729, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.114055871963501, + "step": 5130, + "unclipped_grad_norm": 0.89686119556427 + }, + { + "bits_per_byte": 1.5469013928385495, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.072230339050293, + "step": 5140, + "unclipped_grad_norm": 0.9089201092720032 + }, + { + "bits_per_byte": 1.5509646549478342, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0750467777252197, + "step": 5150, + "unclipped_grad_norm": 0.9353929162025452 + }, + { + "bits_per_byte": 1.5786652125917033, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0942473411560059, + "step": 5160, + "unclipped_grad_norm": 0.9100342988967896 + }, + { + "bits_per_byte": 1.5893422395225683, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1016480922698975, + "step": 5170, + "unclipped_grad_norm": 0.8825995922088623 + }, + { + "bits_per_byte": 1.611850984902191, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1172499656677246, + "step": 5180, + "unclipped_grad_norm": 0.8858869671821594 + }, + { + "bits_per_byte": 1.5765690880429875, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.092794418334961, + "step": 5190, + "unclipped_grad_norm": 0.893822431564331 + }, + { + "bits_per_byte": 1.5180597303062058, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0522388219833374, + "step": 5200, + "unclipped_grad_norm": 0.9365838766098022 + }, + { + "bits_per_byte": 1.6136103674205176, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.118469476699829, + "step": 5210, + "unclipped_grad_norm": 0.9187290668487549 + }, + { + "bits_per_byte": 1.578151672396227, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0938913822174072, + "step": 5220, + "unclipped_grad_norm": 0.9324937462806702 + }, + { + "bits_per_byte": 1.6149896682804372, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1194255352020264, + "step": 5230, + "unclipped_grad_norm": 0.9387596249580383 + }, + { + "bits_per_byte": 1.5729148006774545, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.090261459350586, + "step": 5240, + "unclipped_grad_norm": 0.8688122630119324 + }, + { + "bits_per_byte": 1.5356076361216906, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0644021034240723, + "step": 5250, + "unclipped_grad_norm": 0.890831708908081 + }, + { + "bits_per_byte": 1.5141386978491844, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0495209693908691, + "step": 5260, + "unclipped_grad_norm": 0.8596166968345642 + }, + { + "bits_per_byte": 1.6116535488190005, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1171131134033203, + "step": 5270, + "unclipped_grad_norm": 0.92216956615448 + }, + { + "bits_per_byte": 1.5994467362087108, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1086519956588745, + "step": 5280, + "unclipped_grad_norm": 0.9085080623626709 + }, + { + "bits_per_byte": 1.5769201046333985, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.093037724494934, + "step": 5290, + "unclipped_grad_norm": 0.8931360244750977 + }, + { + "bits_per_byte": 1.537008606795619, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.065373182296753, + "step": 5300, + "unclipped_grad_norm": 0.8922452330589294 + }, + { + "bits_per_byte": 1.5906943671236518, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1025853157043457, + "step": 5310, + "unclipped_grad_norm": 0.887484073638916 + }, + { + "bits_per_byte": 1.5623235930893702, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0829201936721802, + "step": 5320, + "unclipped_grad_norm": 0.9026566743850708 + }, + { + "bits_per_byte": 1.6297041599229574, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.129624843597412, + "step": 5330, + "unclipped_grad_norm": 0.9325461387634277 + }, + { + "bits_per_byte": 1.5387120949524151, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0665539503097534, + "step": 5340, + "unclipped_grad_norm": 0.8923580050468445 + }, + { + "bits_per_byte": 1.5882112816104945, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1008641719818115, + "step": 5350, + "unclipped_grad_norm": 0.9464812278747559 + }, + { + "bits_per_byte": 1.4963152758270173, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.037166714668274, + "step": 5360, + "unclipped_grad_norm": 0.8723886609077454 + }, + { + "bits_per_byte": 1.5320489711100373, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0619354248046875, + "step": 5370, + "unclipped_grad_norm": 0.9227972626686096 + }, + { + "bits_per_byte": 1.5933907111238788, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1044542789459229, + "step": 5380, + "unclipped_grad_norm": 0.9233055710792542 + }, + { + "bits_per_byte": 1.5140970780476757, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0494921207427979, + "step": 5390, + "unclipped_grad_norm": 0.8707529902458191 + }, + { + "bits_per_byte": 1.5792117734561455, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0946261882781982, + "step": 5400, + "unclipped_grad_norm": 0.9207195043563843 + }, + { + "bits_per_byte": 1.5488073045754112, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0735514163970947, + "step": 5410, + "unclipped_grad_norm": 0.8852476477622986 + }, + { + "bits_per_byte": 1.5115954184082252, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0477581024169922, + "step": 5420, + "unclipped_grad_norm": 0.9259113073348999 + }, + { + "bits_per_byte": 1.5742989170515984, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0912208557128906, + "step": 5430, + "unclipped_grad_norm": 0.9123548865318298 + }, + { + "bits_per_byte": 1.5198012266288425, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0534459352493286, + "step": 5440, + "unclipped_grad_norm": 0.8873674273490906 + }, + { + "bits_per_byte": 1.5575636292614403, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0796208381652832, + "step": 5450, + "unclipped_grad_norm": 0.851129949092865 + }, + { + "bits_per_byte": 1.5371788696199733, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0654911994934082, + "step": 5460, + "unclipped_grad_norm": 0.8905500173568726 + }, + { + "bits_per_byte": 1.5449696837040585, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0708913803100586, + "step": 5470, + "unclipped_grad_norm": 0.8742530345916748 + }, + { + "bits_per_byte": 1.5184021477640735, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.052476167678833, + "step": 5480, + "unclipped_grad_norm": 0.9149489998817444 + }, + { + "bits_per_byte": 1.6054506505503316, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1128135919570923, + "step": 5490, + "unclipped_grad_norm": 1.0173310041427612 + }, + { + "bits_per_byte": 1.5652256283400292, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.08493173122406, + "step": 5500, + "unclipped_grad_norm": 0.9233388304710388 + }, + { + "bits_per_byte": 1.5242242764437322, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0565117597579956, + "step": 5510, + "unclipped_grad_norm": 0.8623188734054565 + }, + { + "bits_per_byte": 1.4956853033769069, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0367300510406494, + "step": 5520, + "unclipped_grad_norm": 0.8464505672454834 + }, + { + "bits_per_byte": 1.501370017918526, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.040670394897461, + "step": 5530, + "unclipped_grad_norm": 0.9552764296531677 + }, + { + "bits_per_byte": 1.6303358521995766, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1300626993179321, + "step": 5540, + "unclipped_grad_norm": 0.930858314037323 + }, + { + "bits_per_byte": 1.5750513411491232, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0917423963546753, + "step": 5550, + "unclipped_grad_norm": 0.9512235522270203 + }, + { + "bits_per_byte": 1.6036597952068972, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.111572265625, + "step": 5560, + "unclipped_grad_norm": 0.9797965884208679 + }, + { + "bits_per_byte": 1.5364850916063928, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0650103092193604, + "step": 5570, + "unclipped_grad_norm": 0.8889045715332031 + }, + { + "bits_per_byte": 1.5818145588943031, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0964303016662598, + "step": 5580, + "unclipped_grad_norm": 0.91837477684021 + }, + { + "bits_per_byte": 1.571553042047923, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0893175601959229, + "step": 5590, + "unclipped_grad_norm": 0.9484079480171204 + }, + { + "bits_per_byte": 1.5589711352761015, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0805964469909668, + "step": 5600, + "unclipped_grad_norm": 0.9200659394264221 + }, + { + "bits_per_byte": 1.4898029807694462, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.032652735710144, + "step": 5610, + "unclipped_grad_norm": 0.898281455039978 + }, + { + "bits_per_byte": 1.5617056594248202, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0824918746948242, + "step": 5620, + "unclipped_grad_norm": 0.9515117406845093 + }, + { + "bits_per_byte": 1.481016903084818, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0265626907348633, + "step": 5630, + "unclipped_grad_norm": 0.8537728786468506 + }, + { + "bits_per_byte": 1.471526900410208, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0199847221374512, + "step": 5640, + "unclipped_grad_norm": 0.851723313331604 + }, + { + "bits_per_byte": 1.5525594500692852, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0761522054672241, + "step": 5650, + "unclipped_grad_norm": 0.9591006636619568 + }, + { + "bits_per_byte": 1.4925528113740916, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0345587730407715, + "step": 5660, + "unclipped_grad_norm": 0.9846746325492859 + }, + { + "bits_per_byte": 1.5646704683430441, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0845469236373901, + "step": 5670, + "unclipped_grad_norm": 0.9337268471717834 + }, + { + "bits_per_byte": 1.5783896963850212, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0940563678741455, + "step": 5680, + "unclipped_grad_norm": 1.0177373886108398 + }, + { + "bits_per_byte": 1.5898387534356093, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1019922494888306, + "step": 5690, + "unclipped_grad_norm": 0.9374083876609802 + }, + { + "bits_per_byte": 1.5923674143512452, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1037449836730957, + "step": 5700, + "unclipped_grad_norm": 0.9426813721656799 + }, + { + "bits_per_byte": 1.5582474322812707, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.080094814300537, + "step": 5710, + "unclipped_grad_norm": 0.968785285949707 + }, + { + "bits_per_byte": 1.5819518010496918, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0965254306793213, + "step": 5720, + "unclipped_grad_norm": 0.9166642427444458 + }, + { + "bits_per_byte": 1.5197234904706525, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0533920526504517, + "step": 5730, + "unclipped_grad_norm": 0.9203759431838989 + }, + { + "bits_per_byte": 1.6015581672133536, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1101155281066895, + "step": 5740, + "unclipped_grad_norm": 0.934572160243988 + }, + { + "bits_per_byte": 1.4943809869527633, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0358259677886963, + "step": 5750, + "unclipped_grad_norm": 0.8915490508079529 + }, + { + "bits_per_byte": 1.50218005620409, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0412318706512451, + "step": 5760, + "unclipped_grad_norm": 0.9438080787658691 + }, + { + "bits_per_byte": 1.5178872317073904, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.052119255065918, + "step": 5770, + "unclipped_grad_norm": 0.9708113670349121 + }, + { + "bits_per_byte": 1.6077987296825615, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.114441156387329, + "step": 5780, + "unclipped_grad_norm": 1.0216169357299805 + }, + { + "bits_per_byte": 1.454466909375221, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0081596374511719, + "step": 5790, + "unclipped_grad_norm": 0.8829967975616455 + }, + { + "bits_per_byte": 1.5674770532224742, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0864923000335693, + "step": 5800, + "unclipped_grad_norm": 0.9498817920684814 + }, + { + "bits_per_byte": 1.4688064788388602, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.018099069595337, + "step": 5810, + "unclipped_grad_norm": 0.9071933031082153 + }, + { + "bits_per_byte": 1.4812057400354652, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.02669358253479, + "step": 5820, + "unclipped_grad_norm": 0.9959194660186768 + }, + { + "bits_per_byte": 1.5459386339590195, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0715630054473877, + "step": 5830, + "unclipped_grad_norm": 0.9805806875228882 + }, + { + "bits_per_byte": 1.5386066695874363, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0664808750152588, + "step": 5840, + "unclipped_grad_norm": 0.9813235998153687 + }, + { + "bits_per_byte": 1.4904921152514534, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.033130407333374, + "step": 5850, + "unclipped_grad_norm": 0.8884730339050293 + }, + { + "bits_per_byte": 1.505957139182339, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0438499450683594, + "step": 5860, + "unclipped_grad_norm": 0.9569032192230225 + }, + { + "bits_per_byte": 1.5707541826346652, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0887638330459595, + "step": 5870, + "unclipped_grad_norm": 0.9548988342285156 + }, + { + "bits_per_byte": 1.5056532458382643, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0436393022537231, + "step": 5880, + "unclipped_grad_norm": 0.9100088477134705 + }, + { + "bits_per_byte": 1.5621944341185723, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0828306674957275, + "step": 5890, + "unclipped_grad_norm": 1.0087047815322876 + }, + { + "bits_per_byte": 1.564086587243365, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0841422080993652, + "step": 5900, + "unclipped_grad_norm": 0.9735300540924072 + }, + { + "bits_per_byte": 1.5205851235514751, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.053989291191101, + "step": 5910, + "unclipped_grad_norm": 0.928986668586731 + }, + { + "bits_per_byte": 1.4827504882055151, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0277643203735352, + "step": 5920, + "unclipped_grad_norm": 0.9661484956741333 + }, + { + "bits_per_byte": 1.5936927126587939, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.104663610458374, + "step": 5930, + "unclipped_grad_norm": 0.9932063817977905 + }, + { + "bits_per_byte": 1.5284586612906255, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0594468116760254, + "step": 5940, + "unclipped_grad_norm": 1.0194250345230103 + }, + { + "bits_per_byte": 1.5388254315193337, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0666325092315674, + "step": 5950, + "unclipped_grad_norm": 0.9446147084236145 + }, + { + "bits_per_byte": 1.4754302186541906, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0226902961730957, + "step": 5960, + "unclipped_grad_norm": 0.9241304993629456 + }, + { + "bits_per_byte": 1.5277755462013984, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0589733123779297, + "step": 5970, + "unclipped_grad_norm": 0.9580891728401184 + }, + { + "bits_per_byte": 1.5012076662961116, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.040557861328125, + "step": 5980, + "unclipped_grad_norm": 0.954113245010376 + }, + { + "bits_per_byte": 1.4896813890352862, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0325684547424316, + "step": 5990, + "unclipped_grad_norm": 0.8945097923278809 + }, + { + "bits_per_byte": 1.5812009247960246, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0960049629211426, + "step": 6000, + "unclipped_grad_norm": 1.0032626390457153 + }, + { + "bits_per_byte": 1.556720054358958, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0790361166000366, + "step": 6010, + "unclipped_grad_norm": 0.9715147614479065 + }, + { + "bits_per_byte": 1.5090167105411882, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0459706783294678, + "step": 6020, + "unclipped_grad_norm": 0.9192096590995789 + }, + { + "bits_per_byte": 1.4969094758857484, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0375785827636719, + "step": 6030, + "unclipped_grad_norm": 0.9740291833877563 + }, + { + "bits_per_byte": 1.5263979651679873, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.058018445968628, + "step": 6040, + "unclipped_grad_norm": 0.9185873866081238 + }, + { + "bits_per_byte": 1.5301623214300724, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0606276988983154, + "step": 6050, + "unclipped_grad_norm": 0.9192250370979309 + }, + { + "bits_per_byte": 1.5237477125181913, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0561814308166504, + "step": 6060, + "unclipped_grad_norm": 0.9670439958572388 + }, + { + "bits_per_byte": 1.4068365721927425, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 0.9751448035240173, + "step": 6070, + "unclipped_grad_norm": 0.917002260684967 + }, + { + "bits_per_byte": 1.4967151354902735, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0374438762664795, + "step": 6080, + "unclipped_grad_norm": 0.9365084171295166 + }, + { + "bits_per_byte": 1.4969400887976019, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.037599802017212, + "step": 6090, + "unclipped_grad_norm": 0.9915803074836731 + }, + { + "bits_per_byte": 1.5907201645212812, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.1026031970977783, + "step": 6100, + "unclipped_grad_norm": 1.0093804597854614 + }, + { + "bits_per_byte": 1.5313995646203795, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0614852905273438, + "step": 6110, + "unclipped_grad_norm": 1.0098049640655518 + }, + { + "bits_per_byte": 1.4730523865233613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0210421085357666, + "step": 6120, + "unclipped_grad_norm": 0.9365134239196777 + }, + { + "bits_per_byte": 1.5512489422697104, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0752438306808472, + "step": 6130, + "unclipped_grad_norm": 0.9237281680107117 + }, + { + "bits_per_byte": 1.497698360305256, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0381253957748413, + "step": 6140, + "unclipped_grad_norm": 0.9138990044593811 + }, + { + "bits_per_byte": 1.4838755987074594, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0285441875457764, + "step": 6150, + "unclipped_grad_norm": 0.9429982900619507 + }, + { + "bits_per_byte": 1.551023816979731, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0750877857208252, + "step": 6160, + "unclipped_grad_norm": 1.0046567916870117 + }, + { + "bits_per_byte": 1.4928049379402566, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.034733533859253, + "step": 6170, + "unclipped_grad_norm": 0.964278519153595 + }, + { + "bits_per_byte": 1.4789379768011883, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0251216888427734, + "step": 6180, + "unclipped_grad_norm": 0.9606515169143677 + }, + { + "bits_per_byte": 1.501445346319604, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0407226085662842, + "step": 6190, + "unclipped_grad_norm": 1.0252128839492798 + }, + { + "bits_per_byte": 1.4915814533620186, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0338854789733887, + "step": 6200, + "unclipped_grad_norm": 0.8806238770484924 + }, + { + "bits_per_byte": 1.5186611536362729, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0526556968688965, + "step": 6210, + "unclipped_grad_norm": 0.989686906337738 + }, + { + "bits_per_byte": 1.539449556559315, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0670651197433472, + "step": 6220, + "unclipped_grad_norm": 0.9960981011390686 + }, + { + "bits_per_byte": 1.4808485320696232, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.026445984840393, + "step": 6230, + "unclipped_grad_norm": 0.9705929160118103 + }, + { + "bits_per_byte": 1.552468127281677, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0760889053344727, + "step": 6240, + "unclipped_grad_norm": 1.0799938440322876 + }, + { + "bits_per_byte": 1.4757860507588256, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0229369401931763, + "step": 6250, + "unclipped_grad_norm": 0.9792200922966003 + }, + { + "bits_per_byte": 1.445374530589406, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0018572807312012, + "step": 6260, + "unclipped_grad_norm": 0.9422674179077148 + }, + { + "bits_per_byte": 1.5013631386124917, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.040665626525879, + "step": 6270, + "unclipped_grad_norm": 0.9675420522689819 + }, + { + "bits_per_byte": 1.5119997496204038, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.048038363456726, + "step": 6280, + "unclipped_grad_norm": 0.9240654110908508 + }, + { + "bits_per_byte": 1.5072342823476461, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0447351932525635, + "step": 6290, + "unclipped_grad_norm": 0.9724128842353821 + }, + { + "bits_per_byte": 1.4760084243263911, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0230910778045654, + "step": 6300, + "unclipped_grad_norm": 0.9724760055541992 + }, + { + "bits_per_byte": 1.539534687971492, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0671241283416748, + "step": 6310, + "unclipped_grad_norm": 1.0172570943832397 + }, + { + "bits_per_byte": 1.498671266161187, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.03879976272583, + "step": 6320, + "unclipped_grad_norm": 0.9810369610786438 + }, + { + "bits_per_byte": 1.5103010769778318, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.046860933303833, + "step": 6330, + "unclipped_grad_norm": 0.9735086560249329 + }, + { + "bits_per_byte": 1.4197871237766881, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 0.9841214418411255, + "step": 6340, + "unclipped_grad_norm": 0.9049187302589417 + }, + { + "bits_per_byte": 1.5345103868091858, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0636415481567383, + "step": 6350, + "unclipped_grad_norm": 0.9579975008964539 + }, + { + "bits_per_byte": 1.5509206273892133, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0750162601470947, + "step": 6360, + "unclipped_grad_norm": 0.989890456199646 + }, + { + "bits_per_byte": 1.4729708667468522, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0209856033325195, + "step": 6370, + "unclipped_grad_norm": 1.021422266960144 + }, + { + "bits_per_byte": 1.4625091620948676, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0137341022491455, + "step": 6380, + "unclipped_grad_norm": 1.0111291408538818 + }, + { + "bits_per_byte": 1.576993369242666, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0930885076522827, + "step": 6390, + "unclipped_grad_norm": 1.0084502696990967 + }, + { + "bits_per_byte": 1.5426105976821733, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0692561864852905, + "step": 6400, + "unclipped_grad_norm": 1.060890793800354 + }, + { + "bits_per_byte": 1.542634847235945, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0692729949951172, + "step": 6410, + "unclipped_grad_norm": 1.0786292552947998 + }, + { + "bits_per_byte": 1.4677484415707522, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0173656940460205, + "step": 6420, + "unclipped_grad_norm": 1.0185354948043823 + }, + { + "bits_per_byte": 1.5249625979638863, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.057023525238037, + "step": 6430, + "unclipped_grad_norm": 0.9899823069572449 + }, + { + "bits_per_byte": 1.447091949340922, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0030477046966553, + "step": 6440, + "unclipped_grad_norm": 0.9853183031082153 + }, + { + "bits_per_byte": 1.4600725118974434, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.01204514503479, + "step": 6450, + "unclipped_grad_norm": 0.9329874515533447 + }, + { + "bits_per_byte": 1.4881399085356029, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.031499981880188, + "step": 6460, + "unclipped_grad_norm": 0.9553210735321045 + }, + { + "bits_per_byte": 1.5093300629310604, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0461878776550293, + "step": 6470, + "unclipped_grad_norm": 1.0880736112594604 + }, + { + "bits_per_byte": 1.4620568477230984, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.013420581817627, + "step": 6480, + "unclipped_grad_norm": 0.978254497051239 + }, + { + "bits_per_byte": 1.5301041912940807, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0605874061584473, + "step": 6490, + "unclipped_grad_norm": 1.0214275121688843 + }, + { + "bits_per_byte": 1.5703515712489953, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.088484764099121, + "step": 6500, + "unclipped_grad_norm": 1.024044156074524 + }, + { + "bits_per_byte": 1.5418132861127734, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.068703532218933, + "step": 6510, + "unclipped_grad_norm": 1.0472792387008667 + }, + { + "bits_per_byte": 1.5326505664227554, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0623524188995361, + "step": 6520, + "unclipped_grad_norm": 0.9873340129852295 + }, + { + "bits_per_byte": 1.493115194642413, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0349485874176025, + "step": 6530, + "unclipped_grad_norm": 0.9528921246528625 + }, + { + "bits_per_byte": 1.4717057623671055, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.020108699798584, + "step": 6540, + "unclipped_grad_norm": 0.9596993327140808 + }, + { + "bits_per_byte": 1.532975785615537, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0625778436660767, + "step": 6550, + "unclipped_grad_norm": 0.9932572245597839 + }, + { + "bits_per_byte": 1.5382425823155597, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0662285089492798, + "step": 6560, + "unclipped_grad_norm": 1.0132641792297363 + }, + { + "bits_per_byte": 1.5160394501065202, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0508384704589844, + "step": 6570, + "unclipped_grad_norm": 0.9644629955291748 + }, + { + "bits_per_byte": 1.503854823258192, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0423927307128906, + "step": 6580, + "unclipped_grad_norm": 0.9727261662483215 + }, + { + "bits_per_byte": 1.5231672710715294, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0557790994644165, + "step": 6590, + "unclipped_grad_norm": 1.029261589050293 + }, + { + "bits_per_byte": 1.415425385776831, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9810981154441833, + "step": 6600, + "unclipped_grad_norm": 0.9430753588676453 + }, + { + "bits_per_byte": 1.4673490978554489, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0170888900756836, + "step": 6610, + "unclipped_grad_norm": 0.9699459671974182 + }, + { + "bits_per_byte": 1.5770974187464382, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.093160629272461, + "step": 6620, + "unclipped_grad_norm": 1.053364872932434 + }, + { + "bits_per_byte": 1.520643081704816, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0540294647216797, + "step": 6630, + "unclipped_grad_norm": 1.0116336345672607 + }, + { + "bits_per_byte": 1.4843385760035823, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.028865098953247, + "step": 6640, + "unclipped_grad_norm": 0.9483272433280945 + }, + { + "bits_per_byte": 1.481005552229861, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.026554822921753, + "step": 6650, + "unclipped_grad_norm": 0.9847526550292969 + }, + { + "bits_per_byte": 1.5181830418668742, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0523242950439453, + "step": 6660, + "unclipped_grad_norm": 1.0222764015197754 + }, + { + "bits_per_byte": 1.483456133022005, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.028253436088562, + "step": 6670, + "unclipped_grad_norm": 0.9609436392784119 + }, + { + "bits_per_byte": 1.5070292790278177, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.044593095779419, + "step": 6680, + "unclipped_grad_norm": 1.0481340885162354 + }, + { + "bits_per_byte": 1.4999456576040804, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0396831035614014, + "step": 6690, + "unclipped_grad_norm": 1.0560411214828491 + }, + { + "bits_per_byte": 1.5028304945896531, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0416827201843262, + "step": 6700, + "unclipped_grad_norm": 1.0250314474105835 + }, + { + "bits_per_byte": 1.5313854620430087, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0614755153656006, + "step": 6710, + "unclipped_grad_norm": 0.9873788356781006 + }, + { + "bits_per_byte": 1.5848861690387128, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0985593795776367, + "step": 6720, + "unclipped_grad_norm": 1.065596103668213 + }, + { + "bits_per_byte": 1.4535899698384716, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0075517892837524, + "step": 6730, + "unclipped_grad_norm": 0.9767746329307556 + }, + { + "bits_per_byte": 1.4828168735087481, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0278103351593018, + "step": 6740, + "unclipped_grad_norm": 0.9589064717292786 + }, + { + "bits_per_byte": 1.4599156637198565, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0119364261627197, + "step": 6750, + "unclipped_grad_norm": 0.9927419424057007 + }, + { + "bits_per_byte": 1.4484303183299363, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0039753913879395, + "step": 6760, + "unclipped_grad_norm": 0.9825301766395569 + }, + { + "bits_per_byte": 1.469467236183475, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.018557071685791, + "step": 6770, + "unclipped_grad_norm": 1.0425026416778564 + }, + { + "bits_per_byte": 1.4605650702095145, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0123865604400635, + "step": 6780, + "unclipped_grad_norm": 1.0209197998046875 + }, + { + "bits_per_byte": 1.494792713418929, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0361113548278809, + "step": 6790, + "unclipped_grad_norm": 0.9975507855415344 + }, + { + "bits_per_byte": 1.4806505800384802, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0263087749481201, + "step": 6800, + "unclipped_grad_norm": 1.0428087711334229 + }, + { + "bits_per_byte": 1.4777557680591573, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0243022441864014, + "step": 6810, + "unclipped_grad_norm": 0.9905012249946594 + }, + { + "bits_per_byte": 1.4914748241184836, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0338115692138672, + "step": 6820, + "unclipped_grad_norm": 1.0114696025848389 + }, + { + "bits_per_byte": 1.4751165222990168, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.022472858428955, + "step": 6830, + "unclipped_grad_norm": 0.9909161329269409 + }, + { + "bits_per_byte": 1.462379831141419, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0136444568634033, + "step": 6840, + "unclipped_grad_norm": 1.0066962242126465 + }, + { + "bits_per_byte": 1.542950607382929, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0694918632507324, + "step": 6850, + "unclipped_grad_norm": 1.0904512405395508 + }, + { + "bits_per_byte": 1.5094851912821385, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.046295404434204, + "step": 6860, + "unclipped_grad_norm": 1.0577117204666138 + }, + { + "bits_per_byte": 1.4431473552607326, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0003135204315186, + "step": 6870, + "unclipped_grad_norm": 0.931652843952179 + }, + { + "bits_per_byte": 1.4721159409894133, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0203930139541626, + "step": 6880, + "unclipped_grad_norm": 1.0439471006393433 + }, + { + "bits_per_byte": 1.4601951355275085, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0121301412582397, + "step": 6890, + "unclipped_grad_norm": 0.9945755004882812 + }, + { + "bits_per_byte": 1.5078214311176918, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0451421737670898, + "step": 6900, + "unclipped_grad_norm": 1.0577389001846313 + }, + { + "bits_per_byte": 1.51248938422741, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0483777523040771, + "step": 6910, + "unclipped_grad_norm": 1.042457938194275 + }, + { + "bits_per_byte": 1.4546720846777004, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.008301854133606, + "step": 6920, + "unclipped_grad_norm": 1.0035841464996338 + }, + { + "bits_per_byte": 1.4388924184870613, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9973642230033875, + "step": 6930, + "unclipped_grad_norm": 1.0526496171951294 + }, + { + "bits_per_byte": 1.5114688391771902, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0476703643798828, + "step": 6940, + "unclipped_grad_norm": 1.0117003917694092 + }, + { + "bits_per_byte": 1.4256293744264978, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9881709814071655, + "step": 6950, + "unclipped_grad_norm": 1.020249843597412 + }, + { + "bits_per_byte": 1.4392803253560824, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 0.9976330995559692, + "step": 6960, + "unclipped_grad_norm": 1.007938265800476 + }, + { + "bits_per_byte": 1.425250496646647, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 0.9879083633422852, + "step": 6970, + "unclipped_grad_norm": 1.0789695978164673 + }, + { + "bits_per_byte": 1.5310960152416067, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0612748861312866, + "step": 6980, + "unclipped_grad_norm": 1.0816211700439453 + }, + { + "bits_per_byte": 1.4834984407541174, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0282827615737915, + "step": 6990, + "unclipped_grad_norm": 1.0542293787002563 + }, + { + "bits_per_byte": 1.494992041311279, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0362495183944702, + "step": 7000, + "unclipped_grad_norm": 1.040958046913147 + }, + { + "bits_per_byte": 1.4039343649594325, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9731331467628479, + "step": 7010, + "unclipped_grad_norm": 1.0020650625228882 + }, + { + "bits_per_byte": 1.4906596263533938, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0332465171813965, + "step": 7020, + "unclipped_grad_norm": 1.0739529132843018 + }, + { + "bits_per_byte": 1.4642052549976767, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0149097442626953, + "step": 7030, + "unclipped_grad_norm": 1.0350828170776367 + }, + { + "bits_per_byte": 1.4574465088014192, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0102249383926392, + "step": 7040, + "unclipped_grad_norm": 1.0089728832244873 + }, + { + "bits_per_byte": 1.4534154074478458, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0074307918548584, + "step": 7050, + "unclipped_grad_norm": 1.0608088970184326 + }, + { + "bits_per_byte": 1.4416395833606184, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 0.9992684125900269, + "step": 7060, + "unclipped_grad_norm": 0.9864199757575989 + }, + { + "bits_per_byte": 1.4906718371216052, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0332549810409546, + "step": 7070, + "unclipped_grad_norm": 1.0150113105773926 + }, + { + "bits_per_byte": 1.4931912109740944, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.035001277923584, + "step": 7080, + "unclipped_grad_norm": 1.0090219974517822 + }, + { + "bits_per_byte": 1.4509221749582877, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0057026147842407, + "step": 7090, + "unclipped_grad_norm": 1.0420514345169067 + }, + { + "bits_per_byte": 1.4389093587781714, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9973759651184082, + "step": 7100, + "unclipped_grad_norm": 1.0253499746322632 + }, + { + "bits_per_byte": 1.41304419998431, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 0.979447603225708, + "step": 7110, + "unclipped_grad_norm": 0.9803867936134338 + }, + { + "bits_per_byte": 1.4722433801337027, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0204813480377197, + "step": 7120, + "unclipped_grad_norm": 1.0725315809249878 + }, + { + "bits_per_byte": 1.5197811046586915, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0534319877624512, + "step": 7130, + "unclipped_grad_norm": 1.0701161623001099 + }, + { + "bits_per_byte": 1.465326925846604, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0156872272491455, + "step": 7140, + "unclipped_grad_norm": 1.0326207876205444 + }, + { + "bits_per_byte": 1.5100518741167317, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.046688199043274, + "step": 7150, + "unclipped_grad_norm": 1.0673091411590576 + }, + { + "bits_per_byte": 1.5022746466620645, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.041297435760498, + "step": 7160, + "unclipped_grad_norm": 1.059691071510315 + }, + { + "bits_per_byte": 1.514605802728928, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.049844741821289, + "step": 7170, + "unclipped_grad_norm": 1.085069179534912 + }, + { + "bits_per_byte": 1.4489440305080636, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0043314695358276, + "step": 7180, + "unclipped_grad_norm": 0.9880333542823792 + }, + { + "bits_per_byte": 1.5073161460894569, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0447919368743896, + "step": 7190, + "unclipped_grad_norm": 1.0678606033325195 + }, + { + "bits_per_byte": 1.4434393818018978, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0005159378051758, + "step": 7200, + "unclipped_grad_norm": 1.0179466009140015 + }, + { + "bits_per_byte": 1.4303239848470999, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 0.9914250373840332, + "step": 7210, + "unclipped_grad_norm": 1.005952000617981 + }, + { + "bits_per_byte": 1.4493132772594661, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0045874118804932, + "step": 7220, + "unclipped_grad_norm": 1.0151865482330322 + }, + { + "bits_per_byte": 1.475039818036732, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0224196910858154, + "step": 7230, + "unclipped_grad_norm": 1.0841153860092163 + }, + { + "bits_per_byte": 1.5119313005253603, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0479909181594849, + "step": 7240, + "unclipped_grad_norm": 1.09213387966156 + }, + { + "bits_per_byte": 1.494972951237033, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.03623628616333, + "step": 7250, + "unclipped_grad_norm": 1.0454415082931519 + }, + { + "bits_per_byte": 1.4613747645297765, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0129477977752686, + "step": 7260, + "unclipped_grad_norm": 1.0101300477981567 + }, + { + "bits_per_byte": 1.4686286487778681, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0179758071899414, + "step": 7270, + "unclipped_grad_norm": 1.0528877973556519 + }, + { + "bits_per_byte": 1.4749450555961066, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.022354006767273, + "step": 7280, + "unclipped_grad_norm": 1.0265198945999146 + }, + { + "bits_per_byte": 1.473522931056122, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0213682651519775, + "step": 7290, + "unclipped_grad_norm": 1.066664457321167 + }, + { + "bits_per_byte": 1.4559291058728565, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0091731548309326, + "step": 7300, + "unclipped_grad_norm": 1.0080223083496094 + }, + { + "bits_per_byte": 1.445694074354709, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0020787715911865, + "step": 7310, + "unclipped_grad_norm": 1.0805282592773438 + }, + { + "bits_per_byte": 1.4250067972303744, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9877394437789917, + "step": 7320, + "unclipped_grad_norm": 1.0912657976150513 + }, + { + "bits_per_byte": 1.4783119599520478, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0246877670288086, + "step": 7330, + "unclipped_grad_norm": 1.0849530696868896 + }, + { + "bits_per_byte": 1.4380544330207323, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9967833757400513, + "step": 7340, + "unclipped_grad_norm": 1.042580485343933 + }, + { + "bits_per_byte": 1.4567071553853599, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0097124576568604, + "step": 7350, + "unclipped_grad_norm": 1.0958631038665771 + }, + { + "bits_per_byte": 1.4481006275882324, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0037468671798706, + "step": 7360, + "unclipped_grad_norm": 1.0337251424789429 + }, + { + "bits_per_byte": 1.4027950658887918, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 0.9723434448242188, + "step": 7370, + "unclipped_grad_norm": 1.0347329378128052 + }, + { + "bits_per_byte": 1.4160821015291507, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.981553316116333, + "step": 7380, + "unclipped_grad_norm": 1.0372161865234375 + }, + { + "bits_per_byte": 1.421894943145663, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 0.9855824708938599, + "step": 7390, + "unclipped_grad_norm": 1.038377046585083 + }, + { + "bits_per_byte": 1.4461825050831596, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0024173259735107, + "step": 7400, + "unclipped_grad_norm": 1.057859182357788 + }, + { + "bits_per_byte": 1.475963364871865, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0230598449707031, + "step": 7410, + "unclipped_grad_norm": 1.0708297491073608 + }, + { + "bits_per_byte": 1.469500256852441, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0185799598693848, + "step": 7420, + "unclipped_grad_norm": 1.062455654144287 + }, + { + "bits_per_byte": 1.41574011402791, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9813162684440613, + "step": 7430, + "unclipped_grad_norm": 1.0484251976013184 + }, + { + "bits_per_byte": 1.3821044351027654, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9580017924308777, + "step": 7440, + "unclipped_grad_norm": 1.030645728111267 + }, + { + "bits_per_byte": 1.4987479704234719, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0388529300689697, + "step": 7450, + "unclipped_grad_norm": 1.0465614795684814 + }, + { + "bits_per_byte": 1.4339140366925356, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.993913471698761, + "step": 7460, + "unclipped_grad_norm": 1.0371458530426025 + }, + { + "bits_per_byte": 1.4012647642614147, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9712827205657959, + "step": 7470, + "unclipped_grad_norm": 1.0733681917190552 + }, + { + "bits_per_byte": 1.5010874504231586, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.040474534034729, + "step": 7480, + "unclipped_grad_norm": 1.0303142070770264 + }, + { + "bits_per_byte": 1.4650142613873354, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0154705047607422, + "step": 7490, + "unclipped_grad_norm": 1.0710126161575317 + }, + { + "bits_per_byte": 1.472811954777455, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0208754539489746, + "step": 7500, + "unclipped_grad_norm": 1.0901119709014893 + }, + { + "bits_per_byte": 1.4645168875610401, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0151257514953613, + "step": 7510, + "unclipped_grad_norm": 1.024397611618042 + }, + { + "bits_per_byte": 1.4743771688829574, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0219603776931763, + "step": 7520, + "unclipped_grad_norm": 1.0934079885482788 + }, + { + "bits_per_byte": 1.46691879726299, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0167906284332275, + "step": 7530, + "unclipped_grad_norm": 1.081713318824768 + }, + { + "bits_per_byte": 1.4640198577000465, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0147812366485596, + "step": 7540, + "unclipped_grad_norm": 1.0780924558639526 + }, + { + "bits_per_byte": 1.4520784143200383, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0065040588378906, + "step": 7550, + "unclipped_grad_norm": 1.04312264919281 + }, + { + "bits_per_byte": 1.3812490793886991, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 0.9574089050292969, + "step": 7560, + "unclipped_grad_norm": 1.0638409852981567 + }, + { + "bits_per_byte": 1.4099118799641457, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9772764444351196, + "step": 7570, + "unclipped_grad_norm": 1.0123975276947021 + }, + { + "bits_per_byte": 1.3981073347742243, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9690941572189331, + "step": 7580, + "unclipped_grad_norm": 1.0492113828659058 + }, + { + "bits_per_byte": 1.472455950690169, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0206286907196045, + "step": 7590, + "unclipped_grad_norm": 1.0436540842056274 + }, + { + "bits_per_byte": 1.5055485084038889, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0435667037963867, + "step": 7600, + "unclipped_grad_norm": 1.140743374824524 + }, + { + "bits_per_byte": 1.4701950667619266, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.01906156539917, + "step": 7610, + "unclipped_grad_norm": 1.082565426826477 + }, + { + "bits_per_byte": 1.458967179400348, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0112789869308472, + "step": 7620, + "unclipped_grad_norm": 1.0694279670715332 + }, + { + "bits_per_byte": 1.4490995028244436, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0044392347335815, + "step": 7630, + "unclipped_grad_norm": 1.0964275598526 + }, + { + "bits_per_byte": 1.4314776444690875, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9922246932983398, + "step": 7640, + "unclipped_grad_norm": 1.15219247341156 + }, + { + "bits_per_byte": 1.4066524647649938, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 0.9750171899795532, + "step": 7650, + "unclipped_grad_norm": 1.064915418624878 + }, + { + "bits_per_byte": 1.416584462852321, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9819015264511108, + "step": 7660, + "unclipped_grad_norm": 1.141569972038269 + }, + { + "bits_per_byte": 1.4608237321164121, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0125658512115479, + "step": 7670, + "unclipped_grad_norm": 1.0876517295837402 + }, + { + "bits_per_byte": 1.4539619683122877, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0078096389770508, + "step": 7680, + "unclipped_grad_norm": 1.0938481092453003 + }, + { + "bits_per_byte": 1.433180616677931, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9934051036834717, + "step": 7690, + "unclipped_grad_norm": 1.070209264755249 + }, + { + "bits_per_byte": 1.4952543148538446, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0364313125610352, + "step": 7700, + "unclipped_grad_norm": 1.0864635705947876 + }, + { + "bits_per_byte": 1.412102165014209, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9787946343421936, + "step": 7710, + "unclipped_grad_norm": 1.040504813194275 + }, + { + "bits_per_byte": 1.4180753804526507, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9829349517822266, + "step": 7720, + "unclipped_grad_norm": 1.0190634727478027 + }, + { + "bits_per_byte": 1.4255394275000965, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9881086349487305, + "step": 7730, + "unclipped_grad_norm": 1.0293750762939453 + }, + { + "bits_per_byte": 1.468768986620972, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0180730819702148, + "step": 7740, + "unclipped_grad_norm": 1.0531129837036133 + }, + { + "bits_per_byte": 1.4037740771288283, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9730220437049866, + "step": 7750, + "unclipped_grad_norm": 1.1033527851104736 + }, + { + "bits_per_byte": 1.4111542826339787, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9781376123428345, + "step": 7760, + "unclipped_grad_norm": 1.0707266330718994 + }, + { + "bits_per_byte": 1.4583370349675868, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.010842204093933, + "step": 7770, + "unclipped_grad_norm": 1.0769577026367188 + }, + { + "bits_per_byte": 1.4516912813729461, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0062357187271118, + "step": 7780, + "unclipped_grad_norm": 1.1653907299041748 + }, + { + "bits_per_byte": 1.4586383485718983, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.011051058769226, + "step": 7790, + "unclipped_grad_norm": 1.0739812850952148 + }, + { + "bits_per_byte": 1.4426457678594913, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 0.9999658465385437, + "step": 7800, + "unclipped_grad_norm": 1.0972295999526978 + }, + { + "bits_per_byte": 1.4198621082124645, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9841734170913696, + "step": 7810, + "unclipped_grad_norm": 1.075659990310669 + }, + { + "bits_per_byte": 1.4512271001982673, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0059139728546143, + "step": 7820, + "unclipped_grad_norm": 1.1029958724975586 + }, + { + "bits_per_byte": 1.4583721194283628, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0108665227890015, + "step": 7830, + "unclipped_grad_norm": 1.081362009048462 + }, + { + "bits_per_byte": 1.4242661539444337, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9872260689735413, + "step": 7840, + "unclipped_grad_norm": 1.0629743337631226 + }, + { + "bits_per_byte": 1.423362471105475, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9865996837615967, + "step": 7850, + "unclipped_grad_norm": 1.0624163150787354 + }, + { + "bits_per_byte": 1.4807702799634808, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0263917446136475, + "step": 7860, + "unclipped_grad_norm": 1.0853880643844604 + }, + { + "bits_per_byte": 1.4488509878939468, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0042669773101807, + "step": 7870, + "unclipped_grad_norm": 1.1049087047576904 + }, + { + "bits_per_byte": 1.4220910033676464, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9857183694839478, + "step": 7880, + "unclipped_grad_norm": 1.0985115766525269 + }, + { + "bits_per_byte": 1.4594506226319235, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0116140842437744, + "step": 7890, + "unclipped_grad_norm": 1.0736068487167358 + }, + { + "bits_per_byte": 1.4105420243969071, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 0.9777132272720337, + "step": 7900, + "unclipped_grad_norm": 1.1535842418670654 + }, + { + "bits_per_byte": 1.463574250651661, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0144723653793335, + "step": 7910, + "unclipped_grad_norm": 1.1884382963180542 + }, + { + "bits_per_byte": 1.4830432026772835, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0279672145843506, + "step": 7920, + "unclipped_grad_norm": 1.1013721227645874 + }, + { + "bits_per_byte": 1.435883066062264, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9952782988548279, + "step": 7930, + "unclipped_grad_norm": 1.0824542045593262 + }, + { + "bits_per_byte": 1.4594231054077853, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0115950107574463, + "step": 7940, + "unclipped_grad_norm": 1.1213319301605225 + }, + { + "bits_per_byte": 1.408353201199376, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9761960506439209, + "step": 7950, + "unclipped_grad_norm": 1.0871061086654663 + }, + { + "bits_per_byte": 1.4735662706841393, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0213983058929443, + "step": 7960, + "unclipped_grad_norm": 1.1091856956481934 + }, + { + "bits_per_byte": 1.4267374586460069, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9889390468597412, + "step": 7970, + "unclipped_grad_norm": 1.0478838682174683 + }, + { + "bits_per_byte": 1.472857186214632, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0209068059921265, + "step": 7980, + "unclipped_grad_norm": 1.2172881364822388 + }, + { + "bits_per_byte": 1.4136101948882995, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9798399209976196, + "step": 7990, + "unclipped_grad_norm": 1.1755717992782593 + }, + { + "bits_per_byte": 1.46345644253582, + "learning_rate": 3e-05, + "loss_nats": 1.0143907070159912, + "step": 8000, + "unclipped_grad_norm": 1.0856302976608276 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073003.json b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073003.json new file mode 100644 index 0000000..ac12104 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073003.json @@ -0,0 +1,13072 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "513b85666fcdbf75424596e968cd73733314ec188345e453998de08b18dcef65", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "population_cv": 0.2498932502086342 + }, + "activation_output_rms_by_block": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005511203169589862, + "first_to_last_ratio": 0.6356203088456116, + "imbalance_abs_log_ratio": 0.45315389258709987, + "last_quartile_mean": 0.008670590119436383, + "mean": 0.007157071326219011, + "normalized": [ + 0.4829581625738996, + 0.6886426787339398, + 0.8458438602600115, + 0.9725589429115095, + 0.5050274053272774, + 0.7207530129760953, + 0.8901040788372985, + 1.0544005221525679, + 0.5528397766850007, + 0.7916459680002588, + 0.9438571997073384, + 1.1102770425361632, + 0.6125217730505899, + 0.9295491721602742, + 1.1083022471557418, + 1.3061042379491232, + 0.6819634955048963, + 0.9650194302374437, + 1.1362396940587196, + 1.3715945478756304, + 0.7619466772614072, + 1.0650300118366156, + 1.3201940438400879, + 1.490851576861243, + 0.7782743937317399, + 1.0812722353862223, + 1.3287334463206348, + 1.5181249897087195, + 0.8365475786723071, + 1.138987569259587, + 1.3797450031110483, + 1.6300892253166077 + ], + "population_cv": 0.3025375276561286 + }, + "bits_per_byte": 8.06433666787865, + "branch_output_rms_by_sublayer": [ + 0.0021227982360869646, + 0.0027448106557130814, + 0.00224129855632782, + 0.0027243648655712605, + 0.0024903982412070036, + 0.0027103275060653687, + 0.0023722455371171236, + 0.0027234479784965515, + 0.0023577234242111444, + 0.0027595730498433113, + 0.0023993616923689842, + 0.0027272943407297134, + 0.002688874490559101, + 0.002728298306465149, + 0.0029664887115359306, + 0.0027007071767002344, + 0.0029547461308538914, + 0.002684502862393856, + 0.003052416956052184, + 0.002683063969016075, + 0.0028214550111442804, + 0.0027139110025018454, + 0.0032544767018407583, + 0.002666700165718794, + 0.00348988501355052, + 0.002679243916645646, + 0.003914494998753071, + 0.0026584581937640905, + 0.0035071265883743763, + 0.002684473292902112, + 0.004211232997477055, + 0.0026407993864268064, + 0.004072706215083599, + 0.002663099905475974, + 0.004050548188388348, + 0.0026283918414264917, + 0.003929397091269493, + 0.002597011858597398, + 0.004410331137478352, + 0.0026434636674821377, + 0.004824656527489424, + 0.0025708195753395557, + 0.00433384021744132, + 0.002691218862310052, + 0.004475373309105635, + 0.0026598076801747084, + 0.004755842965096235, + 0.0026596153620630503, + 0.004843301139771938, + 0.0026765079237520695, + 0.0050001065246760845, + 0.002582729561254382, + 0.005346097983419895, + 0.002614464145153761, + 0.004958339501172304, + 0.0025760054122656584, + 0.0054801348596811295, + 0.002634295728057623, + 0.005455824080854654, + 0.00259007909335196, + 0.005286100786179304, + 0.00257018837146461, + 0.005402065813541412, + 0.0025643527042120695 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006159982634052032, + 0.005514663551817784, + 0.005191681274696802, + 0.00489831575692433, + 0.004666415075367917, + 0.004399813239791133, + 0.004483878567781538, + 0.004868281936291656, + 0.00443828064080646, + 0.004266304440803418, + 0.0039471188591477255, + 0.004494388758593749, + 0.0042334459038140095, + 0.00446434979681851, + 0.004322339302329385, + 0.004638471384339534, + 0.004165330760400444, + 0.004499937764754925, + 0.003981478974932938, + 0.004372386299982334, + 0.00458495381977442, + 0.00429501466057645, + 0.003963900484885007, + 0.004271473918508079, + 0.004041799650627023, + 0.004206875397956982, + 0.003955591472942362, + 0.003916464042690177, + 0.003989587235588314, + 0.003922309599766117, + 0.003679536678174494, + 0.003763244111925236 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005022879004590398, + "first_to_last_ratio": 1.2766484804448084, + "imbalance_abs_log_ratio": 0.24423826934058726, + "last_quartile_mean": 0.003934426023708838, + "mean": 0.0043936754999019155, + "normalized": [ + 1.4020112851278042, + 1.255136741878387, + 1.1816260155791434, + 1.1148560600421402, + 1.0620754936207943, + 1.0013969488391563, + 1.0205302070855338, + 1.108020366183241, + 1.0101521245493756, + 0.9710103627130996, + 0.8983637638318627, + 1.0229223252136126, + 0.9635317637610054, + 1.0160854612313022, + 0.9837638902613262, + 1.0557155130011497, + 0.9480287655502622, + 1.0241852783291305, + 0.9061841219320409, + 0.995154580733134, + 1.0435349219296635, + 0.9775448051801577, + 0.9021832597727114, + 0.9721869351988865, + 0.9199131002545027, + 0.9574843199164109, + 0.9002921296829196, + 0.891386731400079, + 0.9080295610536958, + 0.892717179467095, + 0.8374620925593245, + 0.8565138941210488 + ], + "population_cv": 0.11556590700625036 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028097106143832207, + 0.014079716056585312, + 0.014136734418570995, + 0.014167090877890587, + 0.014224153943359852, + 0.014255538582801819, + 0.014337832108139992, + 0.014355218037962914, + 0.014408928342163563, + 0.009655896574258804, + 0.009704872034490108, + 0.009734091348946095, + 0.009773381985723972, + 0.009828771464526653, + 0.00987160298973322, + 0.009915035218000412, + 0.00994801800698042, + 0.007479202933609486, + 0.007515520788729191, + 0.007559895049780607, + 0.007602147292345762, + 0.0076358383521437645, + 0.007658426649868488, + 0.007706487085670233, + 0.007734050042927265, + 0.0062490664422512054, + 0.0062688966281712055, + 0.006315518636256456, + 0.006336068734526634, + 0.0063744354993104935, + 0.006399397272616625, + 0.006439714692533016, + 0.006462901830673218, + 0.0054116579703986645, + 0.005432929843664169, + 0.005481995176523924, + 0.005504137370735407, + 0.005541813559830189, + 0.005560056772083044, + 0.005614723544567823, + 0.005631812382489443, + 0.004862982779741287, + 0.004873365629464388, + 0.004923288244754076, + 0.004937449004501104, + 0.0049925679340958595, + 0.005005750339478254, + 0.005004179198294878, + 0.005022916477173567, + 0.004448203835636377, + 0.0044684382155537605, + 0.004490054678171873, + 0.004500201437622309, + 0.00455933902412653, + 0.004573327023535967, + 0.004625064320862293, + 0.00463611725717783, + 0.004160413984209299, + 0.004166214261204004, + 0.004242690280079842, + 0.0042496914975345135, + 0.004270957782864571, + 0.004283483140170574, + 0.004303553607314825 + ], + "loss_nats": 5.5897722244262695, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0021227982360869646, + 0.0034565660171210766, + 0.00413061399012804, + 0.004928664769977331, + 0.005416445434093475, + 0.0060537648387253284, + 0.006388718727976084, + 0.006960673723369837, + 0.0023577234242111444, + 0.0036145171616226435, + 0.004388594534248114, + 0.0051584807224571705, + 0.00574506726115942, + 0.006370538379997015, + 0.007066869642585516, + 0.007546419743448496, + 0.0029547461308538914, + 0.00395671371370554, + 0.004991721361875534, + 0.005665866658091545, + 0.006190058775246143, + 0.006755253300070763, + 0.0074937604367733, + 0.007946331985294819, + 0.00348988501355052, + 0.004383862018585205, + 0.006057556718587875, + 0.006652849726378918, + 0.0074848332442343235, + 0.007932198233902454, + 0.00897275935858488, + 0.009347881190478802, + 0.004072706215083599, + 0.004880861379206181, + 0.0063449000008404255, + 0.006906712893396616, + 0.007735534105449915, + 0.008132148534059525, + 0.00946879480034113, + 0.009816600009799004, + 0.004824656527489424, + 0.005453306715935469, + 0.007074959110468626, + 0.007622495759278536, + 0.009095527231693268, + 0.009448722936213017, + 0.010346688330173492, + 0.010670131072402, + 0.004843301139771938, + 0.0055701653473079205, + 0.007334613241255283, + 0.007738742511719465, + 0.009146629832684994, + 0.009509840048849583, + 0.010587968863546848, + 0.010865328833460808, + 0.0054801348596811295, + 0.005987230688333511, + 0.0077637480571866035, + 0.00815181527286768, + 0.009536821395158768, + 0.009874933399260044, + 0.011367839761078358, + 0.011666664853692055 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.886429764563218e-05, + 6.52737362543121e-05, + 6.186874816194177e-05, + 5.901425174670294e-05, + 5.411080928752199e-05, + 5.1034174248343334e-05, + 4.815354259335436e-05, + 4.550318044493906e-05, + 4.120725498069078e-05, + 3.8442405639216304e-05, + 3.604330777307041e-05, + 3.37670098815579e-05, + 3.108748569502495e-05, + 2.9203254598542117e-05, + 2.7565200070966966e-05, + 2.5956482204492204e-05, + 2.4566294086980633e-05, + 2.329334893147461e-05, + 2.2260202968027443e-05, + 2.14147730730474e-05, + 2.0902438336634077e-05, + 2.0279332602513023e-05, + 1.9777973648160696e-05, + 1.9328235794091597e-05, + 1.9035323930438608e-05, + 1.8655482563190162e-05, + 1.832095585996285e-05, + 1.8063024981529452e-05, + 1.7733544154907577e-05, + 1.752249227138236e-05, + 1.7320473489235155e-05, + 1.7174717868329026e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.6727842547843466e-05, + "first_to_last_ratio": 3.155359202626439, + "imbalance_abs_log_ratio": 1.1491023413751247, + "last_quartile_mean": 1.79782518898719e-05, + "mean": 3.227324236831919e-05, + "normalized": [ + 2.133789250541258, + 2.0225341944070556, + 1.9170292050567193, + 1.8285814320482991, + 1.67664620337743, + 1.581315371598383, + 1.4920577871848402, + 1.409935200362978, + 1.2768241415105412, + 1.1911541208190792, + 1.1168170635514478, + 1.0462850151897058, + 0.9632588303411923, + 0.904875136661487, + 0.8541193275958588, + 0.8042725273235084, + 0.7611969633115008, + 0.7217542218299198, + 0.6897417592562383, + 0.6635457580819046, + 0.647670850610065, + 0.628363657145899, + 0.6128288389014056, + 0.5988935221787641, + 0.5898175247840765, + 0.5780479801280578, + 0.5676825294116558, + 0.559690432569022, + 0.5494813304632692, + 0.5429417990112825, + 0.5366821620079203, + 0.5321658627392354 + ], + "population_cv": 0.49221598973047537 + }, + "activation_output_rms_by_block": [ + 0.0055193523876369, + 0.009546536952257156, + 0.013977172784507275, + 0.018744563683867455, + 0.00660830270498991, + 0.011765030212700367, + 0.017156247049570084, + 0.022267481312155724, + 0.007449587807059288, + 0.013175037689507008, + 0.019073888659477234, + 0.025712018832564354, + 0.008972279727458954, + 0.01624389924108982, + 0.023743286728858948, + 0.03214457631111145, + 0.00991375558078289, + 0.019281653687357903, + 0.02868141047656536, + 0.03889070823788643, + 0.0127727584913373, + 0.022841427475214005, + 0.034929387271404266, + 0.04634999856352806, + 0.014845813624560833, + 0.029589947313070297, + 0.046351123601198196, + 0.06254568696022034, + 0.01947367936372757, + 0.0370744951069355, + 0.05507569760084152, + 0.07403328269720078 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013198085885960609, + "first_to_last_ratio": 0.31146869331458016, + "imbalance_abs_log_ratio": 1.1664564488083473, + "last_quartile_mean": 0.04237371578346938, + "mean": 0.0251484402542701, + "normalized": [ + 0.21947096248642048, + 0.3796075166385794, + 0.5557868656341027, + 0.7453569085933552, + 0.26277187126417706, + 0.46782345520226504, + 0.6821992487847045, + 0.8854418439877121, + 0.2962246458125521, + 0.5238908479530829, + 0.7584521531604159, + 1.0224100808080359, + 0.35677281122575777, + 0.6459207440641045, + 0.9441256192748351, + 1.2781936369057099, + 0.39420956053525336, + 0.7667137004285568, + 1.140484665711838, + 1.5464461352144077, + 0.5078946591595691, + 0.9082641803733982, + 1.3889285744261377, + 1.8430565909811456, + 0.5903274109431131, + 1.1766116313335198, + 1.8431013268637195, + 2.487060283971303, + 0.7743493897368454, + 1.4742264224772512, + 2.190024392923927, + 2.9438518631242045 + ], + "population_cv": 0.6624204772463126 + }, + "bits_per_byte": 5.18286091123436, + "branch_output_rms_by_sublayer": [ + 0.002723898272961378, + 0.004827107302844524, + 0.0025367254856973886, + 0.004943604115396738, + 0.0027546093333512545, + 0.00549353240057826, + 0.002648916095495224, + 0.005891259294003248, + 0.0027809101156890392, + 0.0059933834709227085, + 0.0025255572982132435, + 0.006044833920896053, + 0.003397067543119192, + 0.0061737713403999805, + 0.0033452848438173532, + 0.0059272525832057, + 0.003559867851436138, + 0.0063678608275949955, + 0.0038245979230850935, + 0.0061998614110052586, + 0.003386941971257329, + 0.006479958072304726, + 0.00461281044408679, + 0.006899545900523663, + 0.004887574352324009, + 0.007097913417965174, + 0.005150536075234413, + 0.006852242164313793, + 0.0054018739610910416, + 0.006816149223595858, + 0.006125702988356352, + 0.007103090640157461, + 0.005625482182949781, + 0.007109561935067177, + 0.006412550341337919, + 0.007626448292285204, + 0.0072620403952896595, + 0.007445307914167643, + 0.008774910122156143, + 0.007442969363182783, + 0.008495661430060863, + 0.007471293676644564, + 0.007638709619641304, + 0.007774693425744772, + 0.009623398073017597, + 0.007449399214237928, + 0.008996927179396152, + 0.007351893465965986, + 0.00961991585791111, + 0.008440978825092316, + 0.011907265521585941, + 0.008385274559259415, + 0.014190610498189926, + 0.008216616697609425, + 0.013635334558784962, + 0.007803319953382015, + 0.014384749345481396, + 0.007909612730145454, + 0.01446358859539032, + 0.007761957123875618, + 0.013604369945824146, + 0.008686874993145466, + 0.015084062702953815, + 0.008338849991559982 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00023786681750428726, + 0.00020576573809540593, + 0.00019740566304489066, + 0.0001813902134209802, + 0.0001696969276866905, + 0.00015923444695871368, + 0.00017314685145041394, + 0.00015953319119459735, + 0.0001633125663305534, + 0.0001601758879769712, + 0.00014581270072009644, + 0.00016837778263142155, + 0.0001603117902727929, + 0.00016340123818181664, + 0.00015138579649161358, + 0.00015756725288262655, + 0.00013845240957374987, + 0.0001515626583813202, + 0.00014303549429743115, + 0.00016115390114098118, + 0.00015144047315466305, + 0.00013355245347473064, + 0.00014934284038847835, + 0.00014161248575260865, + 0.00014194691162247068, + 0.00015146794722523853, + 0.00015520640136475585, + 0.0001429086966261467, + 0.00014288517283478702, + 0.00013975759356060232, + 0.00013469418893153685, + 0.00013332122361590492 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018550498116949745, + "first_to_last_ratio": 1.2992954513055368, + "imbalance_abs_log_ratio": 0.26182215701959494, + "last_quartile_mean": 0.00014277351697268036, + "mean": 0.00015833517864966492, + "normalized": [ + 1.5022992333914333, + 1.2995579368416077, + 1.2467580782011416, + 1.1456090489045851, + 1.0717575786626974, + 1.0056795231275704, + 1.0935463169149642, + 1.0075663068381233, + 1.0314357663491922, + 1.0116253971038178, + 0.9209115874541316, + 1.0634262333071103, + 1.0124837173898131, + 1.0319957925671146, + 0.9561096768430238, + 0.9951499990489321, + 0.874426079880151, + 0.9572266862859976, + 0.9033715407863586, + 1.0178022503612605, + 0.9564549991113648, + 0.8434793494011273, + 0.9432069465681839, + 0.894384223141863, + 0.8964963619142705, + 0.9566285175348117, + 0.9802395316594056, + 0.9025707227220104, + 0.9024221531396832, + 0.8826692510944287, + 0.8506902261408595, + 0.8420189673129665 + ], + "population_cv": 0.13848454603612886 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6920576691627502, + "mean_weights": [ + 0.5229578018188477, + 0.47704219818115234 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922386288642883, + "mean_weights": [ + 0.5206724405288696, + 0.479327529668808 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930604577064514, + "mean_weights": [ + 0.5023424625396729, + 0.4976575970649719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924410462379456, + "mean_weights": [ + 0.5176653861999512, + 0.48233455419540405 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929901838302612, + "mean_weights": [ + 0.5013687014579773, + 0.4986313283443451 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929076313972473, + "mean_weights": [ + 0.5098068714141846, + 0.49019312858581543 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927430629730225, + "mean_weights": [ + 0.5127849578857422, + 0.4872150421142578 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928876638412476, + "mean_weights": [ + 0.5100651383399963, + 0.48993486166000366 + ], + "sources": 2 + }, + { + "entropy_mean": 1.098104476928711, + "mean_weights": [ + 0.33572155237197876, + 0.32083213329315186, + 0.34344637393951416 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098517656326294, + "mean_weights": [ + 0.3379058539867401, + 0.3309776186943054, + 0.33111652731895447 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984238386154175, + "mean_weights": [ + 0.34072375297546387, + 0.3312225341796875, + 0.3280537724494934 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985376834869385, + "mean_weights": [ + 0.3372243344783783, + 0.33110499382019043, + 0.3316706418991089 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983879566192627, + "mean_weights": [ + 0.34103795886039734, + 0.33302855491638184, + 0.3259335160255432 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984759330749512, + "mean_weights": [ + 0.3378061056137085, + 0.3331226110458374, + 0.3290712833404541 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982115268707275, + "mean_weights": [ + 0.3427826464176178, + 0.33246177434921265, + 0.32475557923316956 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098510980606079, + "mean_weights": [ + 0.336602121591568, + 0.33530712127685547, + 0.32809075713157654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860652446746826, + "mean_weights": [ + 0.25563088059425354, + 0.25276094675064087, + 0.24706901609897614, + 0.24453918635845184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861669301986694, + "mean_weights": [ + 0.25406360626220703, + 0.25270599126815796, + 0.24658669531345367, + 0.24664369225502014 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386000394821167, + "mean_weights": [ + 0.2551618814468384, + 0.2541123330593109, + 0.2483193278312683, + 0.24240648746490479 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386232852935791, + "mean_weights": [ + 0.25183945894241333, + 0.25246676802635193, + 0.24847325682640076, + 0.24722051620483398 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385960340499878, + "mean_weights": [ + 0.25454509258270264, + 0.25490614771842957, + 0.24972698092460632, + 0.24082177877426147 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386134147644043, + "mean_weights": [ + 0.25200384855270386, + 0.25437670946121216, + 0.24930089712142944, + 0.24431857466697693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859115839004517, + "mean_weights": [ + 0.2541099786758423, + 0.2563633620738983, + 0.2499358206987381, + 0.2395908385515213 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861732482910156, + "mean_weights": [ + 0.2492598295211792, + 0.25510817766189575, + 0.2504415512084961, + 0.24519044160842896 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6091704368591309, + "mean_weights": [ + 0.20294126868247986, + 0.20561827719211578, + 0.20076946914196014, + 0.19495637714862823, + 0.19571462273597717 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093817949295044, + "mean_weights": [ + 0.1999536156654358, + 0.20290037989616394, + 0.19905918836593628, + 0.1980593204498291, + 0.2000274658203125 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6089024543762207, + "mean_weights": [ + 0.20434537529945374, + 0.20689502358436584, + 0.20309439301490784, + 0.19485914707183838, + 0.1908060759305954 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609398365020752, + "mean_weights": [ + 0.20015040040016174, + 0.2025333195924759, + 0.2003234326839447, + 0.19931097328662872, + 0.19768190383911133 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085304021835327, + "mean_weights": [ + 0.20286625623703003, + 0.21047379076480865, + 0.20388861000537872, + 0.19463706016540527, + 0.18813428282737732 + ], + "sources": 5 + }, + { + "entropy_mean": 1.60928213596344, + "mean_weights": [ + 0.20026028156280518, + 0.20464842021465302, + 0.20178082585334778, + 0.1985689103603363, + 0.19474159181118011 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085617542266846, + "mean_weights": [ + 0.20295967161655426, + 0.2110326737165451, + 0.20332126319408417, + 0.194858580827713, + 0.18782781064510345 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093013286590576, + "mean_weights": [ + 0.20157885551452637, + 0.20404376089572906, + 0.20028473436832428, + 0.19877153635025024, + 0.19532111287117004 + ], + "sources": 5 + }, + { + "entropy_mean": 1.79081130027771, + "mean_weights": [ + 0.1695082187652588, + 0.17698298394680023, + 0.17063108086585999, + 0.16265572607517242, + 0.15738935768604279, + 0.16283263266086578 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917133569717407, + "mean_weights": [ + 0.16738522052764893, + 0.1689736694097519, + 0.16775797307491302, + 0.16576024889945984, + 0.16475272178649902, + 0.16537019610404968 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7902145385742188, + "mean_weights": [ + 0.17057637870311737, + 0.18038460612297058, + 0.17244961857795715, + 0.164292573928833, + 0.15751495957374573, + 0.15478186309337616 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917354106903076, + "mean_weights": [ + 0.16660897433757782, + 0.16850554943084717, + 0.1666441559791565, + 0.16635674238204956, + 0.16625551879405975, + 0.1656290590763092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898752689361572, + "mean_weights": [ + 0.17184747755527496, + 0.180228590965271, + 0.17470473051071167, + 0.16429948806762695, + 0.15632927417755127, + 0.15259043872356415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917178869247437, + "mean_weights": [ + 0.1659121811389923, + 0.16851314902305603, + 0.16693055629730225, + 0.16679996252059937, + 0.16575691103935242, + 0.16608723998069763 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7883799076080322, + "mean_weights": [ + 0.1728566288948059, + 0.18512636423110962, + 0.17585304379463196, + 0.16398008167743683, + 0.1537502557039261, + 0.1484336107969284 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917424440383911, + "mean_weights": [ + 0.1667865514755249, + 0.1680906116962433, + 0.16619735956192017, + 0.16709285974502563, + 0.16617436707019806, + 0.16565825045108795 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9426109790802002, + "mean_weights": [ + 0.14892546832561493, + 0.16000479459762573, + 0.15354622900485992, + 0.14328595995903015, + 0.13467344641685486, + 0.12974688410758972, + 0.12981724739074707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1419999748468399, + 0.1434222161769867, + 0.14279979467391968, + 0.14312142133712769, + 0.14279043674468994, + 0.1428302675485611, + 0.143035888671875 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415357112884521, + "mean_weights": [ + 0.1496729701757431, + 0.16313126683235168, + 0.15414540469646454, + 0.1440294086933136, + 0.1343868225812912, + 0.12868303060531616, + 0.1259511113166809 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458801746368408, + "mean_weights": [ + 0.1429852843284607, + 0.1429750919342041, + 0.14214137196540833, + 0.1414642632007599, + 0.1421438455581665, + 0.14385518431663513, + 0.14443498849868774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9424511194229126, + "mean_weights": [ + 0.14928653836250305, + 0.16045087575912476, + 0.15314346551895142, + 0.1440504789352417, + 0.1356543004512787, + 0.13001246750354767, + 0.1274019032716751 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458882808685303, + "mean_weights": [ + 0.14319148659706116, + 0.14391475915908813, + 0.14284171164035797, + 0.1420421153306961, + 0.14134715497493744, + 0.14342516660690308, + 0.1432376205921173 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395697116851807, + "mean_weights": [ + 0.15016257762908936, + 0.16719450056552887, + 0.15599417686462402, + 0.14574657380580902, + 0.13377557694911957, + 0.12622776627540588, + 0.12089881300926208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1428529918193817, + 0.14342239499092102, + 0.14335417747497559, + 0.14274317026138306, + 0.1419316530227661, + 0.14220726490020752, + 0.14348836243152618 + ], + "sources": 7 + }, + { + "entropy_mean": 2.074862480163574, + "mean_weights": [ + 0.13159659504890442, + 0.14486080408096313, + 0.13573890924453735, + 0.12842535972595215, + 0.12030278891324997, + 0.11495959758758545, + 0.10915482044219971, + 0.11496111750602722 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079418659210205, + "mean_weights": [ + 0.12459750473499298, + 0.12504151463508606, + 0.12489251047372818, + 0.125042125582695, + 0.12461285293102264, + 0.12601442635059357, + 0.1261361837387085, + 0.12366288155317307 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0741164684295654, + "mean_weights": [ + 0.1323997974395752, + 0.14626389741897583, + 0.1369163542985916, + 0.12836164236068726, + 0.12002508342266083, + 0.11502262949943542, + 0.10908686369657516, + 0.11192375421524048 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793213844299316, + "mean_weights": [ + 0.12429114431142807, + 0.12243181467056274, + 0.12357296049594879, + 0.12375843524932861, + 0.12469147145748138, + 0.12560917437076569, + 0.12766826152801514, + 0.12797673046588898 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0734808444976807, + "mean_weights": [ + 0.1330580711364746, + 0.14781729876995087, + 0.1374434530735016, + 0.12885303795337677, + 0.11949820816516876, + 0.11495119333267212, + 0.10982058197259903, + 0.10855817794799805 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07940673828125, + "mean_weights": [ + 0.12457560002803802, + 0.12431882321834564, + 0.12479400634765625, + 0.12437158077955246, + 0.12397614866495132, + 0.12488298863172531, + 0.12619814276695251, + 0.12688270211219788 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07299542427063, + "mean_weights": [ + 0.13274571299552917, + 0.14826728403568268, + 0.13701492547988892, + 0.12918588519096375, + 0.12125477194786072, + 0.1169566735625267, + 0.10889489948749542, + 0.10567984730005264 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079298734664917, + "mean_weights": [ + 0.12391229718923569, + 0.12222036719322205, + 0.12298265099525452, + 0.12416766583919525, + 0.125657856464386, + 0.12553520500659943, + 0.126623272895813, + 0.1289007067680359 + ], + "sources": 8 + }, + { + "entropy_mean": 2.189734935760498, + "mean_weights": [ + 0.1203557699918747, + 0.13466420769691467, + 0.12461012601852417, + 0.11655421555042267, + 0.10905998200178146, + 0.10487955063581467, + 0.09900732338428497, + 0.0950026735663414, + 0.0958661437034607 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197068691253662, + "mean_weights": [ + 0.1099812239408493, + 0.10853179544210434, + 0.10949578881263733, + 0.11001666635274887, + 0.1107073426246643, + 0.11118210852146149, + 0.11218173801898956, + 0.11490923166275024, + 0.11299411952495575 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907877922058105, + "mean_weights": [ + 0.11948458850383759, + 0.13228623569011688, + 0.12348943203687668, + 0.11669500172138214, + 0.11027230322360992, + 0.1054498627781868, + 0.10031206905841827, + 0.09697622060775757, + 0.09503427147865295 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971726417541504, + "mean_weights": [ + 0.11100216209888458, + 0.11012113094329834, + 0.1099993959069252, + 0.11000204086303711, + 0.10997181385755539, + 0.11260797083377838, + 0.11264219880104065, + 0.11185945570468903, + 0.11179381608963013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.190375804901123, + "mean_weights": [ + 0.11983220279216766, + 0.13351142406463623, + 0.12386792153120041, + 0.11615028977394104, + 0.10967587679624557, + 0.10584976524114609, + 0.10062196850776672, + 0.09596225619316101, + 0.09452830255031586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969356536865234, + "mean_weights": [ + 0.10960094630718231, + 0.10724944621324539, + 0.10892992466688156, + 0.1096237450838089, + 0.11032412201166153, + 0.11112386733293533, + 0.11386257410049438, + 0.11429048329591751, + 0.11499489098787308 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188795566558838, + "mean_weights": [ + 0.12045153975486755, + 0.13515985012054443, + 0.12505559623241425, + 0.11722971498966217, + 0.11039333790540695, + 0.10568177700042725, + 0.09951679408550262, + 0.09492656588554382, + 0.09158482402563095 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028617795556783676, + 0.014952295459806919, + 0.015273182652890682, + 0.014778315089643002, + 0.015790818259119987, + 0.01543855108320713, + 0.01658804342150688, + 0.016689632087945938, + 0.017823126167058945, + 0.01178102195262909, + 0.012884888797998428, + 0.012982882559299469, + 0.013985250145196915, + 0.014135614968836308, + 0.015267650596797466, + 0.015412751585245132, + 0.016450626775622368, + 0.012501097284257412, + 0.013461960479617119, + 0.013585822656750679, + 0.014477741904556751, + 0.014657454565167427, + 0.01562186423689127, + 0.01575874350965023, + 0.016906939446926117, + 0.013761760666966438, + 0.014646334573626518, + 0.014892088249325752, + 0.015772713348269463, + 0.016054842621088028, + 0.016963548958301544, + 0.017220331355929375, + 0.018231164664030075, + 0.015453077852725983, + 0.016293151304125786, + 0.016655873507261276, + 0.017603756859898567, + 0.017766626551747322, + 0.018859725445508957, + 0.018939858302474022, + 0.020210443064570427, + 0.017597980797290802, + 0.018590020015835762, + 0.018625281751155853, + 0.019812360405921936, + 0.01988193951547146, + 0.021170100197196007, + 0.0208931565284729, + 0.022594628855586052, + 0.019704710692167282, + 0.021153371781110764, + 0.021027294918894768, + 0.022732628509402275, + 0.02250491827726364, + 0.024445844814181328, + 0.023921677842736244, + 0.02632913924753666, + 0.022959763184189796, + 0.02505173534154892, + 0.02448389306664467, + 0.026562821120023727, + 0.025734128430485725, + 0.028511708602309227, + 0.027005694806575775 + ], + "loss_nats": 3.5924854278564453, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1922647953033447, + "mean_weights": [ + 0.09991011023521423, + 0.0929517149925232, + 0.10029369592666626, + 0.10801893472671509, + 0.11530699580907822, + 0.11817377060651779, + 0.12114627659320831, + 0.12246126681566238, + 0.12173722684383392 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.002723898272961378, + 0.0055193523876369, + 0.006246842443943024, + 0.009546536952257156, + 0.009986688382923603, + 0.013977172784507275, + 0.014312135986983776, + 0.018744563683867455, + 0.0027809101156890392, + 0.00660830270498991, + 0.007167908828705549, + 0.011765030212700367, + 0.012327533215284348, + 0.017156247049570084, + 0.017696699127554893, + 0.022267481312155724, + 0.003559867851436138, + 0.007449587807059288, + 0.008642485365271568, + 0.013175037689507008, + 0.014099182561039925, + 0.019073888659477234, + 0.020258057862520218, + 0.025712018832564354, + 0.004887574352324009, + 0.008972279727458954, + 0.011225050315260887, + 0.01624389924108982, + 0.01857108250260353, + 0.023743286728858948, + 0.026577936485409737, + 0.03214457631111145, + 0.005625482182949781, + 0.00991375558078289, + 0.013686468824744225, + 0.019281653687357903, + 0.023031942546367645, + 0.02868141047656536, + 0.03310495615005493, + 0.03889070823788643, + 0.008495661430060863, + 0.0127727584913373, + 0.017283262684941292, + 0.022841427475214005, + 0.029314789921045303, + 0.034929387271404266, + 0.04071469604969025, + 0.04634999856352806, + 0.00961991585791111, + 0.014845813624560833, + 0.023407738655805588, + 0.029589947313070297, + 0.0400763601064682, + 0.046351123601198196, + 0.05648425966501236, + 0.06254568696022034, + 0.014384749345481396, + 0.01947367936372757, + 0.03149676322937012, + 0.0370744951069355, + 0.04818879812955856, + 0.05507569760084152, + 0.06738673150539398, + 0.07403328269720078 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001494410098530352, + 0.00015486938355024904, + 0.00014897517394274473, + 0.0001431136333849281, + 0.0001401594199705869, + 0.00012453079398255795, + 0.00011206007184227929, + 0.00010208225285168737, + 9.998067253036425e-05, + 9.082697215490043e-05, + 8.336143946507946e-05, + 7.669178739888594e-05, + 7.405834912788123e-05, + 6.847811891930178e-05, + 6.22319639660418e-05, + 5.739524931414053e-05, + 5.947135286987759e-05, + 5.3890027629677206e-05, + 4.890635682386346e-05, + 4.4437034375732765e-05, + 4.31341104558669e-05, + 3.886142803821713e-05, + 3.5836677852785215e-05, + 3.340865805512294e-05, + 3.223997919121757e-05, + 3.0436871384154074e-05, + 2.9292581530171447e-05, + 2.8587397537194192e-05, + 2.2913789507583715e-05, + 2.2530648493557237e-05, + 2.244284223706927e-05, + 2.2519136109622195e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013440396742225857, + "first_to_last_ratio": 5.096772825661455, + "imbalance_abs_log_ratio": 1.6286075601589056, + "last_quartile_mean": 2.6370405748821213e-05, + "mean": 7.053641201082428e-05, + "normalized": [ + 2.1186363977530145, + 2.195594858531836, + 2.112032207136982, + 2.0289327073081966, + 1.9870506023056362, + 1.7654824002594274, + 1.5886840377574483, + 1.4472277500593902, + 1.4174334883240383, + 1.287660792002894, + 1.1818213754945048, + 1.0872652182409996, + 1.0499307664886113, + 0.9708194245660434, + 0.8822672176250175, + 0.8136967514782698, + 0.8431298271983457, + 0.7640029609304118, + 0.6933490863748281, + 0.6299871670381194, + 0.6115155169679979, + 0.5509413780821966, + 0.5080592679889337, + 0.47363704932987183, + 0.4570686014801849, + 0.43150580695093105, + 0.4152831239229507, + 0.4052856775987878, + 0.32485051130850606, + 0.319418692435047, + 0.31817385655546626, + 0.3192554805051111 + ], + "population_cv": 0.606636113722199 + }, + "activation_output_rms_by_block": [ + 0.021243074908852577, + 0.03624089062213898, + 0.04951142147183418, + 0.05851924046874046, + 0.017167888581752777, + 0.02674153819680214, + 0.03352004662156105, + 0.03946679085493088, + 0.020600929856300354, + 0.027865095064044, + 0.035815127193927765, + 0.04293550178408623, + 0.021564343944191933, + 0.029616132378578186, + 0.035480253398418427, + 0.04259813204407692, + 0.021436138078570366, + 0.03586751967668533, + 0.04331304505467415, + 0.051551222801208496, + 0.025608548894524574, + 0.04091007262468338, + 0.05466190353035927, + 0.07478146255016327, + 0.02793189324438572, + 0.05498877540230751, + 0.08125203847885132, + 0.10916247963905334, + 0.026290444657206535, + 0.05761713907122612, + 0.08842677623033524, + 0.12436132878065109 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03530136146582663, + "first_to_last_ratio": 0.495430868506741, + "imbalance_abs_log_ratio": 0.7023274536001348, + "last_quartile_mean": 0.07125385943800211, + "mean": 0.04553272487828508, + "normalized": [ + 0.4665452148018396, + 0.7959306349228079, + 1.0873810342821493, + 1.2852127920121195, + 0.37704505117242126, + 0.5873037088881845, + 0.7361748437231577, + 0.8667785853016872, + 0.452442279950724, + 0.6119795188741951, + 0.7865799222354093, + 0.9429591991003927, + 0.4736009980038805, + 0.6504361963345259, + 0.7792253482140983, + 0.9355498085815704, + 0.47078531179216626, + 0.7877305777891374, + 0.9512508897821419, + 1.1321796123340224, + 0.5624207416309823, + 0.8984762658953829, + 1.2004970859195814, + 1.6423673907077576, + 0.6134465556157976, + 1.2076759198861848, + 1.784475642431869, + 2.397451062523908, + 0.5773966905667149, + 1.2654006371295428, + 1.9420488553389141, + 2.7312516242567333 + ], + "population_cv": 0.5563959277170655 + }, + "bits_per_byte": 3.0745070201808953, + "branch_output_rms_by_sublayer": [ + 0.01049753837287426, + 0.01706349104642868, + 0.013886291533708572, + 0.012710319831967354, + 0.01245343592017889, + 0.013922877609729767, + 0.01395952794700861, + 0.011416541412472725, + 0.014544809237122536, + 0.008325628004968166, + 0.014345151372253895, + 0.00801781378686428, + 0.015135552734136581, + 0.007913725450634956, + 0.014776745811104774, + 0.008828459307551384, + 0.01795734278857708, + 0.008378393016755581, + 0.013709688559174538, + 0.008136541582643986, + 0.016144096851348877, + 0.008199729025363922, + 0.018440017476677895, + 0.009040302596986294, + 0.018456056714057922, + 0.00936646107584238, + 0.012218938209116459, + 0.009016151539981365, + 0.01701919361948967, + 0.009497544728219509, + 0.01305371057242155, + 0.009566655382514, + 0.017418840900063515, + 0.011108046397566795, + 0.026864025741815567, + 0.0114391278475523, + 0.035265542566776276, + 0.012216469272971153, + 0.027374446392059326, + 0.01392423827201128, + 0.01969025284051895, + 0.015176027081906796, + 0.018602686002850533, + 0.017532862722873688, + 0.0133047541603446, + 0.018514130264520645, + 0.015662170946598053, + 0.022848375141620636, + 0.012774706818163395, + 0.0230264812707901, + 0.016374029219150543, + 0.02426965720951557, + 0.010044055990874767, + 0.025484448298811913, + 0.006929588038474321, + 0.027965376153588295, + 0.005331806372851133, + 0.02452327311038971, + 0.01223068032413721, + 0.02779858186841011, + 0.00525518087670207, + 0.03124845214188099, + 0.010387555696070194, + 0.03395458683371544 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004975869912568777, + 0.0004115293940014644, + 0.00043817390286628833, + 0.00039988528646951293, + 0.0003968514715139995, + 0.00031053290415162903, + 0.00024154094413850278, + 0.00022869177850848996, + 0.0002476899080691786, + 0.00021028907646624228, + 0.0002010354933691209, + 0.0001845208571901889, + 0.0002273114134138899, + 0.0001715437435350543, + 0.00015668460984383524, + 0.00015894955615086492, + 0.00016556491904516564, + 0.0001408361881569501, + 0.00014198814555526783, + 0.00012254378127881199, + 0.00011450446233020633, + 0.0001100879077446283, + 0.00010549256620556044, + 9.975908855450905e-05, + 0.00010071115476761671, + 9.794352566848755e-05, + 9.525064309078406e-05, + 8.226123370955787e-05, + 6.340888944803813e-05, + 6.739220497758678e-05, + 6.650164980556211e-05, + 7.228417264196778e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003655990841133456, + "first_to_last_ratio": 4.5292712934136725, + "imbalance_abs_log_ratio": 1.5105610641741143, + "last_quartile_mean": 8.071918426370012e-05, + "mean": 0.0001915421207476825, + "normalized": [ + 2.5977941004023166, + 2.148505991241321, + 2.28761121134026, + 2.0877146233348847, + 2.071875731379053, + 1.621225153712757, + 1.2610330469123472, + 1.1939503312158923, + 1.293135458155751, + 1.0978738026152224, + 1.0495628459389565, + 0.963343501000792, + 1.186743743499249, + 0.8955927963282188, + 0.8180164719499744, + 0.8298412669255573, + 0.8643786463201141, + 0.7352752888356756, + 0.7412894093529857, + 0.6397745874404216, + 0.5978030413531993, + 0.574745164744451, + 0.5507538801062211, + 0.5208206328983963, + 0.5257911647552604, + 0.5113419716048152, + 0.4972830138821386, + 0.4294681158820424, + 0.33104410246958865, + 0.3518401316353921, + 0.3471907356250086, + 0.3773800371417385 + ], + "population_cv": 0.6210714087578444 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6506600379943848, + "mean_weights": [ + 0.3747701942920685, + 0.6252298355102539 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6806463003158569, + "mean_weights": [ + 0.45339685678482056, + 0.5466030836105347 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6869370937347412, + "mean_weights": [ + 0.4668571949005127, + 0.5331428050994873 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6791321635246277, + "mean_weights": [ + 0.4727538824081421, + 0.5272461175918579 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6804035902023315, + "mean_weights": [ + 0.4487338662147522, + 0.5512661933898926 + ], + "sources": 2 + }, + { + "entropy_mean": 0.682766318321228, + "mean_weights": [ + 0.4646179676055908, + 0.5353820323944092 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873732805252075, + "mean_weights": [ + 0.4883750379085541, + 0.5116249322891235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6835889220237732, + "mean_weights": [ + 0.46665582060813904, + 0.5333442091941833 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0856965780258179, + "mean_weights": [ + 0.27463093400001526, + 0.3524433374404907, + 0.372925728559494 + ], + "sources": 3 + }, + { + "entropy_mean": 1.077350378036499, + "mean_weights": [ + 0.2904614210128784, + 0.3131614923477173, + 0.3963770568370819 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0850789546966553, + "mean_weights": [ + 0.27783387899398804, + 0.34068456292152405, + 0.3814815878868103 + ], + "sources": 3 + }, + { + "entropy_mean": 1.062077522277832, + "mean_weights": [ + 0.28390786051750183, + 0.29115602374076843, + 0.42493611574172974 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0936429500579834, + "mean_weights": [ + 0.30466002225875854, + 0.3333379328250885, + 0.36200207471847534 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0627415180206299, + "mean_weights": [ + 0.2850727438926697, + 0.29934102296829224, + 0.4155862331390381 + ], + "sources": 3 + }, + { + "entropy_mean": 1.087097406387329, + "mean_weights": [ + 0.28865373134613037, + 0.3320052921772003, + 0.3793409764766693 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0500720739364624, + "mean_weights": [ + 0.2745887339115143, + 0.286259263753891, + 0.4391520023345947 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3743348121643066, + "mean_weights": [ + 0.2166871875524521, + 0.23293715715408325, + 0.2495419979095459, + 0.30083364248275757 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3609715700149536, + "mean_weights": [ + 0.23144879937171936, + 0.21839870512485504, + 0.24986028671264648, + 0.3002921938896179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3810515403747559, + "mean_weights": [ + 0.2385442852973938, + 0.23454180359840393, + 0.24874086678028107, + 0.27817302942276 + ], + "sources": 4 + }, + { + "entropy_mean": 1.348030686378479, + "mean_weights": [ + 0.23874622583389282, + 0.21300309896469116, + 0.22721335291862488, + 0.32103732228279114 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3825427293777466, + "mean_weights": [ + 0.24534732103347778, + 0.2375141680240631, + 0.25072187185287476, + 0.26641663908958435 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3499886989593506, + "mean_weights": [ + 0.24629786610603333, + 0.22150886058807373, + 0.2235339879989624, + 0.30865928530693054 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3807358741760254, + "mean_weights": [ + 0.24433431029319763, + 0.23110222816467285, + 0.23992134630680084, + 0.2846421003341675 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3399771451950073, + "mean_weights": [ + 0.24593281745910645, + 0.21928201615810394, + 0.21277041733264923, + 0.3220147490501404 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6075894832611084, + "mean_weights": [ + 0.19650746881961823, + 0.19567455351352692, + 0.18970930576324463, + 0.21128275990486145, + 0.20682592689990997 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5706448554992676, + "mean_weights": [ + 0.1975565403699875, + 0.1772933304309845, + 0.16386017203330994, + 0.21728253364562988, + 0.2440074384212494 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6042816638946533, + "mean_weights": [ + 0.21478912234306335, + 0.18552109599113464, + 0.17930489778518677, + 0.20310312509536743, + 0.2172817587852478 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5764570236206055, + "mean_weights": [ + 0.20157726109027863, + 0.17631688714027405, + 0.1703747808933258, + 0.19828173518180847, + 0.25344932079315186 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6011263132095337, + "mean_weights": [ + 0.20813238620758057, + 0.18490281701087952, + 0.17333751916885376, + 0.20533671975135803, + 0.22829055786132812 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5817476511001587, + "mean_weights": [ + 0.20727968215942383, + 0.18177060782909393, + 0.17236655950546265, + 0.18925222754478455, + 0.24933092296123505 + ], + "sources": 5 + }, + { + "entropy_mean": 1.603710412979126, + "mean_weights": [ + 0.2016504406929016, + 0.1979413628578186, + 0.17889465391635895, + 0.19759604334831238, + 0.22391751408576965 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5885388851165771, + "mean_weights": [ + 0.20988406240940094, + 0.18461555242538452, + 0.17252585291862488, + 0.19482356309890747, + 0.2381509691476822 + ], + "sources": 5 + }, + { + "entropy_mean": 1.783994197845459, + "mean_weights": [ + 0.16140900552272797, + 0.16211673617362976, + 0.15125815570354462, + 0.1585075855255127, + 0.17266258597373962, + 0.19404590129852295 + ], + "sources": 6 + }, + { + "entropy_mean": 1.765566349029541, + "mean_weights": [ + 0.1630604863166809, + 0.1509958952665329, + 0.14303800463676453, + 0.14991191029548645, + 0.16761469841003418, + 0.22537899017333984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7848252058029175, + "mean_weights": [ + 0.16126418113708496, + 0.1664336919784546, + 0.15361233055591583, + 0.1546509563922882, + 0.16693615913391113, + 0.19710269570350647 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7731435298919678, + "mean_weights": [ + 0.16123875975608826, + 0.15981441736221313, + 0.1509810984134674, + 0.14913737773895264, + 0.16297483444213867, + 0.2158534824848175 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7815005779266357, + "mean_weights": [ + 0.15527218580245972, + 0.17254948616027832, + 0.15193141996860504, + 0.15522462129592896, + 0.1650869995355606, + 0.19993525743484497 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7801456451416016, + "mean_weights": [ + 0.1661754697561264, + 0.16477231681346893, + 0.15300315618515015, + 0.15292593836784363, + 0.1582789272069931, + 0.2048441767692566 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7743862867355347, + "mean_weights": [ + 0.15205824375152588, + 0.1771804541349411, + 0.15399661660194397, + 0.15122173726558685, + 0.15813913941383362, + 0.20740382373332977 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781898856163025, + "mean_weights": [ + 0.17058727145195007, + 0.1644459068775177, + 0.15141303837299347, + 0.15486885607242584, + 0.15909278392791748, + 0.19959214329719543 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9389331340789795, + "mean_weights": [ + 0.12895603477954865, + 0.15703952312469482, + 0.13832643628120422, + 0.13419194519519806, + 0.1312965601682663, + 0.15331263840198517, + 0.15687686204910278 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941878080368042, + "mean_weights": [ + 0.14007402956485748, + 0.15248695015907288, + 0.14026451110839844, + 0.13737857341766357, + 0.1305495798587799, + 0.14631187915802002, + 0.1529344916343689 + ], + "sources": 7 + }, + { + "entropy_mean": 1.936349630355835, + "mean_weights": [ + 0.12599323689937592, + 0.16029544174671173, + 0.1399340182542801, + 0.13338115811347961, + 0.12978437542915344, + 0.14873966574668884, + 0.16187208890914917 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9385986328125, + "mean_weights": [ + 0.1397915780544281, + 0.15300613641738892, + 0.14345945417881012, + 0.13018865883350372, + 0.1257915198802948, + 0.14249205589294434, + 0.1652706116437912 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939380407333374, + "mean_weights": [ + 0.12855647504329681, + 0.15473899245262146, + 0.1372847855091095, + 0.13797816634178162, + 0.13512183725833893, + 0.15090687572956085, + 0.15541282296180725 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9388402700424194, + "mean_weights": [ + 0.14282521605491638, + 0.15391701459884644, + 0.1462559849023819, + 0.13105326890945435, + 0.12675246596336365, + 0.13615548610687256, + 0.16304057836532593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9313902854919434, + "mean_weights": [ + 0.11694864928722382, + 0.17515113949775696, + 0.15024805068969727, + 0.1394161432981491, + 0.13194763660430908, + 0.1451682150363922, + 0.14112016558647156 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9366493225097656, + "mean_weights": [ + 0.14114168286323547, + 0.16076743602752686, + 0.14226865768432617, + 0.12737974524497986, + 0.12711969017982483, + 0.1412842869758606, + 0.16003848612308502 + ], + "sources": 7 + }, + { + "entropy_mean": 2.064785957336426, + "mean_weights": [ + 0.106104776263237, + 0.15484434366226196, + 0.13487765192985535, + 0.12357068061828613, + 0.12023378908634186, + 0.12704278528690338, + 0.11963208019733429, + 0.11369390785694122 + ], + "sources": 8 + }, + { + "entropy_mean": 2.062044143676758, + "mean_weights": [ + 0.11719764769077301, + 0.14862585067749023, + 0.12573552131652832, + 0.12179906666278839, + 0.11081404983997345, + 0.1260838657617569, + 0.12823458015918732, + 0.12150939553976059 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054962158203125, + "mean_weights": [ + 0.10514847934246063, + 0.16456037759780884, + 0.14191412925720215, + 0.1262327879667282, + 0.1181878000497818, + 0.12522782385349274, + 0.1187765821814537, + 0.09995201230049133 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0610406398773193, + "mean_weights": [ + 0.11198848485946655, + 0.15103809535503387, + 0.13588255643844604, + 0.12388211488723755, + 0.11551118642091751, + 0.12874530255794525, + 0.12429477274417877, + 0.10865749418735504 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0338308811187744, + "mean_weights": [ + 0.0961676836013794, + 0.17635446786880493, + 0.14848342537879944, + 0.13749772310256958, + 0.12027012556791306, + 0.12292921543121338, + 0.10873906314373016, + 0.08955828845500946 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068699598312378, + "mean_weights": [ + 0.11418519914150238, + 0.14292436838150024, + 0.1359846293926239, + 0.13056153059005737, + 0.12100572884082794, + 0.11985942721366882, + 0.12221725285053253, + 0.1132618635892868 + ], + "sources": 8 + }, + { + "entropy_mean": 2.036879539489746, + "mean_weights": [ + 0.09746269136667252, + 0.17786666750907898, + 0.15074899792671204, + 0.13200224936008453, + 0.11842608451843262, + 0.12164927273988724, + 0.1129462867975235, + 0.08889776468276978 + ], + "sources": 8 + }, + { + "entropy_mean": 2.06752347946167, + "mean_weights": [ + 0.11505836993455887, + 0.14705273509025574, + 0.1377231776714325, + 0.12483140081167221, + 0.11869609355926514, + 0.1215740442276001, + 0.12386028468608856, + 0.1112038791179657 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1517844200134277, + "mean_weights": [ + 0.08786984533071518, + 0.16287443041801453, + 0.13886377215385437, + 0.12039706110954285, + 0.10935483872890472, + 0.1119438111782074, + 0.10067413747310638, + 0.07631799578666687, + 0.0917041003704071 + ], + "sources": 9 + }, + { + "entropy_mean": 2.177518129348755, + "mean_weights": [ + 0.10418819636106491, + 0.13366681337356567, + 0.12014994025230408, + 0.11398349702358246, + 0.10732515156269073, + 0.12029910087585449, + 0.11035183072090149, + 0.09321349114179611, + 0.09682197868824005 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1535391807556152, + "mean_weights": [ + 0.08976081013679504, + 0.163266122341156, + 0.13892531394958496, + 0.12143255770206451, + 0.10986581444740295, + 0.11029069125652313, + 0.10418520867824554, + 0.08120252937078476, + 0.0810709297657013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191593647003174, + "mean_weights": [ + 0.10739251971244812, + 0.12338980287313461, + 0.11759916692972183, + 0.1066875234246254, + 0.1060015857219696, + 0.1124890148639679, + 0.11740898340940475, + 0.10738331079483032, + 0.10164809972047806 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1465134620666504, + "mean_weights": [ + 0.08895281702280045, + 0.1655423492193222, + 0.14156755805015564, + 0.12290289252996445, + 0.11169517785310745, + 0.11284246295690536, + 0.1037452220916748, + 0.0791342705488205, + 0.07361726462841034 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1857833862304688, + "mean_weights": [ + 0.10998444259166718, + 0.12038750946521759, + 0.11211743205785751, + 0.11047163605690002, + 0.10957512259483337, + 0.12139561027288437, + 0.11392621695995331, + 0.10294321179389954, + 0.09919881820678711 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1400556564331055, + "mean_weights": [ + 0.08778931945562363, + 0.16802933812141418, + 0.1466498225927353, + 0.12668846547603607, + 0.11206841468811035, + 0.11208955943584442, + 0.10068599879741669, + 0.07686632871627808, + 0.0691327303647995 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04377923160791397, + 0.017109708860516548, + 0.023813940584659576, + 0.02606838569045067, + 0.029579024761915207, + 0.030872320756316185, + 0.034570932388305664, + 0.034764163196086884, + 0.037983279675245285, + 0.025188947096467018, + 0.02627667225897312, + 0.026965361088514328, + 0.02788778953254223, + 0.028485212475061417, + 0.03023744560778141, + 0.030349206179380417, + 0.031768180429935455, + 0.022602196782827377, + 0.02424551732838154, + 0.023387571796774864, + 0.025113960728049278, + 0.02450813725590706, + 0.026638666167855263, + 0.025169003754854202, + 0.027698444202542305, + 0.020798001438379288, + 0.022435806691646576, + 0.02118327096104622, + 0.022681118920445442, + 0.021654823794960976, + 0.023235918954014778, + 0.022377775982022285, + 0.023750189691781998, + 0.018887808546423912, + 0.019677624106407166, + 0.01950881816446781, + 0.020757248625159264, + 0.020426612347364426, + 0.02137788012623787, + 0.021234560757875443, + 0.02207154594361782, + 0.01860968954861164, + 0.019271831959486008, + 0.019469313323497772, + 0.020465917885303497, + 0.020844420418143272, + 0.021911408752202988, + 0.02213398739695549, + 0.024193286895751953, + 0.0209572184830904, + 0.02259150519967079, + 0.022487536072731018, + 0.024306390434503555, + 0.023487187922000885, + 0.02611992135643959, + 0.024889562278985977, + 0.02851196378469467, + 0.023824406787753105, + 0.027104467153549194, + 0.025868752971291542, + 0.03068944811820984, + 0.0271493811160326, + 0.03286270797252655, + 0.028410611674189568 + ], + "loss_nats": 2.1310858726501465, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1613364219665527, + "mean_weights": [ + 0.1255333125591278, + 0.08712290227413177, + 0.09080196917057037, + 0.08838342130184174, + 0.09530800580978394, + 0.09807588160037994, + 0.13817133009433746, + 0.15537576377391815, + 0.12122740596532822 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01049753837287426, + 0.021243074908852577, + 0.028129518032073975, + 0.03624089062213898, + 0.040982723236083984, + 0.04951142147183418, + 0.05135732889175415, + 0.05851924046874046, + 0.014544809237122536, + 0.017167888581752777, + 0.023477500304579735, + 0.02674153819680214, + 0.029818039387464523, + 0.03352004662156105, + 0.03540094941854477, + 0.03946679085493088, + 0.01795734278857708, + 0.020600929856300354, + 0.024303091689944267, + 0.027865095064044, + 0.031844362616539, + 0.035815127193927765, + 0.03825921565294266, + 0.04293550178408623, + 0.018456056714057922, + 0.021564343944191933, + 0.02560909278690815, + 0.029616132378578186, + 0.0305727981030941, + 0.035480253398418427, + 0.037264734506607056, + 0.04259813204407692, + 0.017418840900063515, + 0.021436138078570366, + 0.02998829260468483, + 0.03586751967668533, + 0.03708711639046669, + 0.04331304505467415, + 0.04352690279483795, + 0.051551222801208496, + 0.01969025284051895, + 0.025608548894524574, + 0.030230700969696045, + 0.04091007262468338, + 0.041474223136901855, + 0.05466190353035927, + 0.056570712476968765, + 0.07478146255016327, + 0.012774706818163395, + 0.02793189324438572, + 0.03524070605635643, + 0.05498877540230751, + 0.05920406058430672, + 0.08125203847885132, + 0.08430205285549164, + 0.10916247963905334, + 0.005331806372851133, + 0.026290444657206535, + 0.032364487648010254, + 0.05761713907122612, + 0.059356335550546646, + 0.08842677623033524, + 0.09272631257772446, + 0.12436132878065109 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.294298509601504e-05, + 6.491570093203336e-05, + 6.380436389008537e-05, + 6.04347005719319e-05, + 7.516580808442086e-05, + 7.751413795631379e-05, + 6.965622014831752e-05, + 6.359309190884233e-05, + 6.543538620462641e-05, + 6.0226317145861685e-05, + 5.608194260275923e-05, + 5.3055387979839e-05, + 4.572631951305084e-05, + 4.3727712181862444e-05, + 4.314268517191522e-05, + 4.164052370470017e-05, + 7.694400119362399e-05, + 7.326623017434031e-05, + 6.749018211849034e-05, + 6.173704605316743e-05, + 0.00010233377543045208, + 8.934178913477808e-05, + 7.613583147758618e-05, + 6.664563989033923e-05, + 5.163387322681956e-05, + 4.2875533836195245e-05, + 3.8466609112219885e-05, + 3.59200312232133e-05, + 2.308324837940745e-05, + 2.0679148292401806e-05, + 1.9936938770115376e-05, + 1.9895105651812628e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.725337607349502e-05, + "first_to_last_ratio": 2.130880302861834, + "imbalance_abs_log_ratio": 0.7565351820700446, + "last_quartile_mean": 3.1561311061523156e-05, + "mean": 5.6670258345548064e-05, + "normalized": [ + 1.1106881622493918, + 1.145498588275503, + 1.1258880010928651, + 1.0664271230850946, + 1.3263713679598177, + 1.367809856868302, + 1.2291495077292092, + 1.122159908308202, + 1.154668923611267, + 1.062750001572791, + 0.9896186154790125, + 0.9362122130506743, + 0.8068839078557516, + 0.7716166020495587, + 0.7612932503122151, + 0.7347861986228512, + 1.357749257546284, + 1.2928515294142118, + 1.190927729797305, + 1.089408233799192, + 1.8057756999530474, + 1.5765198843812338, + 1.3434883429213678, + 1.176024987992221, + 0.9111282484717285, + 0.7565791137700626, + 0.678779490957478, + 0.6338427293588494, + 0.4073256246452393, + 0.3649030178459797, + 0.3518060328673549, + 0.35106784815593767 + ], + "population_cv": 0.34914284904214865 + }, + "activation_output_rms_by_block": [ + 0.10241243988275528, + 0.1819411665201187, + 0.24157075583934784, + 0.26444777846336365, + 0.10636615753173828, + 0.13034582138061523, + 0.14029134809970856, + 0.15042904019355774, + 0.04812335968017578, + 0.07954440265893936, + 0.10673297941684723, + 0.14089107513427734, + 0.06885477155447006, + 0.10946234315633774, + 0.13876035809516907, + 0.1687854677438736, + 0.05059739947319031, + 0.07094386219978333, + 0.09166282415390015, + 0.12053482979536057, + 0.048727843910455704, + 0.07109228521585464, + 0.09690473228693008, + 0.11853098124265671, + 0.05259029567241669, + 0.08889466524124146, + 0.12455403059720993, + 0.1728687882423401, + 0.06489895284175873, + 0.14679722487926483, + 0.23576539754867554, + 0.35208994150161743 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16472556348890066, + "first_to_last_ratio": 1.0640676779683806, + "imbalance_abs_log_ratio": 0.0620989960097457, + "last_quartile_mean": 0.15480741206556559, + "mean": 0.127700416254811, + "normalized": [ + 0.8019742056157706, + 1.4247499879489567, + 1.8916990478505735, + 2.0708450780276983, + 0.8329350886335192, + 1.020715712629516, + 1.098597422108428, + 1.177983945591802, + 0.3768457542389787, + 0.6228985385624584, + 0.8358076077361742, + 1.1032937813855357, + 0.5391898755997675, + 0.8571807858317286, + 1.086608502655851, + 1.3217299731199175, + 0.396219534415841, + 0.5555491851978249, + 0.7177958133746243, + 0.9438875246486875, + 0.3815793662976406, + 0.5567114603134765, + 0.7588442945524017, + 0.9281957312194052, + 0.411825561848436, + 0.6961188369493074, + 0.9753611949758815, + 1.3537057537651325, + 0.5082125395132666, + 1.1495438244018596, + 1.8462382853806345, + 2.7571557856089037 + ], + "population_cv": 0.532159231456062 + }, + "bits_per_byte": 2.0533380169036586, + "branch_output_rms_by_sublayer": [ + 0.05045261234045029, + 0.0860176682472229, + 0.09517527371644974, + 0.031032679602503777, + 0.08140470087528229, + 0.011454286053776741, + 0.035610511898994446, + 0.03601690009236336, + 0.1042611226439476, + 0.01171922218054533, + 0.050615254789590836, + 0.008982792496681213, + 0.04264022409915924, + 0.01206296682357788, + 0.03225839138031006, + 0.026524480432271957, + 0.040291935205459595, + 0.024220434948801994, + 0.036216676235198975, + 0.026388581842184067, + 0.02871432714164257, + 0.03031591884791851, + 0.03194425627589226, + 0.032942064106464386, + 0.05663999170064926, + 0.029961559921503067, + 0.03894488140940666, + 0.02937687374651432, + 0.03595346212387085, + 0.031781863421201706, + 0.03414257988333702, + 0.035546254366636276, + 0.03680795058608055, + 0.030760737136006355, + 0.022694969549775124, + 0.03042135201394558, + 0.022243596613407135, + 0.03508196026086807, + 0.02950950525701046, + 0.04756028205156326, + 0.035728659480810165, + 0.0295625701546669, + 0.024367405101656914, + 0.03350908309221268, + 0.0301151555031538, + 0.038334235548973083, + 0.022455131635069847, + 0.043223608285188675, + 0.025910304859280586, + 0.04338310658931732, + 0.025968369096517563, + 0.046161677688360214, + 0.01597527042031288, + 0.050398919731378555, + 0.015805471688508987, + 0.062427736818790436, + 0.013562481850385666, + 0.0627857968211174, + 0.02184087596833706, + 0.08928166329860687, + 0.012465676292777061, + 0.09919292479753494, + 0.01876053772866726, + 0.13455425202846527 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00034319970120439075, + 0.0002207216513437987, + 0.00019628105547348242, + 0.00016658832628724645, + 0.00021517433675904697, + 0.00019244240293185932, + 0.0001402257608174488, + 0.00015224450647613049, + 0.00018347185792801386, + 0.00016427150761043105, + 0.00015319316983403424, + 0.00015483500137268905, + 0.00013970482137555046, + 0.00012521709576159806, + 0.00014171058058500498, + 0.00014304258084243227, + 0.00024403444477986432, + 0.0002443248488541129, + 0.0002486173108247806, + 0.0002822915928542698, + 0.00039666535280576766, + 0.0003607137256748422, + 0.0003593407909791253, + 0.00030791631238161, + 0.0002432000733486066, + 0.00020051667741228585, + 0.0001766506293272091, + 0.00017705575210645458, + 0.00010859706957547057, + 0.0001081942080930384, + 0.00010514694748959741, + 0.00011321633644894032 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002033597176616755, + "first_to_last_ratio": 1.3198987369921267, + "imbalance_abs_log_ratio": 0.2775550192556754, + "last_quartile_mean": 0.00015407221172520036, + "mean": 0.00020340020092372293, + "normalized": [ + 1.6873124984428802, + 1.085159455799021, + 0.9649993194799731, + 0.8190175109498476, + 1.0578865496781857, + 0.9461269067478802, + 0.6894081725614166, + 0.7484973258862397, + 0.9020239758603661, + 0.8076270665633928, + 0.7531613495872759, + 0.7612332764152662, + 0.6868470175599342, + 0.6156193317063423, + 0.6967081642075066, + 0.703256831570558, + 1.1997748461978148, + 1.2012025934317396, + 1.2223061220967748, + 1.3878629006867693, + 1.9501718828415566, + 1.7734187283822467, + 1.7666688102922847, + 1.5138446814862372, + 1.1956727291523621, + 0.985823398903532, + 0.8684879785023163, + 0.8704797306117324, + 0.5339083692262209, + 0.5319277345926137, + 0.516946133839015, + 0.5566186067406962 + ], + "population_cv": 0.38898494230064556 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.39672940969467163, + "mean_weights": [ + 0.854300856590271, + 0.1456991732120514 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6258258819580078, + "mean_weights": [ + 0.5948886871337891, + 0.40511131286621094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5053832530975342, + "mean_weights": [ + 0.778947651386261, + 0.221052348613739 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6737473011016846, + "mean_weights": [ + 0.5406605005264282, + 0.4593394994735718 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6529731750488281, + "mean_weights": [ + 0.5619670748710632, + 0.43803298473358154 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6891955733299255, + "mean_weights": [ + 0.5099825263023376, + 0.49001747369766235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48565131425857544, + "mean_weights": [ + 0.7992967367172241, + 0.20070326328277588 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6875144243240356, + "mean_weights": [ + 0.5191382169723511, + 0.4808618128299713 + ], + "sources": 2 + }, + { + "entropy_mean": 1.048142671585083, + "mean_weights": [ + 0.3059995770454407, + 0.2586204409599304, + 0.4353799819946289 + ], + "sources": 3 + }, + { + "entropy_mean": 1.078475832939148, + "mean_weights": [ + 0.2919672429561615, + 0.28997206687927246, + 0.4180607199668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0040717124938965, + "mean_weights": [ + 0.27878478169441223, + 0.21860124170780182, + 0.5026139616966248 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0727062225341797, + "mean_weights": [ + 0.3306647837162018, + 0.2443503737449646, + 0.42498481273651123 + ], + "sources": 3 + }, + { + "entropy_mean": 1.036304235458374, + "mean_weights": [ + 0.3365095853805542, + 0.20819777250289917, + 0.455292671918869 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071068525314331, + "mean_weights": [ + 0.3291535973548889, + 0.24366843700408936, + 0.42717796564102173 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0272725820541382, + "mean_weights": [ + 0.29237720370292664, + 0.2188395857810974, + 0.48878318071365356 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856674909591675, + "mean_weights": [ + 0.3748946189880371, + 0.26831528544425964, + 0.35679009556770325 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3326265811920166, + "mean_weights": [ + 0.26391589641571045, + 0.14978256821632385, + 0.2545396089553833, + 0.3317619264125824 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3771977424621582, + "mean_weights": [ + 0.27525436878204346, + 0.2067744880914688, + 0.23979949951171875, + 0.2781716585159302 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3576545715332031, + "mean_weights": [ + 0.2848266065120697, + 0.17341844737529755, + 0.23106196522712708, + 0.3106929659843445 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772028684616089, + "mean_weights": [ + 0.2702856659889221, + 0.21318203210830688, + 0.23082655668258667, + 0.2857057452201843 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3277652263641357, + "mean_weights": [ + 0.3119839131832123, + 0.16646917164325714, + 0.1912841796875, + 0.3302627205848694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3750126361846924, + "mean_weights": [ + 0.2621780037879944, + 0.21051618456840515, + 0.23754313588142395, + 0.2897626757621765 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568847179412842, + "mean_weights": [ + 0.2628105878829956, + 0.1954234540462494, + 0.21842113137245178, + 0.3233448266983032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3733241558074951, + "mean_weights": [ + 0.25842776894569397, + 0.22964225709438324, + 0.21348324418067932, + 0.2984467148780823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5855276584625244, + "mean_weights": [ + 0.18060272932052612, + 0.1563272476196289, + 0.18859915435314178, + 0.24991817772388458, + 0.2245527058839798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5906484127044678, + "mean_weights": [ + 0.19773373007774353, + 0.18790484964847565, + 0.15935488045215607, + 0.18813443183898926, + 0.2668721079826355 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5916568040847778, + "mean_weights": [ + 0.2148856818675995, + 0.14808836579322815, + 0.18413493037223816, + 0.22754491865634918, + 0.22534608840942383 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5474352836608887, + "mean_weights": [ + 0.19825294613838196, + 0.1729484647512436, + 0.11353601515293121, + 0.19613605737686157, + 0.3191264867782593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5919160842895508, + "mean_weights": [ + 0.21136541664600372, + 0.1572093367576599, + 0.18091553449630737, + 0.2257765829563141, + 0.2247331142425537 + ], + "sources": 5 + }, + { + "entropy_mean": 1.542572021484375, + "mean_weights": [ + 0.2053474485874176, + 0.19789552688598633, + 0.10896679759025574, + 0.1649853140115738, + 0.32280486822128296 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5774884223937988, + "mean_weights": [ + 0.1874420940876007, + 0.14777487516403198, + 0.18392466008663177, + 0.23523849248886108, + 0.24561989307403564 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585869312286377, + "mean_weights": [ + 0.2123720645904541, + 0.19153764843940735, + 0.14032581448554993, + 0.1918477714061737, + 0.26391667127609253 + ], + "sources": 5 + }, + { + "entropy_mean": 1.747511625289917, + "mean_weights": [ + 0.13209694623947144, + 0.11971477419137955, + 0.14703533053398132, + 0.17972922325134277, + 0.15732964873313904, + 0.2640940546989441 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6791677474975586, + "mean_weights": [ + 0.1429731547832489, + 0.128341943025589, + 0.08682706952095032, + 0.12974600493907928, + 0.16802877187728882, + 0.3440830111503601 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7079570293426514, + "mean_weights": [ + 0.12207645177841187, + 0.11454501748085022, + 0.14774824678897858, + 0.1675407588481903, + 0.1342848241329193, + 0.3138047158718109 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689455270767212, + "mean_weights": [ + 0.14791087806224823, + 0.15263086557388306, + 0.0942135751247406, + 0.12123478949069977, + 0.14674627780914307, + 0.3372636139392853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6673545837402344, + "mean_weights": [ + 0.119766965508461, + 0.10584045946598053, + 0.1391725242137909, + 0.15867865085601807, + 0.1261359602212906, + 0.3504054546356201 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7276721000671387, + "mean_weights": [ + 0.16197484731674194, + 0.16130125522613525, + 0.10335841774940491, + 0.12991490960121155, + 0.15279719233512878, + 0.2906533479690552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.655259609222412, + "mean_weights": [ + 0.12291313707828522, + 0.10535868257284164, + 0.14431551098823547, + 0.1644165962934494, + 0.11736815422773361, + 0.34562790393829346 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7456457614898682, + "mean_weights": [ + 0.17144063115119934, + 0.17272350192070007, + 0.09372985363006592, + 0.13482090830802917, + 0.1837429702281952, + 0.2435421198606491 + ], + "sources": 6 + }, + { + "entropy_mean": 1.8196223974227905, + "mean_weights": [ + 0.097611203789711, + 0.08662202209234238, + 0.10460364818572998, + 0.12231925129890442, + 0.09176328778266907, + 0.22007866203784943, + 0.27700191736221313 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8201665878295898, + "mean_weights": [ + 0.12032124400138855, + 0.12977322936058044, + 0.07435770332813263, + 0.09230788052082062, + 0.1133352667093277, + 0.15118572115898132, + 0.31871896982192993 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7483060359954834, + "mean_weights": [ + 0.09602338075637817, + 0.07951871305704117, + 0.09892410784959793, + 0.11023538559675217, + 0.08284324407577515, + 0.17450374364852905, + 0.35795143246650696 + ], + "sources": 7 + }, + { + "entropy_mean": 1.851926565170288, + "mean_weights": [ + 0.14006076753139496, + 0.14811867475509644, + 0.08932580053806305, + 0.09198777377605438, + 0.11175547540187836, + 0.12843111157417297, + 0.29032039642333984 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7523608207702637, + "mean_weights": [ + 0.10785886645317078, + 0.08607444912195206, + 0.09595336019992828, + 0.10649852454662323, + 0.0838143527507782, + 0.14955362677574158, + 0.3702468276023865 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8693451881408691, + "mean_weights": [ + 0.1378144919872284, + 0.1244172528386116, + 0.09968579560518265, + 0.10974414646625519, + 0.11655184626579285, + 0.130243182182312, + 0.2815432548522949 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7898914813995361, + "mean_weights": [ + 0.10115306824445724, + 0.09615937620401382, + 0.11218561232089996, + 0.1163020133972168, + 0.09238721430301666, + 0.14564841985702515, + 0.33616429567337036 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9094455242156982, + "mean_weights": [ + 0.14588291943073273, + 0.14179158210754395, + 0.1023377850651741, + 0.11511939018964767, + 0.13270339369773865, + 0.13753530383110046, + 0.22462961077690125 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9082129001617432, + "mean_weights": [ + 0.09145341068506241, + 0.07698438316583633, + 0.08525429666042328, + 0.08438773453235626, + 0.06885559856891632, + 0.11482041329145432, + 0.2519546449184418, + 0.2262895107269287 + ], + "sources": 8 + }, + { + "entropy_mean": 2.025789499282837, + "mean_weights": [ + 0.11047068238258362, + 0.12069687247276306, + 0.09073689579963684, + 0.09690164029598236, + 0.10987945646047592, + 0.11353722959756851, + 0.15697816014289856, + 0.20079906284809113 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9302079677581787, + "mean_weights": [ + 0.0971018448472023, + 0.08240298926830292, + 0.0830029845237732, + 0.08926018327474594, + 0.07480385154485703, + 0.10335960984230042, + 0.22211775183677673, + 0.24795079231262207 + ], + "sources": 8 + }, + { + "entropy_mean": 2.010532855987549, + "mean_weights": [ + 0.10700424760580063, + 0.09611141681671143, + 0.09769981354475021, + 0.10596007108688354, + 0.09634587168693542, + 0.11705268919467926, + 0.17551329731941223, + 0.20431260764598846 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9408204555511475, + "mean_weights": [ + 0.08590779453516006, + 0.09530991315841675, + 0.09476494789123535, + 0.1030702069401741, + 0.08048167079687119, + 0.12287819385528564, + 0.22555866837501526, + 0.19202858209609985 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0024678707122803, + "mean_weights": [ + 0.10720442980527878, + 0.09982497990131378, + 0.10896851122379303, + 0.1345594823360443, + 0.09260416775941849, + 0.1016254797577858, + 0.1643495261669159, + 0.19086343050003052 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9389225244522095, + "mean_weights": [ + 0.08632777631282806, + 0.09041796624660492, + 0.09941554069519043, + 0.10805264860391617, + 0.07865813374519348, + 0.11747924983501434, + 0.2661469578742981, + 0.1535017192363739 + ], + "sources": 8 + }, + { + "entropy_mean": 2.008256673812866, + "mean_weights": [ + 0.1016545370221138, + 0.10481878370046616, + 0.10823093354701996, + 0.1270347535610199, + 0.0919865146279335, + 0.14368976652622223, + 0.20522482693195343, + 0.11735989153385162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.094825267791748, + "mean_weights": [ + 0.08519881963729858, + 0.09903685003519058, + 0.08900875598192215, + 0.09697237610816956, + 0.071251779794693, + 0.09201781451702118, + 0.19253811240196228, + 0.1091965064406395, + 0.16477900743484497 + ], + "sources": 9 + }, + { + "entropy_mean": 2.140371322631836, + "mean_weights": [ + 0.10001853853464127, + 0.10924500226974487, + 0.10232416540384293, + 0.11669869720935822, + 0.10123330354690552, + 0.1287832260131836, + 0.14476822316646576, + 0.09069216251373291, + 0.10623672604560852 + ], + "sources": 9 + }, + { + "entropy_mean": 2.117004632949829, + "mean_weights": [ + 0.09184888750314713, + 0.09620077908039093, + 0.09620209038257599, + 0.10222946852445602, + 0.06924223154783249, + 0.0821654349565506, + 0.17978665232658386, + 0.14873988926410675, + 0.13358455896377563 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1423683166503906, + "mean_weights": [ + 0.09596475213766098, + 0.07681103050708771, + 0.11225531250238419, + 0.11592596024274826, + 0.0810609757900238, + 0.11778091639280319, + 0.1649385243654251, + 0.12366649508476257, + 0.11159604042768478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.093705415725708, + "mean_weights": [ + 0.08838655799627304, + 0.09557235240936279, + 0.09992480278015137, + 0.10721968859434128, + 0.07486072182655334, + 0.09290309250354767, + 0.21980023384094238, + 0.12490785866975784, + 0.09642469882965088 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1627297401428223, + "mean_weights": [ + 0.0994885265827179, + 0.09216944873332977, + 0.11588040739297867, + 0.12231601774692535, + 0.09873615205287933, + 0.13436099886894226, + 0.13960200548171997, + 0.1002899631857872, + 0.09715649485588074 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1300058364868164, + "mean_weights": [ + 0.089942067861557, + 0.1065598577260971, + 0.11013364791870117, + 0.12377548217773438, + 0.08726659417152405, + 0.11411209404468536, + 0.1931532621383667, + 0.10132836550474167, + 0.07372863590717316 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05571148172020912, + 0.04887383058667183, + 0.061161335557699203, + 0.06501953303813934, + 0.09595561027526855, + 0.11691275984048843, + 0.12880638241767883, + 0.07614059001207352, + 0.13726800680160522, + 0.09499973803758621, + 0.10176635533571243, + 0.09968593716621399, + 0.09925490617752075, + 0.09547581523656845, + 0.10055620223283768, + 0.09975490719079971, + 0.0990053340792656, + 0.06202486902475357, + 0.07309409230947495, + 0.06455004960298538, + 0.07328634709119797, + 0.062386661767959595, + 0.07434380799531937, + 0.07036563754081726, + 0.07861478626728058, + 0.05747222527861595, + 0.06236656382679939, + 0.057312749326229095, + 0.06600815057754517, + 0.06149744614958763, + 0.07462523877620697, + 0.06584152579307556, + 0.07498960196971893, + 0.04952141270041466, + 0.05233807489275932, + 0.04880477488040924, + 0.05626988410949707, + 0.0513896569609642, + 0.061378348618745804, + 0.057161398231983185, + 0.06856362521648407, + 0.04507989436388016, + 0.04801172763109207, + 0.04335150495171547, + 0.0518198125064373, + 0.047289349138736725, + 0.05105668306350708, + 0.05114771053195, + 0.054886870086193085, + 0.042588189244270325, + 0.04603559896349907, + 0.044108107686042786, + 0.04718422889709473, + 0.05026337504386902, + 0.05210597813129425, + 0.05349528789520264, + 0.05379685014486313, + 0.04645879194140434, + 0.05137190222740173, + 0.05323651432991028, + 0.05757243186235428, + 0.05820684880018234, + 0.06228902190923691, + 0.058245208114385605 + ], + "loss_nats": 1.4232654571533203, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.9258099794387817, + "mean_weights": [ + 0.11242504417896271, + 0.0684729665517807, + 0.07014584541320801, + 0.059657223522663116, + 0.05741160362958908, + 0.05127572640776634, + 0.12547241151332855, + 0.2706507742404938, + 0.18448840081691742 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.05045261234045029, + 0.10241243988275528, + 0.16565440595149994, + 0.1819411665201187, + 0.23719695210456848, + 0.24157075583934784, + 0.24811406433582306, + 0.26444777846336365, + 0.1042611226439476, + 0.10636615753173828, + 0.12873972952365875, + 0.13034582138061523, + 0.1382850706577301, + 0.14029134809970856, + 0.14404986798763275, + 0.15042904019355774, + 0.040291935205459595, + 0.04812335968017578, + 0.06678888201713562, + 0.07954440265893936, + 0.09121427685022354, + 0.10673297941684723, + 0.12122578918933868, + 0.14089107513427734, + 0.05663999170064926, + 0.06885477155447006, + 0.09547740966081619, + 0.10946234315633774, + 0.12204843014478683, + 0.13876035809516907, + 0.14941293001174927, + 0.1687854677438736, + 0.03680795058608055, + 0.05059739947319031, + 0.055156365036964417, + 0.07094386219978333, + 0.07373107224702835, + 0.09166282415390015, + 0.09760652482509613, + 0.12053482979536057, + 0.035728659480810165, + 0.048727843910455704, + 0.05559300258755684, + 0.07109228521585464, + 0.08069392293691635, + 0.09690473228693008, + 0.10087756812572479, + 0.11853098124265671, + 0.025910304859280586, + 0.05259029567241669, + 0.060516636818647385, + 0.08889466524124146, + 0.09090615063905716, + 0.12455403059720993, + 0.1273101419210434, + 0.1728687882423401, + 0.013562481850385666, + 0.06489895284175873, + 0.07113923132419586, + 0.14679722487926483, + 0.14901472628116608, + 0.23576539754867554, + 0.23766301572322845, + 0.35208994150161743 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.705962055595592e-05, + 4.8540012357989326e-05, + 4.8112680815393105e-05, + 4.5881148253101856e-05, + 6.218662019819021e-05, + 6.19510465185158e-05, + 5.5873108067316934e-05, + 5.11586076754611e-05, + 5.377902198233642e-05, + 4.923442247672938e-05, + 4.591418110067025e-05, + 4.340316809248179e-05, + 3.853498856187798e-05, + 3.637185727711767e-05, + 3.548145105014555e-05, + 3.440591899561696e-05, + 7.517015910707414e-05, + 7.005458610365167e-05, + 6.350460171233863e-05, + 5.674128988175653e-05, + 0.00010789759107865393, + 9.314908675150946e-05, + 7.818391168257222e-05, + 6.73785325489007e-05, + 5.735654485761188e-05, + 4.582547626341693e-05, + 3.993835343862884e-05, + 3.59240839316044e-05, + 2.40640074480325e-05, + 2.1051846488262527e-05, + 1.9940822312491946e-05, + 1.993459591176361e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.259535555524053e-05, + "first_to_last_ratio": 1.59358297228639, + "imbalance_abs_log_ratio": 0.46598492272985975, + "last_quartile_mean": 3.300446633147658e-05, + "mean": 5.106260448428657e-05, + "normalized": [ + 0.921606350307445, + 0.9505980521014451, + 0.9422292752458163, + 0.8985273805847644, + 1.2178505351666271, + 1.2132371065714032, + 1.0942079557361923, + 1.0018801076079868, + 1.0531977858451338, + 0.9641972432463806, + 0.8991742893725557, + 0.8499991046449304, + 0.7546616344988103, + 0.712299296999438, + 0.6948617566317875, + 0.6737987484795189, + 1.4721176067352022, + 1.3719352314903852, + 1.2436616258358075, + 1.111210257581308, + 2.1130451955668876, + 1.8242133884920444, + 1.531138344238427, + 1.319527925169485, + 1.1232592899812257, + 0.8974371112918603, + 0.7821448561426008, + 0.7035301918972675, + 0.4712647874324093, + 0.41227521981846393, + 0.3905171409466258, + 0.3903952043397641 + ], + "population_cv": 0.3859968358313239 + }, + "activation_output_rms_by_block": [ + 0.16142085194587708, + 0.2773628532886505, + 0.3768865168094635, + 0.40868353843688965, + 0.14338424801826477, + 0.16827327013015747, + 0.18227216601371765, + 0.19825012981891632, + 0.06985136866569519, + 0.1086883395910263, + 0.14860449731349945, + 0.21192558109760284, + 0.08672910183668137, + 0.13778631389141083, + 0.18114051222801208, + 0.23228809237480164, + 0.062035780400037766, + 0.0909910649061203, + 0.12021689862012863, + 0.1606491208076477, + 0.061457011848688126, + 0.092374786734581, + 0.12395651638507843, + 0.15162576735019684, + 0.06645520776510239, + 0.10725662857294083, + 0.14660172164440155, + 0.20186874270439148, + 0.07922796159982681, + 0.17203646898269653, + 0.2743971645832062, + 0.41855114698410034 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23956669680774212, + "first_to_last_ratio": 1.306969485354029, + "imbalance_abs_log_ratio": 0.26771108728004156, + "last_quartile_mean": 0.18329938035458326, + "mean": 0.1694765428546816, + "normalized": [ + 0.9524672218755847, + 1.6365855039093904, + 2.223827029163714, + 2.4114460417529116, + 0.846041851002414, + 0.9929000633110867, + 1.0755008506988941, + 1.1697791710850909, + 0.41215950885657104, + 0.6413178942658843, + 0.8768440446706599, + 1.250471466598887, + 0.5117469378110197, + 0.8130111198312341, + 1.068823503104685, + 1.370620904003087, + 0.36604346156170187, + 0.5368947429151949, + 0.7093424057110318, + 0.9479136056332998, + 0.36262842522923483, + 0.5450594234376621, + 0.7314081010689808, + 0.8946711137494054, + 0.392120388141766, + 0.6328700524939813, + 0.8650266235965518, + 1.1911308745392848, + 0.46748629789882584, + 1.0151048993854561, + 1.6190863936756676, + 2.4696700790208403 + ], + "population_cv": 0.5508327874312803 + }, + "bits_per_byte": 1.9147359946947653, + "branch_output_rms_by_sublayer": [ + 0.08130212873220444, + 0.13576549291610718, + 0.1432749330997467, + 0.041234951466321945, + 0.12958019971847534, + 0.023600688204169273, + 0.04409155994653702, + 0.0554339736700058, + 0.14100632071495056, + 0.01666521653532982, + 0.0481393076479435, + 0.008341255597770214, + 0.04208913818001747, + 0.01377889234572649, + 0.04236427694559097, + 0.03695881366729736, + 0.05848926678299904, + 0.033706482499837875, + 0.04504085332155228, + 0.0386466309428215, + 0.04397343844175339, + 0.04499480128288269, + 0.05665678158402443, + 0.057638294994831085, + 0.06478945910930634, + 0.04709118604660034, + 0.04678606986999512, + 0.044345736503601074, + 0.04817425087094307, + 0.04982399195432663, + 0.049439266324043274, + 0.060589101165533066, + 0.04376237466931343, + 0.040811967104673386, + 0.02850259095430374, + 0.04198655113577843, + 0.02922668866813183, + 0.05064959079027176, + 0.04021619260311127, + 0.07217346876859665, + 0.04409762844443321, + 0.039372652769088745, + 0.030135832726955414, + 0.04702961444854736, + 0.03803896903991699, + 0.054020222276449203, + 0.030100898817181587, + 0.06186771020293236, + 0.03295852243900299, + 0.054687224328517914, + 0.0306381918489933, + 0.05866995081305504, + 0.02206242084503174, + 0.06297128647565842, + 0.02074088715016842, + 0.07938925921916962, + 0.021799447014927864, + 0.07573848962783813, + 0.026608876883983612, + 0.1105475053191185, + 0.01902451552450657, + 0.12047550827264786, + 0.02519184537231922, + 0.16982334852218628 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00023209204329478853, + 0.00014705811075659293, + 0.0001443553349415038, + 0.0001409677954299832, + 0.0001547598070491459, + 0.00010679063100428474, + 0.00010173131038502872, + 0.00012885752112210694, + 0.00018269174364617834, + 0.00015061935227176197, + 0.00015101151039961197, + 0.0001756105532961918, + 0.00014068815971577485, + 0.0001326364304799064, + 0.00015121016067746042, + 0.00016411832907359437, + 0.0002779928432077, + 0.00028472923376674154, + 0.0003007215693300081, + 0.0003372879732191828, + 0.0005019822813973469, + 0.0004655562424498461, + 0.00043909856222474334, + 0.00038874448014086744, + 0.0003187708239967095, + 0.00026126643200496464, + 0.0002213996139509037, + 0.00021927053392099255, + 0.00013957865743180746, + 0.00013181726813679266, + 0.00012444014246083002, + 0.00012714374135915476 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00014457656924792933, + "first_to_last_ratio": 0.74925318033778, + "imbalance_abs_log_ratio": 0.28867832809930477, + "last_quartile_mean": 0.0001929609016577694, + "mean": 0.00021703122476695333, + "normalized": [ + 1.069394708268395, + 0.6775896459806783, + 0.6651362498484334, + 0.6495277146472977, + 0.7130762276963877, + 0.49205192072687143, + 0.46874043352179906, + 0.5937280281234798, + 0.8417763104933472, + 0.6939985360792947, + 0.6958054563889003, + 0.8091487917683791, + 0.6482392562030871, + 0.6111398515228881, + 0.6967207637510633, + 0.7561968525488604, + 1.2808886993390323, + 1.311927507539442, + 1.3856143034391522, + 1.554098833388423, + 2.3129495856477433, + 2.145111805684907, + 2.0232045536131698, + 1.7911914774397033, + 1.468778625467388, + 1.2038195530873992, + 1.0201279294656385, + 1.0103179123485286, + 0.6431270780584042, + 0.6073654529588411, + 0.573374373178113, + 0.5858315617749513 + ], + "population_cv": 0.5013169471957857 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22879935801029205, + "mean_weights": [ + 0.9326952695846558, + 0.06730473041534424 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5610548257827759, + "mean_weights": [ + 0.6759846210479736, + 0.3240153193473816 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43457120656967163, + "mean_weights": [ + 0.8295697569847107, + 0.1704302579164505 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6696063280105591, + "mean_weights": [ + 0.545992374420166, + 0.45400765538215637 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6690812110900879, + "mean_weights": [ + 0.5202739238739014, + 0.47972607612609863 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865538358688354, + "mean_weights": [ + 0.5361902713775635, + 0.4638097584247589 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4152516722679138, + "mean_weights": [ + 0.8452726602554321, + 0.15472736954689026 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6853918433189392, + "mean_weights": [ + 0.5220736265182495, + 0.4779263138771057 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0265861749649048, + "mean_weights": [ + 0.3228098750114441, + 0.23137259483337402, + 0.4458175599575043 + ], + "sources": 3 + }, + { + "entropy_mean": 1.066486120223999, + "mean_weights": [ + 0.2909930944442749, + 0.26484501361846924, + 0.44416186213493347 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8915231227874756, + "mean_weights": [ + 0.21873247623443604, + 0.16816270351409912, + 0.6131048202514648 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0651111602783203, + "mean_weights": [ + 0.3639928698539734, + 0.22285741567611694, + 0.41314971446990967 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0005524158477783, + "mean_weights": [ + 0.3166183531284332, + 0.18387392163276672, + 0.49950775504112244 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0648009777069092, + "mean_weights": [ + 0.35599708557128906, + 0.22703218460083008, + 0.41697072982788086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0006463527679443, + "mean_weights": [ + 0.28650256991386414, + 0.19727063179016113, + 0.5162267684936523 + ], + "sources": 3 + }, + { + "entropy_mean": 1.072684407234192, + "mean_weights": [ + 0.41332390904426575, + 0.2456190586090088, + 0.34105703234672546 + ], + "sources": 3 + }, + { + "entropy_mean": 1.304518699645996, + "mean_weights": [ + 0.2721603810787201, + 0.12663133442401886, + 0.2481587529182434, + 0.35304954648017883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3646502494812012, + "mean_weights": [ + 0.29890745878219604, + 0.18950369954109192, + 0.2219487726688385, + 0.2896400988101959 + ], + "sources": 4 + }, + { + "entropy_mean": 1.331923484802246, + "mean_weights": [ + 0.31035515666007996, + 0.14580613374710083, + 0.22454026341438293, + 0.31929847598075867 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3639233112335205, + "mean_weights": [ + 0.2952677309513092, + 0.19548878073692322, + 0.21280673146247864, + 0.29643672704696655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2764989137649536, + "mean_weights": [ + 0.3656603693962097, + 0.14041098952293396, + 0.17602922022342682, + 0.3178994059562683 + ], + "sources": 4 + }, + { + "entropy_mean": 1.364103078842163, + "mean_weights": [ + 0.27470695972442627, + 0.18989428877830505, + 0.23374854028224945, + 0.30165016651153564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3429455757141113, + "mean_weights": [ + 0.2830207943916321, + 0.17677831649780273, + 0.23265977203845978, + 0.3075411319732666 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3732160329818726, + "mean_weights": [ + 0.2642192244529724, + 0.22012627124786377, + 0.22637531161308289, + 0.28927919268608093 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5599325895309448, + "mean_weights": [ + 0.18314200639724731, + 0.13039523363113403, + 0.18218961358070374, + 0.26679670810699463, + 0.2374764084815979 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5772080421447754, + "mean_weights": [ + 0.19838765263557434, + 0.18937477469444275, + 0.15325719118118286, + 0.16343195736408234, + 0.2955484390258789 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5662866830825806, + "mean_weights": [ + 0.22973918914794922, + 0.12454323470592499, + 0.16623356938362122, + 0.21617932617664337, + 0.2633046507835388 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5421115159988403, + "mean_weights": [ + 0.2021060585975647, + 0.1677371859550476, + 0.11891372501850128, + 0.17452098429203033, + 0.3367220163345337 + ], + "sources": 5 + }, + { + "entropy_mean": 1.564535140991211, + "mean_weights": [ + 0.22419840097427368, + 0.12987476587295532, + 0.1717071682214737, + 0.21590615808963776, + 0.25831347703933716 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5267750024795532, + "mean_weights": [ + 0.215010404586792, + 0.19819971919059753, + 0.10917140543460846, + 0.13804513216018677, + 0.33957335352897644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.556459903717041, + "mean_weights": [ + 0.2037975788116455, + 0.12750469148159027, + 0.1812792420387268, + 0.22193393111228943, + 0.2654845714569092 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5851528644561768, + "mean_weights": [ + 0.21840542554855347, + 0.1923592984676361, + 0.1442098617553711, + 0.1790001392364502, + 0.26602524518966675 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7158764600753784, + "mean_weights": [ + 0.13390135765075684, + 0.10310791432857513, + 0.14190614223480225, + 0.16341811418533325, + 0.15331929922103882, + 0.3043471872806549 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6410574913024902, + "mean_weights": [ + 0.14665581285953522, + 0.12231245636940002, + 0.08100371062755585, + 0.11105543375015259, + 0.16015371680259705, + 0.3788188695907593 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6537573337554932, + "mean_weights": [ + 0.12217442691326141, + 0.09171081334352493, + 0.1348741054534912, + 0.16064059734344482, + 0.1268058568239212, + 0.363794207572937 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6571812629699707, + "mean_weights": [ + 0.15173450112342834, + 0.14253422617912292, + 0.09320533275604248, + 0.10436245799064636, + 0.14149092137813568, + 0.3666725754737854 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6046829223632812, + "mean_weights": [ + 0.12253664433956146, + 0.08343374729156494, + 0.1302124410867691, + 0.14439499378204346, + 0.11938183009624481, + 0.4000403583049774 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7041404247283936, + "mean_weights": [ + 0.1659325659275055, + 0.15505465865135193, + 0.10219550132751465, + 0.11347830295562744, + 0.1465306580066681, + 0.3168083429336548 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5808616876602173, + "mean_weights": [ + 0.12713032960891724, + 0.08526259660720825, + 0.1294248104095459, + 0.1489948332309723, + 0.10688205063343048, + 0.40230536460876465 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7305450439453125, + "mean_weights": [ + 0.17891491949558258, + 0.17673785984516144, + 0.09075747430324554, + 0.11627361923456192, + 0.17774534225463867, + 0.25957077741622925 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7513642311096191, + "mean_weights": [ + 0.09819091856479645, + 0.06520214676856995, + 0.091708242893219, + 0.11103156208992004, + 0.08388154953718185, + 0.22660189867019653, + 0.3233836889266968 + ], + "sources": 7 + }, + { + "entropy_mean": 1.766061782836914, + "mean_weights": [ + 0.12103402614593506, + 0.1159343495965004, + 0.06956179440021515, + 0.07680170238018036, + 0.10650800168514252, + 0.1487376093864441, + 0.36142250895500183 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6586570739746094, + "mean_weights": [ + 0.09605097770690918, + 0.06012085825204849, + 0.08562932163476944, + 0.10102558881044388, + 0.07161714881658554, + 0.17364460229873657, + 0.4119114875793457 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7821578979492188, + "mean_weights": [ + 0.13399386405944824, + 0.12986589968204498, + 0.0802837461233139, + 0.07629825174808502, + 0.10134232044219971, + 0.12680892646312714, + 0.351406991481781 + ], + "sources": 7 + }, + { + "entropy_mean": 1.655133843421936, + "mean_weights": [ + 0.10968073457479477, + 0.0657510757446289, + 0.08727138489484787, + 0.09466101229190826, + 0.0703636109828949, + 0.14186590909957886, + 0.4304063022136688 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8160101175308228, + "mean_weights": [ + 0.13769462704658508, + 0.11569900065660477, + 0.08787183463573456, + 0.0929793268442154, + 0.10649852454662323, + 0.1315194070339203, + 0.3277372717857361 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6999847888946533, + "mean_weights": [ + 0.10220035165548325, + 0.07546016573905945, + 0.1028895378112793, + 0.10639216750860214, + 0.07665040343999863, + 0.13976991176605225, + 0.3966374397277832 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8706753253936768, + "mean_weights": [ + 0.14694823324680328, + 0.12422746419906616, + 0.09348422288894653, + 0.1027352511882782, + 0.12104703485965729, + 0.14027650654315948, + 0.27128130197525024 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7911043167114258, + "mean_weights": [ + 0.09078209102153778, + 0.05932023376226425, + 0.06744872033596039, + 0.0717800185084343, + 0.05460817739367485, + 0.10924961417913437, + 0.271850049495697, + 0.27496111392974854 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9753689765930176, + "mean_weights": [ + 0.10981250554323196, + 0.10515635460615158, + 0.07847881317138672, + 0.0806870236992836, + 0.10027817636728287, + 0.1099884957075119, + 0.1757785975933075, + 0.23982004821300507 + ], + "sources": 8 + }, + { + "entropy_mean": 1.81705904006958, + "mean_weights": [ + 0.0980612188577652, + 0.060419049113988876, + 0.06808654963970184, + 0.07581339031457901, + 0.06074263155460358, + 0.09064903855323792, + 0.23993313312530518, + 0.3062949776649475 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9814136028289795, + "mean_weights": [ + 0.11045937240123749, + 0.08757394552230835, + 0.08714327216148376, + 0.09700852632522583, + 0.09435248374938965, + 0.114869624376297, + 0.1777447760105133, + 0.23084799945354462 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8373757600784302, + "mean_weights": [ + 0.08649781346321106, + 0.07379771769046783, + 0.0769633948802948, + 0.0880836620926857, + 0.066677525639534, + 0.1187550276517868, + 0.23661984503269196, + 0.25260502099990845 + ], + "sources": 8 + }, + { + "entropy_mean": 1.968585729598999, + "mean_weights": [ + 0.10324696451425552, + 0.0860632136464119, + 0.09797552973031998, + 0.1271316111087799, + 0.08284065127372742, + 0.10640716552734375, + 0.17243653535842896, + 0.22389833629131317 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8529754877090454, + "mean_weights": [ + 0.08385773003101349, + 0.07188580930233002, + 0.08174896240234375, + 0.09558320790529251, + 0.06307300925254822, + 0.108722984790802, + 0.2897399365901947, + 0.20538833737373352 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9637342691421509, + "mean_weights": [ + 0.10303820669651031, + 0.08717727661132812, + 0.09220535308122635, + 0.11336663365364075, + 0.08865514397621155, + 0.14618384838104248, + 0.2349606156349182, + 0.13441291451454163 + ], + "sources": 8 + }, + { + "entropy_mean": 1.962258219718933, + "mean_weights": [ + 0.07841141521930695, + 0.07437640428543091, + 0.07400830090045929, + 0.07925958186388016, + 0.05560165271162987, + 0.08937598764896393, + 0.1956467628479004, + 0.1118645966053009, + 0.24145527184009552 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0993216037750244, + "mean_weights": [ + 0.1043272316455841, + 0.09323035180568695, + 0.08778752386569977, + 0.10311444103717804, + 0.09694567322731018, + 0.13869863748550415, + 0.1680138111114502, + 0.0874539315700531, + 0.12042839825153351 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0489909648895264, + "mean_weights": [ + 0.0919051468372345, + 0.07878251373767853, + 0.07574650645256042, + 0.09311886876821518, + 0.05773407965898514, + 0.07325181365013123, + 0.17863741517066956, + 0.18801960349082947, + 0.1628040373325348 + ], + "sources": 9 + }, + { + "entropy_mean": 2.126072406768799, + "mean_weights": [ + 0.09415344893932343, + 0.07414553314447403, + 0.10936354845762253, + 0.11765885353088379, + 0.07923045754432678, + 0.12059831619262695, + 0.17183929681777954, + 0.12057285010814667, + 0.11243770271539688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.028747081756592, + "mean_weights": [ + 0.08796507865190506, + 0.0757753849029541, + 0.0847562700510025, + 0.09340266138315201, + 0.06450396031141281, + 0.08635804057121277, + 0.24226921796798706, + 0.14922890067100525, + 0.11574049293994904 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1297550201416016, + "mean_weights": [ + 0.10085826367139816, + 0.08009681105613708, + 0.0982624888420105, + 0.11403533071279526, + 0.09301646053791046, + 0.12771156430244446, + 0.17309343814849854, + 0.10872717201709747, + 0.10419845581054688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1051812171936035, + "mean_weights": [ + 0.09203394502401352, + 0.08722756803035736, + 0.10295312106609344, + 0.11847955733537674, + 0.07401065528392792, + 0.11382973194122314, + 0.21416787803173065, + 0.11479790508747101, + 0.08249962329864502 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059127259999513626, + 0.055979277938604355, + 0.0761958509683609, + 0.07538121938705444, + 0.13674014806747437, + 0.18601374328136444, + 0.18700072169303894, + 0.08912894874811172, + 0.2049511820077896, + 0.12973569333553314, + 0.14231130480766296, + 0.13784728944301605, + 0.13109618425369263, + 0.1306019276380539, + 0.13412337005138397, + 0.13534265756607056, + 0.13132649660110474, + 0.08038423955440521, + 0.09715579450130463, + 0.0852823480963707, + 0.09982985258102417, + 0.08248664438724518, + 0.10234102606773376, + 0.09743349254131317, + 0.11231652647256851, + 0.07765212655067444, + 0.08827291429042816, + 0.07436380535364151, + 0.08991213887929916, + 0.08191535621881485, + 0.1065397635102272, + 0.09093168377876282, + 0.10877490043640137, + 0.06605636328458786, + 0.07123783230781555, + 0.06527329236268997, + 0.07851143926382065, + 0.0699063166975975, + 0.08676032721996307, + 0.08022409677505493, + 0.10016339272260666, + 0.05980068817734718, + 0.06517573446035385, + 0.05788872763514519, + 0.07041674107313156, + 0.0643942728638649, + 0.07159053534269333, + 0.07148924469947815, + 0.07631570100784302, + 0.056988880038261414, + 0.06180137023329735, + 0.05852367728948593, + 0.06370948255062103, + 0.06811124831438065, + 0.07006713002920151, + 0.07283915579319, + 0.0730312243103981, + 0.057786133140325546, + 0.06770572811365128, + 0.07090020179748535, + 0.07336394488811493, + 0.07763758301734924, + 0.08089767396450043, + 0.07564623653888702 + ], + "loss_nats": 1.3271938562393188, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.8321013450622559, + "mean_weights": [ + 0.09867185354232788, + 0.05898427963256836, + 0.057430200278759, + 0.05067210644483566, + 0.04944504797458649, + 0.047245003283023834, + 0.1162109300494194, + 0.30178403854370117, + 0.219556525349617 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08130212873220444, + 0.16142085194587708, + 0.2561342120170593, + 0.2773628532886505, + 0.3685380816459656, + 0.3768865168094635, + 0.38441672921180725, + 0.40868353843688965, + 0.14100632071495056, + 0.14338424801826477, + 0.16686734557151794, + 0.16827327013015747, + 0.18023288249969482, + 0.18227216601371765, + 0.18965557217597961, + 0.19825012981891632, + 0.05848926678299904, + 0.06985136866569519, + 0.09072534739971161, + 0.1086883395910263, + 0.12624414265155792, + 0.14860449731349945, + 0.17554737627506256, + 0.21192558109760284, + 0.06478945910930634, + 0.08672910183668137, + 0.11512516438961029, + 0.13778631389141083, + 0.15369470417499542, + 0.18114051222801208, + 0.19735252857208252, + 0.23228809237480164, + 0.04376237466931343, + 0.062035780400037766, + 0.07002050429582596, + 0.0909910649061203, + 0.09636416286230087, + 0.12021689862012863, + 0.12901709973812103, + 0.1606491208076477, + 0.04409762844443321, + 0.061457011848688126, + 0.07137759029865265, + 0.092374786734581, + 0.10297419875860214, + 0.12395651638507843, + 0.1298016905784607, + 0.15162576735019684, + 0.03295852243900299, + 0.06645520776510239, + 0.07529281079769135, + 0.10725662857294083, + 0.11068660020828247, + 0.14660172164440155, + 0.1495535671710968, + 0.20186874270439148, + 0.021799447014927864, + 0.07922796159982681, + 0.08539005368947983, + 0.17203646898269653, + 0.17447474598884583, + 0.2743971645832062, + 0.27695441246032715, + 0.41855114698410034 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "population_cv": 0.42684708327825066 + }, + "activation_output_rms_by_block": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2482638694345951, + "first_to_last_ratio": 1.6504997944655762, + "imbalance_abs_log_ratio": 0.5010781477825214, + "last_quartile_mean": 0.150417388882488, + "mean": 0.1577255482552573, + "normalized": [ + 1.0413313772202522, + 1.8759762186094133, + 2.6842751238870792, + 2.86288304745295, + 0.8807369558111258, + 0.9863695480754758, + 1.0723915340184837, + 1.1882317995492147, + 0.3938410687519575, + 0.6240548714989927, + 0.8686355260833801, + 1.2755653854399962, + 0.532526584341831, + 0.7695868670150526, + 1.006314124418815, + 1.3133324349770026, + 0.3684251942759779, + 0.5299682889456079, + 0.6940544684956076, + 0.9337557124120812, + 0.3540539965769534, + 0.5274131112327584, + 0.7125108234042923, + 0.874443203356988, + 0.3801289540418877, + 0.5954785163594047, + 0.7985322424957776, + 1.083277634082024, + 0.4248333923179985, + 0.8857998846594436, + 1.4024216545081702, + 2.0588504556840044 + ], + "population_cv": 0.6067899489376947 + }, + "bits_per_byte": 1.8565786854442996, + "branch_output_rms_by_sublayer": [ + 0.08626964688301086, + 0.13547764718532562, + 0.16337208449840546, + 0.039667773991823196, + 0.15584443509578705, + 0.026703152805566788, + 0.04305505007505417, + 0.05304057523608208, + 0.1360139101743698, + 0.016798952594399452, + 0.03393339365720749, + 0.00753376679494977, + 0.03730965405702591, + 0.01294450368732214, + 0.037494905292987823, + 0.03526800498366356, + 0.05234000086784363, + 0.030294200405478477, + 0.04223497211933136, + 0.034149833023548126, + 0.041619230061769485, + 0.04367009550333023, + 0.05722156912088394, + 0.05930330231785774, + 0.062187470495700836, + 0.04527241364121437, + 0.0369485504925251, + 0.04281846433877945, + 0.04293901473283768, + 0.04812675341963768, + 0.045369699597358704, + 0.06094835698604584, + 0.04149825870990753, + 0.038031257688999176, + 0.02440357394516468, + 0.040106989443302155, + 0.026009807363152504, + 0.04961257055401802, + 0.03634416684508324, + 0.07085621356964111, + 0.038355328142642975, + 0.03753058984875679, + 0.02631118707358837, + 0.04544757679104805, + 0.033921729773283005, + 0.05368982255458832, + 0.02731483057141304, + 0.06174425780773163, + 0.027557097375392914, + 0.05051252990961075, + 0.024315863847732544, + 0.054937634617090225, + 0.02051021158695221, + 0.05777662619948387, + 0.018631720915436745, + 0.07133159041404724, + 0.019497457891702652, + 0.0636468306183815, + 0.021715763956308365, + 0.09261580556631088, + 0.017114246264100075, + 0.1020733118057251, + 0.020576251670718193, + 0.12878532707691193 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003637111411518744, + 0.0002328437345071155, + 0.00026754486121072013, + 0.00020961054010746675, + 0.00023324217449373482, + 0.00013226952769200274, + 0.0001490970084679736, + 0.0001891597343986376, + 0.0002918746014559555, + 0.00023307226647352922, + 0.00024024223572494834, + 0.00029360747277183056, + 0.00025626387819923106, + 0.00024114885933974834, + 0.0002716298155375107, + 0.0003086967264992385, + 0.0005417594428612147, + 0.0005350465304351142, + 0.0005646380221663597, + 0.0006164043223392351, + 0.0009244274084097353, + 0.0008484364345428017, + 0.0007892136159088089, + 0.0006791712867767833, + 0.0005903268079560613, + 0.00047055631494833115, + 0.0003773406829883918, + 0.0003589161190616393, + 0.00026391753197443733, + 0.00022775345188039026, + 0.00019334088037647755, + 0.0001778215115139631 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022218484025369068, + "first_to_last_ratio": 0.6682317907333766, + "imbalance_abs_log_ratio": 0.4031201735174996, + "last_quartile_mean": 0.00033249666258746146, + "mean": 0.00037728390444285195, + "normalized": [ + 0.9640250667172751, + 0.6171578796900011, + 0.7091340448403511, + 0.5555777430182339, + 0.6182139543911144, + 0.3505835423520907, + 0.3951851820664077, + 0.5013723940277184, + 0.7736206024664017, + 0.6177636091253748, + 0.6367677838780912, + 0.7782136192780627, + 0.6792335299266601, + 0.6391708114234587, + 0.7199613138509996, + 0.81820804668331, + 1.4359463430099129, + 1.4181536082893218, + 1.496586563903859, + 1.633794378929312, + 2.4502169255666204, + 2.2488010343184843, + 2.0918295390158974, + 1.8001597173347186, + 1.5646753042057733, + 1.2472207518187604, + 1.000150492891086, + 0.9513157461399342, + 0.6995197220622846, + 0.6036659640085139, + 0.5124546213069721, + 0.471320163462998 + ], + "population_cv": 0.5587385684613161 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.21598131954669952, + "mean_weights": [ + 0.9387255907058716, + 0.06127442046999931 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5377621054649353, + "mean_weights": [ + 0.7082071900367737, + 0.2917928099632263 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43774232268333435, + "mean_weights": [ + 0.829714834690094, + 0.1702851504087448 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6749181151390076, + "mean_weights": [ + 0.5407299399375916, + 0.45927006006240845 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6663151383399963, + "mean_weights": [ + 0.5011438131332397, + 0.49885618686676025 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865031719207764, + "mean_weights": [ + 0.5308656692504883, + 0.4691343307495117 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4133813679218292, + "mean_weights": [ + 0.8479139804840088, + 0.15208598971366882 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6850745677947998, + "mean_weights": [ + 0.5398513674736023, + 0.4601486027240753 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0074119567871094, + "mean_weights": [ + 0.32744842767715454, + 0.20981493592262268, + 0.4627366065979004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489801168441772, + "mean_weights": [ + 0.28095993399620056, + 0.24264614284038544, + 0.4763939380645752 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8527575731277466, + "mean_weights": [ + 0.20522087812423706, + 0.1493145227432251, + 0.6454645991325378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.049942135810852, + "mean_weights": [ + 0.35788866877555847, + 0.20253834128379822, + 0.4395729899406433 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9791408181190491, + "mean_weights": [ + 0.31465935707092285, + 0.1638563871383667, + 0.5214842557907104 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0485806465148926, + "mean_weights": [ + 0.346283882856369, + 0.20800183713436127, + 0.4457142949104309 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9801348447799683, + "mean_weights": [ + 0.2748759686946869, + 0.1817527413368225, + 0.5433712601661682 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0688585042953491, + "mean_weights": [ + 0.4051227569580078, + 0.23231396079063416, + 0.36256328225135803 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2915732860565186, + "mean_weights": [ + 0.26621368527412415, + 0.1108347550034523, + 0.2563389539718628, + 0.36661261320114136 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568191528320312, + "mean_weights": [ + 0.294538676738739, + 0.1710844486951828, + 0.2252884805202484, + 0.30908840894699097 + ], + "sources": 4 + }, + { + "entropy_mean": 1.320021152496338, + "mean_weights": [ + 0.304654598236084, + 0.12911874055862427, + 0.23493197560310364, + 0.3312946557998657 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3578683137893677, + "mean_weights": [ + 0.29558902978897095, + 0.17812995612621307, + 0.21904321014881134, + 0.30723777413368225 + ], + "sources": 4 + }, + { + "entropy_mean": 1.258887767791748, + "mean_weights": [ + 0.35744374990463257, + 0.12067858874797821, + 0.17883820831775665, + 0.34303945302963257 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3539760112762451, + "mean_weights": [ + 0.2736307978630066, + 0.1722734123468399, + 0.2365075945854187, + 0.3175881803035736 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3203014135360718, + "mean_weights": [ + 0.28207293152809143, + 0.15420743823051453, + 0.22419029474258423, + 0.3395293354988098 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3734662532806396, + "mean_weights": [ + 0.2766023576259613, + 0.21176086366176605, + 0.22839435935020447, + 0.283242404460907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5325813293457031, + "mean_weights": [ + 0.17451074719429016, + 0.10643941164016724, + 0.1691833734512329, + 0.27394938468933105, + 0.27591708302497864 + ], + "sources": 5 + }, + { + "entropy_mean": 1.560360074043274, + "mean_weights": [ + 0.19377154111862183, + 0.17211592197418213, + 0.14605137705802917, + 0.16254930198192596, + 0.3255118727684021 + ], + "sources": 5 + }, + { + "entropy_mean": 1.540909767150879, + "mean_weights": [ + 0.21921303868293762, + 0.10205383598804474, + 0.16381604969501495, + 0.215368390083313, + 0.2995486855506897 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5142419338226318, + "mean_weights": [ + 0.2000826597213745, + 0.15480534732341766, + 0.10827909409999847, + 0.16766759753227234, + 0.3691653311252594 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5333527326583862, + "mean_weights": [ + 0.21770668029785156, + 0.10525082051753998, + 0.15970060229301453, + 0.217601016163826, + 0.29974091053009033 + ], + "sources": 5 + }, + { + "entropy_mean": 1.512608289718628, + "mean_weights": [ + 0.21392285823822021, + 0.1860550343990326, + 0.10291537642478943, + 0.1385909616947174, + 0.35851576924324036 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5262645483016968, + "mean_weights": [ + 0.20246261358261108, + 0.10530520975589752, + 0.16650918126106262, + 0.21146778762340546, + 0.3142552077770233 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5794764757156372, + "mean_weights": [ + 0.2210189402103424, + 0.18839940428733826, + 0.14016607403755188, + 0.17440536618232727, + 0.2760102152824402 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6602650880813599, + "mean_weights": [ + 0.12695768475532532, + 0.08036886155605316, + 0.12672865390777588, + 0.15391480922698975, + 0.1544383019208908, + 0.3575916886329651 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5925545692443848, + "mean_weights": [ + 0.14075040817260742, + 0.11028306186199188, + 0.07537610828876495, + 0.10244421660900116, + 0.1520608514547348, + 0.41908538341522217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5918848514556885, + "mean_weights": [ + 0.11773697286844254, + 0.07344591617584229, + 0.11976177245378494, + 0.14735516905784607, + 0.1265447735786438, + 0.4151553511619568 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6208878755569458, + "mean_weights": [ + 0.14785674214363098, + 0.12492769956588745, + 0.08781817555427551, + 0.10019385814666748, + 0.14005999267101288, + 0.3991435170173645 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5470075607299805, + "mean_weights": [ + 0.12043031305074692, + 0.06855648756027222, + 0.11753211170434952, + 0.13396470248699188, + 0.1162077933549881, + 0.44330859184265137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.683492660522461, + "mean_weights": [ + 0.1658952832221985, + 0.1435280442237854, + 0.09461991488933563, + 0.10784181207418442, + 0.15157341957092285, + 0.3365415334701538 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5540567636489868, + "mean_weights": [ + 0.12987136840820312, + 0.07178180664777756, + 0.12132491171360016, + 0.1441602110862732, + 0.11020088195800781, + 0.42266082763671875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7208337783813477, + "mean_weights": [ + 0.18048501014709473, + 0.1685236394405365, + 0.08726631104946136, + 0.11160479485988617, + 0.18141084909439087, + 0.27070939540863037 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6729483604431152, + "mean_weights": [ + 0.09229261428117752, + 0.051420144736766815, + 0.08079522103071213, + 0.09550285339355469, + 0.07590493559837341, + 0.22372649610042572, + 0.3803577423095703 + ], + "sources": 7 + }, + { + "entropy_mean": 1.710235595703125, + "mean_weights": [ + 0.11657393723726273, + 0.10617929697036743, + 0.061112720519304276, + 0.06796726584434509, + 0.10333786904811859, + 0.14365722239017487, + 0.4011717140674591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5640671253204346, + "mean_weights": [ + 0.0904226303100586, + 0.04561381787061691, + 0.07565715163946152, + 0.08878682553768158, + 0.06555216014385223, + 0.16504976153373718, + 0.4689176678657532 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7556228637695312, + "mean_weights": [ + 0.13281463086605072, + 0.12175846099853516, + 0.07235272228717804, + 0.0707377940416336, + 0.10551950335502625, + 0.12620802223682404, + 0.3706088662147522 + ], + "sources": 7 + }, + { + "entropy_mean": 1.594892978668213, + "mean_weights": [ + 0.10721130669116974, + 0.053925663232803345, + 0.07673212885856628, + 0.08865121006965637, + 0.06888366490602493, + 0.14094024896621704, + 0.4636557698249817 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7775511741638184, + "mean_weights": [ + 0.13637228310108185, + 0.10478455573320389, + 0.07710614800453186, + 0.08548048138618469, + 0.11000139266252518, + 0.12931162118911743, + 0.3569435477256775 + ], + "sources": 7 + }, + { + "entropy_mean": 1.654036045074463, + "mean_weights": [ + 0.10232305526733398, + 0.061968062072992325, + 0.08900615572929382, + 0.1005433052778244, + 0.07864195853471756, + 0.14099234342575073, + 0.4265251159667969 + ], + "sources": 7 + }, + { + "entropy_mean": 1.84956693649292, + "mean_weights": [ + 0.14851760864257812, + 0.11630287766456604, + 0.08512166142463684, + 0.09215595573186874, + 0.1263951063156128, + 0.14375528693199158, + 0.28775155544281006 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7161309719085693, + "mean_weights": [ + 0.0837959572672844, + 0.04434927552938461, + 0.0567089319229126, + 0.06495873630046844, + 0.04885464161634445, + 0.09997115284204483, + 0.2760006785392761, + 0.32536062598228455 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9285151958465576, + "mean_weights": [ + 0.10564712435007095, + 0.09173665940761566, + 0.07135222852230072, + 0.07045470178127289, + 0.09785081446170807, + 0.10425601899623871, + 0.1782122403383255, + 0.2804902195930481 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7219383716583252, + "mean_weights": [ + 0.0923302173614502, + 0.04646994173526764, + 0.05750556290149689, + 0.06549686193466187, + 0.054404959082603455, + 0.08427833020687103, + 0.2333412766456604, + 0.3661728501319885 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9470824003219604, + "mean_weights": [ + 0.10804469138383865, + 0.07850410044193268, + 0.07298290729522705, + 0.08511960506439209, + 0.09968686103820801, + 0.11502514779567719, + 0.18013443052768707, + 0.26050227880477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7855088710784912, + "mean_weights": [ + 0.08493785560131073, + 0.05972691997885704, + 0.07017223536968231, + 0.07963316142559052, + 0.0645163506269455, + 0.11586237698793411, + 0.23710276186466217, + 0.28804832696914673 + ], + "sources": 8 + }, + { + "entropy_mean": 1.973182201385498, + "mean_weights": [ + 0.1049819216132164, + 0.0840599462389946, + 0.09310521930456161, + 0.11566175520420074, + 0.0963074192404747, + 0.1096896231174469, + 0.16087493300437927, + 0.23531918227672577 + ], + "sources": 8 + }, + { + "entropy_mean": 1.802741289138794, + "mean_weights": [ + 0.08190082013607025, + 0.05632811039686203, + 0.07102417945861816, + 0.08421063423156738, + 0.06341388821601868, + 0.10201875865459442, + 0.2913346588611603, + 0.24976897239685059 + ], + "sources": 8 + }, + { + "entropy_mean": 1.945815086364746, + "mean_weights": [ + 0.1064736619591713, + 0.07925166189670563, + 0.07823627442121506, + 0.09447507560253143, + 0.09724453091621399, + 0.1459730863571167, + 0.2451319396495819, + 0.153213769197464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8717057704925537, + "mean_weights": [ + 0.07336360216140747, + 0.05720400810241699, + 0.057025715708732605, + 0.06655597686767578, + 0.05032068490982056, + 0.08329878002405167, + 0.1901058554649353, + 0.11727118492126465, + 0.3048541843891144 + ], + "sources": 9 + }, + { + "entropy_mean": 2.09096622467041, + "mean_weights": [ + 0.10326676070690155, + 0.08232800662517548, + 0.07164021581411362, + 0.08822821080684662, + 0.10803373903036118, + 0.13578416407108307, + 0.17320571839809418, + 0.09955950826406479, + 0.13795366883277893 + ], + "sources": 9 + }, + { + "entropy_mean": 1.96881103515625, + "mean_weights": [ + 0.08843819797039032, + 0.06052382290363312, + 0.0642135739326477, + 0.07744920998811722, + 0.05161681026220322, + 0.06631232053041458, + 0.16417551040649414, + 0.2115624099969864, + 0.21570813655853271 + ], + "sources": 9 + }, + { + "entropy_mean": 2.124063014984131, + "mean_weights": [ + 0.09220977127552032, + 0.06839151680469513, + 0.10039852559566498, + 0.10902463644742966, + 0.0849665105342865, + 0.12137697637081146, + 0.17017139494419098, + 0.12761826813220978, + 0.1258423924446106 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9633058309555054, + "mean_weights": [ + 0.0846441313624382, + 0.05872901901602745, + 0.07210009545087814, + 0.07792744785547256, + 0.05918677896261215, + 0.07445863634347916, + 0.23511825501918793, + 0.18020221590995789, + 0.15763342380523682 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1238560676574707, + "mean_weights": [ + 0.10231245309114456, + 0.07575537264347076, + 0.08897507935762405, + 0.0967075377702713, + 0.09801244735717773, + 0.12438583374023438, + 0.1766587197780609, + 0.12203006446361542, + 0.11516247689723969 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0835556983947754, + "mean_weights": [ + 0.09133295714855194, + 0.07054752856492996, + 0.09000501036643982, + 0.10956106334924698, + 0.07114207744598389, + 0.11565886437892914, + 0.22511571645736694, + 0.13003571331501007, + 0.09660108387470245 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.062413111329078674, + 0.05931045487523079, + 0.07552354782819748, + 0.07934419065713882, + 0.14653873443603516, + 0.214645653963089, + 0.2105521410703659, + 0.09514089673757553, + 0.2186809629201889, + 0.12821291387081146, + 0.14272794127464294, + 0.1321839988231659, + 0.1287408471107483, + 0.12568901479244232, + 0.13253600895404816, + 0.1335759311914444, + 0.13371841609477997, + 0.07788944244384766, + 0.09585829079151154, + 0.08352623134851456, + 0.09883332997560501, + 0.07934392243623734, + 0.10020580142736435, + 0.09343641996383667, + 0.11237040907144547, + 0.07063088566064835, + 0.08538161963224411, + 0.06870647519826889, + 0.08663177490234375, + 0.07622843235731125, + 0.10358677804470062, + 0.08666495233774185, + 0.10874549299478531, + 0.057347409427165985, + 0.06467291712760925, + 0.05777324363589287, + 0.07278503477573395, + 0.06351501494646072, + 0.08346924930810928, + 0.07469023764133453, + 0.09857872873544693, + 0.051319729536771774, + 0.06093711405992508, + 0.05102822557091713, + 0.06834813207387924, + 0.05902284011244774, + 0.0680762529373169, + 0.06656526774168015, + 0.07369214296340942, + 0.050236333161592484, + 0.05633483827114105, + 0.05184732377529144, + 0.05850831791758537, + 0.061248794198036194, + 0.06479237228631973, + 0.06608261168003082, + 0.06705968081951141, + 0.048912134021520615, + 0.061747051775455475, + 0.062179163098335266, + 0.0654391348361969, + 0.07131517678499222, + 0.07205834239721298, + 0.06802479177713394 + ], + "loss_nats": 1.2868822813034058, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.679484486579895, + "mean_weights": [ + 0.08993907272815704, + 0.043493133038282394, + 0.04922422021627426, + 0.037924446165561676, + 0.03939434140920639, + 0.03898489475250244, + 0.09112586826086044, + 0.32717907428741455, + 0.2827349305152893 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.08626964688301086, + 0.16424456238746643, + 0.27676042914390564, + 0.29588937759399414, + 0.41335058212280273, + 0.42337876558303833, + 0.43042075634002686, + 0.45154979825019836, + 0.1360139101743698, + 0.13891471922397614, + 0.15445689857006073, + 0.1555756777524948, + 0.1670466512441635, + 0.16914354264736176, + 0.17834730446338654, + 0.1874145120382309, + 0.05234000086784363, + 0.06211879849433899, + 0.08269349485635757, + 0.09842939674854279, + 0.11548293381929398, + 0.137006014585495, + 0.16410532593727112, + 0.20118924975395203, + 0.062187470495700836, + 0.08399304747581482, + 0.0988159254193306, + 0.12138351052999496, + 0.13199056684970856, + 0.1587214469909668, + 0.17199495434761047, + 0.2071460783481598, + 0.04149825870990753, + 0.0581100657582283, + 0.06434677541255951, + 0.08358953893184662, + 0.08728200942277908, + 0.10947012156248093, + 0.11647984385490417, + 0.1472771316766739, + 0.038355328142642975, + 0.05584336072206497, + 0.06353354454040527, + 0.08318652212619781, + 0.09196189045906067, + 0.11238116025924683, + 0.1169828325510025, + 0.1379220336675644, + 0.027557097375392914, + 0.05995604768395424, + 0.06579430401325226, + 0.09392217546701431, + 0.09665275365114212, + 0.1259489357471466, + 0.12799666821956635, + 0.17086055874824524, + 0.019497457891702652, + 0.06700707972049713, + 0.07176875323057175, + 0.13971327245235443, + 0.14131556451320648, + 0.2211977243423462, + 0.22250843048095703, + 0.32473331689834595 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.061194200882085, + "cross_entropy_nats": 5.587594032287598, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "cross_entropy_nats": 3.598757028579712, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "cross_entropy_nats": 2.1405738294124603, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "cross_entropy_nats": 1.3509249985218048, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "cross_entropy_nats": 1.2388448417186737, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "cross_entropy_nats": 1.1804966628551483, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a", + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.96596761305909, + "measured_steps": 7980, + "median_ms": 109.86474549281411, + "p95_ms": 110.51533734425902, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.082576459898554, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.602415084838867, + "step": 1, + "unclipped_grad_norm": 19.083253860473633 + }, + { + "bits_per_byte": 7.337116283872504, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.0857014656066895, + "step": 10, + "unclipped_grad_norm": 11.73559856414795 + }, + { + "bits_per_byte": 6.794594947933964, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.709654331207275, + "step": 20, + "unclipped_grad_norm": 3.3705036640167236 + }, + { + "bits_per_byte": 6.669727976170927, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.623103141784668, + "step": 30, + "unclipped_grad_norm": 3.037728786468506 + }, + { + "bits_per_byte": 6.488515360401613, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.497496128082275, + "step": 40, + "unclipped_grad_norm": 3.071707010269165 + }, + { + "bits_per_byte": 6.170372909737745, + "learning_rate": 3.75e-05, + "loss_nats": 4.276976585388184, + "step": 50, + "unclipped_grad_norm": 2.8251144886016846 + }, + { + "bits_per_byte": 5.9458358004242795, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.121339321136475, + "step": 60, + "unclipped_grad_norm": 2.6051926612854004 + }, + { + "bits_per_byte": 5.763860269615738, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9952034950256348, + "step": 70, + "unclipped_grad_norm": 2.513765573501587 + }, + { + "bits_per_byte": 5.595572150058764, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8785550594329834, + "step": 80, + "unclipped_grad_norm": 2.595860719680786 + }, + { + "bits_per_byte": 5.4051244099034195, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.746546745300293, + "step": 90, + "unclipped_grad_norm": 2.317351818084717 + }, + { + "bits_per_byte": 5.1779332643218385, + "learning_rate": 7.5e-05, + "loss_nats": 3.5890698432922363, + "step": 100, + "unclipped_grad_norm": 2.214414358139038 + }, + { + "bits_per_byte": 4.995566300999934, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.462662696838379, + "step": 110, + "unclipped_grad_norm": 2.0333311557769775 + }, + { + "bits_per_byte": 4.799066491360726, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3264594078063965, + "step": 120, + "unclipped_grad_norm": 1.8019887208938599 + }, + { + "bits_per_byte": 4.633962458601826, + "learning_rate": 9.75e-05, + "loss_nats": 3.2120180130004883, + "step": 130, + "unclipped_grad_norm": 1.6018544435501099 + }, + { + "bits_per_byte": 4.476944362155892, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.1031813621520996, + "step": 140, + "unclipped_grad_norm": 1.3790134191513062 + }, + { + "bits_per_byte": 4.275779695094682, + "learning_rate": 0.0001125, + "loss_nats": 2.963744640350342, + "step": 150, + "unclipped_grad_norm": 1.2061023712158203 + }, + { + "bits_per_byte": 4.128819144071206, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.861879348754883, + "step": 160, + "unclipped_grad_norm": 1.0463906526565552 + }, + { + "bits_per_byte": 4.021535678532382, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7875161170959473, + "step": 170, + "unclipped_grad_norm": 0.9023508429527283 + }, + { + "bits_per_byte": 3.919193618518149, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7165780067443848, + "step": 180, + "unclipped_grad_norm": 0.9033810496330261 + }, + { + "bits_per_byte": 3.8348498868820102, + "learning_rate": 0.0001425, + "loss_nats": 2.6581153869628906, + "step": 190, + "unclipped_grad_norm": 0.8499347567558289 + }, + { + "bits_per_byte": 3.7665400977858114, + "learning_rate": 0.00015, + "loss_nats": 2.610766649246216, + "step": 200, + "unclipped_grad_norm": 0.8223105072975159 + }, + { + "bits_per_byte": 3.6822345462981643, + "learning_rate": 0.0001575, + "loss_nats": 2.552330493927002, + "step": 210, + "unclipped_grad_norm": 0.8248683214187622 + }, + { + "bits_per_byte": 3.6564082555833983, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5344290733337402, + "step": 220, + "unclipped_grad_norm": 1.0457000732421875 + }, + { + "bits_per_byte": 3.6121584953776043, + "learning_rate": 0.0001725, + "loss_nats": 2.5037574768066406, + "step": 230, + "unclipped_grad_norm": 1.2628414630889893 + }, + { + "bits_per_byte": 3.5855720413460244, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4853291511535645, + "step": 240, + "unclipped_grad_norm": 0.9464337229728699 + }, + { + "bits_per_byte": 3.5024318443006237, + "learning_rate": 0.0001875, + "loss_nats": 2.4277007579803467, + "step": 250, + "unclipped_grad_norm": 0.8258991837501526 + }, + { + "bits_per_byte": 3.520142273721174, + "learning_rate": 0.000195, + "loss_nats": 2.439976692199707, + "step": 260, + "unclipped_grad_norm": 1.1166419982910156 + }, + { + "bits_per_byte": 3.497664141253405, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.42439603805542, + "step": 270, + "unclipped_grad_norm": 0.9347699880599976 + }, + { + "bits_per_byte": 3.454170760746106, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.3942487239837646, + "step": 280, + "unclipped_grad_norm": 1.2552868127822876 + }, + { + "bits_per_byte": 3.4220065653817433, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3719542026519775, + "step": 290, + "unclipped_grad_norm": 0.8032089471817017 + }, + { + "bits_per_byte": 3.435558454304431, + "learning_rate": 0.000225, + "loss_nats": 2.38134765625, + "step": 300, + "unclipped_grad_norm": 1.544488787651062 + }, + { + "bits_per_byte": 3.330305071976392, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.308391571044922, + "step": 310, + "unclipped_grad_norm": 0.8757423162460327 + }, + { + "bits_per_byte": 3.390824390953808, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3503403663635254, + "step": 320, + "unclipped_grad_norm": 0.8823365569114685 + }, + { + "bits_per_byte": 3.3979087003081485, + "learning_rate": 0.0002475, + "loss_nats": 2.355250835418701, + "step": 330, + "unclipped_grad_norm": 0.898759663105011 + }, + { + "bits_per_byte": 3.325636774901372, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3051557540893555, + "step": 340, + "unclipped_grad_norm": 0.9536095261573792 + }, + { + "bits_per_byte": 3.3170438337336625, + "learning_rate": 0.0002625, + "loss_nats": 2.2991995811462402, + "step": 350, + "unclipped_grad_norm": 1.2654696702957153 + }, + { + "bits_per_byte": 3.313786482326321, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2969417572021484, + "step": 360, + "unclipped_grad_norm": 1.3773932456970215 + }, + { + "bits_per_byte": 3.3236555347634327, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3037824630737305, + "step": 370, + "unclipped_grad_norm": 1.2408338785171509 + }, + { + "bits_per_byte": 3.208911117864879, + "learning_rate": 0.000285, + "loss_nats": 2.224247694015503, + "step": 380, + "unclipped_grad_norm": 1.0960922241210938 + }, + { + "bits_per_byte": 3.2804789462990227, + "learning_rate": 0.0002925, + "loss_nats": 2.2738547325134277, + "step": 390, + "unclipped_grad_norm": 0.9866538643836975 + }, + { + "bits_per_byte": 3.27038631641579, + "learning_rate": 0.0003, + "loss_nats": 2.2668590545654297, + "step": 400, + "unclipped_grad_norm": 1.2531954050064087 + }, + { + "bits_per_byte": 3.2889521875417316, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2797279357910156, + "step": 410, + "unclipped_grad_norm": 1.448500633239746 + }, + { + "bits_per_byte": 3.205672684549132, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2220029830932617, + "step": 420, + "unclipped_grad_norm": 0.9654834866523743 + }, + { + "bits_per_byte": 3.218386673996816, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2308156490325928, + "step": 430, + "unclipped_grad_norm": 0.974204421043396 + }, + { + "bits_per_byte": 3.1822768526563587, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2057862281799316, + "step": 440, + "unclipped_grad_norm": 1.4095590114593506 + }, + { + "bits_per_byte": 3.237966210866943, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.244387149810791, + "step": 450, + "unclipped_grad_norm": 0.9198341369628906 + }, + { + "bits_per_byte": 3.1857660366770637, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.208204746246338, + "step": 460, + "unclipped_grad_norm": 1.4268778562545776 + }, + { + "bits_per_byte": 3.1576792059993566, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.1887364387512207, + "step": 470, + "unclipped_grad_norm": 1.2876354455947876 + }, + { + "bits_per_byte": 3.131057323541699, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.170283555984497, + "step": 480, + "unclipped_grad_norm": 1.0718865394592285 + }, + { + "bits_per_byte": 3.1147361699748184, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.158970594406128, + "step": 490, + "unclipped_grad_norm": 0.8680974841117859 + }, + { + "bits_per_byte": 3.059730958749366, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1208438873291016, + "step": 500, + "unclipped_grad_norm": 1.296195387840271 + }, + { + "bits_per_byte": 3.12068642572937, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.163094997406006, + "step": 510, + "unclipped_grad_norm": 1.0469720363616943 + }, + { + "bits_per_byte": 3.1397613655018657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.176316738128662, + "step": 520, + "unclipped_grad_norm": 1.2089059352874756 + }, + { + "bits_per_byte": 2.9844827015867583, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.06868577003479, + "step": 530, + "unclipped_grad_norm": 1.1353814601898193 + }, + { + "bits_per_byte": 3.0015299659055814, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0805020332336426, + "step": 540, + "unclipped_grad_norm": 0.919367253780365 + }, + { + "bits_per_byte": 2.987599371185694, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0708460807800293, + "step": 550, + "unclipped_grad_norm": 0.8667266368865967 + }, + { + "bits_per_byte": 2.995183462123442, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0761029720306396, + "step": 560, + "unclipped_grad_norm": 0.9804732203483582 + }, + { + "bits_per_byte": 2.962294187903042, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0533058643341064, + "step": 570, + "unclipped_grad_norm": 0.9414609670639038 + }, + { + "bits_per_byte": 2.956716790535561, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0494399070739746, + "step": 580, + "unclipped_grad_norm": 0.8727462291717529 + }, + { + "bits_per_byte": 2.9113505189603663, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0179944038391113, + "step": 590, + "unclipped_grad_norm": 1.1642946004867554 + }, + { + "bits_per_byte": 2.8665478345320485, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.986939549446106, + "step": 600, + "unclipped_grad_norm": 0.9213871359825134 + }, + { + "bits_per_byte": 2.875505550902238, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9931485652923584, + "step": 610, + "unclipped_grad_norm": 0.7122263312339783 + }, + { + "bits_per_byte": 2.878308696128651, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9950915575027466, + "step": 620, + "unclipped_grad_norm": 1.4622896909713745 + }, + { + "bits_per_byte": 2.790202156074285, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.934020757675171, + "step": 630, + "unclipped_grad_norm": 1.1472660303115845 + }, + { + "bits_per_byte": 2.7890029210498186, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9331895112991333, + "step": 640, + "unclipped_grad_norm": 1.1472764015197754 + }, + { + "bits_per_byte": 2.7734051265124666, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.9223779439926147, + "step": 650, + "unclipped_grad_norm": 0.9182148575782776 + }, + { + "bits_per_byte": 2.795219749913207, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.937498688697815, + "step": 660, + "unclipped_grad_norm": 1.083983302116394 + }, + { + "bits_per_byte": 2.7702638633944576, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.9202005863189697, + "step": 670, + "unclipped_grad_norm": 1.1721765995025635 + }, + { + "bits_per_byte": 2.755497089026075, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.9099650382995605, + "step": 680, + "unclipped_grad_norm": 1.280714750289917 + }, + { + "bits_per_byte": 2.7128121989604823, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8803781270980835, + "step": 690, + "unclipped_grad_norm": 1.0380759239196777 + }, + { + "bits_per_byte": 2.7091302223881604, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8778259754180908, + "step": 700, + "unclipped_grad_norm": 1.1401211023330688 + }, + { + "bits_per_byte": 2.6870259802033667, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.862504482269287, + "step": 710, + "unclipped_grad_norm": 1.6450653076171875 + }, + { + "bits_per_byte": 2.688485424978589, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.863516092300415, + "step": 720, + "unclipped_grad_norm": 1.0721391439437866 + }, + { + "bits_per_byte": 2.65192277331841, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8381727933883667, + "step": 730, + "unclipped_grad_norm": 1.0645538568496704 + }, + { + "bits_per_byte": 2.6143492076364594, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.8121287822723389, + "step": 740, + "unclipped_grad_norm": 1.1877458095550537 + }, + { + "bits_per_byte": 2.6277774410331767, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.8214365243911743, + "step": 750, + "unclipped_grad_norm": 1.1091431379318237 + }, + { + "bits_per_byte": 2.540277655771055, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7607862949371338, + "step": 760, + "unclipped_grad_norm": 1.222191333770752 + }, + { + "bits_per_byte": 2.5790566478529025, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.787665843963623, + "step": 770, + "unclipped_grad_norm": 1.254854679107666 + }, + { + "bits_per_byte": 2.5138998166949715, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7425025701522827, + "step": 780, + "unclipped_grad_norm": 0.9573470950126648 + }, + { + "bits_per_byte": 2.5258618979930834, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7507940530776978, + "step": 790, + "unclipped_grad_norm": 1.108724594116211 + }, + { + "bits_per_byte": 2.430956195819507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6850104331970215, + "step": 800, + "unclipped_grad_norm": 0.9832104444503784 + }, + { + "bits_per_byte": 2.468076415233783, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7107402086257935, + "step": 810, + "unclipped_grad_norm": 0.9492170214653015 + }, + { + "bits_per_byte": 2.4602348662676934, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7053048610687256, + "step": 820, + "unclipped_grad_norm": 1.192505121231079 + }, + { + "bits_per_byte": 2.4583616312344954, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7040064334869385, + "step": 830, + "unclipped_grad_norm": 0.9113606214523315 + }, + { + "bits_per_byte": 2.4751598646748696, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7156500816345215, + "step": 840, + "unclipped_grad_norm": 1.3671480417251587 + }, + { + "bits_per_byte": 2.386899400147981, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6544725894927979, + "step": 850, + "unclipped_grad_norm": 1.0324512720108032 + }, + { + "bits_per_byte": 2.472668523994471, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.7139232158660889, + "step": 860, + "unclipped_grad_norm": 1.503003478050232 + }, + { + "bits_per_byte": 2.4348980381772827, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6877427101135254, + "step": 870, + "unclipped_grad_norm": 1.1144746541976929 + }, + { + "bits_per_byte": 2.3709108610278693, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.64339017868042, + "step": 880, + "unclipped_grad_norm": 1.0134795904159546 + }, + { + "bits_per_byte": 2.3377405671559597, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6203982830047607, + "step": 890, + "unclipped_grad_norm": 1.0625330209732056 + }, + { + "bits_per_byte": 2.389408283058768, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6562116146087646, + "step": 900, + "unclipped_grad_norm": 0.9719302654266357 + }, + { + "bits_per_byte": 2.3782030974071033, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6484447717666626, + "step": 910, + "unclipped_grad_norm": 1.0672563314437866 + }, + { + "bits_per_byte": 2.3530695528100125, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6310235261917114, + "step": 920, + "unclipped_grad_norm": 1.1197935342788696 + }, + { + "bits_per_byte": 2.297453115243394, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5924731492996216, + "step": 930, + "unclipped_grad_norm": 1.0027272701263428 + }, + { + "bits_per_byte": 2.344061961471073, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6247799396514893, + "step": 940, + "unclipped_grad_norm": 1.0651428699493408 + }, + { + "bits_per_byte": 2.367935217202641, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6413276195526123, + "step": 950, + "unclipped_grad_norm": 0.9822106957435608 + }, + { + "bits_per_byte": 2.309206925551311, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6006202697753906, + "step": 960, + "unclipped_grad_norm": 0.8860561847686768 + }, + { + "bits_per_byte": 2.2419830349126597, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5540242195129395, + "step": 970, + "unclipped_grad_norm": 1.1816905736923218 + }, + { + "bits_per_byte": 2.3851068249780383, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.65323007106781, + "step": 980, + "unclipped_grad_norm": 0.9257189035415649 + }, + { + "bits_per_byte": 2.232579711494085, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.547506332397461, + "step": 990, + "unclipped_grad_norm": 0.9634203910827637 + }, + { + "bits_per_byte": 2.2247954327507324, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5421106815338135, + "step": 1000, + "unclipped_grad_norm": 0.809658944606781 + }, + { + "bits_per_byte": 2.2354428786656486, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5494909286499023, + "step": 1010, + "unclipped_grad_norm": 1.3530596494674683 + }, + { + "bits_per_byte": 2.2067197121797495, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5295815467834473, + "step": 1020, + "unclipped_grad_norm": 1.0303069353103638 + }, + { + "bits_per_byte": 2.1954008459958647, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5217359066009521, + "step": 1030, + "unclipped_grad_norm": 0.9071640968322754 + }, + { + "bits_per_byte": 2.291673982226451, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5884673595428467, + "step": 1040, + "unclipped_grad_norm": 1.3239798545837402 + }, + { + "bits_per_byte": 2.196653911590051, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5226044654846191, + "step": 1050, + "unclipped_grad_norm": 0.9085557460784912 + }, + { + "bits_per_byte": 2.225028985190604, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5422725677490234, + "step": 1060, + "unclipped_grad_norm": 0.9488101005554199 + }, + { + "bits_per_byte": 2.275690258620563, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5773882865905762, + "step": 1070, + "unclipped_grad_norm": 1.047879695892334 + }, + { + "bits_per_byte": 2.235951603346901, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5498435497283936, + "step": 1080, + "unclipped_grad_norm": 0.8919407725334167 + }, + { + "bits_per_byte": 2.1733449309359636, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5064479112625122, + "step": 1090, + "unclipped_grad_norm": 1.0036922693252563 + }, + { + "bits_per_byte": 2.169043472855234, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5034663677215576, + "step": 1100, + "unclipped_grad_norm": 0.9904995560646057 + }, + { + "bits_per_byte": 2.124962083664635, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4729114770889282, + "step": 1110, + "unclipped_grad_norm": 0.9176430702209473 + }, + { + "bits_per_byte": 2.176794214981668, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5088387727737427, + "step": 1120, + "unclipped_grad_norm": 0.9229135513305664 + }, + { + "bits_per_byte": 2.210271497885368, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.53204345703125, + "step": 1130, + "unclipped_grad_norm": 0.871993899345398 + }, + { + "bits_per_byte": 2.123456375556331, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4718677997589111, + "step": 1140, + "unclipped_grad_norm": 0.8673478960990906 + }, + { + "bits_per_byte": 2.1911007637763418, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.518755316734314, + "step": 1150, + "unclipped_grad_norm": 0.8959671258926392 + }, + { + "bits_per_byte": 2.0953000638876755, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4523513317108154, + "step": 1160, + "unclipped_grad_norm": 0.9113589525222778 + }, + { + "bits_per_byte": 2.1570418355474237, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.495147466659546, + "step": 1170, + "unclipped_grad_norm": 0.8980047702789307 + }, + { + "bits_per_byte": 2.1555593450969863, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4941198825836182, + "step": 1180, + "unclipped_grad_norm": 0.948503315448761 + }, + { + "bits_per_byte": 2.1334348089593913, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4787843227386475, + "step": 1190, + "unclipped_grad_norm": 1.1094794273376465 + }, + { + "bits_per_byte": 2.1317098229712372, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4775886535644531, + "step": 1200, + "unclipped_grad_norm": 0.8304017782211304 + }, + { + "bits_per_byte": 2.1385674591917407, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.482342004776001, + "step": 1210, + "unclipped_grad_norm": 0.8620252013206482 + }, + { + "bits_per_byte": 2.0980762078379027, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4542756080627441, + "step": 1220, + "unclipped_grad_norm": 0.88591068983078 + }, + { + "bits_per_byte": 2.108927281211441, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4617969989776611, + "step": 1230, + "unclipped_grad_norm": 1.0479243993759155 + }, + { + "bits_per_byte": 2.081558994049039, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.442826747894287, + "step": 1240, + "unclipped_grad_norm": 0.9365729689598083 + }, + { + "bits_per_byte": 2.0172010223043655, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.3982172012329102, + "step": 1250, + "unclipped_grad_norm": 0.8966301083564758 + }, + { + "bits_per_byte": 2.0690183631134245, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4341342449188232, + "step": 1260, + "unclipped_grad_norm": 0.8974369168281555 + }, + { + "bits_per_byte": 2.101840220134686, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4568846225738525, + "step": 1270, + "unclipped_grad_norm": 0.9220007061958313 + }, + { + "bits_per_byte": 2.1256708241688407, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.473402738571167, + "step": 1280, + "unclipped_grad_norm": 0.9313441514968872 + }, + { + "bits_per_byte": 2.076867651298803, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.439574956893921, + "step": 1290, + "unclipped_grad_norm": 0.8451681137084961 + }, + { + "bits_per_byte": 2.032476865319302, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4088056087493896, + "step": 1300, + "unclipped_grad_norm": 0.9314314126968384 + }, + { + "bits_per_byte": 2.058903719450881, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4271233081817627, + "step": 1310, + "unclipped_grad_norm": 0.8796194195747375 + }, + { + "bits_per_byte": 2.0275063947267156, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4053603410720825, + "step": 1320, + "unclipped_grad_norm": 0.9140937924385071 + }, + { + "bits_per_byte": 2.037489299678698, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4122799634933472, + "step": 1330, + "unclipped_grad_norm": 0.821657657623291 + }, + { + "bits_per_byte": 2.0199990080512524, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4001566171646118, + "step": 1340, + "unclipped_grad_norm": 0.8321346044540405 + }, + { + "bits_per_byte": 2.048903616233812, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.420191764831543, + "step": 1350, + "unclipped_grad_norm": 0.9421875476837158 + }, + { + "bits_per_byte": 2.00944598061166, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3928418159484863, + "step": 1360, + "unclipped_grad_norm": 0.8682821393013 + }, + { + "bits_per_byte": 2.0435301902902543, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4164671897888184, + "step": 1370, + "unclipped_grad_norm": 0.8776147961616516 + }, + { + "bits_per_byte": 2.0135448431296727, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.39568293094635, + "step": 1380, + "unclipped_grad_norm": 0.87440025806427 + }, + { + "bits_per_byte": 2.0533221944997795, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4232544898986816, + "step": 1390, + "unclipped_grad_norm": 0.7801422476768494 + }, + { + "bits_per_byte": 2.1564656936670334, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4947481155395508, + "step": 1400, + "unclipped_grad_norm": 0.8561526536941528 + }, + { + "bits_per_byte": 2.0373921294809607, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.412212610244751, + "step": 1410, + "unclipped_grad_norm": 0.8376067876815796 + }, + { + "bits_per_byte": 2.0325960492963495, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4088882207870483, + "step": 1420, + "unclipped_grad_norm": 0.751315712928772 + }, + { + "bits_per_byte": 1.9860154082234123, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.376600980758667, + "step": 1430, + "unclipped_grad_norm": 0.9057046175003052 + }, + { + "bits_per_byte": 2.015529006972677, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3970582485198975, + "step": 1440, + "unclipped_grad_norm": 0.8342951536178589 + }, + { + "bits_per_byte": 1.9818307263626185, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3737003803253174, + "step": 1450, + "unclipped_grad_norm": 0.8142686486244202 + }, + { + "bits_per_byte": 2.1060192265680016, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.459781289100647, + "step": 1460, + "unclipped_grad_norm": 0.8357279896736145 + }, + { + "bits_per_byte": 2.079546797033944, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.441431999206543, + "step": 1470, + "unclipped_grad_norm": 0.7720605731010437 + }, + { + "bits_per_byte": 2.025708316111945, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.404114007949829, + "step": 1480, + "unclipped_grad_norm": 0.8328851461410522 + }, + { + "bits_per_byte": 2.0422327531721454, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4155678749084473, + "step": 1490, + "unclipped_grad_norm": 0.9698525071144104 + }, + { + "bits_per_byte": 1.9217496191796013, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3320553302764893, + "step": 1500, + "unclipped_grad_norm": 0.7396577000617981 + }, + { + "bits_per_byte": 1.9436662282923034, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.347246766090393, + "step": 1510, + "unclipped_grad_norm": 0.809975266456604 + }, + { + "bits_per_byte": 1.9993110429963143, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3858168125152588, + "step": 1520, + "unclipped_grad_norm": 0.8165730834007263 + }, + { + "bits_per_byte": 2.03089187320895, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4077069759368896, + "step": 1530, + "unclipped_grad_norm": 0.7620540261268616 + }, + { + "bits_per_byte": 2.002577509484152, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3880809545516968, + "step": 1540, + "unclipped_grad_norm": 0.8928612470626831 + }, + { + "bits_per_byte": 2.0629018001354886, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4298945665359497, + "step": 1550, + "unclipped_grad_norm": 0.7963082194328308 + }, + { + "bits_per_byte": 1.9429003895480113, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3467159271240234, + "step": 1560, + "unclipped_grad_norm": 0.7710158228874207 + }, + { + "bits_per_byte": 1.9502758655302628, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3518282175064087, + "step": 1570, + "unclipped_grad_norm": 0.8057935833930969 + }, + { + "bits_per_byte": 1.9693647358974786, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3650596141815186, + "step": 1580, + "unclipped_grad_norm": 0.8588574528694153 + }, + { + "bits_per_byte": 1.972771368245769, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3674209117889404, + "step": 1590, + "unclipped_grad_norm": 0.8608299493789673 + }, + { + "bits_per_byte": 1.9610373359427014, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3592875003814697, + "step": 1600, + "unclipped_grad_norm": 0.8587453365325928 + }, + { + "bits_per_byte": 1.9257736692444893, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3348445892333984, + "step": 1610, + "unclipped_grad_norm": 0.8028180003166199 + }, + { + "bits_per_byte": 1.929578785394829, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3374820947647095, + "step": 1620, + "unclipped_grad_norm": 0.7991352677345276 + }, + { + "bits_per_byte": 1.9716548568763677, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3666470050811768, + "step": 1630, + "unclipped_grad_norm": 0.7573540806770325 + }, + { + "bits_per_byte": 1.9236465878186182, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3333702087402344, + "step": 1640, + "unclipped_grad_norm": 0.8524133563041687 + }, + { + "bits_per_byte": 1.9335809936630575, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3402562141418457, + "step": 1650, + "unclipped_grad_norm": 0.7908568978309631 + }, + { + "bits_per_byte": 1.9524230689262851, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3533165454864502, + "step": 1660, + "unclipped_grad_norm": 0.8385109901428223 + }, + { + "bits_per_byte": 1.8864807930018634, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3076088428497314, + "step": 1670, + "unclipped_grad_norm": 0.8104477524757385 + }, + { + "bits_per_byte": 1.918672677573015, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3299225568771362, + "step": 1680, + "unclipped_grad_norm": 0.7970330119132996 + }, + { + "bits_per_byte": 1.9517956762159374, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.352881669998169, + "step": 1690, + "unclipped_grad_norm": 0.7619603276252747 + }, + { + "bits_per_byte": 1.8990867773448057, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3163466453552246, + "step": 1700, + "unclipped_grad_norm": 0.8769286274909973 + }, + { + "bits_per_byte": 1.938959579086141, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3439843654632568, + "step": 1710, + "unclipped_grad_norm": 0.7246651649475098 + }, + { + "bits_per_byte": 1.847509524316351, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.2805960178375244, + "step": 1720, + "unclipped_grad_norm": 0.7415109276771545 + }, + { + "bits_per_byte": 1.944612132872049, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3479024171829224, + "step": 1730, + "unclipped_grad_norm": 0.8057509660720825 + }, + { + "bits_per_byte": 1.956270664791388, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3559834957122803, + "step": 1740, + "unclipped_grad_norm": 0.8188740611076355 + }, + { + "bits_per_byte": 1.908994697861012, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3232142925262451, + "step": 1750, + "unclipped_grad_norm": 0.7595267295837402 + }, + { + "bits_per_byte": 1.9438418225788343, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3473684787750244, + "step": 1760, + "unclipped_grad_norm": 0.7832275629043579 + }, + { + "bits_per_byte": 1.9277638524802738, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.33622407913208, + "step": 1770, + "unclipped_grad_norm": 0.7276267409324646 + }, + { + "bits_per_byte": 1.987644083927083, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.377729892730713, + "step": 1780, + "unclipped_grad_norm": 0.7699368000030518 + }, + { + "bits_per_byte": 1.8704368754681737, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2964880466461182, + "step": 1790, + "unclipped_grad_norm": 0.7781511545181274 + }, + { + "bits_per_byte": 2.006532250540742, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.390822172164917, + "step": 1800, + "unclipped_grad_norm": 0.8113785982131958 + }, + { + "bits_per_byte": 1.8832052114335305, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3053383827209473, + "step": 1810, + "unclipped_grad_norm": 0.8088275790214539 + }, + { + "bits_per_byte": 1.9263209180395349, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.335223913192749, + "step": 1820, + "unclipped_grad_norm": 0.7881855964660645 + }, + { + "bits_per_byte": 1.968589438107389, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3645222187042236, + "step": 1830, + "unclipped_grad_norm": 0.7948311567306519 + }, + { + "bits_per_byte": 1.9398172565659937, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3445788621902466, + "step": 1840, + "unclipped_grad_norm": 0.8343047499656677 + }, + { + "bits_per_byte": 1.8747907882574166, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2995059490203857, + "step": 1850, + "unclipped_grad_norm": 0.7942258715629578 + }, + { + "bits_per_byte": 1.9135037390013332, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3263397216796875, + "step": 1860, + "unclipped_grad_norm": 0.769598126411438 + }, + { + "bits_per_byte": 1.987024774401326, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3773006200790405, + "step": 1870, + "unclipped_grad_norm": 0.7647593021392822 + }, + { + "bits_per_byte": 1.8769319722606586, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.300990104675293, + "step": 1880, + "unclipped_grad_norm": 0.8446611166000366 + }, + { + "bits_per_byte": 1.8953207012562119, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3137362003326416, + "step": 1890, + "unclipped_grad_norm": 0.7048609852790833 + }, + { + "bits_per_byte": 1.915848206497895, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3279647827148438, + "step": 1900, + "unclipped_grad_norm": 0.7416327595710754 + }, + { + "bits_per_byte": 1.9362319342434569, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3420937061309814, + "step": 1910, + "unclipped_grad_norm": 0.8084686398506165 + }, + { + "bits_per_byte": 1.8860243510464736, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3072924613952637, + "step": 1920, + "unclipped_grad_norm": 0.8338462710380554 + }, + { + "bits_per_byte": 1.930916638435891, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.338409423828125, + "step": 1930, + "unclipped_grad_norm": 0.6878981590270996 + }, + { + "bits_per_byte": 1.9257588787365152, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.334834337234497, + "step": 1940, + "unclipped_grad_norm": 0.6981199383735657 + }, + { + "bits_per_byte": 1.8699291826828268, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.2961361408233643, + "step": 1950, + "unclipped_grad_norm": 0.76193767786026 + }, + { + "bits_per_byte": 1.859464038377825, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.2888822555541992, + "step": 1960, + "unclipped_grad_norm": 0.7367894053459167 + }, + { + "bits_per_byte": 1.8691335909399354, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2955846786499023, + "step": 1970, + "unclipped_grad_norm": 0.8052698373794556 + }, + { + "bits_per_byte": 1.8850715671606937, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3066320419311523, + "step": 1980, + "unclipped_grad_norm": 0.7624203562736511 + }, + { + "bits_per_byte": 1.8715028239382214, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.297226905822754, + "step": 1990, + "unclipped_grad_norm": 0.779162585735321 + }, + { + "bits_per_byte": 1.9325518494802945, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3395428657531738, + "step": 2000, + "unclipped_grad_norm": 0.7498615384101868 + }, + { + "bits_per_byte": 1.9335421255839624, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3402292728424072, + "step": 2010, + "unclipped_grad_norm": 0.7343811392784119 + }, + { + "bits_per_byte": 1.838237939608337, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.2741694450378418, + "step": 2020, + "unclipped_grad_norm": 0.7684847116470337 + }, + { + "bits_per_byte": 1.8631659649376469, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.2914482355117798, + "step": 2030, + "unclipped_grad_norm": 0.6878870129585266 + }, + { + "bits_per_byte": 1.8841837927169398, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3060166835784912, + "step": 2040, + "unclipped_grad_norm": 0.853155255317688 + }, + { + "bits_per_byte": 1.8339112000779305, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2711703777313232, + "step": 2050, + "unclipped_grad_norm": 0.8042898774147034 + }, + { + "bits_per_byte": 1.8136427007084468, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2571213245391846, + "step": 2060, + "unclipped_grad_norm": 0.7331950664520264 + }, + { + "bits_per_byte": 1.9306452498128293, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3382213115692139, + "step": 2070, + "unclipped_grad_norm": 0.7544640898704529 + }, + { + "bits_per_byte": 1.9159075405124426, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3280059099197388, + "step": 2080, + "unclipped_grad_norm": 0.7521310448646545 + }, + { + "bits_per_byte": 1.8394872215842044, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2750353813171387, + "step": 2090, + "unclipped_grad_norm": 0.7430872321128845 + }, + { + "bits_per_byte": 1.7890550290451945, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2400784492492676, + "step": 2100, + "unclipped_grad_norm": 0.7162277698516846 + }, + { + "bits_per_byte": 1.8155914361253733, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2584720849990845, + "step": 2110, + "unclipped_grad_norm": 0.7056337594985962 + }, + { + "bits_per_byte": 1.834290249840432, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2714331150054932, + "step": 2120, + "unclipped_grad_norm": 0.8666519522666931 + }, + { + "bits_per_byte": 1.8865812308699674, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.307678461074829, + "step": 2130, + "unclipped_grad_norm": 0.762921154499054 + }, + { + "bits_per_byte": 1.9030090136803828, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3190653324127197, + "step": 2140, + "unclipped_grad_norm": 0.7508745789527893 + }, + { + "bits_per_byte": 1.8090877402003454, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2539640665054321, + "step": 2150, + "unclipped_grad_norm": 0.7472740411758423 + }, + { + "bits_per_byte": 1.898260228724759, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3157737255096436, + "step": 2160, + "unclipped_grad_norm": 0.7288866639137268 + }, + { + "bits_per_byte": 1.8098697453138184, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2545061111450195, + "step": 2170, + "unclipped_grad_norm": 0.7270416617393494 + }, + { + "bits_per_byte": 1.896411071262682, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3144919872283936, + "step": 2180, + "unclipped_grad_norm": 0.7397056221961975 + }, + { + "bits_per_byte": 1.8517294626205718, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.283521056175232, + "step": 2190, + "unclipped_grad_norm": 0.7186560034751892 + }, + { + "bits_per_byte": 1.8299633383273748, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.268433928489685, + "step": 2200, + "unclipped_grad_norm": 0.6916610598564148 + }, + { + "bits_per_byte": 1.8274437924922347, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2666875123977661, + "step": 2210, + "unclipped_grad_norm": 0.7494430541992188 + }, + { + "bits_per_byte": 1.8262679471082857, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2658724784851074, + "step": 2220, + "unclipped_grad_norm": 0.782909095287323 + }, + { + "bits_per_byte": 1.782249847533204, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2353614568710327, + "step": 2230, + "unclipped_grad_norm": 0.7370295524597168 + }, + { + "bits_per_byte": 1.8783687153259665, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3019859790802002, + "step": 2240, + "unclipped_grad_norm": 0.7077252268791199 + }, + { + "bits_per_byte": 1.844123529886164, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2782490253448486, + "step": 2250, + "unclipped_grad_norm": 0.7420696020126343 + }, + { + "bits_per_byte": 1.897104505310961, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3149726390838623, + "step": 2260, + "unclipped_grad_norm": 0.7444695234298706 + }, + { + "bits_per_byte": 1.7929542197055561, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.242781162261963, + "step": 2270, + "unclipped_grad_norm": 0.7164454460144043 + }, + { + "bits_per_byte": 1.8643507534194408, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2922694683074951, + "step": 2280, + "unclipped_grad_norm": 0.7152382135391235 + }, + { + "bits_per_byte": 1.8035947863144381, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2501566410064697, + "step": 2290, + "unclipped_grad_norm": 0.7178454399108887 + }, + { + "bits_per_byte": 1.7935200426268947, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.243173360824585, + "step": 2300, + "unclipped_grad_norm": 0.7068437337875366 + }, + { + "bits_per_byte": 1.8020768674379228, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2491044998168945, + "step": 2310, + "unclipped_grad_norm": 0.7017168998718262 + }, + { + "bits_per_byte": 1.8564474626816911, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2867913246154785, + "step": 2320, + "unclipped_grad_norm": 0.6947462558746338 + }, + { + "bits_per_byte": 1.7936961528613782, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.243295431137085, + "step": 2330, + "unclipped_grad_norm": 0.6949398517608643 + }, + { + "bits_per_byte": 1.8129625093242843, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2566498517990112, + "step": 2340, + "unclipped_grad_norm": 0.7047421336174011 + }, + { + "bits_per_byte": 1.871945851246844, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2975339889526367, + "step": 2350, + "unclipped_grad_norm": 0.7291849851608276 + }, + { + "bits_per_byte": 1.7987307729827362, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2467851638793945, + "step": 2360, + "unclipped_grad_norm": 0.7353813052177429 + }, + { + "bits_per_byte": 1.7311453748119714, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.1999385356903076, + "step": 2370, + "unclipped_grad_norm": 0.6746760010719299 + }, + { + "bits_per_byte": 1.834468595849377, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2715567350387573, + "step": 2380, + "unclipped_grad_norm": 0.7166123390197754 + }, + { + "bits_per_byte": 1.8260691351638882, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2657346725463867, + "step": 2390, + "unclipped_grad_norm": 0.6613916158676147 + }, + { + "bits_per_byte": 1.8641103216735346, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2921028137207031, + "step": 2400, + "unclipped_grad_norm": 0.824955403804779 + }, + { + "bits_per_byte": 1.793840446305452, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.243395447731018, + "step": 2410, + "unclipped_grad_norm": 0.7929511070251465 + }, + { + "bits_per_byte": 1.8233349549804712, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2638394832611084, + "step": 2420, + "unclipped_grad_norm": 0.725745677947998 + }, + { + "bits_per_byte": 1.825773496987055, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265529751777649, + "step": 2430, + "unclipped_grad_norm": 0.7284960746765137 + }, + { + "bits_per_byte": 1.7725216488871491, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2286183834075928, + "step": 2440, + "unclipped_grad_norm": 0.7458904981613159 + }, + { + "bits_per_byte": 1.9001444706476118, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.317079782485962, + "step": 2450, + "unclipped_grad_norm": 0.8321223258972168 + }, + { + "bits_per_byte": 1.7654523020234334, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.223718285560608, + "step": 2460, + "unclipped_grad_norm": 0.7338651418685913 + }, + { + "bits_per_byte": 1.7862824967306352, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2381566762924194, + "step": 2470, + "unclipped_grad_norm": 0.7279908061027527 + }, + { + "bits_per_byte": 1.8666427662074894, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2938581705093384, + "step": 2480, + "unclipped_grad_norm": 0.7701565027236938 + }, + { + "bits_per_byte": 1.850673661126925, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2827892303466797, + "step": 2490, + "unclipped_grad_norm": 0.6776601672172546 + }, + { + "bits_per_byte": 1.8000481600883451, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2476983070373535, + "step": 2500, + "unclipped_grad_norm": 0.7444605231285095 + }, + { + "bits_per_byte": 1.7603095048146828, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.220153570175171, + "step": 2510, + "unclipped_grad_norm": 0.7512273788452148 + }, + { + "bits_per_byte": 1.7805241736144466, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2341653108596802, + "step": 2520, + "unclipped_grad_norm": 0.6968159079551697 + }, + { + "bits_per_byte": 1.803689032807111, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2502219676971436, + "step": 2530, + "unclipped_grad_norm": 0.7150991559028625 + }, + { + "bits_per_byte": 1.8049077018711248, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2510666847229004, + "step": 2540, + "unclipped_grad_norm": 0.7343016862869263 + }, + { + "bits_per_byte": 1.742795479581418, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2080137729644775, + "step": 2550, + "unclipped_grad_norm": 0.7049802541732788 + }, + { + "bits_per_byte": 1.7179068382791558, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.1907622814178467, + "step": 2560, + "unclipped_grad_norm": 0.7055768370628357 + }, + { + "bits_per_byte": 1.7731971967397384, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2290866374969482, + "step": 2570, + "unclipped_grad_norm": 0.6996883749961853 + }, + { + "bits_per_byte": 1.7458676056737803, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2101432085037231, + "step": 2580, + "unclipped_grad_norm": 0.6933735609054565 + }, + { + "bits_per_byte": 1.7458146350173145, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2101064920425415, + "step": 2590, + "unclipped_grad_norm": 0.7094681859016418 + }, + { + "bits_per_byte": 1.7637799427264436, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2225590944290161, + "step": 2600, + "unclipped_grad_norm": 0.7199745178222656 + }, + { + "bits_per_byte": 1.786311561798631, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2381768226623535, + "step": 2610, + "unclipped_grad_norm": 0.7421150207519531 + }, + { + "bits_per_byte": 1.7252271078304804, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.1958363056182861, + "step": 2620, + "unclipped_grad_norm": 0.7062536478042603 + }, + { + "bits_per_byte": 1.7243496523457782, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.195228099822998, + "step": 2630, + "unclipped_grad_norm": 0.6937817931175232 + }, + { + "bits_per_byte": 1.7124126805146924, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1869540214538574, + "step": 2640, + "unclipped_grad_norm": 0.7252171039581299 + }, + { + "bits_per_byte": 1.781446344588373, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2348045110702515, + "step": 2650, + "unclipped_grad_norm": 0.6726356744766235 + }, + { + "bits_per_byte": 1.6915262474806587, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1724766492843628, + "step": 2660, + "unclipped_grad_norm": 0.6618354320526123 + }, + { + "bits_per_byte": 1.7563157236963467, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2173852920532227, + "step": 2670, + "unclipped_grad_norm": 0.7113842368125916 + }, + { + "bits_per_byte": 1.858153874543552, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2879741191864014, + "step": 2680, + "unclipped_grad_norm": 0.7770810127258301 + }, + { + "bits_per_byte": 1.784997614346039, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2372660636901855, + "step": 2690, + "unclipped_grad_norm": 0.7249491810798645 + }, + { + "bits_per_byte": 1.7001362149308035, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.1784446239471436, + "step": 2700, + "unclipped_grad_norm": 0.6978740096092224 + }, + { + "bits_per_byte": 1.7671420315681605, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2248895168304443, + "step": 2710, + "unclipped_grad_norm": 0.6759880185127258 + }, + { + "bits_per_byte": 1.8404900524213854, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2757304906845093, + "step": 2720, + "unclipped_grad_norm": 0.7600791454315186 + }, + { + "bits_per_byte": 1.708491820040322, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.1842362880706787, + "step": 2730, + "unclipped_grad_norm": 0.6715761423110962 + }, + { + "bits_per_byte": 1.7849604660934528, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2372403144836426, + "step": 2740, + "unclipped_grad_norm": 0.7194302082061768 + }, + { + "bits_per_byte": 1.7485555225241152, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2120063304901123, + "step": 2750, + "unclipped_grad_norm": 0.6991336941719055 + }, + { + "bits_per_byte": 1.8629487508496072, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2912976741790771, + "step": 2760, + "unclipped_grad_norm": 0.8377771973609924 + }, + { + "bits_per_byte": 1.7856594035865592, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.237724781036377, + "step": 2770, + "unclipped_grad_norm": 0.7150954604148865 + }, + { + "bits_per_byte": 1.7901013714930438, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2408037185668945, + "step": 2780, + "unclipped_grad_norm": 0.6968044638633728 + }, + { + "bits_per_byte": 1.752669863480706, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2148581743240356, + "step": 2790, + "unclipped_grad_norm": 0.7223197817802429 + }, + { + "bits_per_byte": 1.7567675221201633, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2176984548568726, + "step": 2800, + "unclipped_grad_norm": 0.7307369709014893 + }, + { + "bits_per_byte": 1.7124591158304254, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.1869862079620361, + "step": 2810, + "unclipped_grad_norm": 0.7025272250175476 + }, + { + "bits_per_byte": 1.7803284573577647, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2340296506881714, + "step": 2820, + "unclipped_grad_norm": 0.7275472283363342 + }, + { + "bits_per_byte": 1.7225665362216327, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.1939921379089355, + "step": 2830, + "unclipped_grad_norm": 0.6614095568656921 + }, + { + "bits_per_byte": 1.7602923065495966, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2201416492462158, + "step": 2840, + "unclipped_grad_norm": 0.7144313454627991 + }, + { + "bits_per_byte": 1.7021394688480533, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.179833173751831, + "step": 2850, + "unclipped_grad_norm": 0.7220690846443176 + }, + { + "bits_per_byte": 1.744293620453084, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2090522050857544, + "step": 2860, + "unclipped_grad_norm": 0.7174125909805298 + }, + { + "bits_per_byte": 1.7433115995166573, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2083715200424194, + "step": 2870, + "unclipped_grad_norm": 0.730305552482605 + }, + { + "bits_per_byte": 1.7421880368585705, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2075927257537842, + "step": 2880, + "unclipped_grad_norm": 0.7075303792953491 + }, + { + "bits_per_byte": 1.7085926018737276, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.1843061447143555, + "step": 2890, + "unclipped_grad_norm": 0.6744402050971985 + }, + { + "bits_per_byte": 1.7378320602775172, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.204573392868042, + "step": 2900, + "unclipped_grad_norm": 0.6889722347259521 + }, + { + "bits_per_byte": 1.7031551983840492, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.180537223815918, + "step": 2910, + "unclipped_grad_norm": 0.7552688717842102 + }, + { + "bits_per_byte": 1.7508614659068837, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2136046886444092, + "step": 2920, + "unclipped_grad_norm": 0.6863148808479309 + }, + { + "bits_per_byte": 1.701510700276499, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.1793973445892334, + "step": 2930, + "unclipped_grad_norm": 0.6827563047409058 + }, + { + "bits_per_byte": 1.7106488264474435, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.1857314109802246, + "step": 2940, + "unclipped_grad_norm": 0.6807736158370972 + }, + { + "bits_per_byte": 1.7703209588867088, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.227092981338501, + "step": 2950, + "unclipped_grad_norm": 0.6866815686225891 + }, + { + "bits_per_byte": 1.7651767858167513, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2235273122787476, + "step": 2960, + "unclipped_grad_norm": 0.741482138633728 + }, + { + "bits_per_byte": 1.7465264712092357, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2105998992919922, + "step": 2970, + "unclipped_grad_norm": 0.6754798293113708 + }, + { + "bits_per_byte": 1.7663952828981142, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2243719100952148, + "step": 2980, + "unclipped_grad_norm": 0.688591480255127 + }, + { + "bits_per_byte": 1.7069632382394535, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.1831767559051514, + "step": 2990, + "unclipped_grad_norm": 0.7298949956893921 + }, + { + "bits_per_byte": 1.79113911480835, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.241523027420044, + "step": 3000, + "unclipped_grad_norm": 0.713433027267456 + }, + { + "bits_per_byte": 1.6766162115641075, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1621417999267578, + "step": 3010, + "unclipped_grad_norm": 0.7228698134422302 + }, + { + "bits_per_byte": 1.7365501015979858, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2036848068237305, + "step": 3020, + "unclipped_grad_norm": 0.6969108581542969 + }, + { + "bits_per_byte": 1.6833617150962485, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.1668174266815186, + "step": 3030, + "unclipped_grad_norm": 0.669261634349823 + }, + { + "bits_per_byte": 1.6671776317021056, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1555994749069214, + "step": 3040, + "unclipped_grad_norm": 0.6839757561683655 + }, + { + "bits_per_byte": 1.8241205717296125, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2643840312957764, + "step": 3050, + "unclipped_grad_norm": 0.7963703870773315 + }, + { + "bits_per_byte": 1.7604708245411922, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2202653884887695, + "step": 3060, + "unclipped_grad_norm": 0.7280165553092957 + }, + { + "bits_per_byte": 1.6594571585222233, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.1502480506896973, + "step": 3070, + "unclipped_grad_norm": 0.7014188766479492 + }, + { + "bits_per_byte": 1.7706030104341237, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2272884845733643, + "step": 3080, + "unclipped_grad_norm": 0.7385008335113525 + }, + { + "bits_per_byte": 1.7493225651469633, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2125380039215088, + "step": 3090, + "unclipped_grad_norm": 0.6976860165596008 + }, + { + "bits_per_byte": 1.706454857523503, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1828243732452393, + "step": 3100, + "unclipped_grad_norm": 0.7350029349327087 + }, + { + "bits_per_byte": 1.8454660264587992, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.279179573059082, + "step": 3110, + "unclipped_grad_norm": 0.8262335658073425 + }, + { + "bits_per_byte": 1.7237298268720687, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.194798469543457, + "step": 3120, + "unclipped_grad_norm": 0.7770306468009949 + }, + { + "bits_per_byte": 1.6658043502349662, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1546475887298584, + "step": 3130, + "unclipped_grad_norm": 0.6939164996147156 + }, + { + "bits_per_byte": 1.7147224075157799, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1885550022125244, + "step": 3140, + "unclipped_grad_norm": 0.744745135307312 + }, + { + "bits_per_byte": 1.7012960659282221, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.179248571395874, + "step": 3150, + "unclipped_grad_norm": 0.6927894353866577 + }, + { + "bits_per_byte": 1.700822425707746, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.1789202690124512, + "step": 3160, + "unclipped_grad_norm": 0.6962236166000366 + }, + { + "bits_per_byte": 1.7730090477196945, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2289562225341797, + "step": 3170, + "unclipped_grad_norm": 0.7581587433815002 + }, + { + "bits_per_byte": 1.7445914944043783, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2092586755752563, + "step": 3180, + "unclipped_grad_norm": 0.7753618359565735 + }, + { + "bits_per_byte": 1.7320080397886992, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2005364894866943, + "step": 3190, + "unclipped_grad_norm": 0.7117310166358948 + }, + { + "bits_per_byte": 1.7215322325593436, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1932752132415771, + "step": 3200, + "unclipped_grad_norm": 0.712298572063446 + }, + { + "bits_per_byte": 1.6960516269728116, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1756134033203125, + "step": 3210, + "unclipped_grad_norm": 0.7590813636779785 + }, + { + "bits_per_byte": 1.6593746068498092, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.150190830230713, + "step": 3220, + "unclipped_grad_norm": 0.6987801194190979 + }, + { + "bits_per_byte": 1.7324070395387008, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2008130550384521, + "step": 3230, + "unclipped_grad_norm": 0.6938228607177734 + }, + { + "bits_per_byte": 1.7474627447605329, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2112488746643066, + "step": 3240, + "unclipped_grad_norm": 0.78226238489151 + }, + { + "bits_per_byte": 1.7110863503112386, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.1860346794128418, + "step": 3250, + "unclipped_grad_norm": 0.6999595165252686 + }, + { + "bits_per_byte": 1.7142417160066183, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.18822181224823, + "step": 3260, + "unclipped_grad_norm": 0.700433075428009 + }, + { + "bits_per_byte": 1.748484321706658, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2119569778442383, + "step": 3270, + "unclipped_grad_norm": 0.7274234294891357 + }, + { + "bits_per_byte": 1.738793787261142, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.20524001121521, + "step": 3280, + "unclipped_grad_norm": 0.7183425426483154 + }, + { + "bits_per_byte": 1.6962500949519075, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.175750970840454, + "step": 3290, + "unclipped_grad_norm": 0.7457373738288879 + }, + { + "bits_per_byte": 1.7210296992535226, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1929268836975098, + "step": 3300, + "unclipped_grad_norm": 0.7834483981132507 + }, + { + "bits_per_byte": 1.6835460804979734, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.166945219039917, + "step": 3310, + "unclipped_grad_norm": 0.6523665189743042 + }, + { + "bits_per_byte": 1.6871995079502522, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1694775819778442, + "step": 3320, + "unclipped_grad_norm": 0.7348048686981201 + }, + { + "bits_per_byte": 1.7008401399207849, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.178932547569275, + "step": 3330, + "unclipped_grad_norm": 0.7228167653083801 + }, + { + "bits_per_byte": 1.643402750046831, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1391199827194214, + "step": 3340, + "unclipped_grad_norm": 0.7094708681106567 + }, + { + "bits_per_byte": 1.686364876145615, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1688990592956543, + "step": 3350, + "unclipped_grad_norm": 0.7186631560325623 + }, + { + "bits_per_byte": 1.7065993229502279, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.182924509048462, + "step": 3360, + "unclipped_grad_norm": 0.6950210332870483 + }, + { + "bits_per_byte": 1.6548356407282379, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1470446586608887, + "step": 3370, + "unclipped_grad_norm": 0.6926314830780029 + }, + { + "bits_per_byte": 1.6926787032240904, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1732754707336426, + "step": 3380, + "unclipped_grad_norm": 0.7107762694358826 + }, + { + "bits_per_byte": 1.7003921251152874, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1786220073699951, + "step": 3390, + "unclipped_grad_norm": 0.700124979019165 + }, + { + "bits_per_byte": 1.6889117672222422, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1706644296646118, + "step": 3400, + "unclipped_grad_norm": 0.7049819827079773 + }, + { + "bits_per_byte": 1.6802605239358903, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.1646678447723389, + "step": 3410, + "unclipped_grad_norm": 0.6922183036804199 + }, + { + "bits_per_byte": 1.664921907253389, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1540359258651733, + "step": 3420, + "unclipped_grad_norm": 0.7144856452941895 + }, + { + "bits_per_byte": 1.698013089105902, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1769729852676392, + "step": 3430, + "unclipped_grad_norm": 0.6953245401382446 + }, + { + "bits_per_byte": 1.7072490734051875, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1833748817443848, + "step": 3440, + "unclipped_grad_norm": 0.727461040019989 + }, + { + "bits_per_byte": 1.7747906159999822, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2301911115646362, + "step": 3450, + "unclipped_grad_norm": 0.7409155368804932 + }, + { + "bits_per_byte": 1.7588486841782542, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2191410064697266, + "step": 3460, + "unclipped_grad_norm": 0.748551070690155 + }, + { + "bits_per_byte": 1.7339454243506687, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2018793821334839, + "step": 3470, + "unclipped_grad_norm": 0.7151429653167725 + }, + { + "bits_per_byte": 1.7067943512763062, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1830596923828125, + "step": 3480, + "unclipped_grad_norm": 0.7189637422561646 + }, + { + "bits_per_byte": 1.6200879219826143, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1229593753814697, + "step": 3490, + "unclipped_grad_norm": 0.6848708391189575 + }, + { + "bits_per_byte": 1.6373195517031627, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1349034309387207, + "step": 3500, + "unclipped_grad_norm": 0.6704034209251404 + }, + { + "bits_per_byte": 1.6849599498707166, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.167925238609314, + "step": 3510, + "unclipped_grad_norm": 0.7942876219749451 + }, + { + "bits_per_byte": 1.6514716600773613, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1447129249572754, + "step": 3520, + "unclipped_grad_norm": 0.7149106860160828 + }, + { + "bits_per_byte": 1.6705435041621417, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1579325199127197, + "step": 3530, + "unclipped_grad_norm": 0.7720932364463806 + }, + { + "bits_per_byte": 1.6828739722984016, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1664793491363525, + "step": 3540, + "unclipped_grad_norm": 0.6750161647796631 + }, + { + "bits_per_byte": 1.7101762181228726, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.185403823852539, + "step": 3550, + "unclipped_grad_norm": 0.6993052363395691 + }, + { + "bits_per_byte": 1.6591575647444203, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1500403881072998, + "step": 3560, + "unclipped_grad_norm": 0.6886851787567139 + }, + { + "bits_per_byte": 1.7182958630354075, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1910319328308105, + "step": 3570, + "unclipped_grad_norm": 0.749891459941864 + }, + { + "bits_per_byte": 1.7607525321233055, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2204606533050537, + "step": 3580, + "unclipped_grad_norm": 0.8161401152610779 + }, + { + "bits_per_byte": 1.690903842267186, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1720452308654785, + "step": 3590, + "unclipped_grad_norm": 0.6925192475318909 + }, + { + "bits_per_byte": 1.6772946831217612, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1626120805740356, + "step": 3600, + "unclipped_grad_norm": 0.7061766982078552 + }, + { + "bits_per_byte": 1.6673109182565242, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1556918621063232, + "step": 3610, + "unclipped_grad_norm": 0.7162109017372131 + }, + { + "bits_per_byte": 1.6598764522250269, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.150538682937622, + "step": 3620, + "unclipped_grad_norm": 0.6953511238098145 + }, + { + "bits_per_byte": 1.6688818077895051, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.15678071975708, + "step": 3630, + "unclipped_grad_norm": 0.7265485525131226 + }, + { + "bits_per_byte": 1.6378186453559664, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.135249376296997, + "step": 3640, + "unclipped_grad_norm": 0.68751460313797 + }, + { + "bits_per_byte": 1.6996694540163617, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1781210899353027, + "step": 3650, + "unclipped_grad_norm": 0.7145417928695679 + }, + { + "bits_per_byte": 1.6073453834148872, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1141269207000732, + "step": 3660, + "unclipped_grad_norm": 0.71860671043396 + }, + { + "bits_per_byte": 1.62879970316207, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1289979219436646, + "step": 3670, + "unclipped_grad_norm": 0.6769548654556274 + }, + { + "bits_per_byte": 1.702227523965295, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.179894208908081, + "step": 3680, + "unclipped_grad_norm": 0.7466121315956116 + }, + { + "bits_per_byte": 1.653425382991163, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1460671424865723, + "step": 3690, + "unclipped_grad_norm": 0.6637522578239441 + }, + { + "bits_per_byte": 1.5989233930021354, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1082892417907715, + "step": 3700, + "unclipped_grad_norm": 0.7335078120231628 + }, + { + "bits_per_byte": 1.6701862961962997, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1576849222183228, + "step": 3710, + "unclipped_grad_norm": 0.7247957587242126 + }, + { + "bits_per_byte": 1.6538059805975223, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1463309526443481, + "step": 3720, + "unclipped_grad_norm": 0.7534144520759583 + }, + { + "bits_per_byte": 1.7119505631318241, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1866337060928345, + "step": 3730, + "unclipped_grad_norm": 0.6951875686645508 + }, + { + "bits_per_byte": 1.6437530787066386, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1393628120422363, + "step": 3740, + "unclipped_grad_norm": 0.7089563608169556 + }, + { + "bits_per_byte": 1.6267918057132467, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1276061534881592, + "step": 3750, + "unclipped_grad_norm": 0.6922760009765625 + }, + { + "bits_per_byte": 1.6201740852906965, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1230190992355347, + "step": 3760, + "unclipped_grad_norm": 0.6971311569213867 + }, + { + "bits_per_byte": 1.6730486034546093, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1596689224243164, + "step": 3770, + "unclipped_grad_norm": 0.6925700902938843 + }, + { + "bits_per_byte": 1.6258194158052683, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.126932144165039, + "step": 3780, + "unclipped_grad_norm": 0.71840900182724 + }, + { + "bits_per_byte": 1.6611288298886098, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1514067649841309, + "step": 3790, + "unclipped_grad_norm": 0.6731990575790405 + }, + { + "bits_per_byte": 1.622545554063444, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1246628761291504, + "step": 3800, + "unclipped_grad_norm": 0.7068047523498535 + }, + { + "bits_per_byte": 1.613386446009094, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.118314266204834, + "step": 3810, + "unclipped_grad_norm": 0.679490327835083 + }, + { + "bits_per_byte": 1.723293162921528, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1944957971572876, + "step": 3820, + "unclipped_grad_norm": 0.7167435884475708 + }, + { + "bits_per_byte": 1.6410942269242994, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1375198364257812, + "step": 3830, + "unclipped_grad_norm": 0.749890148639679 + }, + { + "bits_per_byte": 1.6271811744348, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1278760433197021, + "step": 3840, + "unclipped_grad_norm": 0.6869876980781555 + }, + { + "bits_per_byte": 1.6520392028252087, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.145106315612793, + "step": 3850, + "unclipped_grad_norm": 0.6890377998352051 + }, + { + "bits_per_byte": 1.6079827511189846, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1145687103271484, + "step": 3860, + "unclipped_grad_norm": 0.717820942401886 + }, + { + "bits_per_byte": 1.6751354409401786, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1611154079437256, + "step": 3870, + "unclipped_grad_norm": 0.7368612289428711 + }, + { + "bits_per_byte": 1.6338718155013159, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1325136423110962, + "step": 3880, + "unclipped_grad_norm": 0.7661339640617371 + }, + { + "bits_per_byte": 1.656585564200767, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1482576131820679, + "step": 3890, + "unclipped_grad_norm": 0.7343658804893494 + }, + { + "bits_per_byte": 1.6647341021986468, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1539057493209839, + "step": 3900, + "unclipped_grad_norm": 0.7533658146858215 + }, + { + "bits_per_byte": 1.6762937440937398, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.16191828250885, + "step": 3910, + "unclipped_grad_norm": 0.7147406935691833 + }, + { + "bits_per_byte": 1.6322013480134854, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1313557624816895, + "step": 3920, + "unclipped_grad_norm": 0.6887187361717224 + }, + { + "bits_per_byte": 1.6221845624792828, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1244126558303833, + "step": 3930, + "unclipped_grad_norm": 0.7910217642784119 + }, + { + "bits_per_byte": 1.63472175376188, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1331027746200562, + "step": 3940, + "unclipped_grad_norm": 0.730076789855957 + }, + { + "bits_per_byte": 1.6972579132859635, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1764495372772217, + "step": 3950, + "unclipped_grad_norm": 0.7732967138290405 + }, + { + "bits_per_byte": 1.7135035664691152, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.187710165977478, + "step": 3960, + "unclipped_grad_norm": 0.7405394911766052 + }, + { + "bits_per_byte": 1.685746770498414, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1684706211090088, + "step": 3970, + "unclipped_grad_norm": 0.7160924673080444 + }, + { + "bits_per_byte": 1.6665311489175123, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1551513671875, + "step": 3980, + "unclipped_grad_norm": 0.7374541759490967 + }, + { + "bits_per_byte": 1.7040839046987084, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1811809539794922, + "step": 3990, + "unclipped_grad_norm": 0.7841057181358337 + }, + { + "bits_per_byte": 1.693502844087025, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.17384672164917, + "step": 4000, + "unclipped_grad_norm": 0.7609017491340637 + }, + { + "bits_per_byte": 1.699465998540391, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1779800653457642, + "step": 4010, + "unclipped_grad_norm": 0.7629905343055725 + }, + { + "bits_per_byte": 1.5838037102341824, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.097809076309204, + "step": 4020, + "unclipped_grad_norm": 0.6868616342544556 + }, + { + "bits_per_byte": 1.617556165379263, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1212044954299927, + "step": 4030, + "unclipped_grad_norm": 0.749467670917511 + }, + { + "bits_per_byte": 1.6189757101794846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1221884489059448, + "step": 4040, + "unclipped_grad_norm": 0.7323976755142212 + }, + { + "bits_per_byte": 1.6111234982890412, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1167457103729248, + "step": 4050, + "unclipped_grad_norm": 0.7304336428642273 + }, + { + "bits_per_byte": 1.6428754512392856, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1387544870376587, + "step": 4060, + "unclipped_grad_norm": 0.745829164981842 + }, + { + "bits_per_byte": 1.6097484969953932, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1157926321029663, + "step": 4070, + "unclipped_grad_norm": 0.7523763179779053 + }, + { + "bits_per_byte": 1.6445053308215125, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1398842334747314, + "step": 4080, + "unclipped_grad_norm": 0.7511109113693237 + }, + { + "bits_per_byte": 1.6034616711931031, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1114349365234375, + "step": 4090, + "unclipped_grad_norm": 0.7438745498657227 + }, + { + "bits_per_byte": 1.561935084281071, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.082650899887085, + "step": 4100, + "unclipped_grad_norm": 0.6872318387031555 + }, + { + "bits_per_byte": 1.6907129415247284, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1719129085540771, + "step": 4110, + "unclipped_grad_norm": 0.7602059841156006 + }, + { + "bits_per_byte": 1.6125045189754696, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1177029609680176, + "step": 4120, + "unclipped_grad_norm": 0.7136654257774353 + }, + { + "bits_per_byte": 1.5891117827704122, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1014883518218994, + "step": 4130, + "unclipped_grad_norm": 0.7062129378318787 + }, + { + "bits_per_byte": 1.6146775197691212, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1192091703414917, + "step": 4140, + "unclipped_grad_norm": 0.7150430679321289 + }, + { + "bits_per_byte": 1.6618133208390438, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.151881217956543, + "step": 4150, + "unclipped_grad_norm": 0.7448095679283142 + }, + { + "bits_per_byte": 1.6228898633304711, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.124901533126831, + "step": 4160, + "unclipped_grad_norm": 0.7362097501754761 + }, + { + "bits_per_byte": 1.5905947891688021, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1025162935256958, + "step": 4170, + "unclipped_grad_norm": 0.7651577591896057 + }, + { + "bits_per_byte": 1.5608787668394717, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.081918716430664, + "step": 4180, + "unclipped_grad_norm": 0.7135135531425476 + }, + { + "bits_per_byte": 1.713072749928704, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1874115467071533, + "step": 4190, + "unclipped_grad_norm": 0.7463747262954712 + }, + { + "bits_per_byte": 1.5972510337051453, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1071300506591797, + "step": 4200, + "unclipped_grad_norm": 0.7313898801803589 + }, + { + "bits_per_byte": 1.6169509584308768, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1207849979400635, + "step": 4210, + "unclipped_grad_norm": 0.7539675831794739 + }, + { + "bits_per_byte": 1.6536405332873922, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1462162733078003, + "step": 4220, + "unclipped_grad_norm": 0.7351350784301758 + }, + { + "bits_per_byte": 1.6357479742395782, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1338140964508057, + "step": 4230, + "unclipped_grad_norm": 0.7141795754432678 + }, + { + "bits_per_byte": 1.6650933739562992, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1541547775268555, + "step": 4240, + "unclipped_grad_norm": 0.7625938653945923 + }, + { + "bits_per_byte": 1.6824343846427963, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1661746501922607, + "step": 4250, + "unclipped_grad_norm": 0.702212393283844 + }, + { + "bits_per_byte": 1.6672134040934852, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.155624270439148, + "step": 4260, + "unclipped_grad_norm": 0.775193989276886 + }, + { + "bits_per_byte": 1.643573356836487, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1392382383346558, + "step": 4270, + "unclipped_grad_norm": 0.7224892973899841 + }, + { + "bits_per_byte": 1.6848497089915135, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.167848825454712, + "step": 4280, + "unclipped_grad_norm": 0.792568564414978 + }, + { + "bits_per_byte": 1.6287462165576516, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1289608478546143, + "step": 4290, + "unclipped_grad_norm": 0.7352984547615051 + }, + { + "bits_per_byte": 1.5679441581022175, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0868160724639893, + "step": 4300, + "unclipped_grad_norm": 0.7459802031517029 + }, + { + "bits_per_byte": 1.7134996108681453, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1877074241638184, + "step": 4310, + "unclipped_grad_norm": 0.8436940312385559 + }, + { + "bits_per_byte": 1.585466610485375, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.0989617109298706, + "step": 4320, + "unclipped_grad_norm": 0.7326236367225647 + }, + { + "bits_per_byte": 1.5870087789156617, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1000306606292725, + "step": 4330, + "unclipped_grad_norm": 0.7154185175895691 + }, + { + "bits_per_byte": 1.5845594020020737, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.0983328819274902, + "step": 4340, + "unclipped_grad_norm": 0.7299889326095581 + }, + { + "bits_per_byte": 1.5087386145947432, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0457779169082642, + "step": 4350, + "unclipped_grad_norm": 0.7116599082946777 + }, + { + "bits_per_byte": 1.6642632137005844, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1535793542861938, + "step": 4360, + "unclipped_grad_norm": 0.7841640114784241 + }, + { + "bits_per_byte": 1.6118272512963718, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1172335147857666, + "step": 4370, + "unclipped_grad_norm": 0.7682546973228455 + }, + { + "bits_per_byte": 1.5975416843851036, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1073315143585205, + "step": 4380, + "unclipped_grad_norm": 0.7542603611946106 + }, + { + "bits_per_byte": 1.6190350441940322, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1222295761108398, + "step": 4390, + "unclipped_grad_norm": 0.7354364991188049 + }, + { + "bits_per_byte": 1.5791293937663822, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.0945690870285034, + "step": 4400, + "unclipped_grad_norm": 0.7447035908699036 + }, + { + "bits_per_byte": 1.5777399459300616, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.0936059951782227, + "step": 4410, + "unclipped_grad_norm": 0.7433028221130371 + }, + { + "bits_per_byte": 1.5774630538621723, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.093414068222046, + "step": 4420, + "unclipped_grad_norm": 0.7514091730117798 + }, + { + "bits_per_byte": 1.5840658117940973, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.0979907512664795, + "step": 4430, + "unclipped_grad_norm": 0.748140811920166 + }, + { + "bits_per_byte": 1.5992336497042918, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.108504295349121, + "step": 4440, + "unclipped_grad_norm": 0.7693572044372559 + }, + { + "bits_per_byte": 1.5825853851354703, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.0969645977020264, + "step": 4450, + "unclipped_grad_norm": 0.7571823596954346 + }, + { + "bits_per_byte": 1.5385392523882981, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0664341449737549, + "step": 4460, + "unclipped_grad_norm": 0.7259396314620972 + }, + { + "bits_per_byte": 1.6428487939284018, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1387360095977783, + "step": 4470, + "unclipped_grad_norm": 0.7504101991653442 + }, + { + "bits_per_byte": 1.5687149843433847, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.0873503684997559, + "step": 4480, + "unclipped_grad_norm": 0.7544133067131042 + }, + { + "bits_per_byte": 1.5307804270772734, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.061056137084961, + "step": 4490, + "unclipped_grad_norm": 0.7395938634872437 + }, + { + "bits_per_byte": 1.6123227333135077, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1175769567489624, + "step": 4500, + "unclipped_grad_norm": 0.7624281644821167 + }, + { + "bits_per_byte": 1.5895855949735391, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1018167734146118, + "step": 4510, + "unclipped_grad_norm": 0.7442159652709961 + }, + { + "bits_per_byte": 1.5720557473363947, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.0896660089492798, + "step": 4520, + "unclipped_grad_norm": 0.7299280166625977 + }, + { + "bits_per_byte": 1.5930625682260326, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.10422682762146, + "step": 4530, + "unclipped_grad_norm": 0.7508688569068909 + }, + { + "bits_per_byte": 1.551802554422838, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.0756275653839111, + "step": 4540, + "unclipped_grad_norm": 0.7326505780220032 + }, + { + "bits_per_byte": 1.6245931795046165, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.126082181930542, + "step": 4550, + "unclipped_grad_norm": 0.7591264247894287 + }, + { + "bits_per_byte": 1.6222865481912445, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.124483346939087, + "step": 4560, + "unclipped_grad_norm": 0.7490679621696472 + }, + { + "bits_per_byte": 1.5636793323261218, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.083859920501709, + "step": 4570, + "unclipped_grad_norm": 0.7205959558486938 + }, + { + "bits_per_byte": 1.5242402708302625, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0565228462219238, + "step": 4580, + "unclipped_grad_norm": 0.7278034687042236 + }, + { + "bits_per_byte": 1.649223158899981, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1431543827056885, + "step": 4590, + "unclipped_grad_norm": 0.80039381980896 + }, + { + "bits_per_byte": 1.621592598195013, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1240023374557495, + "step": 4600, + "unclipped_grad_norm": 0.7594562768936157 + }, + { + "bits_per_byte": 1.591440943811047, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1031028032302856, + "step": 4610, + "unclipped_grad_norm": 0.7573962211608887 + }, + { + "bits_per_byte": 1.5204752266375738, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.0539131164550781, + "step": 4620, + "unclipped_grad_norm": 0.7093440890312195 + }, + { + "bits_per_byte": 1.6007937043302685, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1095856428146362, + "step": 4630, + "unclipped_grad_norm": 0.7716385126113892 + }, + { + "bits_per_byte": 1.566827990698118, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.0860424041748047, + "step": 4640, + "unclipped_grad_norm": 0.7477832436561584 + }, + { + "bits_per_byte": 1.5540792607549598, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.0772056579589844, + "step": 4650, + "unclipped_grad_norm": 0.7563350200653076 + }, + { + "bits_per_byte": 1.5711925664117143, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0890676975250244, + "step": 4660, + "unclipped_grad_norm": 0.7477205991744995 + }, + { + "bits_per_byte": 1.6065126434194095, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1135497093200684, + "step": 4670, + "unclipped_grad_norm": 0.7434800267219543 + }, + { + "bits_per_byte": 1.5613855997115644, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.0822700262069702, + "step": 4680, + "unclipped_grad_norm": 0.7484313249588013 + }, + { + "bits_per_byte": 1.533432227570927, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.0628942251205444, + "step": 4690, + "unclipped_grad_norm": 0.7767173647880554 + }, + { + "bits_per_byte": 1.6430371149310967, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1388665437698364, + "step": 4700, + "unclipped_grad_norm": 0.7674224972724915 + }, + { + "bits_per_byte": 1.552380416129737, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.0760281085968018, + "step": 4710, + "unclipped_grad_norm": 0.7836162447929382 + }, + { + "bits_per_byte": 1.5203768525612804, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.053844928741455, + "step": 4720, + "unclipped_grad_norm": 0.7613595128059387 + }, + { + "bits_per_byte": 1.6524877335786587, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1454172134399414, + "step": 4730, + "unclipped_grad_norm": 0.7924392223358154 + }, + { + "bits_per_byte": 1.587415861850254, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1003128290176392, + "step": 4740, + "unclipped_grad_norm": 0.7647171020507812 + }, + { + "bits_per_byte": 1.6396726183322676, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1365344524383545, + "step": 4750, + "unclipped_grad_norm": 0.8181672096252441 + }, + { + "bits_per_byte": 1.5091890371573529, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0460901260375977, + "step": 4760, + "unclipped_grad_norm": 0.7407304048538208 + }, + { + "bits_per_byte": 1.6098291568586478, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1158485412597656, + "step": 4770, + "unclipped_grad_norm": 0.7601819038391113 + }, + { + "bits_per_byte": 1.5582154435082103, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.0800726413726807, + "step": 4780, + "unclipped_grad_norm": 0.7307054996490479 + }, + { + "bits_per_byte": 1.5944504682184955, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1051888465881348, + "step": 4790, + "unclipped_grad_norm": 0.7617009282112122 + }, + { + "bits_per_byte": 1.5823225956449518, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0967824459075928, + "step": 4800, + "unclipped_grad_norm": 0.7428550124168396 + }, + { + "bits_per_byte": 1.542078311377753, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.0688872337341309, + "step": 4810, + "unclipped_grad_norm": 0.773262083530426 + }, + { + "bits_per_byte": 1.6108488419956133, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1165553331375122, + "step": 4820, + "unclipped_grad_norm": 0.8227523565292358 + }, + { + "bits_per_byte": 1.5982464692883394, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1078200340270996, + "step": 4830, + "unclipped_grad_norm": 0.783271312713623 + }, + { + "bits_per_byte": 1.6134836162068316, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1183816194534302, + "step": 4840, + "unclipped_grad_norm": 0.7595719695091248 + }, + { + "bits_per_byte": 1.5430993723759254, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.0695949792861938, + "step": 4850, + "unclipped_grad_norm": 0.8344528079032898 + }, + { + "bits_per_byte": 1.551674427347945, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0755387544631958, + "step": 4860, + "unclipped_grad_norm": 0.7924710512161255 + }, + { + "bits_per_byte": 1.5551434893984988, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.0779433250427246, + "step": 4870, + "unclipped_grad_norm": 0.7731344103813171 + }, + { + "bits_per_byte": 1.7162552888829201, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1896175146102905, + "step": 4880, + "unclipped_grad_norm": 0.8454892039299011 + }, + { + "bits_per_byte": 1.5015098398136777, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0407673120498657, + "step": 4890, + "unclipped_grad_norm": 0.7279435396194458 + }, + { + "bits_per_byte": 1.6017640304464364, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1102582216262817, + "step": 4900, + "unclipped_grad_norm": 0.7495299577713013 + }, + { + "bits_per_byte": 1.5795956387328711, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.0948922634124756, + "step": 4910, + "unclipped_grad_norm": 0.8354763388633728 + }, + { + "bits_per_byte": 1.5130457481029513, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.048763394355774, + "step": 4920, + "unclipped_grad_norm": 0.7128972411155701 + }, + { + "bits_per_byte": 1.5263470583033318, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.057983160018921, + "step": 4930, + "unclipped_grad_norm": 0.7301722168922424 + }, + { + "bits_per_byte": 1.56106777577277, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0820497274398804, + "step": 4940, + "unclipped_grad_norm": 0.7844029664993286 + }, + { + "bits_per_byte": 1.5809642766684375, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0958409309387207, + "step": 4950, + "unclipped_grad_norm": 0.7815173864364624 + }, + { + "bits_per_byte": 1.6004612618661507, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1093552112579346, + "step": 4960, + "unclipped_grad_norm": 0.8055645227432251 + }, + { + "bits_per_byte": 1.5618184800437862, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.0825700759887695, + "step": 4970, + "unclipped_grad_norm": 0.7773133516311646 + }, + { + "bits_per_byte": 1.606830467358204, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1137700080871582, + "step": 4980, + "unclipped_grad_norm": 0.7920810580253601 + }, + { + "bits_per_byte": 1.567882244347907, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.086773157119751, + "step": 4990, + "unclipped_grad_norm": 0.7929123640060425 + }, + { + "bits_per_byte": 1.5708692390280923, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.088843584060669, + "step": 5000, + "unclipped_grad_norm": 0.762309730052948 + }, + { + "bits_per_byte": 1.6157285057485438, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1199376583099365, + "step": 5010, + "unclipped_grad_norm": 0.7734562158584595 + }, + { + "bits_per_byte": 1.5685849654593325, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.0872602462768555, + "step": 5020, + "unclipped_grad_norm": 0.8221466541290283 + }, + { + "bits_per_byte": 1.5125002191344146, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0483852624893188, + "step": 5030, + "unclipped_grad_norm": 0.7515868544578552 + }, + { + "bits_per_byte": 1.5645020973278494, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.08443021774292, + "step": 5040, + "unclipped_grad_norm": 0.7403401732444763 + }, + { + "bits_per_byte": 1.5465288784167808, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.071972131729126, + "step": 5050, + "unclipped_grad_norm": 0.8082988262176514 + }, + { + "bits_per_byte": 1.6139199361920706, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1186840534210205, + "step": 5060, + "unclipped_grad_norm": 0.8066977262496948 + }, + { + "bits_per_byte": 1.5533101543403012, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.0766725540161133, + "step": 5070, + "unclipped_grad_norm": 0.7855141758918762 + }, + { + "bits_per_byte": 1.6163216739113697, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1203488111495972, + "step": 5080, + "unclipped_grad_norm": 0.776370644569397 + }, + { + "bits_per_byte": 1.5614495772576855, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.082314372062683, + "step": 5090, + "unclipped_grad_norm": 0.776827871799469 + }, + { + "bits_per_byte": 1.5419546558517827, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0688015222549438, + "step": 5100, + "unclipped_grad_norm": 0.7785685062408447 + }, + { + "bits_per_byte": 1.5800214677764075, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0951874256134033, + "step": 5110, + "unclipped_grad_norm": 0.8104537725448608 + }, + { + "bits_per_byte": 1.487382496941203, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0309749841690063, + "step": 5120, + "unclipped_grad_norm": 0.7389267086982727 + }, + { + "bits_per_byte": 1.6243671943013827, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1259255409240723, + "step": 5130, + "unclipped_grad_norm": 0.8053697943687439 + }, + { + "bits_per_byte": 1.5876562935961602, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1004794836044312, + "step": 5140, + "unclipped_grad_norm": 0.8099423050880432 + }, + { + "bits_per_byte": 1.5188835272038386, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0528098344802856, + "step": 5150, + "unclipped_grad_norm": 0.7463088035583496 + }, + { + "bits_per_byte": 1.5574305146896725, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.079528570175171, + "step": 5160, + "unclipped_grad_norm": 0.7795574069023132 + }, + { + "bits_per_byte": 1.604379370618107, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1120710372924805, + "step": 5170, + "unclipped_grad_norm": 0.8112488985061646 + }, + { + "bits_per_byte": 1.570943535533265, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0888950824737549, + "step": 5180, + "unclipped_grad_norm": 0.7960402965545654 + }, + { + "bits_per_byte": 1.4686855750353036, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0180152654647827, + "step": 5190, + "unclipped_grad_norm": 0.7735135555267334 + }, + { + "bits_per_byte": 1.5401951013508053, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0675818920135498, + "step": 5200, + "unclipped_grad_norm": 0.7935121655464172 + }, + { + "bits_per_byte": 1.522642552003747, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.055415391921997, + "step": 5210, + "unclipped_grad_norm": 0.7616920471191406 + }, + { + "bits_per_byte": 1.5296819738862126, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0602947473526, + "step": 5220, + "unclipped_grad_norm": 0.799717128276825 + }, + { + "bits_per_byte": 1.60920967535024, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1154191493988037, + "step": 5230, + "unclipped_grad_norm": 0.8263183832168579 + }, + { + "bits_per_byte": 1.5759482306733728, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0923640727996826, + "step": 5240, + "unclipped_grad_norm": 0.9089767336845398 + }, + { + "bits_per_byte": 1.5751560785834986, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0918149948120117, + "step": 5250, + "unclipped_grad_norm": 0.8223497271537781 + }, + { + "bits_per_byte": 1.518346597367845, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.052437663078308, + "step": 5260, + "unclipped_grad_norm": 0.7989961504936218 + }, + { + "bits_per_byte": 1.5652323356634128, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0849363803863525, + "step": 5270, + "unclipped_grad_norm": 0.7858845591545105 + }, + { + "bits_per_byte": 1.4824085866955998, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0275273323059082, + "step": 5280, + "unclipped_grad_norm": 0.7963522672653198 + }, + { + "bits_per_byte": 1.5109090356486319, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.047282338142395, + "step": 5290, + "unclipped_grad_norm": 0.7786324620246887 + }, + { + "bits_per_byte": 1.4973765807654917, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0379023551940918, + "step": 5300, + "unclipped_grad_norm": 0.8200122714042664 + }, + { + "bits_per_byte": 1.4979238295605373, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0382816791534424, + "step": 5310, + "unclipped_grad_norm": 0.798004686832428 + }, + { + "bits_per_byte": 1.4963071926424267, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.037161111831665, + "step": 5320, + "unclipped_grad_norm": 0.7881578207015991 + }, + { + "bits_per_byte": 1.5549873291515153, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0778350830078125, + "step": 5330, + "unclipped_grad_norm": 0.8092938661575317 + }, + { + "bits_per_byte": 1.6144941862633015, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1190820932388306, + "step": 5340, + "unclipped_grad_norm": 0.8349001407623291 + }, + { + "bits_per_byte": 1.4860422361430292, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.030045986175537, + "step": 5350, + "unclipped_grad_norm": 0.7843849062919617 + }, + { + "bits_per_byte": 1.5082066722556244, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0454092025756836, + "step": 5360, + "unclipped_grad_norm": 0.7918040752410889 + }, + { + "bits_per_byte": 1.577472340925319, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0934205055236816, + "step": 5370, + "unclipped_grad_norm": 0.8359649181365967 + }, + { + "bits_per_byte": 1.5013394050066726, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.040649175643921, + "step": 5380, + "unclipped_grad_norm": 0.7734270095825195 + }, + { + "bits_per_byte": 1.5016145772480531, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0408399105072021, + "step": 5390, + "unclipped_grad_norm": 0.7885358333587646 + }, + { + "bits_per_byte": 1.5491196250693782, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073767900466919, + "step": 5400, + "unclipped_grad_norm": 0.8202832341194153 + }, + { + "bits_per_byte": 1.4666546319112648, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0166075229644775, + "step": 5410, + "unclipped_grad_norm": 0.8083077073097229 + }, + { + "bits_per_byte": 1.6009971598062391, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1097266674041748, + "step": 5420, + "unclipped_grad_norm": 0.847724974155426 + }, + { + "bits_per_byte": 1.5289176829857782, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0597649812698364, + "step": 5430, + "unclipped_grad_norm": 0.8225540518760681 + }, + { + "bits_per_byte": 1.530357349756151, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.060762882232666, + "step": 5440, + "unclipped_grad_norm": 0.8045809864997864 + }, + { + "bits_per_byte": 1.5424515137301253, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.069145917892456, + "step": 5450, + "unclipped_grad_norm": 0.8051854968070984 + }, + { + "bits_per_byte": 1.5648894022575925, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0846986770629883, + "step": 5460, + "unclipped_grad_norm": 0.7835660576820374 + }, + { + "bits_per_byte": 1.5450825043230245, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.070969581604004, + "step": 5470, + "unclipped_grad_norm": 0.8599129915237427 + }, + { + "bits_per_byte": 1.4780804713039863, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0245273113250732, + "step": 5480, + "unclipped_grad_norm": 0.8000594973564148 + }, + { + "bits_per_byte": 1.5424891779306642, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0691720247268677, + "step": 5490, + "unclipped_grad_norm": 0.8055532574653625 + }, + { + "bits_per_byte": 1.5569413960306187, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0791895389556885, + "step": 5500, + "unclipped_grad_norm": 0.8300883769989014 + }, + { + "bits_per_byte": 1.5474785666148452, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0726304054260254, + "step": 5510, + "unclipped_grad_norm": 0.826352059841156 + }, + { + "bits_per_byte": 1.5350409532870977, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0640093088150024, + "step": 5520, + "unclipped_grad_norm": 0.853265643119812 + }, + { + "bits_per_byte": 1.5635150888945477, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.083746075630188, + "step": 5530, + "unclipped_grad_norm": 0.8060055375099182 + }, + { + "bits_per_byte": 1.5843347926600468, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.098177194595337, + "step": 5540, + "unclipped_grad_norm": 0.8706573843955994 + }, + { + "bits_per_byte": 1.5165710484803372, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0512069463729858, + "step": 5550, + "unclipped_grad_norm": 0.7865509986877441 + }, + { + "bits_per_byte": 1.5106121935932426, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0470765829086304, + "step": 5560, + "unclipped_grad_norm": 0.8064424991607666 + }, + { + "bits_per_byte": 1.5654490338035, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0850865840911865, + "step": 5570, + "unclipped_grad_norm": 0.8376733064651489 + }, + { + "bits_per_byte": 1.4567845475782482, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0097661018371582, + "step": 5580, + "unclipped_grad_norm": 0.7972759008407593 + }, + { + "bits_per_byte": 1.5257076268074239, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.057539939880371, + "step": 5590, + "unclipped_grad_norm": 0.8159511089324951 + }, + { + "bits_per_byte": 1.5137768463517691, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0492701530456543, + "step": 5600, + "unclipped_grad_norm": 0.7912799715995789 + }, + { + "bits_per_byte": 1.5312812405565859, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0614032745361328, + "step": 5610, + "unclipped_grad_norm": 0.7786760330200195 + }, + { + "bits_per_byte": 1.4626550033827992, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0138351917266846, + "step": 5620, + "unclipped_grad_norm": 0.8055174350738525 + }, + { + "bits_per_byte": 1.4690646247978054, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0182780027389526, + "step": 5630, + "unclipped_grad_norm": 0.7851108312606812 + }, + { + "bits_per_byte": 1.4663743001903584, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0164132118225098, + "step": 5640, + "unclipped_grad_norm": 0.8267287015914917 + }, + { + "bits_per_byte": 1.5738650048234715, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.090920090675354, + "step": 5650, + "unclipped_grad_norm": 0.8359448313713074 + }, + { + "bits_per_byte": 1.5300369460775933, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.060540795326233, + "step": 5660, + "unclipped_grad_norm": 0.8635727167129517 + }, + { + "bits_per_byte": 1.5230305448640935, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0556843280792236, + "step": 5670, + "unclipped_grad_norm": 0.8095638751983643 + }, + { + "bits_per_byte": 1.524155483383387, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0564640760421753, + "step": 5680, + "unclipped_grad_norm": 0.8353701233863831 + }, + { + "bits_per_byte": 1.566590998605229, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.0858781337738037, + "step": 5690, + "unclipped_grad_norm": 0.8297926187515259 + }, + { + "bits_per_byte": 1.499949785187701, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0396859645843506, + "step": 5700, + "unclipped_grad_norm": 0.8335801959037781 + }, + { + "bits_per_byte": 1.5306388853556132, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0609580278396606, + "step": 5710, + "unclipped_grad_norm": 0.8218200206756592 + }, + { + "bits_per_byte": 1.5663932185567369, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0857410430908203, + "step": 5720, + "unclipped_grad_norm": 0.8909743428230286 + }, + { + "bits_per_byte": 1.4614645394735268, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.013010025024414, + "step": 5730, + "unclipped_grad_norm": 0.7809724807739258 + }, + { + "bits_per_byte": 1.5314872757723195, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0615460872650146, + "step": 5740, + "unclipped_grad_norm": 0.8163876533508301 + }, + { + "bits_per_byte": 1.499649159513993, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0394775867462158, + "step": 5750, + "unclipped_grad_norm": 0.8667106628417969 + }, + { + "bits_per_byte": 1.4999243317553734, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.039668321609497, + "step": 5760, + "unclipped_grad_norm": 0.8516601324081421 + }, + { + "bits_per_byte": 1.450708744488567, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0055546760559082, + "step": 5770, + "unclipped_grad_norm": 0.7943668365478516 + }, + { + "bits_per_byte": 1.5466515020468459, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0720571279525757, + "step": 5780, + "unclipped_grad_norm": 0.8325088024139404 + }, + { + "bits_per_byte": 1.494320105094358, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0357837677001953, + "step": 5790, + "unclipped_grad_norm": 0.8366720080375671 + }, + { + "bits_per_byte": 1.4906410522271007, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033233642578125, + "step": 5800, + "unclipped_grad_norm": 0.8333538770675659 + }, + { + "bits_per_byte": 1.5321304908865463, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0619919300079346, + "step": 5810, + "unclipped_grad_norm": 0.8446563482284546 + }, + { + "bits_per_byte": 1.51830377368778, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.05240797996521, + "step": 5820, + "unclipped_grad_norm": 0.8496891856193542 + }, + { + "bits_per_byte": 1.4932847695361637, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0350661277770996, + "step": 5830, + "unclipped_grad_norm": 0.8205961585044861 + }, + { + "bits_per_byte": 1.456290785387621, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.009423851966858, + "step": 5840, + "unclipped_grad_norm": 0.8371089100837708 + }, + { + "bits_per_byte": 1.454681199758196, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0083081722259521, + "step": 5850, + "unclipped_grad_norm": 0.8174164295196533 + }, + { + "bits_per_byte": 1.4966561454410274, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0374029874801636, + "step": 5860, + "unclipped_grad_norm": 0.9058852791786194 + }, + { + "bits_per_byte": 1.5418609253070623, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0687365531921387, + "step": 5870, + "unclipped_grad_norm": 0.8718268275260925 + }, + { + "bits_per_byte": 1.5420466665699941, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0688652992248535, + "step": 5880, + "unclipped_grad_norm": 0.8315351009368896 + }, + { + "bits_per_byte": 1.525186175410008, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0571784973144531, + "step": 5890, + "unclipped_grad_norm": 0.8407886028289795 + }, + { + "bits_per_byte": 1.4989159974733648, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0389693975448608, + "step": 5900, + "unclipped_grad_norm": 0.8770707249641418 + }, + { + "bits_per_byte": 1.4937374278732347, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0353798866271973, + "step": 5910, + "unclipped_grad_norm": 0.8901946544647217 + }, + { + "bits_per_byte": 1.4996551789067731, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.03948175907135, + "step": 5920, + "unclipped_grad_norm": 0.85027015209198 + }, + { + "bits_per_byte": 1.5573538104273876, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.0794754028320312, + "step": 5930, + "unclipped_grad_norm": 0.8851437568664551 + }, + { + "bits_per_byte": 1.5404470559343193, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0677565336227417, + "step": 5940, + "unclipped_grad_norm": 0.8759228587150574 + }, + { + "bits_per_byte": 1.5066732749405318, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0443463325500488, + "step": 5950, + "unclipped_grad_norm": 0.8818607926368713 + }, + { + "bits_per_byte": 1.5044743047665998, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0428221225738525, + "step": 5960, + "unclipped_grad_norm": 0.816109299659729 + }, + { + "bits_per_byte": 1.5173669841885302, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0517586469650269, + "step": 5970, + "unclipped_grad_norm": 0.8590478301048279 + }, + { + "bits_per_byte": 1.4670959393933787, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0169134140014648, + "step": 5980, + "unclipped_grad_norm": 0.8544116616249084 + }, + { + "bits_per_byte": 1.4408131207318973, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 0.9986955523490906, + "step": 5990, + "unclipped_grad_norm": 0.8525132536888123 + }, + { + "bits_per_byte": 1.5172739415744134, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0516941547393799, + "step": 6000, + "unclipped_grad_norm": 0.860197126865387 + }, + { + "bits_per_byte": 1.499894750739425, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0396478176116943, + "step": 6010, + "unclipped_grad_norm": 0.8407278060913086 + }, + { + "bits_per_byte": 1.4776142263374972, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.024204134941101, + "step": 6020, + "unclipped_grad_norm": 0.8550556302070618 + }, + { + "bits_per_byte": 1.5367169242197558, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0651710033416748, + "step": 6030, + "unclipped_grad_norm": 0.8657232522964478 + }, + { + "bits_per_byte": 1.4931389282482321, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0349650382995605, + "step": 6040, + "unclipped_grad_norm": 0.8972119092941284 + }, + { + "bits_per_byte": 1.498401597364634, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0386128425598145, + "step": 6050, + "unclipped_grad_norm": 0.8974336981773376 + }, + { + "bits_per_byte": 1.5272425719663745, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0586038827896118, + "step": 6060, + "unclipped_grad_norm": 0.8948551416397095 + }, + { + "bits_per_byte": 1.5059438965182226, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.043840765953064, + "step": 6070, + "unclipped_grad_norm": 0.843971312046051 + }, + { + "bits_per_byte": 1.4791374766761891, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0252599716186523, + "step": 6080, + "unclipped_grad_norm": 0.8345775008201599 + }, + { + "bits_per_byte": 1.4884991802932552, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0317490100860596, + "step": 6090, + "unclipped_grad_norm": 0.8372970223426819 + }, + { + "bits_per_byte": 1.486896301987214, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0306379795074463, + "step": 6100, + "unclipped_grad_norm": 0.827723503112793 + }, + { + "bits_per_byte": 1.4668594632484424, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0167495012283325, + "step": 6110, + "unclipped_grad_norm": 0.871588945388794 + }, + { + "bits_per_byte": 1.4677823221529722, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.017389178276062, + "step": 6120, + "unclipped_grad_norm": 0.8457695245742798 + }, + { + "bits_per_byte": 1.5061057321926845, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0439529418945312, + "step": 6130, + "unclipped_grad_norm": 0.9012824296951294 + }, + { + "bits_per_byte": 1.5178127632195666, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0520676374435425, + "step": 6140, + "unclipped_grad_norm": 0.8450613617897034 + }, + { + "bits_per_byte": 1.4675592606548031, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0172345638275146, + "step": 6150, + "unclipped_grad_norm": 0.885280191898346 + }, + { + "bits_per_byte": 1.5554258849112155, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.078139066696167, + "step": 6160, + "unclipped_grad_norm": 0.9099593758583069 + }, + { + "bits_per_byte": 1.421598961003528, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 0.985377311706543, + "step": 6170, + "unclipped_grad_norm": 0.7732160091400146 + }, + { + "bits_per_byte": 1.5028951600663776, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0417275428771973, + "step": 6180, + "unclipped_grad_norm": 0.869156002998352 + }, + { + "bits_per_byte": 1.4956368042693635, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.036696434020996, + "step": 6190, + "unclipped_grad_norm": 0.8908746838569641 + }, + { + "bits_per_byte": 1.4533810109176732, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0074069499969482, + "step": 6200, + "unclipped_grad_norm": 0.8600793480873108 + }, + { + "bits_per_byte": 1.475978499345141, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0230703353881836, + "step": 6210, + "unclipped_grad_norm": 0.8739057779312134 + }, + { + "bits_per_byte": 1.4582183669384914, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.010759949684143, + "step": 6220, + "unclipped_grad_norm": 0.8542900681495667 + }, + { + "bits_per_byte": 1.4401650041121208, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 0.9982463121414185, + "step": 6230, + "unclipped_grad_norm": 0.8145909309387207 + }, + { + "bits_per_byte": 1.458188613939892, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0107393264770508, + "step": 6240, + "unclipped_grad_norm": 0.8962635397911072 + }, + { + "bits_per_byte": 1.4595658510080016, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0116939544677734, + "step": 6250, + "unclipped_grad_norm": 0.855821430683136 + }, + { + "bits_per_byte": 1.5147881043388425, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.049971103668213, + "step": 6260, + "unclipped_grad_norm": 0.8663405179977417 + }, + { + "bits_per_byte": 1.4764731214490225, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0234131813049316, + "step": 6270, + "unclipped_grad_norm": 0.8501848578453064 + }, + { + "bits_per_byte": 1.59573105103682, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1060764789581299, + "step": 6280, + "unclipped_grad_norm": 0.9987632036209106 + }, + { + "bits_per_byte": 1.4579987450933396, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0106077194213867, + "step": 6290, + "unclipped_grad_norm": 0.8792169690132141 + }, + { + "bits_per_byte": 1.4898052165439075, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0326542854309082, + "step": 6300, + "unclipped_grad_norm": 0.8601147532463074 + }, + { + "bits_per_byte": 1.537688110249178, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.065844178199768, + "step": 6310, + "unclipped_grad_norm": 0.907733678817749 + }, + { + "bits_per_byte": 1.487509248154889, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0310628414154053, + "step": 6320, + "unclipped_grad_norm": 0.8578711152076721 + }, + { + "bits_per_byte": 1.4237945775157679, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 0.986899197101593, + "step": 6330, + "unclipped_grad_norm": 0.845673143863678 + }, + { + "bits_per_byte": 1.4966876182661353, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0374248027801514, + "step": 6340, + "unclipped_grad_norm": 0.8802160620689392 + }, + { + "bits_per_byte": 1.485631197607467, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0297610759735107, + "step": 6350, + "unclipped_grad_norm": 0.884225606918335 + }, + { + "bits_per_byte": 1.4432266392627804, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0003684759140015, + "step": 6360, + "unclipped_grad_norm": 0.8594306707382202 + }, + { + "bits_per_byte": 1.497669639202562, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0381054878234863, + "step": 6370, + "unclipped_grad_norm": 0.9075121283531189 + }, + { + "bits_per_byte": 1.4643473126672892, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0150082111358643, + "step": 6380, + "unclipped_grad_norm": 0.8724440932273865 + }, + { + "bits_per_byte": 1.4983035672536422, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0385448932647705, + "step": 6390, + "unclipped_grad_norm": 0.9329665303230286 + }, + { + "bits_per_byte": 1.4855978329731998, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.029737949371338, + "step": 6400, + "unclipped_grad_norm": 0.8683961033821106 + }, + { + "bits_per_byte": 1.4585226042478678, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0109708309173584, + "step": 6410, + "unclipped_grad_norm": 0.8944246768951416 + }, + { + "bits_per_byte": 1.4417726979323862, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 0.9993606805801392, + "step": 6420, + "unclipped_grad_norm": 0.8450288772583008 + }, + { + "bits_per_byte": 1.474598510554618, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0221138000488281, + "step": 6430, + "unclipped_grad_norm": 0.8708812594413757 + }, + { + "bits_per_byte": 1.4773634756325393, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.024030327796936, + "step": 6440, + "unclipped_grad_norm": 0.8653560876846313 + }, + { + "bits_per_byte": 1.4567353605401014, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0097320079803467, + "step": 6450, + "unclipped_grad_norm": 0.8851431012153625 + }, + { + "bits_per_byte": 1.4433678370191387, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0004663467407227, + "step": 6460, + "unclipped_grad_norm": 0.902314305305481 + }, + { + "bits_per_byte": 1.478021997202693, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.024486780166626, + "step": 6470, + "unclipped_grad_norm": 0.8634862899780273 + }, + { + "bits_per_byte": 1.49569390250945, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.036736011505127, + "step": 6480, + "unclipped_grad_norm": 0.8877258896827698 + }, + { + "bits_per_byte": 1.4739315618345719, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0216515064239502, + "step": 6490, + "unclipped_grad_norm": 0.9130761027336121 + }, + { + "bits_per_byte": 1.4798123365981748, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0257277488708496, + "step": 6500, + "unclipped_grad_norm": 0.9244931936264038 + }, + { + "bits_per_byte": 1.429752142532981, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 0.9910286664962769, + "step": 6510, + "unclipped_grad_norm": 0.8583832383155823 + }, + { + "bits_per_byte": 1.3992985726054255, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9699198603630066, + "step": 6520, + "unclipped_grad_norm": 0.8968297243118286 + }, + { + "bits_per_byte": 1.5067831718544331, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0444225072860718, + "step": 6530, + "unclipped_grad_norm": 0.9009160399436951 + }, + { + "bits_per_byte": 1.468959715380779, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0182052850723267, + "step": 6540, + "unclipped_grad_norm": 0.8687822222709656 + }, + { + "bits_per_byte": 1.5110841139872102, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0474036931991577, + "step": 6550, + "unclipped_grad_norm": 0.9212287068367004 + }, + { + "bits_per_byte": 1.4657298811975754, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.015966534614563, + "step": 6560, + "unclipped_grad_norm": 0.8736171126365662 + }, + { + "bits_per_byte": 1.3923212364599213, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 0.9650835394859314, + "step": 6570, + "unclipped_grad_norm": 0.9459646344184875 + }, + { + "bits_per_byte": 1.5011586512406159, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.040523886680603, + "step": 6580, + "unclipped_grad_norm": 0.8902279138565063 + }, + { + "bits_per_byte": 1.4631396504929306, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0141711235046387, + "step": 6590, + "unclipped_grad_norm": 0.8934556245803833 + }, + { + "bits_per_byte": 1.3892133379761795, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9629293084144592, + "step": 6600, + "unclipped_grad_norm": 0.8376922607421875 + }, + { + "bits_per_byte": 1.5024242715683152, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0414011478424072, + "step": 6610, + "unclipped_grad_norm": 0.8684409856796265 + }, + { + "bits_per_byte": 1.5173028346597586, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0517141819000244, + "step": 6620, + "unclipped_grad_norm": 0.9338745474815369 + }, + { + "bits_per_byte": 1.4964176055042808, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0372376441955566, + "step": 6630, + "unclipped_grad_norm": 0.9484766721725464 + }, + { + "bits_per_byte": 1.4696447222791655, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0186800956726074, + "step": 6640, + "unclipped_grad_norm": 0.9511991143226624 + }, + { + "bits_per_byte": 1.4476113369465278, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0034077167510986, + "step": 6650, + "unclipped_grad_norm": 0.8744685649871826 + }, + { + "bits_per_byte": 1.5102728718230902, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0468413829803467, + "step": 6660, + "unclipped_grad_norm": 0.9144671559333801 + }, + { + "bits_per_byte": 1.4729612357184039, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0209789276123047, + "step": 6670, + "unclipped_grad_norm": 0.918013870716095 + }, + { + "bits_per_byte": 1.4976474534406006, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0380901098251343, + "step": 6680, + "unclipped_grad_norm": 0.9484038352966309 + }, + { + "bits_per_byte": 1.4621077545877539, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.013455867767334, + "step": 6690, + "unclipped_grad_norm": 0.9366878867149353 + }, + { + "bits_per_byte": 1.513278096664267, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.048924446105957, + "step": 6700, + "unclipped_grad_norm": 0.9588149785995483 + }, + { + "bits_per_byte": 1.448871969777352, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0042815208435059, + "step": 6710, + "unclipped_grad_norm": 0.8966986536979675 + }, + { + "bits_per_byte": 1.4475960304906008, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0033971071243286, + "step": 6720, + "unclipped_grad_norm": 0.9147130250930786 + }, + { + "bits_per_byte": 1.4208115384365525, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 0.9848315119743347, + "step": 6730, + "unclipped_grad_norm": 0.9434075951576233 + }, + { + "bits_per_byte": 1.4530764296429952, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0071958303451538, + "step": 6740, + "unclipped_grad_norm": 0.9062299728393555 + }, + { + "bits_per_byte": 1.4277258429405155, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 0.9896241426467896, + "step": 6750, + "unclipped_grad_norm": 0.9092971086502075 + }, + { + "bits_per_byte": 1.4128754850038137, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 0.9793306589126587, + "step": 6760, + "unclipped_grad_norm": 0.8910818099975586 + }, + { + "bits_per_byte": 1.434429210723195, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9942705631256104, + "step": 6770, + "unclipped_grad_norm": 0.8744387030601501 + }, + { + "bits_per_byte": 1.4315308730995298, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 0.9922615885734558, + "step": 6780, + "unclipped_grad_norm": 0.9047504663467407 + }, + { + "bits_per_byte": 1.53544803622169, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0642914772033691, + "step": 6790, + "unclipped_grad_norm": 0.9123643636703491 + }, + { + "bits_per_byte": 1.504298710480069, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0427004098892212, + "step": 6800, + "unclipped_grad_norm": 0.978726327419281 + }, + { + "bits_per_byte": 1.4819994399691971, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.027243733406067, + "step": 6810, + "unclipped_grad_norm": 0.9510568380355835 + }, + { + "bits_per_byte": 1.4201767504722178, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 0.9843915104866028, + "step": 6820, + "unclipped_grad_norm": 0.9054594039916992 + }, + { + "bits_per_byte": 1.441129998766112, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 0.9989151954650879, + "step": 6830, + "unclipped_grad_norm": 0.9485244154930115 + }, + { + "bits_per_byte": 1.4736460706341397, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.021453619003296, + "step": 6840, + "unclipped_grad_norm": 0.9141860604286194 + }, + { + "bits_per_byte": 1.439767810179953, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 0.9979709982872009, + "step": 6850, + "unclipped_grad_norm": 0.9174992442131042 + }, + { + "bits_per_byte": 1.4496288654237992, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0048061609268188, + "step": 6860, + "unclipped_grad_norm": 0.9317811727523804 + }, + { + "bits_per_byte": 1.4588201342338605, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0111770629882812, + "step": 6870, + "unclipped_grad_norm": 0.9191097617149353 + }, + { + "bits_per_byte": 1.4995917173086046, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0394377708435059, + "step": 6880, + "unclipped_grad_norm": 0.9317643642425537 + }, + { + "bits_per_byte": 1.3990970949299397, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9697802066802979, + "step": 6890, + "unclipped_grad_norm": 0.9130836725234985 + }, + { + "bits_per_byte": 1.4230204836042344, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 0.986362636089325, + "step": 6900, + "unclipped_grad_norm": 0.8910167217254639 + }, + { + "bits_per_byte": 1.5914774041330302, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1031280755996704, + "step": 6910, + "unclipped_grad_norm": 1.0252712965011597 + }, + { + "bits_per_byte": 1.394897880535148, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 0.9668695330619812, + "step": 6920, + "unclipped_grad_norm": 0.8750832676887512 + }, + { + "bits_per_byte": 1.42465509270936, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9874956607818604, + "step": 6930, + "unclipped_grad_norm": 0.8865536451339722 + }, + { + "bits_per_byte": 1.4536008047454758, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0075592994689941, + "step": 6940, + "unclipped_grad_norm": 0.913940966129303 + }, + { + "bits_per_byte": 1.4401447961506444, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9982323050498962, + "step": 6950, + "unclipped_grad_norm": 0.941871702671051 + }, + { + "bits_per_byte": 1.4963969675861772, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0372233390808105, + "step": 6960, + "unclipped_grad_norm": 0.9974261522293091 + }, + { + "bits_per_byte": 1.4704031657694705, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0192058086395264, + "step": 6970, + "unclipped_grad_norm": 0.9634589552879333 + }, + { + "bits_per_byte": 1.5157009882496222, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0506038665771484, + "step": 6980, + "unclipped_grad_norm": 0.9805173873901367 + }, + { + "bits_per_byte": 1.4572141602401034, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.010063886642456, + "step": 6990, + "unclipped_grad_norm": 0.9425977468490601 + }, + { + "bits_per_byte": 1.4146713278441232, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 0.980575442314148, + "step": 7000, + "unclipped_grad_norm": 0.9211949110031128 + }, + { + "bits_per_byte": 1.4423556331274856, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9997647404670715, + "step": 7010, + "unclipped_grad_norm": 0.9508169293403625 + }, + { + "bits_per_byte": 1.4268384124620634, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 0.9890090227127075, + "step": 7020, + "unclipped_grad_norm": 0.9370734095573425 + }, + { + "bits_per_byte": 1.4214180352548202, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 0.9852519035339355, + "step": 7030, + "unclipped_grad_norm": 0.8771427273750305 + }, + { + "bits_per_byte": 1.5093840654834314, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0462253093719482, + "step": 7040, + "unclipped_grad_norm": 0.9992957711219788 + }, + { + "bits_per_byte": 1.466185635222362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0162824392318726, + "step": 7050, + "unclipped_grad_norm": 0.9372990131378174 + }, + { + "bits_per_byte": 1.4783773133593756, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.024733066558838, + "step": 7060, + "unclipped_grad_norm": 1.0182687044143677 + }, + { + "bits_per_byte": 1.4142496263842075, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 0.9802831411361694, + "step": 7070, + "unclipped_grad_norm": 0.8880259394645691 + }, + { + "bits_per_byte": 1.3447492013960853, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9321091175079346, + "step": 7080, + "unclipped_grad_norm": 0.910698413848877 + }, + { + "bits_per_byte": 1.413691198716856, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.979896068572998, + "step": 7090, + "unclipped_grad_norm": 0.9244340062141418 + }, + { + "bits_per_byte": 1.4025504205679395, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9721738696098328, + "step": 7100, + "unclipped_grad_norm": 0.9016135334968567 + }, + { + "bits_per_byte": 1.4533139376838367, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0073604583740234, + "step": 7110, + "unclipped_grad_norm": 0.970280110836029 + }, + { + "bits_per_byte": 1.4606615524766484, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0124534368515015, + "step": 7120, + "unclipped_grad_norm": 0.9771916270256042 + }, + { + "bits_per_byte": 1.396041221198084, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 0.9676620364189148, + "step": 7130, + "unclipped_grad_norm": 0.8866344094276428 + }, + { + "bits_per_byte": 1.5095894127685614, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0463676452636719, + "step": 7140, + "unclipped_grad_norm": 0.9947805404663086 + }, + { + "bits_per_byte": 1.5291790966150898, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0599461793899536, + "step": 7150, + "unclipped_grad_norm": 0.9418985247612 + }, + { + "bits_per_byte": 1.4640222654571586, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0147829055786133, + "step": 7160, + "unclipped_grad_norm": 1.0358314514160156 + }, + { + "bits_per_byte": 1.4559156912260893, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0091638565063477, + "step": 7170, + "unclipped_grad_norm": 0.9539772272109985 + }, + { + "bits_per_byte": 1.3591113005869877, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 0.9420641660690308, + "step": 7180, + "unclipped_grad_norm": 0.8915972709655762 + }, + { + "bits_per_byte": 1.3840849873101015, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9593746066093445, + "step": 7190, + "unclipped_grad_norm": 0.9118176102638245 + }, + { + "bits_per_byte": 1.4715360154907038, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0199910402297974, + "step": 7200, + "unclipped_grad_norm": 0.973518431186676 + }, + { + "bits_per_byte": 1.4678313372084681, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.017423152923584, + "step": 7210, + "unclipped_grad_norm": 0.9181845188140869 + }, + { + "bits_per_byte": 1.4265923052886786, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9888384342193604, + "step": 7220, + "unclipped_grad_norm": 0.926552951335907 + }, + { + "bits_per_byte": 1.4337483314084294, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9937986135482788, + "step": 7230, + "unclipped_grad_norm": 0.9526586532592773 + }, + { + "bits_per_byte": 1.383381664259398, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 0.9588871002197266, + "step": 7240, + "unclipped_grad_norm": 0.8890237212181091 + }, + { + "bits_per_byte": 1.431437056563484, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 0.9921965599060059, + "step": 7250, + "unclipped_grad_norm": 0.9829798340797424 + }, + { + "bits_per_byte": 1.4272578781475178, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9892997741699219, + "step": 7260, + "unclipped_grad_norm": 0.9537968039512634 + }, + { + "bits_per_byte": 1.5069463833901018, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0445356369018555, + "step": 7270, + "unclipped_grad_norm": 0.9953339099884033 + }, + { + "bits_per_byte": 1.4318632295723221, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 0.9924919605255127, + "step": 7280, + "unclipped_grad_norm": 0.9815621972084045 + }, + { + "bits_per_byte": 1.435238647069481, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.9948316216468811, + "step": 7290, + "unclipped_grad_norm": 0.9373346567153931 + }, + { + "bits_per_byte": 1.44098209368637, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 0.9988126754760742, + "step": 7300, + "unclipped_grad_norm": 0.9782799482345581 + }, + { + "bits_per_byte": 1.467732619166873, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0173547267913818, + "step": 7310, + "unclipped_grad_norm": 0.9545702338218689 + }, + { + "bits_per_byte": 1.4680886232541588, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.017601490020752, + "step": 7320, + "unclipped_grad_norm": 0.9820490479469299 + }, + { + "bits_per_byte": 1.4597027491980883, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0117888450622559, + "step": 7330, + "unclipped_grad_norm": 1.0537965297698975 + }, + { + "bits_per_byte": 1.4358094574876945, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9952272772789001, + "step": 7340, + "unclipped_grad_norm": 0.9470505714416504 + }, + { + "bits_per_byte": 1.4763711357370608, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.023342490196228, + "step": 7350, + "unclipped_grad_norm": 0.93792724609375 + }, + { + "bits_per_byte": 1.4524729425211176, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0067775249481201, + "step": 7360, + "unclipped_grad_norm": 0.959013044834137 + }, + { + "bits_per_byte": 1.4678055398108387, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0174052715301514, + "step": 7370, + "unclipped_grad_norm": 0.9769075512886047 + }, + { + "bits_per_byte": 1.5064952728968888, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0442229509353638, + "step": 7380, + "unclipped_grad_norm": 1.0261726379394531 + }, + { + "bits_per_byte": 1.4743861119808024, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.021966576576233, + "step": 7390, + "unclipped_grad_norm": 0.977978527545929 + }, + { + "bits_per_byte": 1.4106443540741704, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9777841567993164, + "step": 7400, + "unclipped_grad_norm": 0.9216173887252808 + }, + { + "bits_per_byte": 1.4343313525948542, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.994202733039856, + "step": 7410, + "unclipped_grad_norm": 0.9778451919555664 + }, + { + "bits_per_byte": 1.5075760118749106, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0449720621109009, + "step": 7420, + "unclipped_grad_norm": 1.0143206119537354 + }, + { + "bits_per_byte": 1.4176087055295346, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9826114773750305, + "step": 7430, + "unclipped_grad_norm": 0.9865745902061462 + }, + { + "bits_per_byte": 1.4384498211350658, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9970574378967285, + "step": 7440, + "unclipped_grad_norm": 0.9753794074058533 + }, + { + "bits_per_byte": 1.385612537215065, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 0.9604334235191345, + "step": 7450, + "unclipped_grad_norm": 0.9882722496986389 + }, + { + "bits_per_byte": 1.40335280562554, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9727300405502319, + "step": 7460, + "unclipped_grad_norm": 0.8938714861869812 + }, + { + "bits_per_byte": 1.4650283639647061, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0154802799224854, + "step": 7470, + "unclipped_grad_norm": 0.95097815990448 + }, + { + "bits_per_byte": 1.378530205661209, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 0.9555243253707886, + "step": 7480, + "unclipped_grad_norm": 0.9593113660812378 + }, + { + "bits_per_byte": 1.4079058743244819, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 0.9758859872817993, + "step": 7490, + "unclipped_grad_norm": 0.9498793482780457 + }, + { + "bits_per_byte": 1.4795925427703722, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0255753993988037, + "step": 7500, + "unclipped_grad_norm": 0.9999135732650757 + }, + { + "bits_per_byte": 1.462181535144974, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0135070085525513, + "step": 7510, + "unclipped_grad_norm": 0.96773761510849 + }, + { + "bits_per_byte": 1.424363066168195, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 0.9872932434082031, + "step": 7520, + "unclipped_grad_norm": 0.9349227547645569 + }, + { + "bits_per_byte": 1.4575968216382733, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0103291273117065, + "step": 7530, + "unclipped_grad_norm": 0.9571959972381592 + }, + { + "bits_per_byte": 1.484410464751643, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0289149284362793, + "step": 7540, + "unclipped_grad_norm": 0.9889418482780457 + }, + { + "bits_per_byte": 1.4190770074112757, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 0.9836292266845703, + "step": 7550, + "unclipped_grad_norm": 0.9660657048225403 + }, + { + "bits_per_byte": 1.458957376389249, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0112721920013428, + "step": 7560, + "unclipped_grad_norm": 0.9621381163597107 + }, + { + "bits_per_byte": 1.4744461339259534, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0220081806182861, + "step": 7570, + "unclipped_grad_norm": 0.9809478521347046 + }, + { + "bits_per_byte": 1.424117474942763, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9871230125427246, + "step": 7580, + "unclipped_grad_norm": 1.0034832954406738 + }, + { + "bits_per_byte": 1.4167208450944553, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9819960594177246, + "step": 7590, + "unclipped_grad_norm": 0.9541960954666138 + }, + { + "bits_per_byte": 1.3862604818521906, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9608825445175171, + "step": 7600, + "unclipped_grad_norm": 0.950609564781189 + }, + { + "bits_per_byte": 1.3616721222583348, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9438391923904419, + "step": 7610, + "unclipped_grad_norm": 0.9224535226821899 + }, + { + "bits_per_byte": 1.4445553772233464, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0012894868850708, + "step": 7620, + "unclipped_grad_norm": 0.9717100858688354 + }, + { + "bits_per_byte": 1.4175787805482842, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9825907349586487, + "step": 7630, + "unclipped_grad_norm": 0.9714697599411011 + }, + { + "bits_per_byte": 1.4152020663046858, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9809433221817017, + "step": 7640, + "unclipped_grad_norm": 0.9795277118682861 + }, + { + "bits_per_byte": 1.5113582543326856, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0475937128067017, + "step": 7650, + "unclipped_grad_norm": 1.0563151836395264 + }, + { + "bits_per_byte": 1.4375178471500403, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9964114427566528, + "step": 7660, + "unclipped_grad_norm": 0.9650751948356628 + }, + { + "bits_per_byte": 1.4172367070557181, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 0.9823536276817322, + "step": 7670, + "unclipped_grad_norm": 0.9894714951515198 + }, + { + "bits_per_byte": 1.3937812831744212, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9660955667495728, + "step": 7680, + "unclipped_grad_norm": 0.9648637175559998 + }, + { + "bits_per_byte": 1.4266868097553278, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9889039397239685, + "step": 7690, + "unclipped_grad_norm": 0.9686183333396912 + }, + { + "bits_per_byte": 1.4503658110827464, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.005316972732544, + "step": 7700, + "unclipped_grad_norm": 0.9993582367897034 + }, + { + "bits_per_byte": 1.3691781330725912, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9490419626235962, + "step": 7710, + "unclipped_grad_norm": 0.9620972275733948 + }, + { + "bits_per_byte": 1.3650053180147068, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9461495876312256, + "step": 7720, + "unclipped_grad_norm": 0.9128993153572083 + }, + { + "bits_per_byte": 1.4252105106803215, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9878806471824646, + "step": 7730, + "unclipped_grad_norm": 0.9594771265983582 + }, + { + "bits_per_byte": 1.4081796707046554, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9760757684707642, + "step": 7740, + "unclipped_grad_norm": 0.9255442023277283 + }, + { + "bits_per_byte": 1.433292233418341, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9934824705123901, + "step": 7750, + "unclipped_grad_norm": 0.9785766005516052 + }, + { + "bits_per_byte": 1.3999133245909345, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9703459739685059, + "step": 7760, + "unclipped_grad_norm": 1.0032256841659546 + }, + { + "bits_per_byte": 1.4048038232508695, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9737358093261719, + "step": 7770, + "unclipped_grad_norm": 1.0204466581344604 + }, + { + "bits_per_byte": 1.536527399338505, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0650396347045898, + "step": 7780, + "unclipped_grad_norm": 1.0556561946868896 + }, + { + "bits_per_byte": 1.4128328333063997, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9793010950088501, + "step": 7790, + "unclipped_grad_norm": 1.0078489780426025 + }, + { + "bits_per_byte": 1.451735308931567, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0062662363052368, + "step": 7800, + "unclipped_grad_norm": 1.0289596319198608 + }, + { + "bits_per_byte": 1.3852072600983067, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9601525068283081, + "step": 7810, + "unclipped_grad_norm": 0.9891387224197388 + }, + { + "bits_per_byte": 1.3953124447150527, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.9671568870544434, + "step": 7820, + "unclipped_grad_norm": 0.9778857231140137 + }, + { + "bits_per_byte": 1.45392653988621, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0077850818634033, + "step": 7830, + "unclipped_grad_norm": 1.0185887813568115 + }, + { + "bits_per_byte": 1.4113820736550464, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9782955050468445, + "step": 7840, + "unclipped_grad_norm": 0.9330953359603882 + }, + { + "bits_per_byte": 1.380841996454107, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9571267366409302, + "step": 7850, + "unclipped_grad_norm": 0.9742368459701538 + }, + { + "bits_per_byte": 1.3846048048723343, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 0.9597349166870117, + "step": 7860, + "unclipped_grad_norm": 0.9645414352416992 + }, + { + "bits_per_byte": 1.4429803601067448, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0001977682113647, + "step": 7870, + "unclipped_grad_norm": 0.9958038926124573 + }, + { + "bits_per_byte": 1.4256185395194934, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9881634712219238, + "step": 7880, + "unclipped_grad_norm": 0.9739922881126404 + }, + { + "bits_per_byte": 1.4834710955126302, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.028263807296753, + "step": 7890, + "unclipped_grad_norm": 0.9912284016609192 + }, + { + "bits_per_byte": 1.4598313922209336, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0118780136108398, + "step": 7900, + "unclipped_grad_norm": 0.9979069232940674 + }, + { + "bits_per_byte": 1.465345499972897, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.015700101852417, + "step": 7910, + "unclipped_grad_norm": 1.0503500699996948 + }, + { + "bits_per_byte": 1.3660396216769959, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 0.946866512298584, + "step": 7920, + "unclipped_grad_norm": 0.9824531078338623 + }, + { + "bits_per_byte": 1.3848080023743288, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.959875762462616, + "step": 7930, + "unclipped_grad_norm": 1.0091050863265991 + }, + { + "bits_per_byte": 1.4375059803471308, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9964032173156738, + "step": 7940, + "unclipped_grad_norm": 1.0658900737762451 + }, + { + "bits_per_byte": 1.4049340141175726, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9738260507583618, + "step": 7950, + "unclipped_grad_norm": 1.0148085355758667 + }, + { + "bits_per_byte": 1.415199744538899, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9809417128562927, + "step": 7960, + "unclipped_grad_norm": 0.9737400412559509 + }, + { + "bits_per_byte": 1.4008050546356583, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9709640741348267, + "step": 7970, + "unclipped_grad_norm": 0.9626508951187134 + }, + { + "bits_per_byte": 1.4372590132604919, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 0.9962320327758789, + "step": 7980, + "unclipped_grad_norm": 0.9673776626586914 + }, + { + "bits_per_byte": 1.4403348369798477, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9983640313148499, + "step": 7990, + "unclipped_grad_norm": 0.9839751720428467 + }, + { + "bits_per_byte": 1.3635582559903472, + "learning_rate": 3e-05, + "loss_nats": 0.9451465606689453, + "step": 8000, + "unclipped_grad_norm": 0.9773924946784973 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/replay-depth-32-block-seed-2026073001.json b/experiments/k3/attnres_gradient/results/raw/replay-depth-32-block-seed-2026073001.json new file mode 100644 index 0000000..4c36216 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/replay-depth-32-block-seed-2026073001.json @@ -0,0 +1,13072 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "9dfebe28b0512ff1872838a4dcfef72cfaea9ff8cbaf3793992bd389f7da76b5", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "population_cv": 0.5072011380661324 + }, + "activation_output_rms_by_block": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013632480811793357, + "first_to_last_ratio": 0.32005493218522185, + "imbalance_abs_log_ratio": 1.1392626348419668, + "last_quartile_mean": 0.04259419068694115, + "mean": 0.025403792154975235, + "normalized": [ + 0.23092935972382672, + 0.3992639012357397, + 0.5833875213879504, + 0.7700561688360058, + 0.2642123711098415, + 0.4687595594389148, + 0.6729013896370168, + 0.9035435305603466, + 0.2950160146386087, + 0.5418894364067922, + 0.8025599005701267, + 1.0558320747576517, + 0.36189473544610173, + 0.6606003816028687, + 0.9564482474371727, + 1.2361787191078082, + 0.389077999650742, + 0.7376930678145923, + 1.1018562250488806, + 1.4948807199009364, + 0.4969232792693754, + 0.9255949835565679, + 1.3737978229029388, + 1.8632120600111322, + 0.6868422339014575, + 1.1618129342705898, + 1.7580201402629136, + 2.3690874226600394, + 0.7468506348084945, + 1.4805931380763324, + 2.2249315569778956, + 2.985352468990339 + ], + "population_cv": 0.6518934572295025 + }, + "bits_per_byte": 5.150682269327325, + "branch_output_rms_by_sublayer": [ + 0.002919268561527133, + 0.005079356953501701, + 0.0026904908008873463, + 0.0052208867855370045, + 0.0028405722696334124, + 0.005689430516213179, + 0.0029144894797354937, + 0.005868426524102688, + 0.0030634324066340923, + 0.00590060418471694, + 0.0032732728868722916, + 0.0059102908708155155, + 0.003848863998427987, + 0.0058776624500751495, + 0.003392934100702405, + 0.006663382984697819, + 0.0037359180860221386, + 0.006277414970099926, + 0.004302032291889191, + 0.006413500756025314, + 0.004344160668551922, + 0.006856496445834637, + 0.004835603758692741, + 0.006499431096017361, + 0.005531937349587679, + 0.006718362681567669, + 0.005483300890773535, + 0.0070298356004059315, + 0.005397035740315914, + 0.006969383917748928, + 0.005739198997616768, + 0.0068367342464625835, + 0.005659298039972782, + 0.007033169735223055, + 0.007230010814964771, + 0.007157569285482168, + 0.006938732694834471, + 0.0075136395171284676, + 0.007343437522649765, + 0.007708695251494646, + 0.007882392033934593, + 0.007887008599936962, + 0.00830139685422182, + 0.008205465972423553, + 0.008992243558168411, + 0.00814912747591734, + 0.009393857792019844, + 0.008287571370601654, + 0.012831038795411587, + 0.008026733994483948, + 0.00905293133109808, + 0.008120592683553696, + 0.012243949808180332, + 0.008479002863168716, + 0.012258118949830532, + 0.00907627958804369, + 0.013431218452751637, + 0.00841403380036354, + 0.01483316533267498, + 0.008773921057581902, + 0.015388047322630882, + 0.008821850642561913, + 0.014634119346737862, + 0.009164483286440372 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00039664652254938686, + 0.00033413090452743513, + 0.0003161152806455274, + 0.0003097753114302562, + 0.00028318594507881776, + 0.00028174654548060826, + 0.00029366070958977545, + 0.0002617199768643518, + 0.00023850476284541366, + 0.0002578321956415913, + 0.00023580491915923377, + 0.00024129375767696618, + 0.00024677521358199467, + 0.00023282043614108414, + 0.00020802360655182162, + 0.00020816264493602737, + 0.00019546204845585216, + 0.00020244473639840523, + 0.00019508742079131332, + 0.00018509966437476374, + 0.00018490270602469487, + 0.0001750081453781441, + 0.00017332318168420213, + 0.00016951527760705314, + 0.0001849276373753642, + 0.00014904354377519617, + 0.00017214209100244258, + 0.0001582098123018746, + 0.0001606899226932654, + 0.0001683991686835421, + 0.00016911992547040899, + 0.00016138660409180947 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030962264952076986, + "first_to_last_ratio": 1.8709465967014842, + "imbalance_abs_log_ratio": 0.6264445042344079, + "last_quartile_mean": 0.00016548983817423794, + "mean": 0.0002234675193377695, + "normalized": [ + 1.774962749507496, + 1.4952101563466982, + 1.4145916220053505, + 1.3862207463001954, + 1.2672353723620313, + 1.2607941696204648, + 1.314109139708619, + 1.1711768119140573, + 1.0672905107292818, + 1.1537792892929548, + 1.0552089174213117, + 1.0797710485712801, + 1.104300143095945, + 1.0418535849461763, + 0.9308896754583627, + 0.9315118615577912, + 0.8746776669606858, + 0.9059246596477921, + 0.8730012369110346, + 0.828306793413619, + 0.8274254199117672, + 0.7831480203331949, + 0.775607937107967, + 0.7585678585836542, + 0.827536985792765, + 0.6669584206996899, + 0.7703226469447271, + 0.7079767689314242, + 0.719075072607683, + 0.7535733568018358, + 0.7567986881117407, + 0.7221926684024036 + ], + "population_cv": 0.268339881652204 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6930912733078003, + "mean_weights": [ + 0.5023526549339294, + 0.49764734506607056 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692903995513916, + "mean_weights": [ + 0.5103387832641602, + 0.48966121673583984 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928629875183105, + "mean_weights": [ + 0.5053091049194336, + 0.4946908950805664 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927823424339294, + "mean_weights": [ + 0.5126834511756897, + 0.4873165488243103 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928155422210693, + "mean_weights": [ + 0.502953290939331, + 0.49704670906066895 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692751407623291, + "mean_weights": [ + 0.5129156708717346, + 0.48708435893058777 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925709247589111, + "mean_weights": [ + 0.5112693905830383, + 0.4887305796146393 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929247379302979, + "mean_weights": [ + 0.5095131397247314, + 0.49048691987991333 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0983518362045288, + "mean_weights": [ + 0.3398353159427643, + 0.32833361625671387, + 0.33183106780052185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098447322845459, + "mean_weights": [ + 0.3408135175704956, + 0.3303682208061218, + 0.3288182020187378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980432033538818, + "mean_weights": [ + 0.3472674489021301, + 0.3313220739364624, + 0.32141047716140747 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985240936279297, + "mean_weights": [ + 0.3373557925224304, + 0.33420175313949585, + 0.32844245433807373 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098268985748291, + "mean_weights": [ + 0.3445510268211365, + 0.33061009645462036, + 0.32483887672424316 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985302925109863, + "mean_weights": [ + 0.3371829390525818, + 0.3332410156726837, + 0.3295760750770569 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983161926269531, + "mean_weights": [ + 0.3423102796077728, + 0.3327076733112335, + 0.32498204708099365 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098489761352539, + "mean_weights": [ + 0.33793729543685913, + 0.3344462811946869, + 0.327616423368454 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3861088752746582, + "mean_weights": [ + 0.2561652958393097, + 0.2513129711151123, + 0.2458309531211853, + 0.2466908097267151 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861844539642334, + "mean_weights": [ + 0.2536832094192505, + 0.25214672088623047, + 0.24721932411193848, + 0.24695077538490295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3860987424850464, + "mean_weights": [ + 0.25705933570861816, + 0.2498370260000229, + 0.2462591528892517, + 0.24684447050094604 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862221240997314, + "mean_weights": [ + 0.2514326870441437, + 0.2528519928455353, + 0.24950753152370453, + 0.24620777368545532 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386016845703125, + "mean_weights": [ + 0.25482749938964844, + 0.2536851465702057, + 0.24858617782592773, + 0.24290117621421814 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862277269363403, + "mean_weights": [ + 0.25171467661857605, + 0.2530350983142853, + 0.24868467450141907, + 0.24656552076339722 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859888315200806, + "mean_weights": [ + 0.25458407402038574, + 0.2546530067920685, + 0.248786062002182, + 0.24197682738304138 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862541913986206, + "mean_weights": [ + 0.2501963675022125, + 0.2521768808364868, + 0.24995486438274384, + 0.24767187237739563 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6090729236602783, + "mean_weights": [ + 0.20360878109931946, + 0.20631682872772217, + 0.19986985623836517, + 0.19557830691337585, + 0.19462622702121735 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093083620071411, + "mean_weights": [ + 0.2017374038696289, + 0.20362424850463867, + 0.20011676847934723, + 0.1968517303466797, + 0.1976698487997055 + ], + "sources": 5 + }, + { + "entropy_mean": 1.608644723892212, + "mean_weights": [ + 0.20431089401245117, + 0.20831988751888275, + 0.20422273874282837, + 0.19545811414718628, + 0.18768835067749023 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609345555305481, + "mean_weights": [ + 0.2002599537372589, + 0.20386311411857605, + 0.2001732438802719, + 0.19764204323291779, + 0.19806164503097534 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6086196899414062, + "mean_weights": [ + 0.20515945553779602, + 0.21025606989860535, + 0.20179016888141632, + 0.1938556730747223, + 0.18893863260746002 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609370470046997, + "mean_weights": [ + 0.2002836912870407, + 0.20340535044670105, + 0.20010054111480713, + 0.1980183869600296, + 0.1981920301914215 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6083989143371582, + "mean_weights": [ + 0.20363344252109528, + 0.21095217764377594, + 0.20392383635044098, + 0.19413790106773376, + 0.18735264241695404 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093392372131348, + "mean_weights": [ + 0.20084109902381897, + 0.20331618189811707, + 0.2011999487876892, + 0.19900763034820557, + 0.19563516974449158 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7908520698547363, + "mean_weights": [ + 0.17160072922706604, + 0.17730243504047394, + 0.16903036832809448, + 0.16099891066551208, + 0.15854957699775696, + 0.1625179797410965 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916775941848755, + "mean_weights": [ + 0.16706335544586182, + 0.16981256008148193, + 0.16858771443367004, + 0.1652718484401703, + 0.16462719440460205, + 0.16463729739189148 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898297309875488, + "mean_weights": [ + 0.17284834384918213, + 0.18101516366004944, + 0.1735592484474182, + 0.16231873631477356, + 0.15689018368721008, + 0.153368279337883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917213439941406, + "mean_weights": [ + 0.16615921258926392, + 0.16885727643966675, + 0.16764026880264282, + 0.16661542654037476, + 0.16609220206737518, + 0.16463561356067657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890949249267578, + "mean_weights": [ + 0.17241932451725006, + 0.18413974344730377, + 0.17484498023986816, + 0.1621202528476715, + 0.15553176403045654, + 0.15094394981861115 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917208671569824, + "mean_weights": [ + 0.16663745045661926, + 0.168935626745224, + 0.16597260534763336, + 0.16514870524406433, + 0.16661295294761658, + 0.16669265925884247 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789101004600525, + "mean_weights": [ + 0.17229264974594116, + 0.18365857005119324, + 0.1747075319290161, + 0.1640234887599945, + 0.155386283993721, + 0.14993149042129517 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917447090148926, + "mean_weights": [ + 0.1658121943473816, + 0.167683407664299, + 0.16629494726657867, + 0.1669958382844925, + 0.16702724993228912, + 0.1661863625049591 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9436898231506348, + "mean_weights": [ + 0.14778709411621094, + 0.15856821835041046, + 0.1500256359577179, + 0.1408001184463501, + 0.13533659279346466, + 0.130964457988739, + 0.13651786744594574 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458942413330078, + "mean_weights": [ + 0.142832413315773, + 0.1429906189441681, + 0.14157448709011078, + 0.14285019040107727, + 0.14392998814582825, + 0.14335544407367706, + 0.14246684312820435 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9425597190856934, + "mean_weights": [ + 0.1483290046453476, + 0.1615946888923645, + 0.15280696749687195, + 0.14239417016506195, + 0.13462956249713898, + 0.1307675987482071, + 0.12947802245616913 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458987712860107, + "mean_weights": [ + 0.1426449418067932, + 0.14291533827781677, + 0.1416938602924347, + 0.14274153113365173, + 0.14325542747974396, + 0.1432831734418869, + 0.14346569776535034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941068410873413, + "mean_weights": [ + 0.14912807941436768, + 0.16456490755081177, + 0.15460123121738434, + 0.14255177974700928, + 0.13539987802505493, + 0.1283380389213562, + 0.1254161298274994 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458861351013184, + "mean_weights": [ + 0.14264336228370667, + 0.1430070698261261, + 0.14314037561416626, + 0.14221979677677155, + 0.14132913947105408, + 0.1431397795677185, + 0.14452049136161804 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415628910064697, + "mean_weights": [ + 0.14895881712436676, + 0.16342055797576904, + 0.15406624972820282, + 0.14293311536312103, + 0.1363803744316101, + 0.12938986718654633, + 0.1248510330915451 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458723068237305, + "mean_weights": [ + 0.14191991090774536, + 0.1418565809726715, + 0.14164230227470398, + 0.14289972186088562, + 0.14397987723350525, + 0.14332248270511627, + 0.14437910914421082 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073223114013672, + "mean_weights": [ + 0.13280923664569855, + 0.14827430248260498, + 0.13796664774417877, + 0.12756125628948212, + 0.12019773572683334, + 0.11310221254825592, + 0.10922321677207947, + 0.11086538434028625 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079413890838623, + "mean_weights": [ + 0.12475188821554184, + 0.12460128962993622, + 0.12424701452255249, + 0.12420951575040817, + 0.12484367191791534, + 0.12469655275344849, + 0.12630701065063477, + 0.12634305655956268 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072925567626953, + "mean_weights": [ + 0.13228744268417358, + 0.14843900501728058, + 0.13848665356636047, + 0.12843599915504456, + 0.12169650197029114, + 0.11442162096500397, + 0.11023421585559845, + 0.10599856078624725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793263912200928, + "mean_weights": [ + 0.12420415878295898, + 0.12273682653903961, + 0.12360996007919312, + 0.1238352432847023, + 0.1241140365600586, + 0.12607696652412415, + 0.1275939792394638, + 0.12782883644104004 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073427200317383, + "mean_weights": [ + 0.13135744631290436, + 0.1465209424495697, + 0.13772746920585632, + 0.12966200709342957, + 0.12215200066566467, + 0.11654195189476013, + 0.11090746521949768, + 0.10513069480657578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079422950744629, + "mean_weights": [ + 0.12477636337280273, + 0.12493626773357391, + 0.1242394968867302, + 0.12470392882823944, + 0.12560302019119263, + 0.12461308389902115, + 0.12591606378555298, + 0.12521176040172577 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0725789070129395, + "mean_weights": [ + 0.13186156749725342, + 0.14827941358089447, + 0.13866659998893738, + 0.12988534569740295, + 0.12183371186256409, + 0.11524046957492828, + 0.1099783331155777, + 0.10425455868244171 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793380737304688, + "mean_weights": [ + 0.12417487055063248, + 0.1225123256444931, + 0.12277349829673767, + 0.1243998259305954, + 0.12567630410194397, + 0.1261572688817978, + 0.12747105956077576, + 0.12683483958244324 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1906280517578125, + "mean_weights": [ + 0.11854428052902222, + 0.13291707634925842, + 0.12378416955471039, + 0.11745279282331467, + 0.11140251904726028, + 0.10409101843833923, + 0.09838292002677917, + 0.09486069530248642, + 0.09856453537940979 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197185516357422, + "mean_weights": [ + 0.11084984987974167, + 0.11021925508975983, + 0.11018812656402588, + 0.11046157777309418, + 0.11131840199232101, + 0.11167057603597641, + 0.11249847710132599, + 0.1118280440568924, + 0.11096569895744324 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1899101734161377, + "mean_weights": [ + 0.1190595030784607, + 0.13352498412132263, + 0.1239522397518158, + 0.11779220402240753, + 0.11234381794929504, + 0.10513705015182495, + 0.09944089502096176, + 0.0946362093091011, + 0.09411308169364929 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197190284729004, + "mean_weights": [ + 0.11067462712526321, + 0.11021950840950012, + 0.11038416624069214, + 0.11103623360395432, + 0.11016053706407547, + 0.11144962906837463, + 0.11242866516113281, + 0.11139006912708282, + 0.11225657165050507 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1896841526031494, + "mean_weights": [ + 0.11968795955181122, + 0.13428261876106262, + 0.12373492866754532, + 0.11790731549263, + 0.11133000254631042, + 0.10504785925149918, + 0.09994535893201828, + 0.09480959177017212, + 0.09325435757637024 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971938610076904, + "mean_weights": [ + 0.11116788536310196, + 0.1117495447397232, + 0.11143069714307785, + 0.11117202788591385, + 0.11217275261878967, + 0.11135637760162354, + 0.11160033196210861, + 0.10969291627407074, + 0.10965746641159058 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188610792160034, + "mean_weights": [ + 0.11987313628196716, + 0.1360916942358017, + 0.12462760508060455, + 0.11755925416946411, + 0.11237476021051407, + 0.10438287258148193, + 0.09920360147953033, + 0.09431086480617523, + 0.09157621115446091 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028571506962180138, + 0.014360599219799042, + 0.01502692885696888, + 0.014914286322891712, + 0.015751885250210762, + 0.015590262599289417, + 0.016715344041585922, + 0.016720104962587357, + 0.01783851347863674, + 0.01203431747853756, + 0.012985615991055965, + 0.01319553330540657, + 0.014028912410140038, + 0.014193638227880001, + 0.015184907242655754, + 0.015358141623437405, + 0.016659293323755264, + 0.012656479142606258, + 0.013609340414404869, + 0.013699201866984367, + 0.014725575223565102, + 0.01495310477912426, + 0.016002535820007324, + 0.01618308201432228, + 0.017315881326794624, + 0.01409733947366476, + 0.014967667870223522, + 0.01521329116076231, + 0.016154561191797256, + 0.016374915838241577, + 0.01737469807267189, + 0.017517324537038803, + 0.018479224294424057, + 0.015613679774105549, + 0.01647813245654106, + 0.016780734062194824, + 0.017671043053269386, + 0.017834074795246124, + 0.01892261393368244, + 0.01904122717678547, + 0.020303834229707718, + 0.017550688236951828, + 0.01861351728439331, + 0.018688000738620758, + 0.01984807848930359, + 0.019847052171826363, + 0.021230308338999748, + 0.021104788407683372, + 0.022779423743486404, + 0.019968591630458832, + 0.02138628251850605, + 0.021149026229977608, + 0.02279111184179783, + 0.02249406836926937, + 0.024287279695272446, + 0.02368815243244171, + 0.02611272968351841, + 0.02278727851808071, + 0.024725768715143204, + 0.024227207526564598, + 0.026407435536384583, + 0.02562388963997364, + 0.028016721829771996, + 0.0269329734146595 + ], + "loss_nats": 3.570180892944336, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1930694580078125, + "mean_weights": [ + 0.10071488469839096, + 0.09411877393722534, + 0.10181896388530731, + 0.10989432036876678, + 0.1145806610584259, + 0.11861918121576309, + 0.11984598636627197, + 0.12094368040561676, + 0.11946355551481247 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.002919268561527133, + 0.005866481456905603, + 0.0065401275642216206, + 0.010142817161977291, + 0.010618876665830612, + 0.01482025533914566, + 0.015120964497327805, + 0.01956234686076641, + 0.0030634324066340923, + 0.0067119961604475975, + 0.007598300464451313, + 0.011908270418643951, + 0.012669830583035946, + 0.017094247043132782, + 0.017725100740790367, + 0.02295343205332756, + 0.0037359180860221386, + 0.007494525518268347, + 0.009108707308769226, + 0.013766046613454819, + 0.015153576619923115, + 0.02038806490600109, + 0.021781234070658684, + 0.02682213857769966, + 0.005531937349587679, + 0.009193498641252518, + 0.0119093032553792, + 0.016781754791736603, + 0.01904802769422531, + 0.024297412484884262, + 0.02626260370016098, + 0.03140362724661827, + 0.005659298039972782, + 0.009884056635200977, + 0.013823030516505241, + 0.018740201368927956, + 0.02223494090139866, + 0.02799132652580738, + 0.0319429449737072, + 0.03797563910484314, + 0.007882392033934593, + 0.012623735703527927, + 0.01784195750951767, + 0.02351362258195877, + 0.028768405318260193, + 0.034899674355983734, + 0.040818072855472565, + 0.047332651913166046, + 0.012831038795411587, + 0.01744839735329151, + 0.023826664313673973, + 0.029514454305171967, + 0.03824474662542343, + 0.044660378247499466, + 0.053081437945365906, + 0.0601838044822216, + 0.013431218452751637, + 0.018972838297486305, + 0.03125866502523422, + 0.0376126803457737, + 0.04969283565878868, + 0.0565216988325119, + 0.06860727071762085, + 0.07583927363157272 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "population_cv": 0.5868596491376813 + }, + "activation_output_rms_by_block": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.036073930095881224, + "first_to_last_ratio": 0.5177916953272103, + "imbalance_abs_log_ratio": 0.6581822501725932, + "last_quartile_mean": 0.0696688077878207, + "mean": 0.04647241474594921, + "normalized": [ + 0.46329906589537145, + 0.8300990481085756, + 1.06603234209322, + 1.2414755802942754, + 0.4606131805713009, + 0.5586764120892566, + 0.7186330436105184, + 0.8711228081179162, + 0.4630334112674251, + 0.7541501214633191, + 0.8015850238486076, + 0.9304351337283648, + 0.4571868851775181, + 0.6517799422454194, + 0.8562272308804426, + 1.1237390363826774, + 0.8427218914862018, + 0.8798161433899512, + 1.0235622313138022, + 1.1926338466716728, + 0.4420161942758587, + 0.7919574083446129, + 1.1031381372276345, + 1.482919724824311, + 0.6253151219931478, + 1.1530933116414785, + 1.6957169376190813, + 2.3138065984257823, + 0.6272181919070765, + 1.163956934601208, + 1.8566997944110037, + 2.5573392660929675 + ], + "population_cv": 0.5056023373965206 + }, + "bits_per_byte": 3.0609520355704922, + "branch_output_rms_by_sublayer": [ + 0.010549778118729591, + 0.017611408606171608, + 0.013795096427202225, + 0.014558257535099983, + 0.012347182258963585, + 0.01080919150263071, + 0.012820021249353886, + 0.010327119380235672, + 0.01913420483469963, + 0.008616750128567219, + 0.018582463264465332, + 0.007770510856062174, + 0.013614931143820286, + 0.007606723811477423, + 0.013612493872642517, + 0.008487755432724953, + 0.019129201769828796, + 0.00865182839334011, + 0.02525762841105461, + 0.009020541794598103, + 0.027986859902739525, + 0.008682658895850182, + 0.014458909630775452, + 0.008706990629434586, + 0.018231205642223358, + 0.009003226645290852, + 0.016914349049329758, + 0.009712434373795986, + 0.022478405386209488, + 0.00981286820024252, + 0.03932488337159157, + 0.011337134055793285, + 0.03611915931105614, + 0.010840619914233685, + 0.0266753900796175, + 0.010956712998449802, + 0.018954744562506676, + 0.012231985107064247, + 0.014813725836575031, + 0.014082341454923153, + 0.013802599161863327, + 0.014436324127018452, + 0.016533223912119865, + 0.016183899715542793, + 0.016679473221302032, + 0.017385432496666908, + 0.01826235093176365, + 0.02254617027938366, + 0.015529165975749493, + 0.022119678556919098, + 0.011229476891458035, + 0.022372635081410408, + 0.008122270926833153, + 0.024847008287906647, + 0.008894859813153744, + 0.026897024363279343, + 0.009738311171531677, + 0.026162855327129364, + 0.007465044502168894, + 0.024674534797668457, + 0.008605006150901318, + 0.030845578759908676, + 0.009693782776594162, + 0.029862264171242714 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006151943096300597, + 0.0003715258289130363, + 0.0003398511463891951, + 0.0003030496529751275, + 0.00028984271802693696, + 0.0002579116693919001, + 0.00022400787473702318, + 0.00028477639847268295, + 0.000278695861793727, + 0.00022792247112618847, + 0.00023417680672174036, + 0.00019296669269337194, + 0.00020274366236211453, + 0.00017465616055496573, + 0.00015881266908702134, + 0.00017808900330691153, + 0.00016226510475368574, + 0.00014508607560593053, + 0.00013101353173913878, + 0.00011930204103094746, + 0.00012786456583361604, + 0.00012508100764064805, + 0.00012354146515800217, + 0.00012514897623388166, + 0.00011397604830293932, + 9.556324385288696e-05, + 9.289013346881659e-05, + 9.744025747256133e-05, + 8.302691520063822e-05, + 7.404706874242776e-05, + 8.914255818565412e-05, + 8.643783624226864e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003357699498169952, + "first_to_last_ratio": 3.666991625028822, + "imbalance_abs_log_ratio": 1.2993716052111575, + "last_quartile_mean": 9.156550768352412e-05, + "mean": 0.00019143905486393895, + "normalized": [ + 3.2135256312631473, + 1.940700288021637, + 1.7752445896201103, + 1.5830085098911153, + 1.514020836724963, + 1.3472259857070703, + 1.1701263093689624, + 1.487556437609251, + 1.4557941794677367, + 1.1905745736583337, + 1.223244686870066, + 1.0079797606111183, + 1.0590506859021536, + 0.9123329650739171, + 0.8295729896906037, + 0.9302647436986373, + 0.8476071137574935, + 0.7578708310540252, + 0.6843615678747147, + 0.6231854890619823, + 0.6679126467924371, + 0.6533724674390322, + 0.645330521746498, + 0.6537275078109245, + 0.5953646625760102, + 0.49918363795102527, + 0.4852203931681348, + 0.5089883960292994, + 0.4336989401647838, + 0.386791863316788, + 0.46564457941463516, + 0.45151620866339115 + ], + "population_cv": 0.578773657781326 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.658259391784668, + "mean_weights": [ + 0.3887256383895874, + 0.6112743616104126 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6694332361221313, + "mean_weights": [ + 0.42782124876976013, + 0.5721787214279175 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6890689730644226, + "mean_weights": [ + 0.504529595375061, + 0.49547040462493896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740086078643799, + "mean_weights": [ + 0.45225459337234497, + 0.547745406627655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.679557204246521, + "mean_weights": [ + 0.4461178183555603, + 0.5538821816444397 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740646362304688, + "mean_weights": [ + 0.44189155101776123, + 0.5581084489822388 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6792672276496887, + "mean_weights": [ + 0.4455937147140503, + 0.5544062852859497 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674970269203186, + "mean_weights": [ + 0.44674086570739746, + 0.5532591342926025 + ], + "sources": 2 + }, + { + "entropy_mean": 1.093062162399292, + "mean_weights": [ + 0.3082374036312103, + 0.33985745906829834, + 0.35190510749816895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0853323936462402, + "mean_weights": [ + 0.30180174112319946, + 0.3268374502658844, + 0.3713608384132385 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0921990871429443, + "mean_weights": [ + 0.3034760355949402, + 0.3359980285167694, + 0.3605259656906128 + ], + "sources": 3 + }, + { + "entropy_mean": 1.076724648475647, + "mean_weights": [ + 0.29931002855300903, + 0.30481812357902527, + 0.3958718776702881 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856893062591553, + "mean_weights": [ + 0.2801036834716797, + 0.3310527801513672, + 0.3888435363769531 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0752873420715332, + "mean_weights": [ + 0.29463040828704834, + 0.31052330136299133, + 0.3948462903499603 + ], + "sources": 3 + }, + { + "entropy_mean": 1.088913917541504, + "mean_weights": [ + 0.29400017857551575, + 0.32166045904159546, + 0.3843393623828888 + ], + "sources": 3 + }, + { + "entropy_mean": 1.054490327835083, + "mean_weights": [ + 0.2850497364997864, + 0.2890807092189789, + 0.42586958408355713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3700100183486938, + "mean_weights": [ + 0.21610887348651886, + 0.2282472848892212, + 0.2728016972541809, + 0.28284209966659546 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3620216846466064, + "mean_weights": [ + 0.22791308164596558, + 0.2214205265045166, + 0.2562018632888794, + 0.2944645285606384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3784711360931396, + "mean_weights": [ + 0.23347607254981995, + 0.23307976126670837, + 0.2570483684539795, + 0.2763957977294922 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3619427680969238, + "mean_weights": [ + 0.24715131521224976, + 0.22003202140331268, + 0.23014183342456818, + 0.3026748299598694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804662227630615, + "mean_weights": [ + 0.24302782118320465, + 0.23108476400375366, + 0.24228686094284058, + 0.2836005389690399 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3502743244171143, + "mean_weights": [ + 0.2455797642469406, + 0.21441739797592163, + 0.2208172082901001, + 0.31918564438819885 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3820215463638306, + "mean_weights": [ + 0.24522465467453003, + 0.22846508026123047, + 0.25027886033058167, + 0.27603137493133545 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3491160869598389, + "mean_weights": [ + 0.24498313665390015, + 0.21178750693798065, + 0.2205154001712799, + 0.3227139711380005 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6032840013504028, + "mean_weights": [ + 0.20074918866157532, + 0.17959122359752655, + 0.18619459867477417, + 0.2124267816543579, + 0.22103822231292725 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5699318647384644, + "mean_weights": [ + 0.1938496232032776, + 0.17019830644130707, + 0.16257275640964508, + 0.20397180318832397, + 0.2694075107574463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6068798303604126, + "mean_weights": [ + 0.19717518985271454, + 0.1942940056324005, + 0.19104912877082825, + 0.20343774557113647, + 0.21404394507408142 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5761134624481201, + "mean_weights": [ + 0.1940101534128189, + 0.18250694870948792, + 0.16849923133850098, + 0.18937936425209045, + 0.26560431718826294 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6027262210845947, + "mean_weights": [ + 0.20452985167503357, + 0.18702910840511322, + 0.17977532744407654, + 0.20095866918563843, + 0.22770702838897705 + ], + "sources": 5 + }, + { + "entropy_mean": 1.577937126159668, + "mean_weights": [ + 0.1962222307920456, + 0.1818992793560028, + 0.17167484760284424, + 0.18149182200431824, + 0.2687118649482727 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5977532863616943, + "mean_weights": [ + 0.2153339385986328, + 0.18402469158172607, + 0.17019470036029816, + 0.1919252574443817, + 0.23852142691612244 + ], + "sources": 5 + }, + { + "entropy_mean": 1.578121542930603, + "mean_weights": [ + 0.20422281324863434, + 0.1835813820362091, + 0.16642534732818604, + 0.17722904682159424, + 0.26854145526885986 + ], + "sources": 5 + }, + { + "entropy_mean": 1.782798409461975, + "mean_weights": [ + 0.15961389243602753, + 0.16475999355316162, + 0.14714640378952026, + 0.15768738090991974, + 0.1760568618774414, + 0.19473549723625183 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7707297801971436, + "mean_weights": [ + 0.16306114196777344, + 0.1578725278377533, + 0.139655202627182, + 0.14299748837947845, + 0.18966570496559143, + 0.20674791932106018 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781312108039856, + "mean_weights": [ + 0.15919369459152222, + 0.16844786703586578, + 0.14742718636989594, + 0.1539241373538971, + 0.1669306755065918, + 0.20407646894454956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7736209630966187, + "mean_weights": [ + 0.15695160627365112, + 0.16567477583885193, + 0.1478763222694397, + 0.14320015907287598, + 0.17375096678733826, + 0.2125461995601654 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781569480895996, + "mean_weights": [ + 0.15368077158927917, + 0.1725173443555832, + 0.14978237450122833, + 0.15674246847629547, + 0.16688688099384308, + 0.20039016008377075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7793617248535156, + "mean_weights": [ + 0.1678246557712555, + 0.1683025360107422, + 0.14878368377685547, + 0.14351718127727509, + 0.16739681363105774, + 0.20417512953281403 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7772380113601685, + "mean_weights": [ + 0.1536770462989807, + 0.18459689617156982, + 0.14832255244255066, + 0.15016385912895203, + 0.1652815043926239, + 0.19795814156532288 + ], + "sources": 6 + }, + { + "entropy_mean": 1.780207633972168, + "mean_weights": [ + 0.16951784491539001, + 0.17200809717178345, + 0.15038040280342102, + 0.14669056236743927, + 0.16062122583389282, + 0.20078186690807343 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937079668045044, + "mean_weights": [ + 0.13008031249046326, + 0.15163566172122955, + 0.1323973685503006, + 0.13641127943992615, + 0.13806134462356567, + 0.1444631665945053, + 0.16695083677768707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939245343208313, + "mean_weights": [ + 0.14692582190036774, + 0.14671875536441803, + 0.13336658477783203, + 0.1307726800441742, + 0.131200909614563, + 0.147438645362854, + 0.16357660293579102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9312366247177124, + "mean_weights": [ + 0.12334205210208893, + 0.16950437426567078, + 0.13870283961296082, + 0.1300538182258606, + 0.12885476648807526, + 0.14288470149040222, + 0.1666574478149414 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9345850944519043, + "mean_weights": [ + 0.1465124636888504, + 0.153631791472435, + 0.13159149885177612, + 0.13120220601558685, + 0.1253916621208191, + 0.14393125474452972, + 0.1677391082048416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9284589290618896, + "mean_weights": [ + 0.12412485480308533, + 0.1698783040046692, + 0.13807369768619537, + 0.13034150004386902, + 0.13028821349143982, + 0.1419651210308075, + 0.16532832384109497 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9329564571380615, + "mean_weights": [ + 0.1454196274280548, + 0.1566689908504486, + 0.1286500096321106, + 0.12666445970535278, + 0.12782412767410278, + 0.14222374558448792, + 0.17254900932312012 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9259672164916992, + "mean_weights": [ + 0.12082186341285706, + 0.17791953682899475, + 0.1424643099308014, + 0.13298648595809937, + 0.12929362058639526, + 0.1344166398048401, + 0.16209754347801208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9201786518096924, + "mean_weights": [ + 0.13756656646728516, + 0.17355048656463623, + 0.14306491613388062, + 0.13006147742271423, + 0.11658851057291031, + 0.139857679605484, + 0.15931037068367004 + ], + "sources": 7 + }, + { + "entropy_mean": 2.043663501739502, + "mean_weights": [ + 0.10298450291156769, + 0.1751018464565277, + 0.1414714753627777, + 0.12628036737442017, + 0.11767096072435379, + 0.12069195508956909, + 0.12702946364879608, + 0.08876943588256836 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0607244968414307, + "mean_weights": [ + 0.1220984011888504, + 0.1498527079820633, + 0.12913095951080322, + 0.1179434061050415, + 0.10775719583034515, + 0.12020602822303772, + 0.13844498991966248, + 0.11456632614135742 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0388383865356445, + "mean_weights": [ + 0.1025313287973404, + 0.18260905146598816, + 0.14208021759986877, + 0.12606360018253326, + 0.11357533931732178, + 0.11596501618623734, + 0.12797299027442932, + 0.08920245617628098 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0634398460388184, + "mean_weights": [ + 0.12216806411743164, + 0.15090107917785645, + 0.12766355276107788, + 0.1134132519364357, + 0.10988184064626694, + 0.11335282772779465, + 0.13999530673027039, + 0.12262405455112457 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0386834144592285, + "mean_weights": [ + 0.10029368102550507, + 0.17931804060935974, + 0.14185777306556702, + 0.13114282488822937, + 0.11872631311416626, + 0.11815309524536133, + 0.12660562992095947, + 0.08390261977910995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0626261234283447, + "mean_weights": [ + 0.11644256860017776, + 0.15398627519607544, + 0.1359611600637436, + 0.12036904692649841, + 0.10964122414588928, + 0.11886263638734818, + 0.13317911326885223, + 0.1115579903125763 + ], + "sources": 8 + }, + { + "entropy_mean": 2.020051956176758, + "mean_weights": [ + 0.09590239822864532, + 0.1913527548313141, + 0.14906750619411469, + 0.1341754049062729, + 0.11847006529569626, + 0.11603336781263351, + 0.12013453245162964, + 0.07486395537853241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0452136993408203, + "mean_weights": [ + 0.11420649290084839, + 0.15673109889030457, + 0.13583534955978394, + 0.12758757174015045, + 0.1112617775797844, + 0.13417255878448486, + 0.12519711256027222, + 0.09500804543495178 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1403071880340576, + "mean_weights": [ + 0.08488726615905762, + 0.1673685610294342, + 0.13726134598255157, + 0.12220048159360886, + 0.10616928339004517, + 0.1037440225481987, + 0.10401669889688492, + 0.06658264249563217, + 0.10776971280574799 + ], + "sources": 9 + }, + { + "entropy_mean": 2.18704891204834, + "mean_weights": [ + 0.10885682702064514, + 0.12684625387191772, + 0.11236055195331573, + 0.10673345625400543, + 0.10181289166212082, + 0.11441421508789062, + 0.11840946972370148, + 0.10466625541448593, + 0.1059001013636589 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1283018589019775, + "mean_weights": [ + 0.08667369186878204, + 0.1812124401330948, + 0.14178472757339478, + 0.12440058588981628, + 0.11160320788621902, + 0.10330478847026825, + 0.11197598278522491, + 0.06831350177526474, + 0.07073107361793518 + ], + "sources": 9 + }, + { + "entropy_mean": 2.173035144805908, + "mean_weights": [ + 0.10752685368061066, + 0.13639645278453827, + 0.11839058995246887, + 0.11235916614532471, + 0.10138258337974548, + 0.11953261494636536, + 0.11256720125675201, + 0.09452059864997864, + 0.097323939204216 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1028335094451904, + "mean_weights": [ + 0.08200174570083618, + 0.19083696603775024, + 0.15512439608573914, + 0.1322997808456421, + 0.11209174990653992, + 0.10435720533132553, + 0.1024213582277298, + 0.06072331964969635, + 0.06014348566532135 + ], + "sources": 9 + }, + { + "entropy_mean": 2.147671937942505, + "mean_weights": [ + 0.09926989674568176, + 0.14984291791915894, + 0.12505242228507996, + 0.13004425168037415, + 0.1122550517320633, + 0.12226949632167816, + 0.10689133405685425, + 0.07532045990228653, + 0.07905416935682297 + ], + "sources": 9 + }, + { + "entropy_mean": 2.109609603881836, + "mean_weights": [ + 0.08386656641960144, + 0.18956288695335388, + 0.1487923562526703, + 0.12990039587020874, + 0.11159682273864746, + 0.10738210380077362, + 0.10535432398319244, + 0.0630122721195221, + 0.06053228676319122 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04393457621335983, + 0.018067941069602966, + 0.024121688678860664, + 0.027584951370954514, + 0.030756350606679916, + 0.03231721371412277, + 0.03544071316719055, + 0.036101218312978745, + 0.03840360417962074, + 0.025345133617520332, + 0.026969486847519875, + 0.027166159823536873, + 0.027978016063570976, + 0.028526438400149345, + 0.029923241585493088, + 0.02979987859725952, + 0.03152087330818176, + 0.02248864620923996, + 0.024161184206604958, + 0.023894935846328735, + 0.025710709393024445, + 0.02451697364449501, + 0.02641460672020912, + 0.025183402001857758, + 0.02742084674537182, + 0.020499059930443764, + 0.021717891097068787, + 0.02105838619172573, + 0.0224718376994133, + 0.0216353852301836, + 0.023445574566721916, + 0.022715488448739052, + 0.024849263951182365, + 0.01881815865635872, + 0.01992032676935196, + 0.0193756353110075, + 0.02058842033147812, + 0.02001863345503807, + 0.020979218184947968, + 0.021088071167469025, + 0.02214338816702366, + 0.018417155370116234, + 0.01903437450528145, + 0.019219009205698967, + 0.02008410543203354, + 0.020342160016298294, + 0.021635910496115685, + 0.02150510996580124, + 0.023218298330903053, + 0.02099662832915783, + 0.022257626056671143, + 0.02211105264723301, + 0.024358727037906647, + 0.02328667603433132, + 0.025900650769472122, + 0.024160878732800484, + 0.027197737246751785, + 0.02260657213628292, + 0.02823900803923607, + 0.024989914149045944, + 0.02913021855056286, + 0.02536177635192871, + 0.029048049822449684, + 0.027097374200820923 + ], + "loss_nats": 2.121690273284912, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1622862815856934, + "mean_weights": [ + 0.12356589734554291, + 0.09235234558582306, + 0.08736804127693176, + 0.08735492080450058, + 0.08889827132225037, + 0.10504580289125443, + 0.14004790782928467, + 0.14956718683242798, + 0.12579965591430664 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.010549778118729591, + 0.021530626341700554, + 0.02910016104578972, + 0.03857670724391937, + 0.04255657270550728, + 0.04954109713435173, + 0.05177471786737442, + 0.057694368064403534, + 0.01913420483469963, + 0.02140580676496029, + 0.02246284671127796, + 0.025963041931390762, + 0.029843268916010857, + 0.03339661285281181, + 0.036163609474897385, + 0.040483180433511734, + 0.019129201769828796, + 0.02151828072965145, + 0.0314205139875412, + 0.035047177225351334, + 0.03302976116538048, + 0.03725159168243408, + 0.03871368616819382, + 0.043239567428827286, + 0.018231205642223358, + 0.02124657854437828, + 0.02580643631517887, + 0.03028978779911995, + 0.0349617674946785, + 0.03979094699025154, + 0.04659247398376465, + 0.05222286656498909, + 0.03611915931105614, + 0.03916332125663757, + 0.03664228320121765, + 0.04088718071579933, + 0.04177439957857132, + 0.04756740853190422, + 0.04776441678404808, + 0.05542457476258278, + 0.013802599161863327, + 0.020541559904813766, + 0.02693362347781658, + 0.03680417314171791, + 0.03932841867208481, + 0.05126549303531647, + 0.0514267161488533, + 0.06891486048698425, + 0.015529165975749493, + 0.02905990369617939, + 0.03558400645852089, + 0.05358703061938286, + 0.05707564577460289, + 0.07880406081676483, + 0.08332383632659912, + 0.10752817988395691, + 0.009738311171531677, + 0.029148343950510025, + 0.03187238425016403, + 0.054091889411211014, + 0.05786185711622238, + 0.08628532290458679, + 0.09122034162282944, + 0.11884573101997375 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "population_cv": 0.5384894543542103 + }, + "activation_output_rms_by_block": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14976411033421755, + "first_to_last_ratio": 1.036636701844262, + "imbalance_abs_log_ratio": 0.035981532131895505, + "last_quartile_mean": 0.14447116339579225, + "mean": 0.12951817945577204, + "normalized": [ + 0.7297249643070838, + 1.5686735699681784, + 1.7647660354854744, + 1.8765703797812938, + 0.636469098825333, + 0.749653246535227, + 0.8711234054002422, + 1.0535576234146717, + 0.42020993720306404, + 0.6840666215997374, + 0.9431112804194469, + 1.192130706187048, + 0.4699981697046636, + 0.7312780916470571, + 0.9733494312076132, + 1.2651958587237788, + 0.6835788065012903, + 0.9366911620158862, + 1.2091399212742004, + 1.6196823469097898, + 0.34397948019882096, + 0.5722161994862464, + 0.7727653276358534, + 1.008461533628578, + 0.3829837776042769, + 0.608007158744095, + 0.8769750006177236, + 1.2205488119398642, + 0.5370088925855727, + 1.0271058497013745, + 1.7339584411053826, + 2.5370188696411313 + ], + "population_cv": 0.4929365839545065 + }, + "bits_per_byte": 2.073383110809672, + "branch_output_rms_by_sublayer": [ + 0.04804428666830063, + 0.07808538526296616, + 0.12356939166784286, + 0.03881331905722618, + 0.05923242121934891, + 0.01480295043438673, + 0.04508936405181885, + 0.02500341460108757, + 0.0799286738038063, + 0.015180142596364021, + 0.03290615975856781, + 0.013721407391130924, + 0.03170505166053772, + 0.016902483999729156, + 0.03961387276649475, + 0.03325798362493515, + 0.04508088901638985, + 0.026172807440161705, + 0.0312613770365715, + 0.027887089177966118, + 0.029994890093803406, + 0.03240044787526131, + 0.03058313950896263, + 0.03682815283536911, + 0.0497574545443058, + 0.029301028698682785, + 0.03759258985519409, + 0.0338764488697052, + 0.030690157786011696, + 0.0393521711230278, + 0.03000541403889656, + 0.046694882214069366, + 0.079553984105587, + 0.029503758996725082, + 0.0414140485227108, + 0.03615153953433037, + 0.04122982919216156, + 0.030687781050801277, + 0.05830265209078789, + 0.05279085040092468, + 0.03440643846988678, + 0.024021649733185768, + 0.033897366374731064, + 0.03336174041032791, + 0.02957344427704811, + 0.04255227372050285, + 0.03406442329287529, + 0.0520954355597496, + 0.025225287303328514, + 0.03984410688281059, + 0.01915411651134491, + 0.04249191656708717, + 0.014526215381920338, + 0.05309459939599037, + 0.011566666886210442, + 0.06231515854597092, + 0.0242468249052763, + 0.06388300657272339, + 0.013597902841866016, + 0.0749494656920433, + 0.020146220922470093, + 0.10156872868537903, + 0.02710210345685482, + 0.11044161766767502 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002879944983597963, + 0.00022853159560210997, + 0.00014071406888316784, + 0.00014318671869519129, + 0.00013886536811003923, + 0.00010286844600842235, + 9.677924556041764e-05, + 0.0001337288949114584, + 0.00010758657043361882, + 9.13787469069405e-05, + 9.373485859236601e-05, + 9.381017142422257e-05, + 0.00010148426099734976, + 9.552468088000852e-05, + 0.00010200246393672037, + 0.00010753255944705701, + 0.00013288008273706816, + 0.00012966970691820486, + 0.0001395588859415414, + 0.00019785772756123382, + 0.0004949688435464942, + 0.0005919211037074493, + 0.000516172452953512, + 0.0004666614668327092, + 0.0003441119880144902, + 0.00027788329882105144, + 0.00022798541204965527, + 0.00021216050695355764, + 0.00012431994828037904, + 0.00011340006302052245, + 0.00011536177905162644, + 0.00011797314476595472 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015908360451632537, + "first_to_last_ratio": 0.8300756844691923, + "imbalance_abs_log_ratio": 0.18623839624123317, + "last_quartile_mean": 0.00019164951761965464, + "mean": 0.00019589404874701053, + "normalized": [ + 1.4701544033720488, + 1.1666081591751138, + 0.7183172219024098, + 0.730939605419618, + 0.7088799736299207, + 0.5251228746681984, + 0.4940387223575344, + 0.682659303673712, + 0.5492079576779928, + 0.4664702551783621, + 0.4784977348312449, + 0.4788821917983569, + 0.5180568866000248, + 0.4876344201930039, + 0.5207022091235275, + 0.5489322423772612, + 0.6783262870260932, + 0.661937959563377, + 0.7124202436684344, + 1.01002418821186, + 2.526717104028653, + 3.0216390313719645, + 2.6349572958192744, + 2.3822135987162336, + 1.7566229817369148, + 1.4185387488719823, + 1.1638200012093756, + 1.0830370208313709, + 0.6346285100316312, + 0.5788846764149236, + 0.5888988450109153, + 0.6022293455086652 + ], + "population_cv": 0.7015245062301059 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4738388657569885, + "mean_weights": [ + 0.8067666292190552, + 0.19323334097862244 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6344536542892456, + "mean_weights": [ + 0.4895974397659302, + 0.5104025602340698 + ], + "sources": 2 + }, + { + "entropy_mean": 0.41773349046707153, + "mean_weights": [ + 0.8461704254150391, + 0.15382957458496094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868985891342163, + "mean_weights": [ + 0.5008196830749512, + 0.49918028712272644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5841774344444275, + "mean_weights": [ + 0.684928297996521, + 0.315071702003479 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6896515488624573, + "mean_weights": [ + 0.5092520117759705, + 0.49074798822402954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611414909362793, + "mean_weights": [ + 0.7280503511428833, + 0.2719496488571167 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6817618608474731, + "mean_weights": [ + 0.5313050746917725, + 0.46869492530822754 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0218309164047241, + "mean_weights": [ + 0.4974328279495239, + 0.24236315488815308, + 0.2602040767669678 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0932960510253906, + "mean_weights": [ + 0.3533782958984375, + 0.29999595880508423, + 0.34662577509880066 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0456881523132324, + "mean_weights": [ + 0.4338536858558655, + 0.23105540871620178, + 0.3350909352302551 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0891339778900146, + "mean_weights": [ + 0.33998236060142517, + 0.2821658253669739, + 0.37785181403160095 + ], + "sources": 3 + }, + { + "entropy_mean": 1.038641333580017, + "mean_weights": [ + 0.34165239334106445, + 0.23904743790626526, + 0.4193001389503479 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0754427909851074, + "mean_weights": [ + 0.3781457543373108, + 0.24021697044372559, + 0.3816372752189636 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0433809757232666, + "mean_weights": [ + 0.37802231311798096, + 0.21331989765167236, + 0.4086577892303467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0922105312347412, + "mean_weights": [ + 0.3460429906845093, + 0.3074926435947418, + 0.3464643955230713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3404325246810913, + "mean_weights": [ + 0.2880150079727173, + 0.16260021924972534, + 0.2385387420654297, + 0.3108460605144501 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765960931777954, + "mean_weights": [ + 0.28703436255455017, + 0.2552259564399719, + 0.2101576328277588, + 0.24758204817771912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3274900913238525, + "mean_weights": [ + 0.33539247512817383, + 0.1987970769405365, + 0.20543015003204346, + 0.2603802978992462 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3747133016586304, + "mean_weights": [ + 0.27988845109939575, + 0.23608103394508362, + 0.20066317915916443, + 0.2833673357963562 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3239198923110962, + "mean_weights": [ + 0.3361120820045471, + 0.20296622812747955, + 0.1885247528553009, + 0.2723969519138336 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3692121505737305, + "mean_weights": [ + 0.28167375922203064, + 0.21946553885936737, + 0.20140981674194336, + 0.2974509000778198 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3342361450195312, + "mean_weights": [ + 0.36468517780303955, + 0.20593920350074768, + 0.18943950533866882, + 0.23993611335754395 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3645737171173096, + "mean_weights": [ + 0.2597482204437256, + 0.22607669234275818, + 0.209133118391037, + 0.30504196882247925 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5910131931304932, + "mean_weights": [ + 0.2617476284503937, + 0.18299485743045807, + 0.17935843765735626, + 0.17898105084896088, + 0.19691801071166992 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5884685516357422, + "mean_weights": [ + 0.2046908289194107, + 0.17294223606586456, + 0.15044087171554565, + 0.20923088490962982, + 0.26269516348838806 + ], + "sources": 5 + }, + { + "entropy_mean": 1.599259853363037, + "mean_weights": [ + 0.21657392382621765, + 0.20580387115478516, + 0.19366897642612457, + 0.1731838881969452, + 0.21076934039592743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5786192417144775, + "mean_weights": [ + 0.20599734783172607, + 0.20593789219856262, + 0.13473063707351685, + 0.17552703619003296, + 0.2778070569038391 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580681324005127, + "mean_weights": [ + 0.21277281641960144, + 0.18522506952285767, + 0.18489867448806763, + 0.17205308377742767, + 0.2450503408908844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5705549716949463, + "mean_weights": [ + 0.2125053107738495, + 0.20224159955978394, + 0.12915469706058502, + 0.1714920997619629, + 0.28460633754730225 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5599372386932373, + "mean_weights": [ + 0.2665032148361206, + 0.181569442152977, + 0.15844659507274628, + 0.14596089720726013, + 0.2475198209285736 + ], + "sources": 5 + }, + { + "entropy_mean": 1.562828540802002, + "mean_weights": [ + 0.21806953847408295, + 0.22324788570404053, + 0.11591021716594696, + 0.17848318815231323, + 0.26428914070129395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.761409044265747, + "mean_weights": [ + 0.1612042486667633, + 0.1651904582977295, + 0.1752963364124298, + 0.14996090531349182, + 0.2072838991880417, + 0.14106415212154388 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7366840839385986, + "mean_weights": [ + 0.1851401925086975, + 0.1663871705532074, + 0.09063514322042465, + 0.11971023678779602, + 0.18503433465957642, + 0.2530929446220398 + ], + "sources": 6 + }, + { + "entropy_mean": 1.777672290802002, + "mean_weights": [ + 0.15483248233795166, + 0.16576308012008667, + 0.16957050561904907, + 0.14225167036056519, + 0.16980668902397156, + 0.19777557253837585 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7414222955703735, + "mean_weights": [ + 0.18020270764827728, + 0.1900298297405243, + 0.09944288432598114, + 0.11960913985967636, + 0.15577411651611328, + 0.25494131445884705 + ], + "sources": 6 + }, + { + "entropy_mean": 1.733978509902954, + "mean_weights": [ + 0.15204599499702454, + 0.14166373014450073, + 0.1616208702325821, + 0.1256796270608902, + 0.1571422517299652, + 0.26184749603271484 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7344707250595093, + "mean_weights": [ + 0.1829337775707245, + 0.18416841328144073, + 0.09014187753200531, + 0.12335793673992157, + 0.15779981017112732, + 0.2615981996059418 + ], + "sources": 6 + }, + { + "entropy_mean": 1.735896348953247, + "mean_weights": [ + 0.17276549339294434, + 0.15101772546768188, + 0.13220663368701935, + 0.11932596564292908, + 0.15458261966705322, + 0.27010154724121094 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7644665241241455, + "mean_weights": [ + 0.19160008430480957, + 0.1915181279182434, + 0.11520625650882721, + 0.1341419219970703, + 0.1580837219953537, + 0.2094498574733734 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6835086345672607, + "mean_weights": [ + 0.10065847635269165, + 0.09339895844459534, + 0.09682720899581909, + 0.08155890554189682, + 0.09126688539981842, + 0.10349369794130325, + 0.4327958822250366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7937740087509155, + "mean_weights": [ + 0.13723835349082947, + 0.12937892973423004, + 0.07450413703918457, + 0.08537043631076813, + 0.09880411624908447, + 0.12944120168685913, + 0.3452628552913666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5790722370147705, + "mean_weights": [ + 0.09500400722026825, + 0.08892714232206345, + 0.09081116318702698, + 0.07538282871246338, + 0.07685283571481705, + 0.08106325566768646, + 0.49195876717567444 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8034920692443848, + "mean_weights": [ + 0.142653226852417, + 0.13419847190380096, + 0.07437048852443695, + 0.09070360660552979, + 0.1013825386762619, + 0.12073953449726105, + 0.33595213294029236 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5278494358062744, + "mean_weights": [ + 0.10045769065618515, + 0.08844951540231705, + 0.08561251312494278, + 0.06598107516765594, + 0.06767018139362335, + 0.07361933588981628, + 0.51820969581604 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8435695171356201, + "mean_weights": [ + 0.14839640259742737, + 0.1482572704553604, + 0.08113027364015579, + 0.09534022957086563, + 0.10612697899341583, + 0.12550383806228638, + 0.2952450215816498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6421003341674805, + "mean_weights": [ + 0.11422763019800186, + 0.10543526709079742, + 0.09285669028759003, + 0.07746599614620209, + 0.07571054995059967, + 0.08126918226480484, + 0.4530346691608429 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8733552694320679, + "mean_weights": [ + 0.14798465371131897, + 0.16052685678005219, + 0.09668852388858795, + 0.10477185249328613, + 0.10857058316469193, + 0.11694246530532837, + 0.26451510190963745 + ], + "sources": 7 + }, + { + "entropy_mean": 1.784506916999817, + "mean_weights": [ + 0.08589775860309601, + 0.08786045759916306, + 0.07714233547449112, + 0.06314891576766968, + 0.05684572085738182, + 0.056072384119033813, + 0.29875272512435913, + 0.2742796838283539 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9675533771514893, + "mean_weights": [ + 0.11549945175647736, + 0.12592051923274994, + 0.07530814409255981, + 0.08277833461761475, + 0.08296641707420349, + 0.09016790241003036, + 0.17876628041267395, + 0.24859298765659332 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7119876146316528, + "mean_weights": [ + 0.08804735541343689, + 0.07527872174978256, + 0.05666808784008026, + 0.05041901767253876, + 0.05142981559038162, + 0.05313319340348244, + 0.2591783106327057, + 0.3658455014228821 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9094492197036743, + "mean_weights": [ + 0.10337945818901062, + 0.10583682358264923, + 0.08642245084047318, + 0.08043928444385529, + 0.0717030018568039, + 0.0645938515663147, + 0.21096080541610718, + 0.2766643166542053 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7860023975372314, + "mean_weights": [ + 0.09247160702943802, + 0.0784597396850586, + 0.07236532866954803, + 0.05986309051513672, + 0.05389959365129471, + 0.06059400737285614, + 0.2579019069671631, + 0.3244447112083435 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9232951402664185, + "mean_weights": [ + 0.1160542219877243, + 0.11346358060836792, + 0.08816123008728027, + 0.08138670027256012, + 0.07161372900009155, + 0.06013647839426994, + 0.21822679042816162, + 0.2509572505950928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294003009796143, + "mean_weights": [ + 0.08831655979156494, + 0.10072161257266998, + 0.0818098857998848, + 0.07514279335737228, + 0.06651484966278076, + 0.06491033732891083, + 0.31431835889816284, + 0.20826560258865356 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9393219947814941, + "mean_weights": [ + 0.11041727662086487, + 0.12080886960029602, + 0.09995828568935394, + 0.09934850037097931, + 0.08031868189573288, + 0.075287364423275, + 0.2604207992553711, + 0.15344023704528809 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9925593137741089, + "mean_weights": [ + 0.08698172867298126, + 0.08857265114784241, + 0.07299254834651947, + 0.060478877276182175, + 0.054115522652864456, + 0.056821923702955246, + 0.2503892779350281, + 0.14502599835395813, + 0.18462148308753967 + ], + "sources": 9 + }, + { + "entropy_mean": 2.068819999694824, + "mean_weights": [ + 0.09429103136062622, + 0.09028280526399612, + 0.09188265353441238, + 0.0735517293214798, + 0.06746295094490051, + 0.060166649520397186, + 0.21415996551513672, + 0.15325874090194702, + 0.15494348108768463 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0139355659484863, + "mean_weights": [ + 0.0956040769815445, + 0.09831400215625763, + 0.0714857280254364, + 0.0677608773112297, + 0.05654233694076538, + 0.05388312786817551, + 0.24272149801254272, + 0.17641445994377136, + 0.137273907661438 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0349321365356445, + "mean_weights": [ + 0.10256819427013397, + 0.0938093364238739, + 0.0912778377532959, + 0.08394120633602142, + 0.06161238253116608, + 0.053358763456344604, + 0.2261163890361786, + 0.15197426080703735, + 0.13534162938594818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.046657085418701, + "mean_weights": [ + 0.08380690217018127, + 0.1217331811785698, + 0.10242430865764618, + 0.09225839376449585, + 0.06788390129804611, + 0.06336672604084015, + 0.2572976052761078, + 0.11757771670818329, + 0.09365126490592957 + ], + "sources": 9 + }, + { + "entropy_mean": 2.094015121459961, + "mean_weights": [ + 0.10284869372844696, + 0.13057856261730194, + 0.10152992606163025, + 0.10079468786716461, + 0.0796234980225563, + 0.07438632845878601, + 0.2018478661775589, + 0.1111527606844902, + 0.09723767638206482 + ], + "sources": 9 + }, + { + "entropy_mean": 2.057201385498047, + "mean_weights": [ + 0.09265108406543732, + 0.11481083929538727, + 0.09007447212934494, + 0.08496075868606567, + 0.06770835071802139, + 0.05925413966178894, + 0.24551761150360107, + 0.15022093057632446, + 0.09480181336402893 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05595625191926956, + 0.047033343464136124, + 0.0632273480296135, + 0.060984931886196136, + 0.11160287261009216, + 0.09079429507255554, + 0.12218338996171951, + 0.08531255275011063, + 0.12365243583917618, + 0.0786692351102829, + 0.0926642045378685, + 0.08076046407222748, + 0.093480683863163, + 0.08904207497835159, + 0.08749273419380188, + 0.08403915911912918, + 0.09927791357040405, + 0.06012389436364174, + 0.07820390164852142, + 0.06841155141592026, + 0.07669360190629959, + 0.07131828367710114, + 0.07840896397829056, + 0.07358616590499878, + 0.08920575678348541, + 0.06597626209259033, + 0.06850729882717133, + 0.07235056161880493, + 0.07440610229969025, + 0.07261744141578674, + 0.07867594808340073, + 0.07352335751056671, + 0.09058429300785065, + 0.07520031183958054, + 0.07234266400337219, + 0.07378771901130676, + 0.07771008461713791, + 0.07715436071157455, + 0.08416424691677094, + 0.08649503439664841, + 0.08947562426328659, + 0.04556623473763466, + 0.056279782205820084, + 0.04546353965997696, + 0.057985786348581314, + 0.05167657509446144, + 0.06238175556063652, + 0.058827146887779236, + 0.06515856087207794, + 0.04798740893602371, + 0.05041429027915001, + 0.046526361256837845, + 0.05083387345075607, + 0.051699329167604446, + 0.05600840970873833, + 0.059578344225883484, + 0.06128782033920288, + 0.05090610682964325, + 0.056301049888134, + 0.05821637064218521, + 0.06424970924854279, + 0.06188955530524254, + 0.06844019144773483, + 0.06796324998140335 + ], + "loss_nats": 1.4371596574783325, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.8435449600219727, + "mean_weights": [ + 0.1112452894449234, + 0.06898629665374756, + 0.05835210159420967, + 0.042265649884939194, + 0.041034020483493805, + 0.04826357960700989, + 0.10493715852499008, + 0.31774088740348816, + 0.20717504620552063 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04804428666830063, + 0.09451264888048172, + 0.18500657379627228, + 0.2031717449426651, + 0.22299833595752716, + 0.22856928408145905, + 0.2331935465335846, + 0.2430499792098999, + 0.0799286738038063, + 0.08243431895971298, + 0.09402555972337723, + 0.09709372371435165, + 0.10843002796173096, + 0.11282631754875183, + 0.12457633018493652, + 0.13645486533641815, + 0.04508088901638985, + 0.05442482605576515, + 0.0760042667388916, + 0.0885990634560585, + 0.10498681664466858, + 0.12215005606412888, + 0.13159531354904175, + 0.15440259873867035, + 0.0497574545443058, + 0.060873307287693024, + 0.07735268771648407, + 0.09471380710601807, + 0.10224150121212006, + 0.1260664463043213, + 0.13362281024456024, + 0.163865864276886, + 0.079553984105587, + 0.08853588253259659, + 0.10536550730466843, + 0.12131853401660919, + 0.14260481297969818, + 0.15660560131072998, + 0.19080305099487305, + 0.2097783088684082, + 0.03440643846988678, + 0.04455159604549408, + 0.060867179185152054, + 0.07411240041255951, + 0.08364810794591904, + 0.10008715838193893, + 0.11119203269481659, + 0.13061410188674927, + 0.025225287303328514, + 0.04960336163640022, + 0.05530283972620964, + 0.07874798029661179, + 0.08037404716014862, + 0.11358420550823212, + 0.11502843350172043, + 0.1580832600593567, + 0.0242468249052763, + 0.06955241411924362, + 0.07164724916219711, + 0.13302887976169586, + 0.1371813863515854, + 0.22457914054393768, + 0.23144805431365967, + 0.32859006524086 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "population_cv": 0.6027314599912318 + }, + "activation_output_rms_by_block": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22688884753733873, + "first_to_last_ratio": 1.3228372499356764, + "imbalance_abs_log_ratio": 0.2797788616441042, + "last_quartile_mean": 0.17151682684198022, + "mean": 0.17372163233812898, + "normalized": [ + 0.9268367409782202, + 1.919387740182172, + 2.055289136269615, + 2.170611306021289, + 0.6406640960204442, + 0.7450635822219737, + 0.8704616111421293, + 1.1200724150644035, + 0.4345583639666029, + 0.677281348405711, + 0.9287495476341928, + 1.2372530952113672, + 0.4794804996179952, + 0.7087278040538177, + 0.9675973641757485, + 1.32977546738468, + 0.6641820538556276, + 0.930123337383164, + 1.1558946518222766, + 1.5614450026347633, + 0.3131140120506511, + 0.5373702635317168, + 0.742418848247821, + 0.9851745133746388, + 0.3553571676266622, + 0.5579738073870804, + 0.7927337262517246, + 1.0584297759573136, + 0.4595608055861626, + 0.8937492173058054, + 1.5039930398498347, + 2.276669658784396 + ], + "population_cv": 0.5173518438224235 + }, + "bits_per_byte": 1.9433676664104056, + "branch_output_rms_by_sublayer": [ + 0.08528144657611847, + 0.13266965746879578, + 0.2063225358724594, + 0.045500293374061584, + 0.07479097694158554, + 0.018729008734226227, + 0.053203217685222626, + 0.03815881907939911, + 0.1070452407002449, + 0.02209169790148735, + 0.03943018242716789, + 0.01640710048377514, + 0.041509829461574554, + 0.02340921014547348, + 0.06257687509059906, + 0.05481670796871185, + 0.061002109199762344, + 0.037687551230192184, + 0.04113767296075821, + 0.04149388521909714, + 0.040399398654699326, + 0.048693906515836716, + 0.03907898813486099, + 0.06190986558794975, + 0.06714949756860733, + 0.04065176844596863, + 0.04437125846743584, + 0.049332331866025925, + 0.0412614606320858, + 0.0589165985584259, + 0.046277787536382675, + 0.08173932880163193, + 0.10073095560073853, + 0.043543290346860886, + 0.05776162073016167, + 0.05228494107723236, + 0.05185067281126976, + 0.04403071105480194, + 0.07827401161193848, + 0.0687810555100441, + 0.04170985147356987, + 0.030229199677705765, + 0.041845910251140594, + 0.04699210822582245, + 0.038240060210227966, + 0.06082639843225479, + 0.046402644366025925, + 0.07667754590511322, + 0.030182205140590668, + 0.050203632563352585, + 0.025695007294416428, + 0.05539015680551529, + 0.024306239560246468, + 0.06946423649787903, + 0.017157377675175667, + 0.07630576938390732, + 0.030125634744763374, + 0.07239200174808502, + 0.01829664409160614, + 0.09628339111804962, + 0.024464696645736694, + 0.12468792498111725, + 0.032671861350536346, + 0.15056027472019196 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002458031044227025, + 0.00017276993155813553, + 0.00010939491852011586, + 0.000125258738441463, + 0.00011215544691093282, + 7.31087164016768e-05, + 9.049562402296399e-05, + 0.00013630654451635742, + 9.25970647966133e-05, + 7.580293890923953e-05, + 7.957502622146584e-05, + 8.776523713070475e-05, + 0.00010318467043462154, + 0.00010632361913360551, + 0.00011315585331317908, + 0.00012408116098017353, + 0.00012849590289080713, + 0.00014012462333747852, + 0.00013373629199672077, + 0.00019202792118308513, + 0.0005640243510171648, + 0.0006705621295490249, + 0.000579819507581129, + 0.0005486985299520294, + 0.0004559472160958734, + 0.00038131537611027277, + 0.00030455316817913544, + 0.000254991251417659, + 0.00017923215935365346, + 0.0001463414879582755, + 0.00013146832622599984, + 0.0001387805380572583 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0001331616280992935, + "first_to_last_ratio": 0.5346167023449758, + "imbalance_abs_log_ratio": 0.6262052330699333, + "last_quartile_mean": 0.000249078690424766, + "mean": 0.00021243429301935995, + "normalized": [ + 1.1570782707870122, + 0.8132864477883067, + 0.5149588466403882, + 0.5896352074852985, + 0.5279535865742339, + 0.34414743195447317, + 0.4259934812630124, + 0.6416409637964399, + 0.4358856730733892, + 0.3568300476907056, + 0.37458653727914765, + 0.4131406275478619, + 0.4857251104237578, + 0.5005012026185237, + 0.5326628375526298, + 0.584091952464876, + 0.6048736344046712, + 0.6596139509580424, + 0.6295419166835406, + 0.9039403123373536, + 2.655053207279312, + 3.156562530551101, + 2.7294063464992813, + 2.5829093888434693, + 2.146297613325176, + 1.7949803239890372, + 1.433634672869791, + 1.200329983419488, + 0.8437063376453977, + 0.6888788334421074, + 0.6188658354422025, + 0.6532868873699723 + ], + "population_cv": 0.7868568919954686 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.24809812009334564, + "mean_weights": [ + 0.9288002252578735, + 0.07119979709386826 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6067080497741699, + "mean_weights": [ + 0.5604246854782104, + 0.43957534432411194 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3847596049308777, + "mean_weights": [ + 0.8648147583007812, + 0.13518527150154114 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6843481659889221, + "mean_weights": [ + 0.49438929557800293, + 0.5056107044219971 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5728601813316345, + "mean_weights": [ + 0.6954652070999146, + 0.3045347332954407 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686518132686615, + "mean_weights": [ + 0.5263072848320007, + 0.4736926555633545 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044870376586914, + "mean_weights": [ + 0.777843713760376, + 0.2221563160419464 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6724032163619995, + "mean_weights": [ + 0.5694063901901245, + 0.4305936098098755 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9718273282051086, + "mean_weights": [ + 0.5544795989990234, + 0.20597517490386963, + 0.23954525589942932 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856361389160156, + "mean_weights": [ + 0.3703838884830475, + 0.26754841208457947, + 0.36206769943237305 + ], + "sources": 3 + }, + { + "entropy_mean": 1.022249460220337, + "mean_weights": [ + 0.46441033482551575, + 0.21309645473957062, + 0.32249322533607483 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0742855072021484, + "mean_weights": [ + 0.38770636916160583, + 0.24731171131134033, + 0.36498188972473145 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0207322835922241, + "mean_weights": [ + 0.3404029607772827, + 0.22699853777885437, + 0.4325985312461853 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0628657341003418, + "mean_weights": [ + 0.4176652133464813, + 0.22035521268844604, + 0.361979603767395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.029329538345337, + "mean_weights": [ + 0.42153143882751465, + 0.19259268045425415, + 0.3858758807182312 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0849454402923584, + "mean_weights": [ + 0.37970077991485596, + 0.2838303744792938, + 0.3364688754081726 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3144235610961914, + "mean_weights": [ + 0.3205832540988922, + 0.13048580288887024, + 0.2396528720855713, + 0.30927807092666626 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647149801254272, + "mean_weights": [ + 0.3188267648220062, + 0.2379944771528244, + 0.19448311626911163, + 0.24869562685489655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2778496742248535, + "mean_weights": [ + 0.37896889448165894, + 0.16353651881217957, + 0.1984466314315796, + 0.2590479254722595 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3567583560943604, + "mean_weights": [ + 0.29770219326019287, + 0.21492166817188263, + 0.17857596278190613, + 0.3088001608848572 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2773678302764893, + "mean_weights": [ + 0.3856483995914459, + 0.18032220005989075, + 0.1928499937057495, + 0.24117937684059143 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655540943145752, + "mean_weights": [ + 0.29819539189338684, + 0.20063453912734985, + 0.20929867029190063, + 0.29187142848968506 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2754871845245361, + "mean_weights": [ + 0.42651012539863586, + 0.1713070571422577, + 0.1726072132587433, + 0.22957557439804077 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3596348762512207, + "mean_weights": [ + 0.2750997543334961, + 0.22746169567108154, + 0.18702520430088043, + 0.31041333079338074 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5752925872802734, + "mean_weights": [ + 0.28211507201194763, + 0.1675817370414734, + 0.1759152114391327, + 0.16564446687698364, + 0.20874354243278503 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5704567432403564, + "mean_weights": [ + 0.21000899374485016, + 0.18079039454460144, + 0.13544103503227234, + 0.1749160885810852, + 0.29884350299835205 + ], + "sources": 5 + }, + { + "entropy_mean": 1.579707384109497, + "mean_weights": [ + 0.21335941553115845, + 0.18788449466228485, + 0.20250459015369415, + 0.1511598825454712, + 0.24509160220623016 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5663042068481445, + "mean_weights": [ + 0.21676909923553467, + 0.19467836618423462, + 0.13713446259498596, + 0.15313217043876648, + 0.29828590154647827 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5384478569030762, + "mean_weights": [ + 0.21904908120632172, + 0.15864285826683044, + 0.18178997933864594, + 0.14981383085250854, + 0.29070425033569336 + ], + "sources": 5 + }, + { + "entropy_mean": 1.552668571472168, + "mean_weights": [ + 0.22710487246513367, + 0.1922147125005722, + 0.13047492504119873, + 0.14686013758182526, + 0.30334532260894775 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5210195779800415, + "mean_weights": [ + 0.2812899947166443, + 0.14511609077453613, + 0.15909329056739807, + 0.13197475671768188, + 0.282525897026062 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5376527309417725, + "mean_weights": [ + 0.23905543982982635, + 0.21673071384429932, + 0.0950920507311821, + 0.16199833154678345, + 0.2871234714984894 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7209875583648682, + "mean_weights": [ + 0.1703755110502243, + 0.14151470363140106, + 0.17087554931640625, + 0.12742051482200623, + 0.25328749418258667, + 0.1365262269973755 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7200093269348145, + "mean_weights": [ + 0.18992580473423004, + 0.16036251187324524, + 0.08439569175243378, + 0.11250905692577362, + 0.1775888204574585, + 0.27521812915802 + ], + "sources": 6 + }, + { + "entropy_mean": 1.753357172012329, + "mean_weights": [ + 0.15539541840553284, + 0.1388990879058838, + 0.15745995938777924, + 0.12427005916833878, + 0.17903099954128265, + 0.24494445323944092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7199265956878662, + "mean_weights": [ + 0.19207912683486938, + 0.17904852330684662, + 0.08410340547561646, + 0.11219910532236099, + 0.15951165556907654, + 0.2730581760406494 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6756346225738525, + "mean_weights": [ + 0.15863081812858582, + 0.11242729425430298, + 0.15970338881015778, + 0.09774985909461975, + 0.15905189514160156, + 0.3124367296695709 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7271867990493774, + "mean_weights": [ + 0.19560003280639648, + 0.18333327770233154, + 0.08475087583065033, + 0.11598999053239822, + 0.15889793634414673, + 0.2614278793334961 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6916961669921875, + "mean_weights": [ + 0.19065657258033752, + 0.12983909249305725, + 0.11857748031616211, + 0.09883742034435272, + 0.1529499739408493, + 0.3091394901275635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7504222393035889, + "mean_weights": [ + 0.20809361338615417, + 0.19103100895881653, + 0.10743936896324158, + 0.1194620355963707, + 0.15873855352401733, + 0.2152354121208191 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5379457473754883, + "mean_weights": [ + 0.09571269154548645, + 0.07078731805086136, + 0.0824701339006424, + 0.060012347996234894, + 0.08145605027675629, + 0.09452696144580841, + 0.5150344967842102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7319592237472534, + "mean_weights": [ + 0.13678354024887085, + 0.11740625649690628, + 0.06668869405984879, + 0.07360264658927917, + 0.09287553280591965, + 0.12232495844364166, + 0.390318363904953 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4663677215576172, + "mean_weights": [ + 0.09485072642564774, + 0.07167200744152069, + 0.0841417908668518, + 0.05784624442458153, + 0.07052354514598846, + 0.0724712610244751, + 0.5484944581985474 + ], + "sources": 7 + }, + { + "entropy_mean": 1.749481201171875, + "mean_weights": [ + 0.14530614018440247, + 0.12230368703603745, + 0.0660901889204979, + 0.07630561292171478, + 0.09453228116035461, + 0.12125740200281143, + 0.3742046654224396 + ], + "sources": 7 + }, + { + "entropy_mean": 1.472255825996399, + "mean_weights": [ + 0.10688216984272003, + 0.07465942203998566, + 0.08327433466911316, + 0.054589346051216125, + 0.0637175589799881, + 0.07299347221851349, + 0.5438836812973022 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7924723625183105, + "mean_weights": [ + 0.15522946417331696, + 0.13724805414676666, + 0.06889298558235168, + 0.08198928833007812, + 0.09816981852054596, + 0.123870849609375, + 0.3345995247364044 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755163431167603, + "mean_weights": [ + 0.12274101376533508, + 0.08900541067123413, + 0.08551642298698425, + 0.06285787373781204, + 0.07429315149784088, + 0.07904880493879318, + 0.4865373373031616 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8390922546386719, + "mean_weights": [ + 0.15884171426296234, + 0.14726482331752777, + 0.08053631335496902, + 0.09097073972225189, + 0.10738551616668701, + 0.12150256335735321, + 0.29349833726882935 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6550344228744507, + "mean_weights": [ + 0.0844201147556305, + 0.06695197522640228, + 0.061310961842536926, + 0.0447731651365757, + 0.04748314991593361, + 0.045562684535980225, + 0.2905467748641968, + 0.3589511513710022 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8679088354110718, + "mean_weights": [ + 0.11674155294895172, + 0.1088850349187851, + 0.05779466778039932, + 0.06273136287927628, + 0.07066237181425095, + 0.08129343390464783, + 0.17753683030605316, + 0.32435476779937744 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5923268795013428, + "mean_weights": [ + 0.08656083047389984, + 0.05770102143287659, + 0.05105586349964142, + 0.039375390857458115, + 0.04410157352685928, + 0.04500962793827057, + 0.2357216477394104, + 0.4404740333557129 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8469831943511963, + "mean_weights": [ + 0.10824646055698395, + 0.09116974472999573, + 0.0701109915971756, + 0.0629766583442688, + 0.07049155235290527, + 0.07297641038894653, + 0.18795298039913177, + 0.33607521653175354 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6590619087219238, + "mean_weights": [ + 0.0944472923874855, + 0.0654030293226242, + 0.060137998312711716, + 0.0449567511677742, + 0.044405773282051086, + 0.05244143307209015, + 0.2202538251876831, + 0.4179539084434509 + ], + "sources": 8 + }, + { + "entropy_mean": 1.884438157081604, + "mean_weights": [ + 0.11613816022872925, + 0.09775452315807343, + 0.08394932001829147, + 0.07057420164346695, + 0.06920696794986725, + 0.06340071558952332, + 0.21605131030082703, + 0.2829247713088989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7424441576004028, + "mean_weights": [ + 0.08763163536787033, + 0.07771545648574829, + 0.07502467930316925, + 0.06006643921136856, + 0.06314834952354431, + 0.054570816457271576, + 0.3030073344707489, + 0.278835266828537 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8953055143356323, + "mean_weights": [ + 0.1135263741016388, + 0.10274903476238251, + 0.0858360230922699, + 0.08475605398416519, + 0.08078759908676147, + 0.07204630225896835, + 0.2779633104801178, + 0.18233531713485718 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801485300064087, + "mean_weights": [ + 0.07824120670557022, + 0.06023171544075012, + 0.054100219160318375, + 0.039871927350759506, + 0.04207881540060043, + 0.045095670968294144, + 0.2047339826822281, + 0.15412676334381104, + 0.321519672870636 + ], + "sources": 9 + }, + { + "entropy_mean": 2.008213520050049, + "mean_weights": [ + 0.09438058733940125, + 0.07153058052062988, + 0.07423336803913116, + 0.06403130292892456, + 0.0645662397146225, + 0.05577655881643295, + 0.21496367454528809, + 0.1609123945236206, + 0.1996053010225296 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8966764211654663, + "mean_weights": [ + 0.09610478579998016, + 0.07267358899116516, + 0.060228414833545685, + 0.046177469193935394, + 0.04206448048353195, + 0.04096413776278496, + 0.21562573313713074, + 0.21000833809375763, + 0.21615305542945862 + ], + "sources": 9 + }, + { + "entropy_mean": 1.981720209121704, + "mean_weights": [ + 0.10501760244369507, + 0.06946457922458649, + 0.07816961407661438, + 0.07385893911123276, + 0.06364607810974121, + 0.05328303575515747, + 0.23084522783756256, + 0.15472577512264252, + 0.17098914086818695 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0028607845306396, + "mean_weights": [ + 0.0842779204249382, + 0.10298711061477661, + 0.09207960218191147, + 0.07771934568881989, + 0.06265503168106079, + 0.0552704893052578, + 0.2620389461517334, + 0.1382768750190735, + 0.12469469010829926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0655057430267334, + "mean_weights": [ + 0.10472239553928375, + 0.11792458593845367, + 0.0928182601928711, + 0.10183010995388031, + 0.08287280797958374, + 0.06570011377334595, + 0.21871674060821533, + 0.11616810411214828, + 0.09924688935279846 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0258567333221436, + "mean_weights": [ + 0.09674200415611267, + 0.09482249617576599, + 0.08996410667896271, + 0.06836152076721191, + 0.06293784081935883, + 0.05016916245222092, + 0.22868779301643372, + 0.1848844438791275, + 0.12343062460422516 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059412047266960144, + 0.056020382791757584, + 0.08935670554637909, + 0.0742802768945694, + 0.17747226357460022, + 0.1285802572965622, + 0.17946027219295502, + 0.10598119348287582, + 0.17260213196277618, + 0.10133238881826401, + 0.12674333155155182, + 0.10985831171274185, + 0.12412907183170319, + 0.12611564993858337, + 0.11767174303531647, + 0.1125042662024498, + 0.13582420349121094, + 0.07848236709833145, + 0.10855290293693542, + 0.0899009183049202, + 0.10545514523983002, + 0.09790628403425217, + 0.11005345731973648, + 0.09713682532310486, + 0.13049271702766418, + 0.09147685021162033, + 0.09666567295789719, + 0.10080859810113907, + 0.10358717292547226, + 0.09896469116210938, + 0.11041814833879471, + 0.10007023066282272, + 0.13209649920463562, + 0.10974904894828796, + 0.10239993035793304, + 0.10291008651256561, + 0.11153820157051086, + 0.10674569010734558, + 0.11912636458873749, + 0.12148815393447876, + 0.127323716878891, + 0.055056143552064896, + 0.07468672841787338, + 0.05981758236885071, + 0.07975661009550095, + 0.07101839780807495, + 0.08746611326932907, + 0.08339047431945801, + 0.09389390796422958, + 0.06143829599022865, + 0.06575146317481995, + 0.06018613278865814, + 0.0681111291050911, + 0.06816941499710083, + 0.07568862289190292, + 0.08121132105588913, + 0.08424623310565948, + 0.05916989967226982, + 0.0719028189778328, + 0.07458337396383286, + 0.08439797908067703, + 0.08333029597997665, + 0.08961156010627747, + 0.09104464948177338 + ], + "loss_nats": 1.347039818763733, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7394249439239502, + "mean_weights": [ + 0.10602211207151413, + 0.05902107432484627, + 0.049884356558322906, + 0.034865088760852814, + 0.03282830864191055, + 0.043302543461322784, + 0.08693542331457138, + 0.3314756751060486, + 0.2556654214859009 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08528144657611847, + 0.16101159155368805, + 0.3133305609226227, + 0.3334391713142395, + 0.3510810136795044, + 0.3570481836795807, + 0.36267346143722534, + 0.37708213925361633, + 0.1070452407002449, + 0.11129721254110336, + 0.1260577291250229, + 0.12943366169929504, + 0.14502130448818207, + 0.1512180119752884, + 0.17329032719135284, + 0.1945808082818985, + 0.061002109199762344, + 0.07549218833446503, + 0.0987050011754036, + 0.11765842139720917, + 0.13416865468025208, + 0.16134388744831085, + 0.17569874227046967, + 0.21493762731552124, + 0.06714949756860733, + 0.08329613506793976, + 0.09735818952322006, + 0.12312135100364685, + 0.13312570750713348, + 0.16809259355068207, + 0.17941679060459137, + 0.23101076483726501, + 0.10073095560073853, + 0.11538279056549072, + 0.13780468702316284, + 0.16158254444599152, + 0.17965665459632874, + 0.20080390572547913, + 0.24453195929527283, + 0.27125677466392517, + 0.04170985147356987, + 0.0543946772813797, + 0.07352267950773239, + 0.09335283935070038, + 0.10462787002325058, + 0.12897421419620514, + 0.14256280660629272, + 0.17114612460136414, + 0.030182205140590668, + 0.06173322722315788, + 0.06915032118558884, + 0.09693212062120438, + 0.10093247890472412, + 0.13771499693393707, + 0.1392749696969986, + 0.18387214839458466, + 0.030125634744763374, + 0.07983565330505371, + 0.08267941325902939, + 0.15526357293128967, + 0.15948866307735443, + 0.26127612590789795, + 0.2670789062976837, + 0.39550676941871643 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "population_cv": 0.6420341358971389 + }, + "activation_output_rms_by_block": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23796202801167965, + "first_to_last_ratio": 1.7210004980371292, + "imbalance_abs_log_ratio": 0.542905806617582, + "last_quartile_mean": 0.13826958695426583, + "mean": 0.15974999987520278, + "normalized": [ + 1.0588703589137207, + 2.3439518181213, + 2.5018026404306735, + 2.625406629974646, + 0.6192476728139908, + 0.7301872144203714, + 0.8733231516990753, + 1.1639318005158055, + 0.41362918444307983, + 0.6526334563206251, + 0.9108721190988135, + 1.2435622068278698, + 0.40590548584340497, + 0.6529420199592338, + 0.9214830517691831, + 1.2794736788600816, + 0.6292358348878756, + 0.8817240486108118, + 1.0928942572485663, + 1.5151234871428112, + 0.30958389010160414, + 0.5326764501933169, + 0.737604494775539, + 0.9796364773984299, + 0.3464072479049888, + 0.5325333617345424, + 0.7475662125351719, + 0.9760454607844148, + 0.41495429177381643, + 0.7752670281456541, + 1.247487625087717, + 1.8840373416628649 + ], + "population_cv": 0.591409330949146 + }, + "bits_per_byte": 1.8740129107275643, + "branch_output_rms_by_sublayer": [ + 0.0933690071105957, + 0.13583384454250336, + 0.24537226557731628, + 0.0415157787501812, + 0.07698935270309448, + 0.018140379339456558, + 0.0523640513420105, + 0.039518751204013824, + 0.09380833059549332, + 0.022877270355820656, + 0.03647654876112938, + 0.015532293356955051, + 0.039120711386203766, + 0.02314678393304348, + 0.06371714919805527, + 0.05432315170764923, + 0.05156813934445381, + 0.03743875026702881, + 0.03728010877966881, + 0.04006272554397583, + 0.03983496502041817, + 0.045583780854940414, + 0.040284499526023865, + 0.05995568633079529, + 0.046871740370988846, + 0.038651175796985626, + 0.04314620420336723, + 0.04559725895524025, + 0.04033593460917473, + 0.05632574483752251, + 0.04471700266003609, + 0.07952121645212173, + 0.08397205173969269, + 0.042337141931056976, + 0.0528092160820961, + 0.05044656619429588, + 0.04443390294909477, + 0.04418695345520973, + 0.07560808956623077, + 0.06310147047042847, + 0.03708760067820549, + 0.029010163620114326, + 0.03599191829562187, + 0.04586305841803551, + 0.033304885029792786, + 0.06033914536237717, + 0.04061548039317131, + 0.07656749337911606, + 0.025487789884209633, + 0.045884523540735245, + 0.021565141156315804, + 0.051604680716991425, + 0.02342291921377182, + 0.06502768397331238, + 0.017244853079319, + 0.06792232394218445, + 0.024166326969861984, + 0.060306549072265625, + 0.016819356009364128, + 0.08025261014699936, + 0.021590016782283783, + 0.09663844108581543, + 0.026201345026493073, + 0.12230555713176727 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003065916678101079, + 0.00023586292031870838, + 0.0001322871105193671, + 0.00016391009158296018, + 0.0001640597280926294, + 0.00011452252186811768, + 0.00012670462239247783, + 0.00021513822087305528, + 0.00015124888511926465, + 0.00012797922905715274, + 0.0001317390201682035, + 0.00014536342393434786, + 0.00017054401801644127, + 0.00018163687748006386, + 0.00019727429589294548, + 0.0002118946661868627, + 0.0002209032617907253, + 0.0002444386191807451, + 0.00023590577825104205, + 0.00032383840230939837, + 0.0010316902394974224, + 0.0010821684520459678, + 0.0009765210521856827, + 0.0008761640307174725, + 0.0008326776627851861, + 0.0006767392019523396, + 0.00052552098450176, + 0.0004066875433884613, + 0.0003364842674135882, + 0.00025378360531775436, + 0.000205309661176443, + 0.00020659565641729775 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018238461043217796, + "first_to_last_ratio": 0.42368241008054586, + "imbalance_abs_log_ratio": 0.8587711372443304, + "last_quartile_mean": 0.0004304748228691038, + "mean": 0.00035038080369512474, + "normalized": [ + 0.875024158220954, + 0.6731616510701846, + 0.37755239192405493, + 0.46780556998089007, + 0.46823263821091626, + 0.3268515873596963, + 0.3616197607181892, + 0.6140125788975943, + 0.43167001024082974, + 0.3652575361078021, + 0.37598812143496585, + 0.4148726825252455, + 0.486739045683667, + 0.5183984840622454, + 0.5630282647122382, + 0.6047553517550511, + 0.630466222638552, + 0.69763701836085, + 0.6732839692219831, + 0.9242469875466649, + 2.944482770223686, + 3.0885494885375917, + 2.7870278333951726, + 2.500605117282182, + 2.3764933866346176, + 1.9314391508194257, + 1.499856667259172, + 1.160701554136312, + 0.9603387624693381, + 0.7243079604856947, + 0.5859614996348036, + 0.5896317784494322 + ], + "population_cv": 0.8248093894360681 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22886337339878082, + "mean_weights": [ + 0.9364937543869019, + 0.06350618600845337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.603729784488678, + "mean_weights": [ + 0.5866844654083252, + 0.4133155643939972 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3967982828617096, + "mean_weights": [ + 0.858893632888794, + 0.14110638201236725 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6852496862411499, + "mean_weights": [ + 0.48102498054504395, + 0.518975019454956 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5890734195709229, + "mean_weights": [ + 0.6876576542854309, + 0.3123423457145691 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873323321342468, + "mean_weights": [ + 0.5168514847755432, + 0.4831485152244568 + ], + "sources": 2 + }, + { + "entropy_mean": 0.487301766872406, + "mean_weights": [ + 0.7939615249633789, + 0.2060384750366211 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674697756767273, + "mean_weights": [ + 0.5704737305641174, + 0.42952626943588257 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9691112041473389, + "mean_weights": [ + 0.5506306886672974, + 0.18125343322753906, + 0.2681158185005188 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0745112895965576, + "mean_weights": [ + 0.342629998922348, + 0.24457570910453796, + 0.412794291973114 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0074045658111572, + "mean_weights": [ + 0.4399106502532959, + 0.18251104652881622, + 0.3775783181190491 + ], + "sources": 3 + }, + { + "entropy_mean": 1.060497522354126, + "mean_weights": [ + 0.3786153495311737, + 0.2188822329044342, + 0.4025024175643921 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9941019415855408, + "mean_weights": [ + 0.32296958565711975, + 0.19686520099639893, + 0.4801652133464813 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0503522157669067, + "mean_weights": [ + 0.40130966901779175, + 0.19853851199150085, + 0.4001518487930298 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0129402875900269, + "mean_weights": [ + 0.4093267321586609, + 0.16906049847602844, + 0.42161279916763306 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0774019956588745, + "mean_weights": [ + 0.38510820269584656, + 0.25884830951690674, + 0.3560434877872467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2740377187728882, + "mean_weights": [ + 0.29550763964653015, + 0.09541245549917221, + 0.24569378793239594, + 0.3633860945701599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.360302448272705, + "mean_weights": [ + 0.3109527826309204, + 0.20984822511672974, + 0.1935039907693863, + 0.28569501638412476 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2722704410552979, + "mean_weights": [ + 0.350364625453949, + 0.12631991505622864, + 0.22418734431266785, + 0.29912811517715454 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3558036088943481, + "mean_weights": [ + 0.2989768385887146, + 0.1941717267036438, + 0.1926535815000534, + 0.314197838306427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2696614265441895, + "mean_weights": [ + 0.3667730689048767, + 0.15310388803482056, + 0.19384080171585083, + 0.2862822115421295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3606936931610107, + "mean_weights": [ + 0.2982274293899536, + 0.18611131608486176, + 0.21284276247024536, + 0.3028184771537781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.269316554069519, + "mean_weights": [ + 0.4081852436065674, + 0.14420126378536224, + 0.18228203058242798, + 0.2653314471244812 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3636136054992676, + "mean_weights": [ + 0.27427661418914795, + 0.21341991424560547, + 0.19867971539497375, + 0.3136237859725952 + ], + "sources": 4 + }, + { + "entropy_mean": 1.570773959159851, + "mean_weights": [ + 0.26587405800819397, + 0.13419361412525177, + 0.1907704770565033, + 0.17973750829696655, + 0.2294243425130844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5560038089752197, + "mean_weights": [ + 0.20477107167243958, + 0.15721666812896729, + 0.13811248540878296, + 0.1752161681652069, + 0.32468360662460327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5659973621368408, + "mean_weights": [ + 0.20531430840492249, + 0.15986156463623047, + 0.21557101607322693, + 0.15222224593162537, + 0.26703086495399475 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5552496910095215, + "mean_weights": [ + 0.2162160575389862, + 0.17982497811317444, + 0.1307673156261444, + 0.1554446816444397, + 0.31774696707725525 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5068919658660889, + "mean_weights": [ + 0.20646695792675018, + 0.1197461187839508, + 0.19215428829193115, + 0.15179987251758575, + 0.3298327624797821 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5435223579406738, + "mean_weights": [ + 0.22410975396633148, + 0.16986176371574402, + 0.13080401718616486, + 0.15279728174209595, + 0.3224271535873413 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4949920177459717, + "mean_weights": [ + 0.27174079418182373, + 0.12178683280944824, + 0.1529826819896698, + 0.1350194811820984, + 0.31847020983695984 + ], + "sources": 5 + }, + { + "entropy_mean": 1.544264793395996, + "mean_weights": [ + 0.24016115069389343, + 0.20730282366275787, + 0.09692574292421341, + 0.1695065051317215, + 0.2861037850379944 + ], + "sources": 5 + }, + { + "entropy_mean": 1.686682939529419, + "mean_weights": [ + 0.16755425930023193, + 0.10962115228176117, + 0.1783454418182373, + 0.13310004770755768, + 0.2885603904724121, + 0.12281867861747742 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7070834636688232, + "mean_weights": [ + 0.181177020072937, + 0.14557164907455444, + 0.08318156003952026, + 0.11424209922552109, + 0.17698991298675537, + 0.2988377511501312 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7419235706329346, + "mean_weights": [ + 0.1532188206911087, + 0.11420352011919022, + 0.1659742295742035, + 0.1250365972518921, + 0.18140509724617004, + 0.26016172766685486 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7127439975738525, + "mean_weights": [ + 0.18731769919395447, + 0.16715484857559204, + 0.08496320992708206, + 0.11362075805664062, + 0.1561521738767624, + 0.2907913327217102 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6076834201812744, + "mean_weights": [ + 0.15032559633255005, + 0.08624927699565887, + 0.15736976265907288, + 0.08584291487932205, + 0.14632999897003174, + 0.3738824725151062 + ], + "sources": 6 + }, + { + "entropy_mean": 1.709290862083435, + "mean_weights": [ + 0.18991519510746002, + 0.17091524600982666, + 0.07822999358177185, + 0.116765096783638, + 0.1540035903453827, + 0.2901708781719208 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6690887212753296, + "mean_weights": [ + 0.19029225409030914, + 0.11002412438392639, + 0.11471512168645859, + 0.0999990925192833, + 0.1507222205400467, + 0.3342471718788147 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7458693981170654, + "mean_weights": [ + 0.20528709888458252, + 0.1722964346408844, + 0.10435444861650467, + 0.12075909972190857, + 0.16116347908973694, + 0.2361394464969635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4612125158309937, + "mean_weights": [ + 0.08934466540813446, + 0.054379623383283615, + 0.07567165046930313, + 0.05758609622716904, + 0.07884950935840607, + 0.08982288837432861, + 0.5543456077575684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7241814136505127, + "mean_weights": [ + 0.13360008597373962, + 0.10609637945890427, + 0.06363911926746368, + 0.07426095008850098, + 0.09383077174425125, + 0.13373199105262756, + 0.39484071731567383 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3979300260543823, + "mean_weights": [ + 0.08959861099720001, + 0.05414731800556183, + 0.07668881118297577, + 0.05603286623954773, + 0.06979228556156158, + 0.07073447853326797, + 0.5830056071281433 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7459560632705688, + "mean_weights": [ + 0.14318783581256866, + 0.1094653457403183, + 0.06349228322505951, + 0.07859592139720917, + 0.09567482024431229, + 0.13266830146312714, + 0.37691545486450195 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4055393934249878, + "mean_weights": [ + 0.10154954344034195, + 0.05619275942444801, + 0.07791627943515778, + 0.053915880620479584, + 0.06398355215787888, + 0.06915920972824097, + 0.5772827863693237 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7853620052337646, + "mean_weights": [ + 0.15284112095832825, + 0.12206703424453735, + 0.06508444249629974, + 0.08347312361001968, + 0.09811970591545105, + 0.13767032325267792, + 0.3407442569732666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.509842872619629, + "mean_weights": [ + 0.11788781732320786, + 0.06822571903467178, + 0.08556114137172699, + 0.06067090481519699, + 0.07228855788707733, + 0.07258446514606476, + 0.5227814316749573 + ], + "sources": 7 + }, + { + "entropy_mean": 1.839367389678955, + "mean_weights": [ + 0.15906387567520142, + 0.13139380514621735, + 0.07717745006084442, + 0.09607303887605667, + 0.11078508198261261, + 0.12967455387115479, + 0.29583218693733215 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5841834545135498, + "mean_weights": [ + 0.07896879315376282, + 0.048752427101135254, + 0.055707160383462906, + 0.04139826446771622, + 0.04449019208550453, + 0.04285799711942673, + 0.2801477909088135, + 0.40767738223075867 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8125314712524414, + "mean_weights": [ + 0.10889512300491333, + 0.08656036853790283, + 0.048123255372047424, + 0.06561170518398285, + 0.06917259097099304, + 0.08286689221858978, + 0.16869932413101196, + 0.3700707256793976 + ], + "sources": 8 + }, + { + "entropy_mean": 1.50089693069458, + "mean_weights": [ + 0.07974998652935028, + 0.04166398197412491, + 0.04607272148132324, + 0.03586764261126518, + 0.04223340004682541, + 0.04095568507909775, + 0.22249138355255127, + 0.49096518754959106 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8059929609298706, + "mean_weights": [ + 0.10711923241615295, + 0.07651708275079727, + 0.058284737169742584, + 0.06609588116407394, + 0.07139427959918976, + 0.07493151724338531, + 0.17425751686096191, + 0.37139976024627686 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5622732639312744, + "mean_weights": [ + 0.08775827288627625, + 0.047765426337718964, + 0.053928621113300323, + 0.04293380677700043, + 0.04174989461898804, + 0.04734005779027939, + 0.2016349583864212, + 0.4768889844417572 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8757455348968506, + "mean_weights": [ + 0.11563669890165329, + 0.08108697831630707, + 0.07280858606100082, + 0.07553227990865707, + 0.07606013864278793, + 0.06935927271842957, + 0.19856664538383484, + 0.3109493851661682 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6924852132797241, + "mean_weights": [ + 0.08638577163219452, + 0.059912849217653275, + 0.06609142571687698, + 0.056745607405900955, + 0.06277773529291153, + 0.05352145433425903, + 0.2830069661140442, + 0.3315581977367401 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8980522155761719, + "mean_weights": [ + 0.11603894829750061, + 0.08804173022508621, + 0.07975024729967117, + 0.09026999771595001, + 0.08430293202400208, + 0.07433462888002396, + 0.26602888107299805, + 0.2012326419353485 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7512003183364868, + "mean_weights": [ + 0.07744461297988892, + 0.046086352318525314, + 0.05025433003902435, + 0.037484876811504364, + 0.04005061089992523, + 0.041623204946517944, + 0.198423832654953, + 0.17380405962467194, + 0.33482813835144043 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0160694122314453, + "mean_weights": [ + 0.0938527062535286, + 0.06208505481481552, + 0.06560768187046051, + 0.07029052078723907, + 0.06935097277164459, + 0.056832704693078995, + 0.1942581683397293, + 0.1706070452928543, + 0.21711516380310059 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7837426662445068, + "mean_weights": [ + 0.08836328238248825, + 0.04953558370471001, + 0.04813014343380928, + 0.0373263955116272, + 0.037015400826931, + 0.03390160948038101, + 0.1840541511774063, + 0.23140482604503632, + 0.29026859998703003 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9973840713500977, + "mean_weights": [ + 0.10261696577072144, + 0.06039813905954361, + 0.06671924889087677, + 0.08377155661582947, + 0.07002593576908112, + 0.05768897384405136, + 0.20831166207790375, + 0.1594475358724594, + 0.19101998209953308 + ], + "sources": 9 + }, + { + "entropy_mean": 1.969987154006958, + "mean_weights": [ + 0.08120635896921158, + 0.07894632965326309, + 0.08615730702877045, + 0.07609778642654419, + 0.058728985488414764, + 0.05076971277594566, + 0.24503237009048462, + 0.1544169783592224, + 0.16864418983459473 + ], + "sources": 9 + }, + { + "entropy_mean": 2.071648359298706, + "mean_weights": [ + 0.10569609701633453, + 0.09922641515731812, + 0.08180510997772217, + 0.10646798461675644, + 0.0826798751950264, + 0.06615597754716873, + 0.20955613255500793, + 0.13320739567279816, + 0.11520501971244812 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781701564788818, + "mean_weights": [ + 0.09446848928928375, + 0.07056106626987457, + 0.08033449202775955, + 0.0658978745341301, + 0.058234505355358124, + 0.04316519945859909, + 0.2036968618631363, + 0.22251945734024048, + 0.16112206876277924 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0626930519938469, + 0.059514161199331284, + 0.09030644595623016, + 0.08157797157764435, + 0.20229491591453552, + 0.14292451739311218, + 0.20300470292568207, + 0.10997853428125381, + 0.19049881398677826, + 0.0991024449467659, + 0.12695962190628052, + 0.1065012738108635, + 0.12132706493139267, + 0.12155740708112717, + 0.11585728824138641, + 0.1095810979604721, + 0.13427142798900604, + 0.06964563578367233, + 0.10485932976007462, + 0.08311934769153595, + 0.10419609397649765, + 0.09363401681184769, + 0.10852954536676407, + 0.0941167026758194, + 0.1286265105009079, + 0.08564268052577972, + 0.09096142649650574, + 0.09493856877088547, + 0.10080574452877045, + 0.09063401818275452, + 0.10566592961549759, + 0.09473779052495956, + 0.1287592649459839, + 0.10505300015211105, + 0.09650235623121262, + 0.09355910867452621, + 0.1065765842795372, + 0.09618847072124481, + 0.11446042358875275, + 0.11416846513748169, + 0.12395420670509338, + 0.047205954790115356, + 0.07221997529268265, + 0.053352247923612595, + 0.07677769660949707, + 0.06512638926506042, + 0.08416415005922318, + 0.07829347252845764, + 0.09094838052988052, + 0.05368548259139061, + 0.059439923614263535, + 0.05310077592730522, + 0.062182098627090454, + 0.061235878616571426, + 0.06896103918552399, + 0.07320034503936768, + 0.0776243805885315, + 0.05304349213838577, + 0.06285123527050018, + 0.0639304369688034, + 0.07231374084949493, + 0.07404901832342148, + 0.08016443997621536, + 0.08157160878181458 + ], + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.6036605834960938, + "mean_weights": [ + 0.09759747982025146, + 0.04215940088033676, + 0.041145484894514084, + 0.029554029926657677, + 0.02899969555437565, + 0.03647579252719879, + 0.06642461568117142, + 0.341248482465744, + 0.31639501452445984 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0933690071105957, + 0.16915453970432281, + 0.35791492462158203, + 0.374446302652359, + 0.3940892517566681, + 0.39966297149658203, + 0.4057151973247528, + 0.4194087088108063, + 0.09380833059549332, + 0.09892481565475464, + 0.11286329478025436, + 0.11664740741252899, + 0.13231655955314636, + 0.13951337337493896, + 0.1621972620487213, + 0.18593810498714447, + 0.05156813934445381, + 0.06607726216316223, + 0.08533531427383423, + 0.10425819456577301, + 0.12003090232610703, + 0.14551182091236115, + 0.16063299775123596, + 0.19865906238555908, + 0.046871740370988846, + 0.06484340131282806, + 0.079384446144104, + 0.10430748760700226, + 0.11459959298372269, + 0.14720691740512848, + 0.15625107288360596, + 0.20439592003822327, + 0.08397205173969269, + 0.10052042454481125, + 0.11654024571180344, + 0.14085541665554047, + 0.15294702351093292, + 0.1745898574590683, + 0.21572710573673248, + 0.2420409768819809, + 0.03708760067820549, + 0.049456026405096054, + 0.06553087383508682, + 0.08509506285190582, + 0.09390194714069366, + 0.11783231794834137, + 0.12852425873279572, + 0.15649692714214325, + 0.025487789884209633, + 0.05533855780959129, + 0.06059965491294861, + 0.08507220447063446, + 0.08860157430171967, + 0.11942370235919952, + 0.12100546807050705, + 0.1559232622385025, + 0.024166326969861984, + 0.06628894805908203, + 0.06877554953098297, + 0.12384890764951706, + 0.12682415544986725, + 0.19928614795207977, + 0.2025238573551178, + 0.30097496509552 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "replay", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 110.1352149358973, + "measured_steps": 7980, + "median_ms": 110.01293901063036, + "p95_ms": 110.78795030334732, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + }, + { + "bits_per_byte": 6.611855126224988, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.582988739013672, + "step": 30, + "unclipped_grad_norm": 3.075686454772949 + }, + { + "bits_per_byte": 6.3915955694037825, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43031644821167, + "step": 40, + "unclipped_grad_norm": 3.2318925857543945 + }, + { + "bits_per_byte": 6.093941755902502, + "learning_rate": 3.75e-05, + "loss_nats": 4.223998546600342, + "step": 50, + "unclipped_grad_norm": 2.8704335689544678 + }, + { + "bits_per_byte": 5.8858000966610895, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.079725742340088, + "step": 60, + "unclipped_grad_norm": 2.669306755065918 + }, + { + "bits_per_byte": 5.715495996470702, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9616799354553223, + "step": 70, + "unclipped_grad_norm": 3.1737496852874756 + }, + { + "bits_per_byte": 5.565897919513593, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8579864501953125, + "step": 80, + "unclipped_grad_norm": 2.5544967651367188 + }, + { + "bits_per_byte": 5.354669171689194, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.711573839187622, + "step": 90, + "unclipped_grad_norm": 2.425950050354004 + }, + { + "bits_per_byte": 5.180876231443403, + "learning_rate": 7.5e-05, + "loss_nats": 3.5911097526550293, + "step": 100, + "unclipped_grad_norm": 2.2783501148223877 + }, + { + "bits_per_byte": 4.992272489270609, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4603796005249023, + "step": 110, + "unclipped_grad_norm": 2.1105899810791016 + }, + { + "bits_per_byte": 4.775783480086919, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3103208541870117, + "step": 120, + "unclipped_grad_norm": 1.8969556093215942 + }, + { + "bits_per_byte": 4.582375918510225, + "learning_rate": 9.75e-05, + "loss_nats": 3.1762609481811523, + "step": 130, + "unclipped_grad_norm": 1.7015109062194824 + }, + { + "bits_per_byte": 4.427903165297061, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0691885948181152, + "step": 140, + "unclipped_grad_norm": 1.446854829788208 + }, + { + "bits_per_byte": 4.271120341117507, + "learning_rate": 0.0001125, + "loss_nats": 2.960515022277832, + "step": 150, + "unclipped_grad_norm": 1.2307006120681763 + }, + { + "bits_per_byte": 4.166711049639903, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.888144016265869, + "step": 160, + "unclipped_grad_norm": 1.0598198175430298 + }, + { + "bits_per_byte": 4.026318516052877, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7908313274383545, + "step": 170, + "unclipped_grad_norm": 0.9654788374900818 + }, + { + "bits_per_byte": 3.94755837315965, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.736238956451416, + "step": 180, + "unclipped_grad_norm": 0.8419675230979919 + }, + { + "bits_per_byte": 3.838975750676209, + "learning_rate": 0.0001425, + "loss_nats": 2.660975217819214, + "step": 190, + "unclipped_grad_norm": 0.817121684551239 + }, + { + "bits_per_byte": 3.774983758012572, + "learning_rate": 0.00015, + "loss_nats": 2.616619348526001, + "step": 200, + "unclipped_grad_norm": 0.955824613571167 + }, + { + "bits_per_byte": 3.7344016998190783, + "learning_rate": 0.0001575, + "loss_nats": 2.5884900093078613, + "step": 210, + "unclipped_grad_norm": 0.8557147979736328 + }, + { + "bits_per_byte": 3.63166201591605, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5172762870788574, + "step": 220, + "unclipped_grad_norm": 0.6891153454780579 + }, + { + "bits_per_byte": 3.6208855830129867, + "learning_rate": 0.0001725, + "loss_nats": 2.5098066329956055, + "step": 230, + "unclipped_grad_norm": 1.036860466003418 + }, + { + "bits_per_byte": 3.4987607026353063, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4251561164855957, + "step": 240, + "unclipped_grad_norm": 0.6406477689743042 + }, + { + "bits_per_byte": 3.5387748741156506, + "learning_rate": 0.0001875, + "loss_nats": 2.4528918266296387, + "step": 250, + "unclipped_grad_norm": 1.106186866760254 + }, + { + "bits_per_byte": 3.5260464381252943, + "learning_rate": 0.000195, + "loss_nats": 2.4440691471099854, + "step": 260, + "unclipped_grad_norm": 0.8556521534919739 + }, + { + "bits_per_byte": 3.5219934949750615, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4412598609924316, + "step": 270, + "unclipped_grad_norm": 0.9917709827423096 + }, + { + "bits_per_byte": 3.474740573719901, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4085066318511963, + "step": 280, + "unclipped_grad_norm": 1.2901222705841064 + }, + { + "bits_per_byte": 3.363479149466621, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.331386089324951, + "step": 290, + "unclipped_grad_norm": 0.7895627617835999 + }, + { + "bits_per_byte": 3.4037392561376993, + "learning_rate": 0.000225, + "loss_nats": 2.3592922687530518, + "step": 300, + "unclipped_grad_norm": 1.5281883478164673 + }, + { + "bits_per_byte": 3.3541442751430894, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.324915647506714, + "step": 310, + "unclipped_grad_norm": 0.8648361563682556 + }, + { + "bits_per_byte": 3.3434108379027414, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174757957458496, + "step": 320, + "unclipped_grad_norm": 1.4613573551177979 + }, + { + "bits_per_byte": 3.3325662998699364, + "learning_rate": 0.0002475, + "loss_nats": 2.3099589347839355, + "step": 330, + "unclipped_grad_norm": 1.1215400695800781 + }, + { + "bits_per_byte": 3.4180712583647006, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3692264556884766, + "step": 340, + "unclipped_grad_norm": 1.1166281700134277 + }, + { + "bits_per_byte": 3.3577222022116393, + "learning_rate": 0.0002625, + "loss_nats": 2.3273956775665283, + "step": 350, + "unclipped_grad_norm": 1.0810235738754272 + }, + { + "bits_per_byte": 3.2923828974611427, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2821059226989746, + "step": 360, + "unclipped_grad_norm": 1.128420114517212 + }, + { + "bits_per_byte": 3.291989401155969, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2818331718444824, + "step": 370, + "unclipped_grad_norm": 1.0846341848373413 + }, + { + "bits_per_byte": 3.3389695579268603, + "learning_rate": 0.000285, + "loss_nats": 2.3143973350524902, + "step": 380, + "unclipped_grad_norm": 1.0868512392044067 + }, + { + "bits_per_byte": 3.256548592327368, + "learning_rate": 0.0002925, + "loss_nats": 2.257267475128174, + "step": 390, + "unclipped_grad_norm": 1.1476690769195557 + }, + { + "bits_per_byte": 3.2058776878689605, + "learning_rate": 0.0003, + "loss_nats": 2.2221450805664062, + "step": 400, + "unclipped_grad_norm": 1.1263198852539062 + }, + { + "bits_per_byte": 3.195324144481415, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.214829921722412, + "step": 410, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 3.227808915506986, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.237346649169922, + "step": 420, + "unclipped_grad_norm": 1.2929860353469849 + }, + { + "bits_per_byte": 3.1876103786249166, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2094831466674805, + "step": 430, + "unclipped_grad_norm": 1.071694016456604 + }, + { + "bits_per_byte": 3.2374495749837515, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2440290451049805, + "step": 440, + "unclipped_grad_norm": 0.9089367985725403 + }, + { + "bits_per_byte": 3.155426921203657, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1871752738952637, + "step": 450, + "unclipped_grad_norm": 1.1129212379455566 + }, + { + "bits_per_byte": 3.188086770567806, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.209813356399536, + "step": 460, + "unclipped_grad_norm": 1.3905333280563354 + }, + { + "bits_per_byte": 3.128408790718412, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.168447732925415, + "step": 470, + "unclipped_grad_norm": 1.059533953666687 + }, + { + "bits_per_byte": 3.136691131218663, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1741886138916016, + "step": 480, + "unclipped_grad_norm": 1.0161854028701782 + }, + { + "bits_per_byte": 3.0887341129908705, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1409473419189453, + "step": 490, + "unclipped_grad_norm": 1.385905385017395 + }, + { + "bits_per_byte": 3.033835874974255, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1028947830200195, + "step": 500, + "unclipped_grad_norm": 1.1716443300247192 + }, + { + "bits_per_byte": 3.0607071322756636, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.121520519256592, + "step": 510, + "unclipped_grad_norm": 0.947066605091095 + }, + { + "bits_per_byte": 2.997100036784657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0774314403533936, + "step": 520, + "unclipped_grad_norm": 1.352820634841919 + }, + { + "bits_per_byte": 3.0643050093317137, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.124014377593994, + "step": 530, + "unclipped_grad_norm": 1.2646857500076294 + }, + { + "bits_per_byte": 3.008989197438803, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.085672378540039, + "step": 540, + "unclipped_grad_norm": 1.0451104640960693 + }, + { + "bits_per_byte": 2.978105584892765, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.064265489578247, + "step": 550, + "unclipped_grad_norm": 1.2619587182998657 + }, + { + "bits_per_byte": 2.968300166036473, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.057468891143799, + "step": 560, + "unclipped_grad_norm": 0.9861598610877991 + }, + { + "bits_per_byte": 2.9221661639078262, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.025491237640381, + "step": 570, + "unclipped_grad_norm": 1.1261875629425049 + }, + { + "bits_per_byte": 2.96999178739036, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0586414337158203, + "step": 580, + "unclipped_grad_norm": 0.8640331625938416 + }, + { + "bits_per_byte": 2.883698804389342, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9988276958465576, + "step": 590, + "unclipped_grad_norm": 1.0034456253051758 + }, + { + "bits_per_byte": 2.9404706214044567, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0381789207458496, + "step": 600, + "unclipped_grad_norm": 0.8714175224304199 + }, + { + "bits_per_byte": 2.8380133330141457, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.967160940170288, + "step": 610, + "unclipped_grad_norm": 0.9427422881126404 + }, + { + "bits_per_byte": 2.894644124255553, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0064144134521484, + "step": 620, + "unclipped_grad_norm": 1.0238139629364014 + }, + { + "bits_per_byte": 2.81883675149499, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9538687467575073, + "step": 630, + "unclipped_grad_norm": 0.9875481128692627 + }, + { + "bits_per_byte": 2.7787291934352267, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9260683059692383, + "step": 640, + "unclipped_grad_norm": 1.0125378370285034 + }, + { + "bits_per_byte": 2.728423580127252, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8911991119384766, + "step": 650, + "unclipped_grad_norm": 1.5507988929748535 + }, + { + "bits_per_byte": 2.649404087396524, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8364269733428955, + "step": 660, + "unclipped_grad_norm": 0.9040125012397766 + }, + { + "bits_per_byte": 2.7024973674923345, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8732284307479858, + "step": 670, + "unclipped_grad_norm": 1.205861210823059 + }, + { + "bits_per_byte": 2.641403626461037, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8308814764022827, + "step": 680, + "unclipped_grad_norm": 1.3370649814605713 + }, + { + "bits_per_byte": 2.7325929555321187, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8940891027450562, + "step": 690, + "unclipped_grad_norm": 1.6124995946884155 + }, + { + "bits_per_byte": 2.6130706886099455, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8112425804138184, + "step": 700, + "unclipped_grad_norm": 1.08383309841156 + }, + { + "bits_per_byte": 2.628106443844277, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.821664571762085, + "step": 710, + "unclipped_grad_norm": 1.0767779350280762 + }, + { + "bits_per_byte": 2.6725746220166666, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.852487564086914, + "step": 720, + "unclipped_grad_norm": 1.296352505683899 + }, + { + "bits_per_byte": 2.608218370098502, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8078792095184326, + "step": 730, + "unclipped_grad_norm": 1.0864766836166382 + }, + { + "bits_per_byte": 2.555699340073923, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7714757919311523, + "step": 740, + "unclipped_grad_norm": 1.0339913368225098 + }, + { + "bits_per_byte": 2.515936435246489, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743914246559143, + "step": 750, + "unclipped_grad_norm": 1.1017143726348877 + }, + { + "bits_per_byte": 2.4657125136976736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.709101676940918, + "step": 760, + "unclipped_grad_norm": 1.7173939943313599 + }, + { + "bits_per_byte": 2.601046349592221, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.802907943725586, + "step": 770, + "unclipped_grad_norm": 0.9487742781639099 + }, + { + "bits_per_byte": 2.4794518637098015, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7186250686645508, + "step": 780, + "unclipped_grad_norm": 0.9247332811355591 + }, + { + "bits_per_byte": 2.466098098800908, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7093689441680908, + "step": 790, + "unclipped_grad_norm": 1.9121034145355225 + }, + { + "bits_per_byte": 2.4025570446478333, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.66532564163208, + "step": 800, + "unclipped_grad_norm": 1.1767630577087402 + }, + { + "bits_per_byte": 2.455517898102479, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7020353078842163, + "step": 810, + "unclipped_grad_norm": 1.1299240589141846 + }, + { + "bits_per_byte": 2.400107839716896, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6636279821395874, + "step": 820, + "unclipped_grad_norm": 1.0006046295166016 + }, + { + "bits_per_byte": 2.4500607166079513, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6982526779174805, + "step": 830, + "unclipped_grad_norm": 1.0554603338241577 + }, + { + "bits_per_byte": 2.5153210813217015, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7434877157211304, + "step": 840, + "unclipped_grad_norm": 1.0736713409423828 + }, + { + "bits_per_byte": 2.399556119372928, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6632455587387085, + "step": 850, + "unclipped_grad_norm": 0.8279580473899841 + }, + { + "bits_per_byte": 2.414819407654351, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6738252639770508, + "step": 860, + "unclipped_grad_norm": 1.2342168092727661 + }, + { + "bits_per_byte": 2.3222158652452247, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6096373796463013, + "step": 870, + "unclipped_grad_norm": 1.0648512840270996 + }, + { + "bits_per_byte": 2.3629336177502487, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6378607749938965, + "step": 880, + "unclipped_grad_norm": 0.9258757829666138 + }, + { + "bits_per_byte": 2.2961121665146167, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5915436744689941, + "step": 890, + "unclipped_grad_norm": 1.0717874765396118 + }, + { + "bits_per_byte": 2.3158979105831285, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6052581071853638, + "step": 900, + "unclipped_grad_norm": 0.9598665833473206 + }, + { + "bits_per_byte": 2.3301258633063577, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6151201725006104, + "step": 910, + "unclipped_grad_norm": 1.0995051860809326 + }, + { + "bits_per_byte": 2.3123376977276173, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.602790355682373, + "step": 920, + "unclipped_grad_norm": 1.1837356090545654 + }, + { + "bits_per_byte": 2.3340250539667196, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6178228855133057, + "step": 930, + "unclipped_grad_norm": 1.1502224206924438 + }, + { + "bits_per_byte": 2.330418921743428, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6153233051300049, + "step": 940, + "unclipped_grad_norm": 1.0088714361190796 + }, + { + "bits_per_byte": 2.2995397807463123, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5939195156097412, + "step": 950, + "unclipped_grad_norm": 0.9483612179756165 + }, + { + "bits_per_byte": 2.3536873144919115, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6314517259597778, + "step": 960, + "unclipped_grad_norm": 0.9582105278968811 + }, + { + "bits_per_byte": 2.2526236015215417, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5613996982574463, + "step": 970, + "unclipped_grad_norm": 1.2002545595169067 + }, + { + "bits_per_byte": 2.3230659754884395, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6102266311645508, + "step": 980, + "unclipped_grad_norm": 0.9191399812698364 + }, + { + "bits_per_byte": 2.285470911975131, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.584167718887329, + "step": 990, + "unclipped_grad_norm": 1.053741216659546 + }, + { + "bits_per_byte": 2.248061417742104, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5582374334335327, + "step": 1000, + "unclipped_grad_norm": 0.9024814963340759 + }, + { + "bits_per_byte": 2.2402749032242903, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.552840232849121, + "step": 1010, + "unclipped_grad_norm": 1.048907995223999 + }, + { + "bits_per_byte": 2.2041726491204714, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278160572052002, + "step": 1020, + "unclipped_grad_norm": 1.0486118793487549 + }, + { + "bits_per_byte": 2.216881995036582, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5366255044937134, + "step": 1030, + "unclipped_grad_norm": 0.9243784546852112 + }, + { + "bits_per_byte": 2.2610579746851553, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5672459602355957, + "step": 1040, + "unclipped_grad_norm": 1.102870225906372 + }, + { + "bits_per_byte": 2.2528270569975124, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5615407228469849, + "step": 1050, + "unclipped_grad_norm": 1.0764544010162354 + }, + { + "bits_per_byte": 2.14889054582713, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.489497423171997, + "step": 1060, + "unclipped_grad_norm": 0.971049427986145 + }, + { + "bits_per_byte": 2.2350196293618754, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5491975545883179, + "step": 1070, + "unclipped_grad_norm": 1.0203380584716797 + }, + { + "bits_per_byte": 2.217042282867186, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5367366075515747, + "step": 1080, + "unclipped_grad_norm": 1.023221492767334 + }, + { + "bits_per_byte": 2.186512266651322, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5155748128890991, + "step": 1090, + "unclipped_grad_norm": 1.111006498336792 + }, + { + "bits_per_byte": 2.2323391077655277, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5473395586013794, + "step": 1100, + "unclipped_grad_norm": 0.9547863602638245 + }, + { + "bits_per_byte": 2.116467344590568, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4670233726501465, + "step": 1110, + "unclipped_grad_norm": 1.1085364818572998 + }, + { + "bits_per_byte": 2.143238164023873, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.485579490661621, + "step": 1120, + "unclipped_grad_norm": 0.901618242263794 + }, + { + "bits_per_byte": 2.077939275196329, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4403177499771118, + "step": 1130, + "unclipped_grad_norm": 0.8683149814605713 + }, + { + "bits_per_byte": 2.224320932617002, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5417817831039429, + "step": 1140, + "unclipped_grad_norm": 1.030465006828308 + }, + { + "bits_per_byte": 2.0642131678483175, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4308035373687744, + "step": 1150, + "unclipped_grad_norm": 0.827274739742279 + }, + { + "bits_per_byte": 2.103388407957753, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4579577445983887, + "step": 1160, + "unclipped_grad_norm": 0.9158841967582703 + }, + { + "bits_per_byte": 2.1566975262803965, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4949088096618652, + "step": 1170, + "unclipped_grad_norm": 0.9397322535514832 + }, + { + "bits_per_byte": 2.1322636071070153, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4779725074768066, + "step": 1180, + "unclipped_grad_norm": 0.8510220050811768 + }, + { + "bits_per_byte": 2.068760561119781, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4339555501937866, + "step": 1190, + "unclipped_grad_norm": 0.9270830154418945 + }, + { + "bits_per_byte": 2.0310528489701576, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4078185558319092, + "step": 1200, + "unclipped_grad_norm": 0.8815417885780334 + }, + { + "bits_per_byte": 2.1008361854189492, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.456188678741455, + "step": 1210, + "unclipped_grad_norm": 1.2037686109542847 + }, + { + "bits_per_byte": 2.1123913557651197, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.464198112487793, + "step": 1220, + "unclipped_grad_norm": 0.8964646458625793 + }, + { + "bits_per_byte": 2.0711254945517954, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4355947971343994, + "step": 1230, + "unclipped_grad_norm": 0.8661414980888367 + }, + { + "bits_per_byte": 2.0159510523978943, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.397350788116455, + "step": 1240, + "unclipped_grad_norm": 0.9392534494400024 + }, + { + "bits_per_byte": 2.104896867788471, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4590033292770386, + "step": 1250, + "unclipped_grad_norm": 0.8430818915367126 + }, + { + "bits_per_byte": 2.08180441329182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.442996859550476, + "step": 1260, + "unclipped_grad_norm": 0.8917548656463623 + }, + { + "bits_per_byte": 2.087352401626003, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4468424320220947, + "step": 1270, + "unclipped_grad_norm": 0.9866762161254883 + }, + { + "bits_per_byte": 1.9607164163161914, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.359065055847168, + "step": 1280, + "unclipped_grad_norm": 0.8596939444541931 + }, + { + "bits_per_byte": 2.088993460080536, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4479799270629883, + "step": 1290, + "unclipped_grad_norm": 1.007126808166504 + }, + { + "bits_per_byte": 2.034349412421896, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4101035594940186, + "step": 1300, + "unclipped_grad_norm": 0.8474438786506653 + }, + { + "bits_per_byte": 2.0247273270714237, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4034340381622314, + "step": 1310, + "unclipped_grad_norm": 0.8226730823516846 + }, + { + "bits_per_byte": 2.020511000402871, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4005115032196045, + "step": 1320, + "unclipped_grad_norm": 0.9414357542991638 + }, + { + "bits_per_byte": 2.085571349293668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4456079006195068, + "step": 1330, + "unclipped_grad_norm": 0.9917200207710266 + }, + { + "bits_per_byte": 2.0422444479924042, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4155759811401367, + "step": 1340, + "unclipped_grad_norm": 0.8823419213294983 + }, + { + "bits_per_byte": 2.0447034559344406, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4172804355621338, + "step": 1350, + "unclipped_grad_norm": 0.8861486315727234 + }, + { + "bits_per_byte": 2.0988803987133373, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548330307006836, + "step": 1360, + "unclipped_grad_norm": 0.9090827107429504 + }, + { + "bits_per_byte": 2.082760980795919, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4436599016189575, + "step": 1370, + "unclipped_grad_norm": 0.8366561532020569 + }, + { + "bits_per_byte": 2.0317923743688677, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4083311557769775, + "step": 1380, + "unclipped_grad_norm": 0.8379060626029968 + }, + { + "bits_per_byte": 2.0432778917414387, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4162923097610474, + "step": 1390, + "unclipped_grad_norm": 0.8665340542793274 + }, + { + "bits_per_byte": 1.9746810636009497, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3687446117401123, + "step": 1400, + "unclipped_grad_norm": 0.8112426996231079 + }, + { + "bits_per_byte": 2.0034977886489185, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.388718843460083, + "step": 1410, + "unclipped_grad_norm": 0.8745980262756348 + }, + { + "bits_per_byte": 2.019574210903621, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.3998621702194214, + "step": 1420, + "unclipped_grad_norm": 0.8596643209457397 + }, + { + "bits_per_byte": 2.0775151659793014, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4400237798690796, + "step": 1430, + "unclipped_grad_norm": 0.913244903087616 + }, + { + "bits_per_byte": 2.027370700415185, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.405266284942627, + "step": 1440, + "unclipped_grad_norm": 1.0067864656448364 + }, + { + "bits_per_byte": 1.9631205617926026, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3607314825057983, + "step": 1450, + "unclipped_grad_norm": 0.8125852942466736 + }, + { + "bits_per_byte": 1.9959419028659118, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.383481502532959, + "step": 1460, + "unclipped_grad_norm": 0.9857028126716614 + }, + { + "bits_per_byte": 1.9364156117145783, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3422210216522217, + "step": 1470, + "unclipped_grad_norm": 0.8698804974555969 + }, + { + "bits_per_byte": 1.996789089404062, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3840687274932861, + "step": 1480, + "unclipped_grad_norm": 0.8397032022476196 + }, + { + "bits_per_byte": 2.017818268038312, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.398645043373108, + "step": 1490, + "unclipped_grad_norm": 0.8181726932525635 + }, + { + "bits_per_byte": 1.996446843928845, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.38383150100708, + "step": 1500, + "unclipped_grad_norm": 0.8242432475090027 + }, + { + "bits_per_byte": 2.0162594172908914, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3975645303726196, + "step": 1510, + "unclipped_grad_norm": 0.9299814701080322 + }, + { + "bits_per_byte": 1.980743108078562, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3729465007781982, + "step": 1520, + "unclipped_grad_norm": 0.8446369171142578 + }, + { + "bits_per_byte": 2.047233664693934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4190342426300049, + "step": 1530, + "unclipped_grad_norm": 0.9717140197753906 + }, + { + "bits_per_byte": 1.993557363411699, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3818286657333374, + "step": 1540, + "unclipped_grad_norm": 0.8417568802833557 + }, + { + "bits_per_byte": 2.049996737962696, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209494590759277, + "step": 1550, + "unclipped_grad_norm": 0.8546327352523804 + }, + { + "bits_per_byte": 1.966114779744124, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3628069162368774, + "step": 1560, + "unclipped_grad_norm": 0.9258830547332764 + }, + { + "bits_per_byte": 2.0427641795633114, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4159362316131592, + "step": 1570, + "unclipped_grad_norm": 0.8567822575569153 + }, + { + "bits_per_byte": 1.9830621221427962, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.374553918838501, + "step": 1580, + "unclipped_grad_norm": 0.8522606492042542 + }, + { + "bits_per_byte": 1.9876789964052082, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3777540922164917, + "step": 1590, + "unclipped_grad_norm": 0.8731296062469482 + }, + { + "bits_per_byte": 2.0122188568915207, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3947638273239136, + "step": 1600, + "unclipped_grad_norm": 0.8627029657363892 + }, + { + "bits_per_byte": 1.9498011934138815, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3514991998672485, + "step": 1610, + "unclipped_grad_norm": 0.7740172147750854 + }, + { + "bits_per_byte": 1.9095543294069197, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3236021995544434, + "step": 1620, + "unclipped_grad_norm": 0.7910023331642151 + }, + { + "bits_per_byte": 1.9407452749500496, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3452221155166626, + "step": 1630, + "unclipped_grad_norm": 0.8484107851982117 + }, + { + "bits_per_byte": 1.9911346438089947, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3801493644714355, + "step": 1640, + "unclipped_grad_norm": 0.8722788691520691 + }, + { + "bits_per_byte": 2.0368601871418424, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4118438959121704, + "step": 1650, + "unclipped_grad_norm": 0.8482462763786316 + }, + { + "bits_per_byte": 1.9665676100638458, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3631207942962646, + "step": 1660, + "unclipped_grad_norm": 0.8444085121154785 + }, + { + "bits_per_byte": 1.9945896032821775, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3825441598892212, + "step": 1670, + "unclipped_grad_norm": 0.8560684323310852 + }, + { + "bits_per_byte": 2.0061820938635853, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3905794620513916, + "step": 1680, + "unclipped_grad_norm": 0.8526912927627563 + }, + { + "bits_per_byte": 2.0514603103215387, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4219639301300049, + "step": 1690, + "unclipped_grad_norm": 0.8917502760887146 + }, + { + "bits_per_byte": 1.9422809080396035, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3462865352630615, + "step": 1700, + "unclipped_grad_norm": 0.7474871873855591 + }, + { + "bits_per_byte": 1.878015978909047, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3017414808273315, + "step": 1710, + "unclipped_grad_norm": 0.8072315454483032 + }, + { + "bits_per_byte": 1.8905051870320533, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3103983402252197, + "step": 1720, + "unclipped_grad_norm": 0.845556914806366 + }, + { + "bits_per_byte": 1.8762399140735866, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3005104064941406, + "step": 1730, + "unclipped_grad_norm": 0.9344940185546875 + }, + { + "bits_per_byte": 1.9632232354351677, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3608026504516602, + "step": 1740, + "unclipped_grad_norm": 0.7860697507858276 + }, + { + "bits_per_byte": 1.989613629244764, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3790950775146484, + "step": 1750, + "unclipped_grad_norm": 0.8348115086555481 + }, + { + "bits_per_byte": 1.9869124697303127, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3772227764129639, + "step": 1760, + "unclipped_grad_norm": 0.8862095475196838 + }, + { + "bits_per_byte": 1.9928130224987648, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3813127279281616, + "step": 1770, + "unclipped_grad_norm": 0.816246509552002 + }, + { + "bits_per_byte": 1.8514405317671223, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.2833207845687866, + "step": 1780, + "unclipped_grad_norm": 0.8118871450424194 + }, + { + "bits_per_byte": 1.9588221993995885, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3577520847320557, + "step": 1790, + "unclipped_grad_norm": 0.839783251285553 + }, + { + "bits_per_byte": 1.8827697513615458, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3050365447998047, + "step": 1800, + "unclipped_grad_norm": 0.8949951529502869 + }, + { + "bits_per_byte": 1.92937567388416, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.33734130859375, + "step": 1810, + "unclipped_grad_norm": 0.7295374274253845 + }, + { + "bits_per_byte": 1.8974312723476003, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3151991367340088, + "step": 1820, + "unclipped_grad_norm": 0.7755441069602966 + }, + { + "bits_per_byte": 1.8886371114783813, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3091034889221191, + "step": 1830, + "unclipped_grad_norm": 0.7666956186294556 + }, + { + "bits_per_byte": 1.8876932706904463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3084492683410645, + "step": 1840, + "unclipped_grad_norm": 0.8784642219543457 + }, + { + "bits_per_byte": 1.8964055678178544, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.314488172531128, + "step": 1850, + "unclipped_grad_norm": 0.8603965044021606 + }, + { + "bits_per_byte": 1.8927124123732264, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3119282722473145, + "step": 1860, + "unclipped_grad_norm": 0.8056252002716064 + }, + { + "bits_per_byte": 1.8818489562488263, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3043982982635498, + "step": 1870, + "unclipped_grad_norm": 0.8151036500930786 + }, + { + "bits_per_byte": 1.8811881989042114, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3039402961730957, + "step": 1880, + "unclipped_grad_norm": 0.79389888048172 + }, + { + "bits_per_byte": 1.8949089747900463, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.313450813293457, + "step": 1890, + "unclipped_grad_norm": 0.8192400932312012 + }, + { + "bits_per_byte": 1.9233574849825177, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3331698179244995, + "step": 1900, + "unclipped_grad_norm": 0.7950363159179688 + }, + { + "bits_per_byte": 1.8611783614416253, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2900705337524414, + "step": 1910, + "unclipped_grad_norm": 0.8037525415420532 + }, + { + "bits_per_byte": 1.855479888287937, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2861206531524658, + "step": 1920, + "unclipped_grad_norm": 0.7507341504096985 + }, + { + "bits_per_byte": 1.9602173226633874, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3587191104888916, + "step": 1930, + "unclipped_grad_norm": 0.8329893350601196 + }, + { + "bits_per_byte": 1.8066889261861108, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2523013353347778, + "step": 1940, + "unclipped_grad_norm": 0.7595474123954773 + }, + { + "bits_per_byte": 1.9033319970987033, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.319289207458496, + "step": 1950, + "unclipped_grad_norm": 0.7690061330795288 + }, + { + "bits_per_byte": 1.816397862775269, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259031057357788, + "step": 1960, + "unclipped_grad_norm": 0.7758741974830627 + }, + { + "bits_per_byte": 1.909067618504978, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3232648372650146, + "step": 1970, + "unclipped_grad_norm": 0.7503845691680908 + }, + { + "bits_per_byte": 1.9405849871194454, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3451110124588013, + "step": 1980, + "unclipped_grad_norm": 0.8655099868774414 + }, + { + "bits_per_byte": 1.9436471382180576, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.347233533859253, + "step": 1990, + "unclipped_grad_norm": 0.8436980247497559 + }, + { + "bits_per_byte": 1.9015801818170146, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3180749416351318, + "step": 2000, + "unclipped_grad_norm": 0.8021923303604126 + }, + { + "bits_per_byte": 1.8746163978494417, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.2993850708007812, + "step": 2010, + "unclipped_grad_norm": 0.7083144187927246 + }, + { + "bits_per_byte": 1.9072580170525995, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3220105171203613, + "step": 2020, + "unclipped_grad_norm": 0.7678324580192566 + }, + { + "bits_per_byte": 1.983610230851096, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3749338388442993, + "step": 2030, + "unclipped_grad_norm": 0.8317387104034424 + }, + { + "bits_per_byte": 1.909348466173837, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.323459506034851, + "step": 2040, + "unclipped_grad_norm": 0.8302199840545654 + }, + { + "bits_per_byte": 1.961244747019642, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.359431266784668, + "step": 2050, + "unclipped_grad_norm": 0.7990598082542419 + }, + { + "bits_per_byte": 1.9220292629699043, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3322491645812988, + "step": 2060, + "unclipped_grad_norm": 0.8404503464698792 + }, + { + "bits_per_byte": 1.8663830724046866, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2936781644821167, + "step": 2070, + "unclipped_grad_norm": 0.7720745801925659 + }, + { + "bits_per_byte": 1.8672120287818452, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2942527532577515, + "step": 2080, + "unclipped_grad_norm": 0.8028935790061951 + }, + { + "bits_per_byte": 1.857879218250124, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2877837419509888, + "step": 2090, + "unclipped_grad_norm": 0.7713416814804077 + }, + { + "bits_per_byte": 1.8603779541845098, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.289515733718872, + "step": 2100, + "unclipped_grad_norm": 0.7443708181381226 + }, + { + "bits_per_byte": 1.897931741861611, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3155460357666016, + "step": 2110, + "unclipped_grad_norm": 0.7534405589103699 + }, + { + "bits_per_byte": 1.8710701155886507, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969269752502441, + "step": 2120, + "unclipped_grad_norm": 0.7505779266357422 + }, + { + "bits_per_byte": 1.8349618420920515, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.271898627281189, + "step": 2130, + "unclipped_grad_norm": 0.7835680246353149 + }, + { + "bits_per_byte": 1.9616896661374241, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3597396612167358, + "step": 2140, + "unclipped_grad_norm": 0.7786996960639954 + }, + { + "bits_per_byte": 1.8168395142226847, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2593371868133545, + "step": 2150, + "unclipped_grad_norm": 0.7194197773933411 + }, + { + "bits_per_byte": 1.8739360344826286, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.2989134788513184, + "step": 2160, + "unclipped_grad_norm": 0.8012002110481262 + }, + { + "bits_per_byte": 1.9634492206384013, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3609592914581299, + "step": 2170, + "unclipped_grad_norm": 0.8270084261894226 + }, + { + "bits_per_byte": 1.8175344961148212, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2598189115524292, + "step": 2180, + "unclipped_grad_norm": 0.7824540734291077 + }, + { + "bits_per_byte": 1.8458584908680682, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.279451608657837, + "step": 2190, + "unclipped_grad_norm": 0.7973959445953369 + }, + { + "bits_per_byte": 1.8557901449900935, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2863357067108154, + "step": 2200, + "unclipped_grad_norm": 0.7451788783073425 + }, + { + "bits_per_byte": 1.8380872828061812, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2740650177001953, + "step": 2210, + "unclipped_grad_norm": 0.7804975509643555 + }, + { + "bits_per_byte": 1.8298283319464475, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2683403491973877, + "step": 2220, + "unclipped_grad_norm": 0.7694668173789978 + }, + { + "bits_per_byte": 1.9031531351418058, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3191652297973633, + "step": 2230, + "unclipped_grad_norm": 0.8418469429016113 + }, + { + "bits_per_byte": 1.8400618156207371, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2754336595535278, + "step": 2240, + "unclipped_grad_norm": 0.7987614870071411 + }, + { + "bits_per_byte": 1.852736936989326, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2842193841934204, + "step": 2250, + "unclipped_grad_norm": 0.7795421481132507 + }, + { + "bits_per_byte": 1.8339105121473271, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.271169900894165, + "step": 2260, + "unclipped_grad_norm": 0.7681040167808533 + }, + { + "bits_per_byte": 1.825916414569922, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2656288146972656, + "step": 2270, + "unclipped_grad_norm": 0.7933830618858337 + }, + { + "bits_per_byte": 1.8451832869807807, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2789835929870605, + "step": 2280, + "unclipped_grad_norm": 0.7935850024223328 + }, + { + "bits_per_byte": 1.8711039961708706, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2969504594802856, + "step": 2290, + "unclipped_grad_norm": 0.7818295955657959 + }, + { + "bits_per_byte": 1.7944778140095496, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2438372373580933, + "step": 2300, + "unclipped_grad_norm": 0.7245032787322998 + }, + { + "bits_per_byte": 1.8221434591752936, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2630136013031006, + "step": 2310, + "unclipped_grad_norm": 0.7396021485328674 + }, + { + "bits_per_byte": 1.7928696042413317, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.242722511291504, + "step": 2320, + "unclipped_grad_norm": 0.7618010640144348 + }, + { + "bits_per_byte": 1.8702191454321815, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2963371276855469, + "step": 2330, + "unclipped_grad_norm": 0.7263991236686707 + }, + { + "bits_per_byte": 1.7994940319872652, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.247314214706421, + "step": 2340, + "unclipped_grad_norm": 0.7788637280464172 + }, + { + "bits_per_byte": 1.849551646312696, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2820115089416504, + "step": 2350, + "unclipped_grad_norm": 0.7732409238815308 + }, + { + "bits_per_byte": 1.8613248906601605, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2901721000671387, + "step": 2360, + "unclipped_grad_norm": 0.8177725672721863 + }, + { + "bits_per_byte": 1.8822589628884834, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3046824932098389, + "step": 2370, + "unclipped_grad_norm": 0.7931574583053589 + }, + { + "bits_per_byte": 1.8382422391746087, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2741724252700806, + "step": 2380, + "unclipped_grad_norm": 0.8604201674461365 + }, + { + "bits_per_byte": 1.8797817247854554, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3029654026031494, + "step": 2390, + "unclipped_grad_norm": 0.7636454105377197 + }, + { + "bits_per_byte": 1.817860919186159, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2600451707839966, + "step": 2400, + "unclipped_grad_norm": 0.779283881187439 + }, + { + "bits_per_byte": 1.8354538844561699, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2722396850585938, + "step": 2410, + "unclipped_grad_norm": 0.7781111001968384 + }, + { + "bits_per_byte": 1.8069649583407457, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2524926662445068, + "step": 2420, + "unclipped_grad_norm": 0.7548633813858032 + }, + { + "bits_per_byte": 1.8169932667125561, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.259443759918213, + "step": 2430, + "unclipped_grad_norm": 0.7562755942344666 + }, + { + "bits_per_byte": 1.8760644917697067, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3003888130187988, + "step": 2440, + "unclipped_grad_norm": 0.7478408813476562 + }, + { + "bits_per_byte": 1.838183937055966, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2741320133209229, + "step": 2450, + "unclipped_grad_norm": 0.760445237159729 + }, + { + "bits_per_byte": 1.8195946762895068, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.261246919631958, + "step": 2460, + "unclipped_grad_norm": 0.7331485748291016 + }, + { + "bits_per_byte": 1.8163858239897086, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2590227127075195, + "step": 2470, + "unclipped_grad_norm": 0.7682369351387024 + }, + { + "bits_per_byte": 1.8266683227194942, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2661499977111816, + "step": 2480, + "unclipped_grad_norm": 0.7132375240325928 + }, + { + "bits_per_byte": 1.8042156436840528, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.250586986541748, + "step": 2490, + "unclipped_grad_norm": 0.7770085334777832 + }, + { + "bits_per_byte": 1.8313916542427904, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2694239616394043, + "step": 2500, + "unclipped_grad_norm": 0.7582318782806396 + }, + { + "bits_per_byte": 1.7788244690759694, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2329871654510498, + "step": 2510, + "unclipped_grad_norm": 0.8006963133811951 + }, + { + "bits_per_byte": 1.8448090527325032, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.278724193572998, + "step": 2520, + "unclipped_grad_norm": 0.7802237272262573 + }, + { + "bits_per_byte": 1.855150369528884, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2858922481536865, + "step": 2530, + "unclipped_grad_norm": 0.7603861689567566 + }, + { + "bits_per_byte": 1.7974587892969547, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.245903491973877, + "step": 2540, + "unclipped_grad_norm": 0.7829954624176025 + }, + { + "bits_per_byte": 1.8048358131230642, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2510168552398682, + "step": 2550, + "unclipped_grad_norm": 0.8207260966300964 + }, + { + "bits_per_byte": 1.8410618947355044, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2761268615722656, + "step": 2560, + "unclipped_grad_norm": 0.8064310550689697 + }, + { + "bits_per_byte": 1.8072459779922554, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2526874542236328, + "step": 2570, + "unclipped_grad_norm": 0.7300699353218079 + }, + { + "bits_per_byte": 1.8352736466380657, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2721147537231445, + "step": 2580, + "unclipped_grad_norm": 0.7476873993873596 + }, + { + "bits_per_byte": 1.793617556789934, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2432409524917603, + "step": 2590, + "unclipped_grad_norm": 0.7210070490837097 + }, + { + "bits_per_byte": 1.797604974550188, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2460048198699951, + "step": 2600, + "unclipped_grad_norm": 0.7853913903236389 + }, + { + "bits_per_byte": 1.7385629865436842, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2050800323486328, + "step": 2610, + "unclipped_grad_norm": 0.7092626094818115 + }, + { + "bits_per_byte": 1.8079149905041119, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2531511783599854, + "step": 2620, + "unclipped_grad_norm": 0.7522015571594238 + }, + { + "bits_per_byte": 1.8198622812942493, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.261432409286499, + "step": 2630, + "unclipped_grad_norm": 0.7675716876983643 + }, + { + "bits_per_byte": 1.680096624469618, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.164554238319397, + "step": 2640, + "unclipped_grad_norm": 0.6825466156005859 + }, + { + "bits_per_byte": 1.7638260340768748, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2225910425186157, + "step": 2650, + "unclipped_grad_norm": 0.7630164623260498 + }, + { + "bits_per_byte": 1.8201589513669878, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2616380453109741, + "step": 2660, + "unclipped_grad_norm": 0.7772946953773499 + }, + { + "bits_per_byte": 1.811244230659514, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2554588317871094, + "step": 2670, + "unclipped_grad_norm": 0.7582975625991821 + }, + { + "bits_per_byte": 1.74257860945868, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.207863450050354, + "step": 2680, + "unclipped_grad_norm": 0.7818799614906311 + }, + { + "bits_per_byte": 1.9030688636428832, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3191068172454834, + "step": 2690, + "unclipped_grad_norm": 0.7075229287147522 + }, + { + "bits_per_byte": 1.8200497423836899, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2615623474121094, + "step": 2700, + "unclipped_grad_norm": 0.8272333741188049 + }, + { + "bits_per_byte": 1.7354167359288, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2028992176055908, + "step": 2710, + "unclipped_grad_norm": 0.7310962677001953 + }, + { + "bits_per_byte": 1.728331394678554, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1979880332946777, + "step": 2720, + "unclipped_grad_norm": 0.7193791270256042 + }, + { + "bits_per_byte": 1.76859700479446, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.225898027420044, + "step": 2730, + "unclipped_grad_norm": 0.773657500743866 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2331104278564453, + "step": 2740, + "unclipped_grad_norm": 0.7759174704551697 + }, + { + "bits_per_byte": 1.7909771071512373, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.241410732269287, + "step": 2750, + "unclipped_grad_norm": 0.770866870880127 + }, + { + "bits_per_byte": 1.81886375002334, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2607402801513672, + "step": 2760, + "unclipped_grad_norm": 0.7793042659759521 + }, + { + "bits_per_byte": 1.704546881994831, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.181501865386963, + "step": 2770, + "unclipped_grad_norm": 0.7179723978042603 + }, + { + "bits_per_byte": 1.7222445846992174, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.1937689781188965, + "step": 2780, + "unclipped_grad_norm": 0.711125910282135 + }, + { + "bits_per_byte": 1.8073385046584196, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2527515888214111, + "step": 2790, + "unclipped_grad_norm": 0.7564381957054138 + }, + { + "bits_per_byte": 1.8564130661515186, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2867674827575684, + "step": 2800, + "unclipped_grad_norm": 0.7982874512672424 + }, + { + "bits_per_byte": 1.7924368958917607, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2424225807189941, + "step": 2810, + "unclipped_grad_norm": 0.827074408531189 + }, + { + "bits_per_byte": 1.781587886310033, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2349026203155518, + "step": 2820, + "unclipped_grad_norm": 0.6945767998695374 + }, + { + "bits_per_byte": 1.862601517877515, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2910569906234741, + "step": 2830, + "unclipped_grad_norm": 0.7473636269569397 + }, + { + "bits_per_byte": 1.7520161574247766, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2144050598144531, + "step": 2840, + "unclipped_grad_norm": 0.7269325852394104 + }, + { + "bits_per_byte": 1.7726036846116109, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2286752462387085, + "step": 2850, + "unclipped_grad_norm": 0.7537488341331482 + }, + { + "bits_per_byte": 1.7375121725469125, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2043516635894775, + "step": 2860, + "unclipped_grad_norm": 0.7839256525039673 + }, + { + "bits_per_byte": 1.836773851301542, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.273154616355896, + "step": 2870, + "unclipped_grad_norm": 0.7615643739700317 + }, + { + "bits_per_byte": 1.8235200083127996, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.263967752456665, + "step": 2880, + "unclipped_grad_norm": 0.7500733137130737 + }, + { + "bits_per_byte": 1.7346435019305209, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2023632526397705, + "step": 2890, + "unclipped_grad_norm": 0.7335168123245239 + }, + { + "bits_per_byte": 1.710140445731493, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1853790283203125, + "step": 2900, + "unclipped_grad_norm": 0.7237004041671753 + }, + { + "bits_per_byte": 1.7640040361205178, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2227144241333008, + "step": 2910, + "unclipped_grad_norm": 0.7653093338012695 + }, + { + "bits_per_byte": 1.7372325287566095, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.204157829284668, + "step": 2920, + "unclipped_grad_norm": 0.7592934966087341 + }, + { + "bits_per_byte": 1.7605243111456106, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2203024625778198, + "step": 2930, + "unclipped_grad_norm": 0.7871323227882385 + }, + { + "bits_per_byte": 1.7773966691085064, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2319974899291992, + "step": 2940, + "unclipped_grad_norm": 0.7994735240936279 + }, + { + "bits_per_byte": 1.761262116717812, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208138704299927, + "step": 2950, + "unclipped_grad_norm": 0.7672474384307861 + }, + { + "bits_per_byte": 1.8009139207527887, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2482984066009521, + "step": 2960, + "unclipped_grad_norm": 0.7721540927886963 + }, + { + "bits_per_byte": 1.7577087831683356, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218350887298584, + "step": 2970, + "unclipped_grad_norm": 0.7659915685653687 + }, + { + "bits_per_byte": 1.7511909846659368, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2138330936431885, + "step": 2980, + "unclipped_grad_norm": 0.7457980513572693 + }, + { + "bits_per_byte": 1.7677185174138528, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2252891063690186, + "step": 2990, + "unclipped_grad_norm": 0.7551211714744568 + }, + { + "bits_per_byte": 1.7804380103063644, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2341055870056152, + "step": 3000, + "unclipped_grad_norm": 0.7369211316108704 + }, + { + "bits_per_byte": 1.77081558099059, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.227435827255249, + "step": 3010, + "unclipped_grad_norm": 0.7526711225509644 + }, + { + "bits_per_byte": 1.726040929734363, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1964004039764404, + "step": 3020, + "unclipped_grad_norm": 0.8113403916358948 + }, + { + "bits_per_byte": 1.8045116258261877, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.250792145729065, + "step": 3030, + "unclipped_grad_norm": 0.7493385672569275 + }, + { + "bits_per_byte": 1.7229166928987893, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.194234848022461, + "step": 3040, + "unclipped_grad_norm": 0.7606903314590454 + }, + { + "bits_per_byte": 1.7403560776615798, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2063229084014893, + "step": 3050, + "unclipped_grad_norm": 0.7690193057060242 + }, + { + "bits_per_byte": 1.759874388708, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2198519706726074, + "step": 3060, + "unclipped_grad_norm": 0.7480750679969788 + }, + { + "bits_per_byte": 1.6873952242069339, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.169613242149353, + "step": 3070, + "unclipped_grad_norm": 0.7001752853393555 + }, + { + "bits_per_byte": 1.737272084766308, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2041852474212646, + "step": 3080, + "unclipped_grad_norm": 0.8322204351425171 + }, + { + "bits_per_byte": 1.8015143121869506, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.248714566230774, + "step": 3090, + "unclipped_grad_norm": 0.8183287382125854 + }, + { + "bits_per_byte": 1.7001176408045102, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.178431749343872, + "step": 3100, + "unclipped_grad_norm": 0.7750749588012695 + }, + { + "bits_per_byte": 1.7770231227908324, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.231738567352295, + "step": 3110, + "unclipped_grad_norm": 0.7082787752151489 + }, + { + "bits_per_byte": 1.749435729731231, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2126164436340332, + "step": 3120, + "unclipped_grad_norm": 0.736611545085907 + }, + { + "bits_per_byte": 1.7966217497352055, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2453233003616333, + "step": 3130, + "unclipped_grad_norm": 0.7366204857826233 + }, + { + "bits_per_byte": 1.7636881039908827, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.222495436668396, + "step": 3140, + "unclipped_grad_norm": 0.7583194375038147 + }, + { + "bits_per_byte": 1.7359698321339747, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2032825946807861, + "step": 3150, + "unclipped_grad_norm": 0.7600930333137512 + }, + { + "bits_per_byte": 1.7415977924008093, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.207183599472046, + "step": 3160, + "unclipped_grad_norm": 0.7639689445495605 + }, + { + "bits_per_byte": 1.654884311818432, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1470783948898315, + "step": 3170, + "unclipped_grad_norm": 0.7011478543281555 + }, + { + "bits_per_byte": 1.7584833930278216, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2188878059387207, + "step": 3180, + "unclipped_grad_norm": 0.7645173072814941 + }, + { + "bits_per_byte": 1.7305350083840592, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.199515461921692, + "step": 3190, + "unclipped_grad_norm": 0.7665988206863403 + }, + { + "bits_per_byte": 1.7003714871971838, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178607702255249, + "step": 3200, + "unclipped_grad_norm": 0.7379894852638245 + }, + { + "bits_per_byte": 1.6959066455981344, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755129098892212, + "step": 3210, + "unclipped_grad_norm": 0.8013631701469421 + }, + { + "bits_per_byte": 1.7098007799960389, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1851435899734497, + "step": 3220, + "unclipped_grad_norm": 0.7296364903450012 + }, + { + "bits_per_byte": 1.7987765203678656, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.246816873550415, + "step": 3230, + "unclipped_grad_norm": 0.7610434889793396 + }, + { + "bits_per_byte": 1.6862874839527267, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1688454151153564, + "step": 3240, + "unclipped_grad_norm": 0.723595917224884 + }, + { + "bits_per_byte": 1.8345631863073515, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2716223001480103, + "step": 3250, + "unclipped_grad_norm": 0.8353017568588257 + }, + { + "bits_per_byte": 1.6872093109613513, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1694843769073486, + "step": 3260, + "unclipped_grad_norm": 0.7602720260620117 + }, + { + "bits_per_byte": 1.7198954736710823, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1921406984329224, + "step": 3270, + "unclipped_grad_norm": 0.7516167163848877 + }, + { + "bits_per_byte": 1.7692164863028679, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2263274192810059, + "step": 3280, + "unclipped_grad_norm": 0.7487898468971252 + }, + { + "bits_per_byte": 1.7624494849393688, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2216368913650513, + "step": 3290, + "unclipped_grad_norm": 0.7708044052124023 + }, + { + "bits_per_byte": 1.7002387165907176, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1785156726837158, + "step": 3300, + "unclipped_grad_norm": 0.7703046202659607 + }, + { + "bits_per_byte": 1.7332599015043295, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2014042139053345, + "step": 3310, + "unclipped_grad_norm": 0.8132672905921936 + }, + { + "bits_per_byte": 1.7262779218272521, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1965646743774414, + "step": 3320, + "unclipped_grad_norm": 0.7545798420906067 + }, + { + "bits_per_byte": 1.7272033604715449, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1972061395645142, + "step": 3330, + "unclipped_grad_norm": 0.7775130867958069 + }, + { + "bits_per_byte": 1.6975052243379043, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1766209602355957, + "step": 3340, + "unclipped_grad_norm": 0.7105540633201599 + }, + { + "bits_per_byte": 1.7083408192728646, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1841316223144531, + "step": 3350, + "unclipped_grad_norm": 0.7163253426551819 + }, + { + "bits_per_byte": 1.732425613664994, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2008259296417236, + "step": 3360, + "unclipped_grad_norm": 0.7626740336418152 + }, + { + "bits_per_byte": 1.6756070173688444, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1614422798156738, + "step": 3370, + "unclipped_grad_norm": 0.8061453700065613 + }, + { + "bits_per_byte": 1.7092863798873084, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1847870349884033, + "step": 3380, + "unclipped_grad_norm": 0.7406735420227051 + }, + { + "bits_per_byte": 1.7539413312185348, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2157394886016846, + "step": 3390, + "unclipped_grad_norm": 0.7695116400718689 + }, + { + "bits_per_byte": 1.8092177590843976, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2540541887283325, + "step": 3400, + "unclipped_grad_norm": 0.7858016490936279 + }, + { + "bits_per_byte": 1.8255297975707825, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2653608322143555, + "step": 3410, + "unclipped_grad_norm": 0.7603964805603027 + }, + { + "bits_per_byte": 1.6857877023693193, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1684989929199219, + "step": 3420, + "unclipped_grad_norm": 0.7168062329292297 + }, + { + "bits_per_byte": 1.6176447364444573, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1212658882141113, + "step": 3430, + "unclipped_grad_norm": 0.7431134581565857 + }, + { + "bits_per_byte": 1.711189711884407, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1861063241958618, + "step": 3440, + "unclipped_grad_norm": 0.7753933668136597 + }, + { + "bits_per_byte": 1.77535626693867, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2305831909179688, + "step": 3450, + "unclipped_grad_norm": 0.8042635321617126 + }, + { + "bits_per_byte": 1.7083845028661837, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.184161901473999, + "step": 3460, + "unclipped_grad_norm": 0.7639697194099426 + }, + { + "bits_per_byte": 1.6937178224006033, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1739957332611084, + "step": 3470, + "unclipped_grad_norm": 0.748127281665802 + }, + { + "bits_per_byte": 1.6501064617948125, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1437666416168213, + "step": 3480, + "unclipped_grad_norm": 0.7506517767906189 + }, + { + "bits_per_byte": 1.8358463488654388, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2725117206573486, + "step": 3490, + "unclipped_grad_norm": 0.836082398891449 + }, + { + "bits_per_byte": 1.714104301868579, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.188126564025879, + "step": 3500, + "unclipped_grad_norm": 0.7544166445732117 + }, + { + "bits_per_byte": 1.744907598516664, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2094777822494507, + "step": 3510, + "unclipped_grad_norm": 0.7650617361068726 + }, + { + "bits_per_byte": 1.738600994709525, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2051063776016235, + "step": 3520, + "unclipped_grad_norm": 0.7794367671012878 + }, + { + "bits_per_byte": 1.7051804660806096, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.181941032409668, + "step": 3530, + "unclipped_grad_norm": 0.7445787787437439 + }, + { + "bits_per_byte": 1.7158172490711725, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1893138885498047, + "step": 3540, + "unclipped_grad_norm": 0.7629107236862183 + }, + { + "bits_per_byte": 1.7173026632266746, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1903434991836548, + "step": 3550, + "unclipped_grad_norm": 0.7763495445251465 + }, + { + "bits_per_byte": 1.6560372835098163, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.14787757396698, + "step": 3560, + "unclipped_grad_norm": 0.7450886964797974 + }, + { + "bits_per_byte": 1.6838996768281473, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1671903133392334, + "step": 3570, + "unclipped_grad_norm": 0.7487124800682068 + }, + { + "bits_per_byte": 1.6946102403759304, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1746143102645874, + "step": 3580, + "unclipped_grad_norm": 0.757170557975769 + }, + { + "bits_per_byte": 1.7307594457434352, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1996710300445557, + "step": 3590, + "unclipped_grad_norm": 0.7700981497764587 + }, + { + "bits_per_byte": 1.7491381997452384, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2124102115631104, + "step": 3600, + "unclipped_grad_norm": 0.7877120971679688 + }, + { + "bits_per_byte": 1.688025540622346, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1700501441955566, + "step": 3610, + "unclipped_grad_norm": 0.780867874622345 + }, + { + "bits_per_byte": 1.6436277033541595, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1392759084701538, + "step": 3620, + "unclipped_grad_norm": 0.7423334717750549 + }, + { + "bits_per_byte": 1.6950637586262558, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1749286651611328, + "step": 3630, + "unclipped_grad_norm": 0.7513464689254761 + }, + { + "bits_per_byte": 1.6758522646289749, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1616122722625732, + "step": 3640, + "unclipped_grad_norm": 0.7453336715698242 + }, + { + "bits_per_byte": 1.736063734661346, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2033476829528809, + "step": 3650, + "unclipped_grad_norm": 0.7682862877845764 + }, + { + "bits_per_byte": 1.6360068081291268, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1339935064315796, + "step": 3660, + "unclipped_grad_norm": 0.7531437277793884 + }, + { + "bits_per_byte": 1.7183529612754938, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1910715103149414, + "step": 3670, + "unclipped_grad_norm": 0.7578480243682861 + }, + { + "bits_per_byte": 1.6583936178092877, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1495108604431152, + "step": 3680, + "unclipped_grad_norm": 0.7224898934364319 + }, + { + "bits_per_byte": 1.6372521345040243, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1348567008972168, + "step": 3690, + "unclipped_grad_norm": 0.7246464490890503 + }, + { + "bits_per_byte": 1.6659130432703113, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1547229290008545, + "step": 3700, + "unclipped_grad_norm": 0.7878638505935669 + }, + { + "bits_per_byte": 1.694639821391879, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1746348142623901, + "step": 3710, + "unclipped_grad_norm": 0.7455682754516602 + }, + { + "bits_per_byte": 1.6891824679147003, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1708520650863647, + "step": 3720, + "unclipped_grad_norm": 0.7515487670898438 + }, + { + "bits_per_byte": 1.6091924770851538, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1154072284698486, + "step": 3730, + "unclipped_grad_norm": 0.7353127598762512 + }, + { + "bits_per_byte": 1.6633680440028435, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.152958869934082, + "step": 3740, + "unclipped_grad_norm": 0.7253232002258301 + }, + { + "bits_per_byte": 1.6922951819126664, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1730096340179443, + "step": 3750, + "unclipped_grad_norm": 0.7686370611190796 + }, + { + "bits_per_byte": 1.7713081393026613, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2277772426605225, + "step": 3760, + "unclipped_grad_norm": 0.7601476907730103 + }, + { + "bits_per_byte": 1.7305630415561497, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1995348930358887, + "step": 3770, + "unclipped_grad_norm": 0.7877833247184753 + }, + { + "bits_per_byte": 1.5644979697442287, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0844273567199707, + "step": 3780, + "unclipped_grad_norm": 0.7386898994445801 + }, + { + "bits_per_byte": 1.6644766441703052, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1537272930145264, + "step": 3790, + "unclipped_grad_norm": 0.7420179843902588 + }, + { + "bits_per_byte": 1.6653939996300076, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1543631553649902, + "step": 3800, + "unclipped_grad_norm": 0.7838358283042908 + }, + { + "bits_per_byte": 1.6952983429620325, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17509126663208, + "step": 3810, + "unclipped_grad_norm": 0.8048396110534668 + }, + { + "bits_per_byte": 1.6744323758634516, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.160628080368042, + "step": 3820, + "unclipped_grad_norm": 0.7709245681762695 + }, + { + "bits_per_byte": 1.720191799778519, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1923460960388184, + "step": 3830, + "unclipped_grad_norm": 0.7672807574272156 + }, + { + "bits_per_byte": 1.6467005173771254, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1414058208465576, + "step": 3840, + "unclipped_grad_norm": 0.7319741249084473 + }, + { + "bits_per_byte": 1.5964396195583748, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106567621231079, + "step": 3850, + "unclipped_grad_norm": 0.7756319642066956 + }, + { + "bits_per_byte": 1.6360811046342996, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1340450048446655, + "step": 3860, + "unclipped_grad_norm": 0.766754150390625 + }, + { + "bits_per_byte": 1.7433616464680584, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2084062099456787, + "step": 3870, + "unclipped_grad_norm": 0.7916337847709656 + }, + { + "bits_per_byte": 1.6358671582166262, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1338967084884644, + "step": 3880, + "unclipped_grad_norm": 0.7497678399085999 + }, + { + "bits_per_byte": 1.600699113872294, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1095200777053833, + "step": 3890, + "unclipped_grad_norm": 0.7555440664291382 + }, + { + "bits_per_byte": 1.6778517349279058, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1629981994628906, + "step": 3900, + "unclipped_grad_norm": 0.826478123664856 + }, + { + "bits_per_byte": 1.6805711246033486, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1648831367492676, + "step": 3910, + "unclipped_grad_norm": 0.7849931120872498 + }, + { + "bits_per_byte": 1.6375147520118918, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1350387334823608, + "step": 3920, + "unclipped_grad_norm": 0.7588120102882385 + }, + { + "bits_per_byte": 1.673005091843941, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.15963876247406, + "step": 3930, + "unclipped_grad_norm": 0.742434024810791 + }, + { + "bits_per_byte": 1.6996766772876981, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1781260967254639, + "step": 3940, + "unclipped_grad_norm": 0.7588658928871155 + }, + { + "bits_per_byte": 1.7339400928884918, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2018756866455078, + "step": 3950, + "unclipped_grad_norm": 0.7937244176864624 + }, + { + "bits_per_byte": 1.713467450112434, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1876851320266724, + "step": 3960, + "unclipped_grad_norm": 0.7884036302566528 + }, + { + "bits_per_byte": 1.6516962694193882, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1448686122894287, + "step": 3970, + "unclipped_grad_norm": 0.8004661798477173 + }, + { + "bits_per_byte": 1.6740500584305835, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1603630781173706, + "step": 3980, + "unclipped_grad_norm": 0.7692103385925293 + }, + { + "bits_per_byte": 1.691941929547794, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.172764778137207, + "step": 3990, + "unclipped_grad_norm": 0.7891059517860413 + }, + { + "bits_per_byte": 1.5813677479673616, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0961205959320068, + "step": 4000, + "unclipped_grad_norm": 0.7436221837997437 + }, + { + "bits_per_byte": 1.6793110077204767, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.164009690284729, + "step": 4010, + "unclipped_grad_norm": 0.8044730424880981 + }, + { + "bits_per_byte": 1.6584589712166156, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1495561599731445, + "step": 4020, + "unclipped_grad_norm": 0.8183668851852417 + }, + { + "bits_per_byte": 1.6850787898824628, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1680076122283936, + "step": 4030, + "unclipped_grad_norm": 0.793128252029419 + }, + { + "bits_per_byte": 1.617821878574846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1213886737823486, + "step": 4040, + "unclipped_grad_norm": 0.7582541108131409 + }, + { + "bits_per_byte": 1.6450140555027646, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1402368545532227, + "step": 4050, + "unclipped_grad_norm": 0.7559582591056824 + }, + { + "bits_per_byte": 1.6581645369183384, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1493520736694336, + "step": 4060, + "unclipped_grad_norm": 0.743245005607605 + }, + { + "bits_per_byte": 1.680131880913045, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1645786762237549, + "step": 4070, + "unclipped_grad_norm": 0.7624887824058533 + }, + { + "bits_per_byte": 1.7564567494700543, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2174830436706543, + "step": 4080, + "unclipped_grad_norm": 0.8132302761077881 + }, + { + "bits_per_byte": 1.6669108866106175, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1554145812988281, + "step": 4090, + "unclipped_grad_norm": 0.8276432752609253 + }, + { + "bits_per_byte": 1.638365894151012, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1356287002563477, + "step": 4100, + "unclipped_grad_norm": 0.7751885652542114 + }, + { + "bits_per_byte": 1.6303674970073352, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1300846338272095, + "step": 4110, + "unclipped_grad_norm": 0.7898787260055542 + }, + { + "bits_per_byte": 1.6896222275529564, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.171156883239746, + "step": 4120, + "unclipped_grad_norm": 0.8148247599601746 + }, + { + "bits_per_byte": 1.6213521664491068, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1238356828689575, + "step": 4130, + "unclipped_grad_norm": 0.8133904933929443 + }, + { + "bits_per_byte": 1.6881359534842002, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1701266765594482, + "step": 4140, + "unclipped_grad_norm": 0.8217440247535706 + }, + { + "bits_per_byte": 1.6129334437067215, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1180002689361572, + "step": 4150, + "unclipped_grad_norm": 0.7947918176651001 + }, + { + "bits_per_byte": 1.651558683298698, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.144773244857788, + "step": 4160, + "unclipped_grad_norm": 0.7860805988311768 + }, + { + "bits_per_byte": 1.6828089628563754, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1664342880249023, + "step": 4170, + "unclipped_grad_norm": 0.8207855224609375 + }, + { + "bits_per_byte": 1.7311515661874024, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1999428272247314, + "step": 4180, + "unclipped_grad_norm": 0.8373809456825256 + }, + { + "bits_per_byte": 1.6433404923272186, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.139076828956604, + "step": 4190, + "unclipped_grad_norm": 0.7621071338653564 + }, + { + "bits_per_byte": 1.682679287937625, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.166344404220581, + "step": 4200, + "unclipped_grad_norm": 0.7672834396362305 + }, + { + "bits_per_byte": 1.661693964879345, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1517984867095947, + "step": 4210, + "unclipped_grad_norm": 0.7330533862113953 + }, + { + "bits_per_byte": 1.6972419188994332, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1764384508132935, + "step": 4220, + "unclipped_grad_norm": 0.7866858243942261 + }, + { + "bits_per_byte": 1.6670219874030747, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.155491590499878, + "step": 4230, + "unclipped_grad_norm": 0.7902593612670898 + }, + { + "bits_per_byte": 1.6114956687455084, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1170036792755127, + "step": 4240, + "unclipped_grad_norm": 0.740251362323761 + }, + { + "bits_per_byte": 1.6335233786506678, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1322721242904663, + "step": 4250, + "unclipped_grad_norm": 0.7777352333068848 + }, + { + "bits_per_byte": 1.6115895712728796, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1170687675476074, + "step": 4260, + "unclipped_grad_norm": 0.774424135684967 + }, + { + "bits_per_byte": 1.619611874005026, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1226294040679932, + "step": 4270, + "unclipped_grad_norm": 0.7803477048873901 + }, + { + "bits_per_byte": 1.6299772683725275, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1298141479492188, + "step": 4280, + "unclipped_grad_norm": 0.7580373287200928 + }, + { + "bits_per_byte": 1.6795242662075467, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.164157509803772, + "step": 4290, + "unclipped_grad_norm": 0.7638103365898132 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.109092354774475, + "step": 4300, + "unclipped_grad_norm": 0.7825598120689392 + }, + { + "bits_per_byte": 1.6771565810531184, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1625163555145264, + "step": 4310, + "unclipped_grad_norm": 0.7940689921379089 + }, + { + "bits_per_byte": 1.6577068910843924, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.149034857749939, + "step": 4320, + "unclipped_grad_norm": 0.7976187467575073 + }, + { + "bits_per_byte": 1.6195226150092283, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1225675344467163, + "step": 4330, + "unclipped_grad_norm": 0.7892199158668518 + }, + { + "bits_per_byte": 1.655279183984813, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1473520994186401, + "step": 4340, + "unclipped_grad_norm": 0.8144169449806213 + }, + { + "bits_per_byte": 1.6394932404274176, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.136410117149353, + "step": 4350, + "unclipped_grad_norm": 0.8119622468948364 + }, + { + "bits_per_byte": 1.6287904160989233, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1289914846420288, + "step": 4360, + "unclipped_grad_norm": 0.7554351091384888 + }, + { + "bits_per_byte": 1.6321829458698431, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1313430070877075, + "step": 4370, + "unclipped_grad_norm": 0.808085024356842 + }, + { + "bits_per_byte": 1.6820371047193032, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1658992767333984, + "step": 4380, + "unclipped_grad_norm": 0.7708722949028015 + }, + { + "bits_per_byte": 1.6522423143358775, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1452471017837524, + "step": 4390, + "unclipped_grad_norm": 0.8123121857643127 + }, + { + "bits_per_byte": 1.6025224739367414, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1107839345932007, + "step": 4400, + "unclipped_grad_norm": 0.7717524766921997 + }, + { + "bits_per_byte": 1.6853694405624209, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1682090759277344, + "step": 4410, + "unclipped_grad_norm": 0.81525057554245 + }, + { + "bits_per_byte": 1.6142167782474597, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1188898086547852, + "step": 4420, + "unclipped_grad_norm": 0.7956154346466064 + }, + { + "bits_per_byte": 1.6755413199962148, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1613967418670654, + "step": 4430, + "unclipped_grad_norm": 0.7907513380050659 + }, + { + "bits_per_byte": 1.6011154838700328, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1098086833953857, + "step": 4440, + "unclipped_grad_norm": 0.7607184648513794 + }, + { + "bits_per_byte": 1.5990663105850023, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1083883047103882, + "step": 4450, + "unclipped_grad_norm": 0.7442869544029236 + }, + { + "bits_per_byte": 1.5859970049806358, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0993293523788452, + "step": 4460, + "unclipped_grad_norm": 0.7537500262260437 + }, + { + "bits_per_byte": 1.6451918855637568, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1403601169586182, + "step": 4470, + "unclipped_grad_norm": 0.7931647896766663 + }, + { + "bits_per_byte": 1.6505943765753102, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1441048383712769, + "step": 4480, + "unclipped_grad_norm": 0.9106901288032532 + }, + { + "bits_per_byte": 1.6301040195862135, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1299020051956177, + "step": 4490, + "unclipped_grad_norm": 0.7690432667732239 + }, + { + "bits_per_byte": 1.6520481459230536, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1451125144958496, + "step": 4500, + "unclipped_grad_norm": 0.7990603446960449 + }, + { + "bits_per_byte": 1.5968599451570833, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1068589687347412, + "step": 4510, + "unclipped_grad_norm": 0.8130239248275757 + }, + { + "bits_per_byte": 1.6430763269754933, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.138893723487854, + "step": 4520, + "unclipped_grad_norm": 0.7745712399482727 + }, + { + "bits_per_byte": 1.660709880151108, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1511163711547852, + "step": 4530, + "unclipped_grad_norm": 0.7695059776306152 + }, + { + "bits_per_byte": 1.707295852686222, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1834073066711426, + "step": 4540, + "unclipped_grad_norm": 0.8436031937599182 + }, + { + "bits_per_byte": 1.5623325361872151, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.0829263925552368, + "step": 4550, + "unclipped_grad_norm": 0.7549967169761658 + }, + { + "bits_per_byte": 1.6257084869954619, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1268552541732788, + "step": 4560, + "unclipped_grad_norm": 0.8449703454971313 + }, + { + "bits_per_byte": 1.6175644205465043, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1212102174758911, + "step": 4570, + "unclipped_grad_norm": 0.7528937458992004 + }, + { + "bits_per_byte": 1.570274179056107, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0884311199188232, + "step": 4580, + "unclipped_grad_norm": 0.790579617023468 + }, + { + "bits_per_byte": 1.609637052237634, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1157153844833374, + "step": 4590, + "unclipped_grad_norm": 0.7935193181037903 + }, + { + "bits_per_byte": 1.6088013885370918, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1151361465454102, + "step": 4600, + "unclipped_grad_norm": 0.7915687561035156 + }, + { + "bits_per_byte": 1.6615916352020816, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.151727557182312, + "step": 4610, + "unclipped_grad_norm": 0.8493216633796692 + }, + { + "bits_per_byte": 1.6675298521710726, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1558436155319214, + "step": 4620, + "unclipped_grad_norm": 0.8179065585136414 + }, + { + "bits_per_byte": 1.6286588493710132, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1289002895355225, + "step": 4630, + "unclipped_grad_norm": 0.8027536273002625 + }, + { + "bits_per_byte": 1.7031861552612044, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.180558681488037, + "step": 4640, + "unclipped_grad_norm": 0.8179929256439209 + }, + { + "bits_per_byte": 1.6059254946493637, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.113142728805542, + "step": 4650, + "unclipped_grad_norm": 0.8228299021720886 + }, + { + "bits_per_byte": 1.5739184914278899, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0909571647644043, + "step": 4660, + "unclipped_grad_norm": 0.820624589920044 + }, + { + "bits_per_byte": 1.7237126286069824, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.194786548614502, + "step": 4670, + "unclipped_grad_norm": 0.8582949638366699 + }, + { + "bits_per_byte": 1.7009104808249877, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1789813041687012, + "step": 4680, + "unclipped_grad_norm": 0.8158925175666809 + }, + { + "bits_per_byte": 1.6679322915740917, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1561225652694702, + "step": 4690, + "unclipped_grad_norm": 0.849236249923706 + }, + { + "bits_per_byte": 1.589747430648001, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.101928949356079, + "step": 4700, + "unclipped_grad_norm": 0.7403190732002258 + }, + { + "bits_per_byte": 1.6742342518496576, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1604907512664795, + "step": 4710, + "unclipped_grad_norm": 0.8582251071929932 + }, + { + "bits_per_byte": 1.607032031025015, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1139097213745117, + "step": 4720, + "unclipped_grad_norm": 0.8400667905807495 + }, + { + "bits_per_byte": 1.5176990826873464, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0519888401031494, + "step": 4730, + "unclipped_grad_norm": 0.7827285528182983 + }, + { + "bits_per_byte": 1.5868581221135059, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.099926233291626, + "step": 4740, + "unclipped_grad_norm": 0.7815871238708496 + }, + { + "bits_per_byte": 1.6377919880450826, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1352308988571167, + "step": 4750, + "unclipped_grad_norm": 0.7846289873123169 + }, + { + "bits_per_byte": 1.6125860387519786, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1177594661712646, + "step": 4760, + "unclipped_grad_norm": 0.8307337164878845 + }, + { + "bits_per_byte": 1.5936545325103024, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1046371459960938, + "step": 4770, + "unclipped_grad_norm": 0.8213374614715576 + }, + { + "bits_per_byte": 1.6121588338472355, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1174633502960205, + "step": 4780, + "unclipped_grad_norm": 0.8224621415138245 + }, + { + "bits_per_byte": 1.5705015401205475, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0885887145996094, + "step": 4790, + "unclipped_grad_norm": 0.8122685551643372 + }, + { + "bits_per_byte": 1.5722774329733569, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0898196697235107, + "step": 4800, + "unclipped_grad_norm": 0.8455460667610168 + }, + { + "bits_per_byte": 1.6480244398234671, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1423234939575195, + "step": 4810, + "unclipped_grad_norm": 0.8817458748817444 + }, + { + "bits_per_byte": 1.574972745077679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0916879177093506, + "step": 4820, + "unclipped_grad_norm": 0.7954387664794922 + }, + { + "bits_per_byte": 1.608869665649484, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1151834726333618, + "step": 4830, + "unclipped_grad_norm": 0.8687117695808411 + }, + { + "bits_per_byte": 1.5985519104762718, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1080317497253418, + "step": 4840, + "unclipped_grad_norm": 0.8505465388298035 + }, + { + "bits_per_byte": 1.6228387844831649, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1248661279678345, + "step": 4850, + "unclipped_grad_norm": 0.7912812232971191 + }, + { + "bits_per_byte": 1.5432041098103009, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0696675777435303, + "step": 4860, + "unclipped_grad_norm": 0.7933070063591003 + }, + { + "bits_per_byte": 1.635918581029234, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.13393235206604, + "step": 4870, + "unclipped_grad_norm": 0.8805916905403137 + }, + { + "bits_per_byte": 1.4766798445953597, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0235564708709717, + "step": 4880, + "unclipped_grad_norm": 0.8066675066947937 + }, + { + "bits_per_byte": 1.5344280071194227, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0635844469070435, + "step": 4890, + "unclipped_grad_norm": 0.8088182806968689 + }, + { + "bits_per_byte": 1.6101516243290155, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1160720586776733, + "step": 4900, + "unclipped_grad_norm": 0.8653903603553772 + }, + { + "bits_per_byte": 1.6779198400576476, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1630454063415527, + "step": 4910, + "unclipped_grad_norm": 0.8391038179397583 + }, + { + "bits_per_byte": 1.6284904783558185, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1287835836410522, + "step": 4920, + "unclipped_grad_norm": 0.8723531365394592 + }, + { + "bits_per_byte": 1.5831391692712484, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0973484516143799, + "step": 4930, + "unclipped_grad_norm": 0.7553638815879822 + }, + { + "bits_per_byte": 1.5755332365368409, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0920764207839966, + "step": 4940, + "unclipped_grad_norm": 0.8158645033836365 + }, + { + "bits_per_byte": 1.530142715407874, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0606141090393066, + "step": 4950, + "unclipped_grad_norm": 0.7773033380508423 + }, + { + "bits_per_byte": 1.5220387209165676, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0549968481063843, + "step": 4960, + "unclipped_grad_norm": 0.8325244784355164 + }, + { + "bits_per_byte": 1.6031579498316795, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1112244129180908, + "step": 4970, + "unclipped_grad_norm": 0.8591333627700806 + }, + { + "bits_per_byte": 1.6532000857185325, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1459109783172607, + "step": 4980, + "unclipped_grad_norm": 0.8351151347160339 + }, + { + "bits_per_byte": 1.585272098107249, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.0988268852233887, + "step": 4990, + "unclipped_grad_norm": 0.8779996037483215 + }, + { + "bits_per_byte": 1.5394700224947675, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670793056488037, + "step": 5000, + "unclipped_grad_norm": 0.7928086519241333 + }, + { + "bits_per_byte": 1.5552827953456976, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0780398845672607, + "step": 5010, + "unclipped_grad_norm": 0.8201308846473694 + }, + { + "bits_per_byte": 1.5898205232746179, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1019796133041382, + "step": 5020, + "unclipped_grad_norm": 0.7774789929389954 + }, + { + "bits_per_byte": 1.5697566832596608, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.088072419166565, + "step": 5030, + "unclipped_grad_norm": 0.7855319380760193 + }, + { + "bits_per_byte": 1.539816567536256, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0673195123672485, + "step": 5040, + "unclipped_grad_norm": 0.7920348048210144 + }, + { + "bits_per_byte": 1.5092143186070297, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0461076498031616, + "step": 5050, + "unclipped_grad_norm": 0.8045132160186768 + }, + { + "bits_per_byte": 1.5339026001210367, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0632202625274658, + "step": 5060, + "unclipped_grad_norm": 0.8044583797454834 + }, + { + "bits_per_byte": 1.6138325690054323, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1186234951019287, + "step": 5070, + "unclipped_grad_norm": 0.8723475337028503 + }, + { + "bits_per_byte": 1.574335549356232, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.091246247291565, + "step": 5080, + "unclipped_grad_norm": 0.8741123080253601 + }, + { + "bits_per_byte": 1.5594206979254568, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0809080600738525, + "step": 5090, + "unclipped_grad_norm": 0.8465037941932678 + }, + { + "bits_per_byte": 1.5640659493252613, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0841279029846191, + "step": 5100, + "unclipped_grad_norm": 0.783299446105957 + }, + { + "bits_per_byte": 1.488738924108558, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0319151878356934, + "step": 5110, + "unclipped_grad_norm": 0.8348572254180908 + }, + { + "bits_per_byte": 1.5847128105266433, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0984392166137695, + "step": 5120, + "unclipped_grad_norm": 0.8110998868942261 + }, + { + "bits_per_byte": 1.590178763136365, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1022279262542725, + "step": 5130, + "unclipped_grad_norm": 0.8208310604095459 + }, + { + "bits_per_byte": 1.5494388248693793, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0739891529083252, + "step": 5140, + "unclipped_grad_norm": 0.8678956627845764 + }, + { + "bits_per_byte": 1.6273239200350162, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1279749870300293, + "step": 5150, + "unclipped_grad_norm": 0.848126232624054 + }, + { + "bits_per_byte": 1.5085432423033631, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0456424951553345, + "step": 5160, + "unclipped_grad_norm": 0.7922812104225159 + }, + { + "bits_per_byte": 1.6700107019097687, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1575632095336914, + "step": 5170, + "unclipped_grad_norm": 0.8759046196937561 + }, + { + "bits_per_byte": 1.5869253673299932, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0999728441238403, + "step": 5180, + "unclipped_grad_norm": 0.8647724390029907 + }, + { + "bits_per_byte": 1.5237559676854329, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0561871528625488, + "step": 5190, + "unclipped_grad_norm": 0.813337504863739 + }, + { + "bits_per_byte": 1.556985939537192, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0792204141616821, + "step": 5200, + "unclipped_grad_norm": 0.8388134241104126 + }, + { + "bits_per_byte": 1.5602121620847276, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0814566612243652, + "step": 5210, + "unclipped_grad_norm": 0.8162683248519897 + }, + { + "bits_per_byte": 1.6432956048553435, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1390457153320312, + "step": 5220, + "unclipped_grad_norm": 0.8667425513267517 + }, + { + "bits_per_byte": 1.5501931407760636, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.074512004852295, + "step": 5230, + "unclipped_grad_norm": 0.8668702840805054 + }, + { + "bits_per_byte": 1.5510165937083948, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.075082778930664, + "step": 5240, + "unclipped_grad_norm": 0.8049055337905884 + }, + { + "bits_per_byte": 1.5393878147876552, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0670223236083984, + "step": 5250, + "unclipped_grad_norm": 0.8209428191184998 + }, + { + "bits_per_byte": 1.540701762240247, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0679330825805664, + "step": 5260, + "unclipped_grad_norm": 0.8524174690246582 + }, + { + "bits_per_byte": 1.5376201771020872, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0657970905303955, + "step": 5270, + "unclipped_grad_norm": 0.8064137697219849 + }, + { + "bits_per_byte": 1.5550475230793173, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0778768062591553, + "step": 5280, + "unclipped_grad_norm": 0.8169121146202087 + }, + { + "bits_per_byte": 1.5423493560355128, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.069075107574463, + "step": 5290, + "unclipped_grad_norm": 0.8263237476348877 + }, + { + "bits_per_byte": 1.5542627662434303, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.077332854270935, + "step": 5300, + "unclipped_grad_norm": 0.8492859601974487 + }, + { + "bits_per_byte": 1.5509104804128124, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0750092267990112, + "step": 5310, + "unclipped_grad_norm": 0.8885292410850525 + }, + { + "bits_per_byte": 1.5517654061702515, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0756018161773682, + "step": 5320, + "unclipped_grad_norm": 0.8326203227043152 + }, + { + "bits_per_byte": 1.5477100552629064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0727908611297607, + "step": 5330, + "unclipped_grad_norm": 0.8806997537612915 + }, + { + "bits_per_byte": 1.503639328996661, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0422433614730835, + "step": 5340, + "unclipped_grad_norm": 0.8234722018241882 + }, + { + "bits_per_byte": 1.5869049013945407, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0999586582183838, + "step": 5350, + "unclipped_grad_norm": 0.8706242442131042 + }, + { + "bits_per_byte": 1.5072631754329913, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.044755220413208, + "step": 5360, + "unclipped_grad_norm": 0.8708282709121704 + }, + { + "bits_per_byte": 1.471025742965594, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0196373462677002, + "step": 5370, + "unclipped_grad_norm": 0.8303158283233643 + }, + { + "bits_per_byte": 1.5968417149960918, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1068463325500488, + "step": 5380, + "unclipped_grad_norm": 0.883955180644989 + }, + { + "bits_per_byte": 1.531507741707772, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0615602731704712, + "step": 5390, + "unclipped_grad_norm": 0.8507125377655029 + }, + { + "bits_per_byte": 1.5483677169198058, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073246717453003, + "step": 5400, + "unclipped_grad_norm": 0.8522453904151917 + }, + { + "bits_per_byte": 1.571479089508052, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.089266300201416, + "step": 5410, + "unclipped_grad_norm": 0.8313266038894653 + }, + { + "bits_per_byte": 1.5408727129952047, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0680515766143799, + "step": 5420, + "unclipped_grad_norm": 0.8902517557144165 + }, + { + "bits_per_byte": 1.5778336764747818, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0936709642410278, + "step": 5430, + "unclipped_grad_norm": 0.8956695795059204 + }, + { + "bits_per_byte": 1.513927503153925, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0493745803833008, + "step": 5440, + "unclipped_grad_norm": 0.897573709487915 + }, + { + "bits_per_byte": 1.5078418970531444, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0451563596725464, + "step": 5450, + "unclipped_grad_norm": 0.8440007567405701 + }, + { + "bits_per_byte": 1.5030907043404085, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0418630838394165, + "step": 5460, + "unclipped_grad_norm": 0.8436890244483948 + }, + { + "bits_per_byte": 1.5464133060754008, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0718920230865479, + "step": 5470, + "unclipped_grad_norm": 0.8681005239486694 + }, + { + "bits_per_byte": 1.553998600891705, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.077149748802185, + "step": 5480, + "unclipped_grad_norm": 0.8523774147033691 + }, + { + "bits_per_byte": 1.543775436176467, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.070063591003418, + "step": 5490, + "unclipped_grad_norm": 0.8906207084655762 + }, + { + "bits_per_byte": 1.5444891641775478, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0705583095550537, + "step": 5500, + "unclipped_grad_norm": 0.8658390045166016 + }, + { + "bits_per_byte": 1.6370650173798855, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1347270011901855, + "step": 5510, + "unclipped_grad_norm": 0.9161565899848938 + }, + { + "bits_per_byte": 1.5520985365649729, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.075832724571228, + "step": 5520, + "unclipped_grad_norm": 0.845907986164093 + }, + { + "bits_per_byte": 1.5215033389244317, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0546257495880127, + "step": 5530, + "unclipped_grad_norm": 0.8585748672485352 + }, + { + "bits_per_byte": 1.5943249208833656, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1051018238067627, + "step": 5540, + "unclipped_grad_norm": 0.9360165596008301 + }, + { + "bits_per_byte": 1.5176287417831436, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0519400835037231, + "step": 5550, + "unclipped_grad_norm": 0.8308430910110474 + }, + { + "bits_per_byte": 1.5164790377621256, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0511431694030762, + "step": 5560, + "unclipped_grad_norm": 0.894058346748352 + }, + { + "bits_per_byte": 1.4905304673825959, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0331569910049438, + "step": 5570, + "unclipped_grad_norm": 0.8399901390075684 + }, + { + "bits_per_byte": 1.5655759569998369, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.085174560546875, + "step": 5580, + "unclipped_grad_norm": 0.8579773902893066 + }, + { + "bits_per_byte": 1.482571282283316, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0276401042938232, + "step": 5590, + "unclipped_grad_norm": 0.8161733746528625 + }, + { + "bits_per_byte": 1.5496305855250914, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0741220712661743, + "step": 5600, + "unclipped_grad_norm": 0.8440633416175842 + }, + { + "bits_per_byte": 1.5466313800766949, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0720431804656982, + "step": 5610, + "unclipped_grad_norm": 0.8384022116661072 + }, + { + "bits_per_byte": 1.55366271877457, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0769169330596924, + "step": 5620, + "unclipped_grad_norm": 0.8643615245819092 + }, + { + "bits_per_byte": 1.5359743031333302, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0646562576293945, + "step": 5630, + "unclipped_grad_norm": 0.8559057712554932 + }, + { + "bits_per_byte": 1.5154779267514533, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.050449252128601, + "step": 5640, + "unclipped_grad_norm": 0.8564271330833435 + }, + { + "bits_per_byte": 1.5399061704973558, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0673816204071045, + "step": 5650, + "unclipped_grad_norm": 0.899230420589447 + }, + { + "bits_per_byte": 1.6017691899259623, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1102617979049683, + "step": 5660, + "unclipped_grad_norm": 0.8796955943107605 + }, + { + "bits_per_byte": 1.5203328250026595, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.05381441116333, + "step": 5670, + "unclipped_grad_norm": 0.8509314060211182 + }, + { + "bits_per_byte": 1.5301421994599214, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.060613751411438, + "step": 5680, + "unclipped_grad_norm": 0.8624072074890137 + }, + { + "bits_per_byte": 1.6028342784827556, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1110000610351562, + "step": 5690, + "unclipped_grad_norm": 0.8914096355438232 + }, + { + "bits_per_byte": 1.526354281574668, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.057988166809082, + "step": 5700, + "unclipped_grad_norm": 0.8579604625701904 + }, + { + "bits_per_byte": 1.5347007716036911, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.063773512840271, + "step": 5710, + "unclipped_grad_norm": 0.8810272216796875 + }, + { + "bits_per_byte": 1.5491378552303696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0737805366516113, + "step": 5720, + "unclipped_grad_norm": 0.8394830226898193 + }, + { + "bits_per_byte": 1.5456280332915613, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.071347713470459, + "step": 5730, + "unclipped_grad_norm": 0.865943968296051 + }, + { + "bits_per_byte": 1.5433466834278662, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0697664022445679, + "step": 5740, + "unclipped_grad_norm": 0.886944591999054 + }, + { + "bits_per_byte": 1.5488365416260579, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0735716819763184, + "step": 5750, + "unclipped_grad_norm": 0.9145897626876831 + }, + { + "bits_per_byte": 1.5636783004302166, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0838592052459717, + "step": 5760, + "unclipped_grad_norm": 0.9009037017822266 + }, + { + "bits_per_byte": 1.511110599315443, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0474220514297485, + "step": 5770, + "unclipped_grad_norm": 0.844070315361023 + }, + { + "bits_per_byte": 1.494012772097266, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.035570740699768, + "step": 5780, + "unclipped_grad_norm": 0.8943845629692078 + }, + { + "bits_per_byte": 1.5135893852623288, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.049140214920044, + "step": 5790, + "unclipped_grad_norm": 0.8841454386711121 + }, + { + "bits_per_byte": 1.4905120652389536, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033144235610962, + "step": 5800, + "unclipped_grad_norm": 0.8580894470214844 + }, + { + "bits_per_byte": 1.5735937881830608, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0907320976257324, + "step": 5810, + "unclipped_grad_norm": 0.933175265789032 + }, + { + "bits_per_byte": 1.5042643139498963, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.042676568031311, + "step": 5820, + "unclipped_grad_norm": 0.828717052936554 + }, + { + "bits_per_byte": 1.5156406223391694, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0505620241165161, + "step": 5830, + "unclipped_grad_norm": 0.9511673450469971 + }, + { + "bits_per_byte": 1.5092205099824607, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0461119413375854, + "step": 5840, + "unclipped_grad_norm": 0.8395604491233826 + }, + { + "bits_per_byte": 1.477936693807865, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0244276523590088, + "step": 5850, + "unclipped_grad_norm": 0.9143134355545044 + }, + { + "bits_per_byte": 1.5336375748560571, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.063036561012268, + "step": 5860, + "unclipped_grad_norm": 0.9014923572540283 + }, + { + "bits_per_byte": 1.5402322496033916, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0676076412200928, + "step": 5870, + "unclipped_grad_norm": 0.8728002905845642 + }, + { + "bits_per_byte": 1.555285031120159, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.078041434288025, + "step": 5880, + "unclipped_grad_norm": 0.8936865925788879 + }, + { + "bits_per_byte": 1.564054942435606, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.084120273590088, + "step": 5890, + "unclipped_grad_norm": 0.8752698302268982 + }, + { + "bits_per_byte": 1.5458715607251832, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071516513824463, + "step": 5900, + "unclipped_grad_norm": 0.8957662582397461 + }, + { + "bits_per_byte": 1.5568008862048637, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0790921449661255, + "step": 5910, + "unclipped_grad_norm": 0.8808028101921082 + }, + { + "bits_per_byte": 1.4896764015384112, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0325649976730347, + "step": 5920, + "unclipped_grad_norm": 0.8660398721694946 + }, + { + "bits_per_byte": 1.5908711652887386, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.102707862854004, + "step": 5930, + "unclipped_grad_norm": 0.9604227542877197 + }, + { + "bits_per_byte": 1.4574508083676907, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.010227918624878, + "step": 5940, + "unclipped_grad_norm": 0.8380697965621948 + }, + { + "bits_per_byte": 1.5131755950043528, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0488533973693848, + "step": 5950, + "unclipped_grad_norm": 0.8392192125320435 + }, + { + "bits_per_byte": 1.5452194025131114, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0710644721984863, + "step": 5960, + "unclipped_grad_norm": 0.932554304599762 + }, + { + "bits_per_byte": 1.5213482105733536, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.054518222808838, + "step": 5970, + "unclipped_grad_norm": 0.927780270576477 + }, + { + "bits_per_byte": 1.5601101763727658, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0813859701156616, + "step": 5980, + "unclipped_grad_norm": 0.8920019268989563 + }, + { + "bits_per_byte": 1.5382338112003657, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0662224292755127, + "step": 5990, + "unclipped_grad_norm": 0.9537093639373779 + }, + { + "bits_per_byte": 1.4675097296513546, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.017200231552124, + "step": 6000, + "unclipped_grad_norm": 0.8773758411407471 + }, + { + "bits_per_byte": 1.506333093257125, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0441105365753174, + "step": 6010, + "unclipped_grad_norm": 0.8785988092422485 + }, + { + "bits_per_byte": 1.4794742187065786, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0254933834075928, + "step": 6020, + "unclipped_grad_norm": 0.8875186443328857 + }, + { + "bits_per_byte": 1.5245675538148544, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.056749701499939, + "step": 6030, + "unclipped_grad_norm": 0.88789963722229 + }, + { + "bits_per_byte": 1.5628784091210537, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.083304762840271, + "step": 6040, + "unclipped_grad_norm": 0.9149079322814941 + }, + { + "bits_per_byte": 1.4881958028971334, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.031538724899292, + "step": 6050, + "unclipped_grad_norm": 0.9299342632293701 + }, + { + "bits_per_byte": 1.4785307218839452, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0248394012451172, + "step": 6060, + "unclipped_grad_norm": 0.8783867359161377 + }, + { + "bits_per_byte": 1.5509557118499895, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.075040578842163, + "step": 6070, + "unclipped_grad_norm": 0.9337034821510315 + }, + { + "bits_per_byte": 1.4914187577743023, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0337727069854736, + "step": 6080, + "unclipped_grad_norm": 0.9420101046562195 + }, + { + "bits_per_byte": 1.4968499698885498, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0375373363494873, + "step": 6090, + "unclipped_grad_norm": 0.8664546012878418 + }, + { + "bits_per_byte": 1.4906561867003767, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0332441329956055, + "step": 6100, + "unclipped_grad_norm": 0.8687640428543091 + }, + { + "bits_per_byte": 1.5538661742505406, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.077057957649231, + "step": 6110, + "unclipped_grad_norm": 0.8986173868179321 + }, + { + "bits_per_byte": 1.5476213122150613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0727293491363525, + "step": 6120, + "unclipped_grad_norm": 0.9189077019691467 + }, + { + "bits_per_byte": 1.4789527673091625, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0251319408416748, + "step": 6130, + "unclipped_grad_norm": 0.8851786851882935 + }, + { + "bits_per_byte": 1.4768556108645414, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0236783027648926, + "step": 6140, + "unclipped_grad_norm": 0.9043483138084412 + }, + { + "bits_per_byte": 1.552988202817886, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0764493942260742, + "step": 6150, + "unclipped_grad_norm": 0.9005390405654907 + }, + { + "bits_per_byte": 1.3994871515820966, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.970050573348999, + "step": 6160, + "unclipped_grad_norm": 0.8554620742797852 + }, + { + "bits_per_byte": 1.5302097886417105, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0606606006622314, + "step": 6170, + "unclipped_grad_norm": 0.8954014778137207 + }, + { + "bits_per_byte": 1.4938230752333643, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0354392528533936, + "step": 6180, + "unclipped_grad_norm": 0.892278790473938 + }, + { + "bits_per_byte": 1.516749394489282, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.05133056640625, + "step": 6190, + "unclipped_grad_norm": 0.9185703992843628 + }, + { + "bits_per_byte": 1.5086576107661869, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0457217693328857, + "step": 6200, + "unclipped_grad_norm": 0.8927735090255737 + }, + { + "bits_per_byte": 1.5088584865023946, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.045861005783081, + "step": 6210, + "unclipped_grad_norm": 0.8906724452972412 + }, + { + "bits_per_byte": 1.5118191678369979, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0479131937026978, + "step": 6220, + "unclipped_grad_norm": 0.9077256917953491 + }, + { + "bits_per_byte": 1.4471287536282067, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0030732154846191, + "step": 6230, + "unclipped_grad_norm": 0.8839235305786133 + }, + { + "bits_per_byte": 1.5076786855174757, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0450432300567627, + "step": 6240, + "unclipped_grad_norm": 0.9515005946159363 + }, + { + "bits_per_byte": 1.5408861276419719, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0680608749389648, + "step": 6250, + "unclipped_grad_norm": 0.901561975479126 + }, + { + "bits_per_byte": 1.4200397662908057, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9842965602874756, + "step": 6260, + "unclipped_grad_norm": 0.889621376991272 + }, + { + "bits_per_byte": 1.5216213190229235, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0547075271606445, + "step": 6270, + "unclipped_grad_norm": 0.9411186575889587 + }, + { + "bits_per_byte": 1.5224643779774532, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0552918910980225, + "step": 6280, + "unclipped_grad_norm": 0.9406285881996155 + }, + { + "bits_per_byte": 1.452789390598705, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0069968700408936, + "step": 6290, + "unclipped_grad_norm": 0.8839805126190186 + }, + { + "bits_per_byte": 1.5074908804627336, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0449130535125732, + "step": 6300, + "unclipped_grad_norm": 0.9184122681617737 + }, + { + "bits_per_byte": 1.478450577968643, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0247838497161865, + "step": 6310, + "unclipped_grad_norm": 0.8948619365692139 + }, + { + "bits_per_byte": 1.4214500240278807, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.985274076461792, + "step": 6320, + "unclipped_grad_norm": 0.8749227523803711 + }, + { + "bits_per_byte": 1.5185149683830395, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0525543689727783, + "step": 6330, + "unclipped_grad_norm": 0.9816636443138123 + }, + { + "bits_per_byte": 1.4759477144506365, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.023048996925354, + "step": 6340, + "unclipped_grad_norm": 0.8950188159942627 + }, + { + "bits_per_byte": 1.587707716408768, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1005151271820068, + "step": 6350, + "unclipped_grad_norm": 1.01009202003479 + }, + { + "bits_per_byte": 1.4638088349874379, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0146349668502808, + "step": 6360, + "unclipped_grad_norm": 0.9816907644271851 + }, + { + "bits_per_byte": 1.4382974445064014, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9969518184661865, + "step": 6370, + "unclipped_grad_norm": 0.8970034122467041 + }, + { + "bits_per_byte": 1.4818757844432269, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0271580219268799, + "step": 6380, + "unclipped_grad_norm": 0.9058489799499512 + }, + { + "bits_per_byte": 1.527050123380059, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0584704875946045, + "step": 6390, + "unclipped_grad_norm": 0.929339587688446 + }, + { + "bits_per_byte": 1.5156593684481134, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0505750179290771, + "step": 6400, + "unclipped_grad_norm": 0.9009133577346802 + }, + { + "bits_per_byte": 1.4969940913499729, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0376372337341309, + "step": 6410, + "unclipped_grad_norm": 0.957128643989563 + }, + { + "bits_per_byte": 1.4783625228514012, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0247228145599365, + "step": 6420, + "unclipped_grad_norm": 0.9414587616920471 + }, + { + "bits_per_byte": 1.523227980947284, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.055821180343628, + "step": 6430, + "unclipped_grad_norm": 0.9234302639961243 + }, + { + "bits_per_byte": 1.4623229048839832, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.013604998588562, + "step": 6440, + "unclipped_grad_norm": 0.9177728295326233 + }, + { + "bits_per_byte": 1.5306256426914968, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0609488487243652, + "step": 6450, + "unclipped_grad_norm": 0.9453822374343872 + }, + { + "bits_per_byte": 1.5847898587542297, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0984926223754883, + "step": 6460, + "unclipped_grad_norm": 0.9647909998893738 + }, + { + "bits_per_byte": 1.5690988496201106, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0876164436340332, + "step": 6470, + "unclipped_grad_norm": 0.9592546820640564 + }, + { + "bits_per_byte": 1.4544314809491432, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0081350803375244, + "step": 6480, + "unclipped_grad_norm": 0.9334172010421753 + }, + { + "bits_per_byte": 1.5573938823850386, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0795031785964966, + "step": 6490, + "unclipped_grad_norm": 0.967822790145874 + }, + { + "bits_per_byte": 1.5280063469188563, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0591332912445068, + "step": 6500, + "unclipped_grad_norm": 0.9625112414360046 + }, + { + "bits_per_byte": 1.4687796495453256, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.018080472946167, + "step": 6510, + "unclipped_grad_norm": 0.9510544538497925 + }, + { + "bits_per_byte": 1.4284500618832987, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9901261329650879, + "step": 6520, + "unclipped_grad_norm": 0.8985839486122131 + }, + { + "bits_per_byte": 1.4546447394362132, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0082828998565674, + "step": 6530, + "unclipped_grad_norm": 0.907886803150177 + }, + { + "bits_per_byte": 1.4970680438898438, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0376884937286377, + "step": 6540, + "unclipped_grad_norm": 0.9381445050239563 + }, + { + "bits_per_byte": 1.4280130539674563, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9898232221603394, + "step": 6550, + "unclipped_grad_norm": 0.9031082391738892 + }, + { + "bits_per_byte": 1.5210671909218436, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.054323434829712, + "step": 6560, + "unclipped_grad_norm": 0.9433532357215881 + }, + { + "bits_per_byte": 1.479243074023819, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0253331661224365, + "step": 6570, + "unclipped_grad_norm": 0.9311387538909912 + }, + { + "bits_per_byte": 1.5051876888023787, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0433166027069092, + "step": 6580, + "unclipped_grad_norm": 0.9569925665855408 + }, + { + "bits_per_byte": 1.494823842278735, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0361329317092896, + "step": 6590, + "unclipped_grad_norm": 0.8958868384361267 + }, + { + "bits_per_byte": 1.4302109062541577, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9913466572761536, + "step": 6600, + "unclipped_grad_norm": 0.9136191010475159 + }, + { + "bits_per_byte": 1.436879103584736, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 0.9959686994552612, + "step": 6610, + "unclipped_grad_norm": 0.8710504770278931 + }, + { + "bits_per_byte": 1.4278971376607748, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9897428750991821, + "step": 6620, + "unclipped_grad_norm": 0.9145034551620483 + }, + { + "bits_per_byte": 1.4582168190946336, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.010758876800537, + "step": 6630, + "unclipped_grad_norm": 0.8999127149581909 + }, + { + "bits_per_byte": 1.5240701799885592, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.056404948234558, + "step": 6640, + "unclipped_grad_norm": 0.9801992774009705 + }, + { + "bits_per_byte": 1.5169175935218258, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0514471530914307, + "step": 6650, + "unclipped_grad_norm": 0.8882671594619751 + }, + { + "bits_per_byte": 1.5152221885496202, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.050271987915039, + "step": 6660, + "unclipped_grad_norm": 0.9029646515846252 + }, + { + "bits_per_byte": 1.4896082964086694, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0325177907943726, + "step": 6670, + "unclipped_grad_norm": 0.9797500371932983 + }, + { + "bits_per_byte": 1.4764526555135697, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.023398995399475, + "step": 6680, + "unclipped_grad_norm": 0.9356769919395447 + }, + { + "bits_per_byte": 1.4958203097578342, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0368236303329468, + "step": 6690, + "unclipped_grad_norm": 0.939789891242981 + }, + { + "bits_per_byte": 1.5480882451121538, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.073053002357483, + "step": 6700, + "unclipped_grad_norm": 1.0080333948135376 + }, + { + "bits_per_byte": 1.525887864625528, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0576648712158203, + "step": 6710, + "unclipped_grad_norm": 0.9197862148284912 + }, + { + "bits_per_byte": 1.534906462854123, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0639160871505737, + "step": 6720, + "unclipped_grad_norm": 0.9543269872665405 + }, + { + "bits_per_byte": 1.4820493149379474, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0272783041000366, + "step": 6730, + "unclipped_grad_norm": 0.9817740321159363 + }, + { + "bits_per_byte": 1.4384499931177168, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9970575571060181, + "step": 6740, + "unclipped_grad_norm": 0.9710240364074707 + }, + { + "bits_per_byte": 1.511069151496585, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0473933219909668, + "step": 6750, + "unclipped_grad_norm": 0.9612263441085815 + }, + { + "bits_per_byte": 1.4780942299160553, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0245368480682373, + "step": 6760, + "unclipped_grad_norm": 0.9238441586494446 + }, + { + "bits_per_byte": 1.3767874194686909, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9543163180351257, + "step": 6770, + "unclipped_grad_norm": 0.9215719699859619 + }, + { + "bits_per_byte": 1.4785716537548506, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0248677730560303, + "step": 6780, + "unclipped_grad_norm": 0.9403936266899109 + }, + { + "bits_per_byte": 1.4459957319243224, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0022878646850586, + "step": 6790, + "unclipped_grad_norm": 0.9651554822921753 + }, + { + "bits_per_byte": 1.5175859181030786, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.051910400390625, + "step": 6800, + "unclipped_grad_norm": 0.9926899075508118 + }, + { + "bits_per_byte": 1.5110634760691066, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0473893880844116, + "step": 6810, + "unclipped_grad_norm": 0.9793192148208618 + }, + { + "bits_per_byte": 1.5052492585913877, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0433592796325684, + "step": 6820, + "unclipped_grad_norm": 1.0081095695495605 + }, + { + "bits_per_byte": 1.466343515295854, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0163918733596802, + "step": 6830, + "unclipped_grad_norm": 0.960361659526825 + }, + { + "bits_per_byte": 1.4376153613130795, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9964790344238281, + "step": 6840, + "unclipped_grad_norm": 0.9702624678611755 + }, + { + "bits_per_byte": 1.4922177891702109, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0343265533447266, + "step": 6850, + "unclipped_grad_norm": 0.9893983006477356 + }, + { + "bits_per_byte": 1.4659078832412185, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.016089916229248, + "step": 6860, + "unclipped_grad_norm": 0.9693014621734619 + }, + { + "bits_per_byte": 1.413224953750367, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9795728921890259, + "step": 6870, + "unclipped_grad_norm": 0.9252057075500488 + }, + { + "bits_per_byte": 1.4200107012228098, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9842764139175415, + "step": 6880, + "unclipped_grad_norm": 0.8884434700012207 + }, + { + "bits_per_byte": 1.4366857090938405, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9958346486091614, + "step": 6890, + "unclipped_grad_norm": 0.9679457545280457 + }, + { + "bits_per_byte": 1.5416700245646047, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0686042308807373, + "step": 6900, + "unclipped_grad_norm": 0.9985603094100952 + }, + { + "bits_per_byte": 1.4893800754309745, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0323596000671387, + "step": 6910, + "unclipped_grad_norm": 0.9936915636062622 + }, + { + "bits_per_byte": 1.4499423897963222, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.00502347946167, + "step": 6920, + "unclipped_grad_norm": 0.923648476600647 + }, + { + "bits_per_byte": 1.4406846496917027, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9986065030097961, + "step": 6930, + "unclipped_grad_norm": 0.9748247861862183 + }, + { + "bits_per_byte": 1.408974058568991, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9766263961791992, + "step": 6940, + "unclipped_grad_norm": 0.9357670545578003 + }, + { + "bits_per_byte": 1.434796565665438, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9945251941680908, + "step": 6950, + "unclipped_grad_norm": 0.9624595642089844 + }, + { + "bits_per_byte": 1.49675830313564, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0374737977981567, + "step": 6960, + "unclipped_grad_norm": 0.9387544989585876 + }, + { + "bits_per_byte": 1.4929531869853003, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0348362922668457, + "step": 6970, + "unclipped_grad_norm": 0.9343914985656738 + }, + { + "bits_per_byte": 1.4660633555575984, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.016197681427002, + "step": 6980, + "unclipped_grad_norm": 0.9326976537704468 + }, + { + "bits_per_byte": 1.492015881538098, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034186601638794, + "step": 6990, + "unclipped_grad_norm": 1.0069916248321533 + }, + { + "bits_per_byte": 1.52302985693349, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0556838512420654, + "step": 7000, + "unclipped_grad_norm": 0.9719612002372742 + }, + { + "bits_per_byte": 1.512785538352196, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0485830307006836, + "step": 7010, + "unclipped_grad_norm": 0.9873834848403931 + }, + { + "bits_per_byte": 1.4473323810868282, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0032143592834473, + "step": 7020, + "unclipped_grad_norm": 0.939332127571106 + }, + { + "bits_per_byte": 1.5166639191118032, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0512713193893433, + "step": 7030, + "unclipped_grad_norm": 0.9837853312492371 + }, + { + "bits_per_byte": 1.5072451172546506, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0447427034378052, + "step": 7040, + "unclipped_grad_norm": 0.9734936952590942 + }, + { + "bits_per_byte": 1.4796651194490362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0256257057189941, + "step": 7050, + "unclipped_grad_norm": 1.005554437637329 + }, + { + "bits_per_byte": 1.4927048440374544, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0346641540527344, + "step": 7060, + "unclipped_grad_norm": 0.9903765320777893 + }, + { + "bits_per_byte": 1.4561590466770602, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.009332537651062, + "step": 7070, + "unclipped_grad_norm": 1.009922981262207 + }, + { + "bits_per_byte": 1.4353391709289103, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9949012994766235, + "step": 7080, + "unclipped_grad_norm": 0.9552373290061951 + }, + { + "bits_per_byte": 1.4780041110070032, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0244743824005127, + "step": 7090, + "unclipped_grad_norm": 1.0046122074127197 + }, + { + "bits_per_byte": 1.4728850474040718, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0209261178970337, + "step": 7100, + "unclipped_grad_norm": 0.9868813157081604 + }, + { + "bits_per_byte": 1.4862149067244954, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.030165672302246, + "step": 7110, + "unclipped_grad_norm": 0.9593554735183716 + }, + { + "bits_per_byte": 1.5328834309320238, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.062513828277588, + "step": 7120, + "unclipped_grad_norm": 1.0129024982452393 + }, + { + "bits_per_byte": 1.4811555211014134, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0266587734222412, + "step": 7130, + "unclipped_grad_norm": 1.0546875 + }, + { + "bits_per_byte": 1.4724865636020226, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0206499099731445, + "step": 7140, + "unclipped_grad_norm": 0.9776664972305298 + }, + { + "bits_per_byte": 1.4357171887955067, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.9951633214950562, + "step": 7150, + "unclipped_grad_norm": 0.9291411638259888 + }, + { + "bits_per_byte": 1.5232358921492237, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0558266639709473, + "step": 7160, + "unclipped_grad_norm": 1.0189123153686523 + }, + { + "bits_per_byte": 1.4618840051589814, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0133007764816284, + "step": 7170, + "unclipped_grad_norm": 0.9592419266700745 + }, + { + "bits_per_byte": 1.4762450724539784, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0232551097869873, + "step": 7180, + "unclipped_grad_norm": 0.9680334329605103 + }, + { + "bits_per_byte": 1.4441101141402626, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0009808540344238, + "step": 7190, + "unclipped_grad_norm": 1.0148857831954956 + }, + { + "bits_per_byte": 1.472608843266786, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0207346677780151, + "step": 7200, + "unclipped_grad_norm": 0.9935786128044128 + }, + { + "bits_per_byte": 1.5299951542934338, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.060511827468872, + "step": 7210, + "unclipped_grad_norm": 0.9725626111030579 + }, + { + "bits_per_byte": 1.5238232129019202, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0562337636947632, + "step": 7220, + "unclipped_grad_norm": 1.0141370296478271 + }, + { + "bits_per_byte": 1.4390708504873315, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9974879026412964, + "step": 7230, + "unclipped_grad_norm": 1.0297694206237793 + }, + { + "bits_per_byte": 1.4632025961431465, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0142147541046143, + "step": 7240, + "unclipped_grad_norm": 0.9345177412033081 + }, + { + "bits_per_byte": 1.5031187375124992, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0418825149536133, + "step": 7250, + "unclipped_grad_norm": 1.0408201217651367 + }, + { + "bits_per_byte": 1.3820238612308362, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9579459428787231, + "step": 7260, + "unclipped_grad_norm": 0.9544695615768433 + }, + { + "bits_per_byte": 1.4274823155068936, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9894553422927856, + "step": 7270, + "unclipped_grad_norm": 0.9940797686576843 + }, + { + "bits_per_byte": 1.4990891840027836, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0390894412994385, + "step": 7280, + "unclipped_grad_norm": 1.013985514640808 + }, + { + "bits_per_byte": 1.4370805812602216, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.99610835313797, + "step": 7290, + "unclipped_grad_norm": 0.9820091724395752 + }, + { + "bits_per_byte": 1.4451605841717328, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.001708984375, + "step": 7300, + "unclipped_grad_norm": 0.9996907114982605 + }, + { + "bits_per_byte": 1.4502488628801598, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0052359104156494, + "step": 7310, + "unclipped_grad_norm": 0.9723894000053406 + }, + { + "bits_per_byte": 1.4141343980081293, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9802032709121704, + "step": 7320, + "unclipped_grad_norm": 0.9815637469291687 + }, + { + "bits_per_byte": 1.429772436485783, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9910427331924438, + "step": 7330, + "unclipped_grad_norm": 1.0089058876037598 + }, + { + "bits_per_byte": 1.4033304478809279, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9727145433425903, + "step": 7340, + "unclipped_grad_norm": 0.9830491542816162 + }, + { + "bits_per_byte": 1.4800763299672492, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.02591073513031, + "step": 7350, + "unclipped_grad_norm": 0.9842677712440491 + }, + { + "bits_per_byte": 1.4409101189469837, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9987627863883972, + "step": 7360, + "unclipped_grad_norm": 1.0266367197036743 + }, + { + "bits_per_byte": 1.4700960047550296, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0189929008483887, + "step": 7370, + "unclipped_grad_norm": 0.9935671091079712 + }, + { + "bits_per_byte": 1.4111584102175994, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9781404733657837, + "step": 7380, + "unclipped_grad_norm": 0.9791932702064514 + }, + { + "bits_per_byte": 1.4805810990475317, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0262606143951416, + "step": 7390, + "unclipped_grad_norm": 0.9806758761405945 + }, + { + "bits_per_byte": 1.422500752033327, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9860023856163025, + "step": 7400, + "unclipped_grad_norm": 1.014562726020813 + }, + { + "bits_per_byte": 1.419007526420327, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9835810661315918, + "step": 7410, + "unclipped_grad_norm": 0.9609370231628418 + }, + { + "bits_per_byte": 1.483305992167802, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0281493663787842, + "step": 7420, + "unclipped_grad_norm": 0.9924190640449524 + }, + { + "bits_per_byte": 1.4892452410326982, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0322661399841309, + "step": 7430, + "unclipped_grad_norm": 1.0244429111480713 + }, + { + "bits_per_byte": 1.50888548777858, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0458797216415405, + "step": 7440, + "unclipped_grad_norm": 1.0394790172576904 + }, + { + "bits_per_byte": 1.4553271665948369, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0087559223175049, + "step": 7450, + "unclipped_grad_norm": 0.99248868227005 + }, + { + "bits_per_byte": 1.4425173828106221, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.999876856803894, + "step": 7460, + "unclipped_grad_norm": 1.0004469156265259 + }, + { + "bits_per_byte": 1.437790267669007, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9966002702713013, + "step": 7470, + "unclipped_grad_norm": 1.0109968185424805 + }, + { + "bits_per_byte": 1.4682612938356252, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.017721176147461, + "step": 7480, + "unclipped_grad_norm": 1.0299010276794434 + }, + { + "bits_per_byte": 1.5222762289574092, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.055161476135254, + "step": 7490, + "unclipped_grad_norm": 1.0327476263046265 + }, + { + "bits_per_byte": 1.4747042798848986, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0221871137619019, + "step": 7500, + "unclipped_grad_norm": 0.9886637926101685 + }, + { + "bits_per_byte": 1.4352613487793948, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9948473572731018, + "step": 7510, + "unclipped_grad_norm": 0.9710773825645447 + }, + { + "bits_per_byte": 1.4570311706995855, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.009937047958374, + "step": 7520, + "unclipped_grad_norm": 1.0144232511520386 + }, + { + "bits_per_byte": 1.502727820947088, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0416115522384644, + "step": 7530, + "unclipped_grad_norm": 1.0307852029800415 + }, + { + "bits_per_byte": 1.4572943041554056, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0101194381713867, + "step": 7540, + "unclipped_grad_norm": 0.9732069969177246 + }, + { + "bits_per_byte": 1.4747278415080667, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0222034454345703, + "step": 7550, + "unclipped_grad_norm": 1.0250486135482788 + }, + { + "bits_per_byte": 1.4513507557242378, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0059996843338013, + "step": 7560, + "unclipped_grad_norm": 1.0031380653381348 + }, + { + "bits_per_byte": 1.4127096077370564, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.979215681552887, + "step": 7570, + "unclipped_grad_norm": 0.9720339775085449 + }, + { + "bits_per_byte": 1.587541409185384, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1003998517990112, + "step": 7580, + "unclipped_grad_norm": 1.080464482307434 + }, + { + "bits_per_byte": 1.4421111597892842, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9995952844619751, + "step": 7590, + "unclipped_grad_norm": 0.9815735220909119 + }, + { + "bits_per_byte": 1.4267507013101233, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9889482259750366, + "step": 7600, + "unclipped_grad_norm": 1.029758334159851 + }, + { + "bits_per_byte": 1.4539179407536669, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0077791213989258, + "step": 7610, + "unclipped_grad_norm": 1.031221628189087 + }, + { + "bits_per_byte": 1.4028547438686412, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9723848104476929, + "step": 7620, + "unclipped_grad_norm": 0.9562389850616455 + }, + { + "bits_per_byte": 1.3938100042771153, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9661154747009277, + "step": 7630, + "unclipped_grad_norm": 1.023301124572754 + }, + { + "bits_per_byte": 1.4184527103886437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.983196496963501, + "step": 7640, + "unclipped_grad_norm": 0.9981763958930969 + }, + { + "bits_per_byte": 1.5608753271864544, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.081916332244873, + "step": 7650, + "unclipped_grad_norm": 1.0295648574829102 + }, + { + "bits_per_byte": 1.4665378556913289, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0165265798568726, + "step": 7660, + "unclipped_grad_norm": 1.0587892532348633 + }, + { + "bits_per_byte": 1.4931012640476933, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.034938931465149, + "step": 7670, + "unclipped_grad_norm": 1.0342050790786743 + }, + { + "bits_per_byte": 1.4465245785757257, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0026544332504272, + "step": 7680, + "unclipped_grad_norm": 1.0567405223846436 + }, + { + "bits_per_byte": 1.4541416901824393, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0079342126846313, + "step": 7690, + "unclipped_grad_norm": 1.0190099477767944 + }, + { + "bits_per_byte": 1.3949941048283057, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9669362306594849, + "step": 7700, + "unclipped_grad_norm": 1.002891182899475 + }, + { + "bits_per_byte": 1.4762163513512843, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0232352018356323, + "step": 7710, + "unclipped_grad_norm": 0.9984713792800903 + }, + { + "bits_per_byte": 1.480895139368007, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0264782905578613, + "step": 7720, + "unclipped_grad_norm": 1.0342158079147339 + }, + { + "bits_per_byte": 1.486845911070511, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.030603051185608, + "step": 7730, + "unclipped_grad_norm": 1.027107834815979 + }, + { + "bits_per_byte": 1.4387832095037634, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9972885251045227, + "step": 7740, + "unclipped_grad_norm": 1.0455139875411987 + }, + { + "bits_per_byte": 1.4381068877292456, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9968197345733643, + "step": 7750, + "unclipped_grad_norm": 1.0159157514572144 + }, + { + "bits_per_byte": 1.4538223183997872, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0077128410339355, + "step": 7760, + "unclipped_grad_norm": 0.9867706894874573 + }, + { + "bits_per_byte": 1.3986834766546148, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9694935083389282, + "step": 7770, + "unclipped_grad_norm": 0.9943742156028748 + }, + { + "bits_per_byte": 1.4760859885019304, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0231448411941528, + "step": 7780, + "unclipped_grad_norm": 1.0422483682632446 + }, + { + "bits_per_byte": 1.4162891686407895, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9816968441009521, + "step": 7790, + "unclipped_grad_norm": 0.9937946796417236 + }, + { + "bits_per_byte": 1.45477768202533, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0083750486373901, + "step": 7800, + "unclipped_grad_norm": 1.0428839921951294 + }, + { + "bits_per_byte": 1.4292601001688625, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9906876087188721, + "step": 7810, + "unclipped_grad_norm": 1.028254508972168 + }, + { + "bits_per_byte": 1.4070698666586379, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.975306510925293, + "step": 7820, + "unclipped_grad_norm": 1.0253064632415771 + }, + { + "bits_per_byte": 1.4530960356651936, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0072094202041626, + "step": 7830, + "unclipped_grad_norm": 1.0885143280029297 + }, + { + "bits_per_byte": 1.4330722676078875, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9933300018310547, + "step": 7840, + "unclipped_grad_norm": 1.0074925422668457 + }, + { + "bits_per_byte": 1.3858685333908742, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9606108665466309, + "step": 7850, + "unclipped_grad_norm": 0.9880329966545105 + }, + { + "bits_per_byte": 1.4517502714221922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0062766075134277, + "step": 7860, + "unclipped_grad_norm": 1.0082415342330933 + }, + { + "bits_per_byte": 1.4675547891058807, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0172314643859863, + "step": 7870, + "unclipped_grad_norm": 1.023923397064209 + }, + { + "bits_per_byte": 1.3867994754799946, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9612561464309692, + "step": 7880, + "unclipped_grad_norm": 1.0151392221450806 + }, + { + "bits_per_byte": 1.4568015738606837, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0097779035568237, + "step": 7890, + "unclipped_grad_norm": 1.0505098104476929 + }, + { + "bits_per_byte": 1.446776189193938, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.00282883644104, + "step": 7900, + "unclipped_grad_norm": 1.0279992818832397 + }, + { + "bits_per_byte": 1.461830518554563, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0132637023925781, + "step": 7910, + "unclipped_grad_norm": 1.0125948190689087 + }, + { + "bits_per_byte": 1.4593548282953928, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0115476846694946, + "step": 7920, + "unclipped_grad_norm": 1.0411789417266846 + }, + { + "bits_per_byte": 1.4317973602170415, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9924463033676147, + "step": 7930, + "unclipped_grad_norm": 1.0330348014831543 + }, + { + "bits_per_byte": 1.431196452834927, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9920297861099243, + "step": 7940, + "unclipped_grad_norm": 1.0142537355422974 + }, + { + "bits_per_byte": 1.49434246283897, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.035799264907837, + "step": 7950, + "unclipped_grad_norm": 1.0716009140014648 + }, + { + "bits_per_byte": 1.4186909923514142, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9833616614341736, + "step": 7960, + "unclipped_grad_norm": 1.0115852355957031 + }, + { + "bits_per_byte": 1.4420433126335188, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9995482563972473, + "step": 7970, + "unclipped_grad_norm": 1.0335451364517212 + }, + { + "bits_per_byte": 1.4872165336831205, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0308599472045898, + "step": 7980, + "unclipped_grad_norm": 1.0748283863067627 + }, + { + "bits_per_byte": 1.5234063269562288, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.055944800376892, + "step": 7990, + "unclipped_grad_norm": 1.076164722442627 + }, + { + "bits_per_byte": 1.4309937712808851, + "learning_rate": 3e-05, + "loss_nats": 0.9918892979621887, + "step": 8000, + "unclipped_grad_norm": 1.0322442054748535 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-baseline.json b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-baseline.json new file mode 100644 index 0000000..cd8ae6f --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-baseline.json @@ -0,0 +1,700 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "0ae9ad1697ca15ec4c84270ad82b9fa98056f14f20fc7372c4b9886c29845415", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002865509013645351, + 0.00024785567075014114, + 0.00021551651298068464, + 0.00019683866412378848, + 0.0001823508064262569, + 0.0001679604029050097, + 0.00015584587526973337, + 0.00014737028686795384, + 0.00014130656199995428, + 0.00013612695329356939, + 0.00013129493163432926, + 0.00012807638267986476, + 0.00012550120300147682, + 0.00012324050476308912, + 0.00012108208466088399, + 0.00011888353037647903 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023669043730478734, + "first_to_last_ratio": 1.9372775995887173, + "imbalance_abs_log_ratio": 0.6612836883477485, + "last_quartile_mean": 0.00012217683070048224, + "mean": 0.00016411257956860936, + "normalized": [ + 1.7460629899168627, + 1.5102783187106137, + 1.313223602646409, + 1.1994124072706904, + 1.1111324123086055, + 1.0234462424910682, + 0.9496278449793059, + 0.8979828801383491, + 0.8610343117596252, + 0.8294729974472175, + 0.8000296624393728, + 0.7804178266926869, + 0.7647262832098098, + 0.7509509940495869, + 0.7377989242455608, + 0.7244023016942358 + ], + "population_cv": 0.29361872380036635 + }, + "activation_output_rms_by_block": [ + 0.02866268903017044, + 0.029099803417921066, + 0.02953805774450302, + 0.030088091269135475, + 0.030644793063402176, + 0.03131929785013199, + 0.03213750571012497, + 0.03286394104361534, + 0.033869802951812744, + 0.03499744459986687, + 0.03588006645441055, + 0.037365153431892395, + 0.03789033368229866, + 0.039445169270038605, + 0.04050002992153168, + 0.04121527820825577 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0293471603654325, + "first_to_last_ratio": 0.7380574840395957, + "imbalance_abs_log_ratio": 0.3037335657624931, + "last_quartile_mean": 0.03976270277053118, + "mean": 0.034094841103069484, + "normalized": [ + 0.8406752488894867, + 0.8534957922212247, + 0.8663497699023965, + 0.8824822259232266, + 0.8988102619619861, + 0.9185934539320196, + 0.942591449919727, + 0.9638977622528551, + 0.9933996421752943, + 1.0264733158329964, + 1.0523605710888722, + 1.0959180985456622, + 1.1113216092650298, + 1.1569248600043318, + 1.1878638706395246, + 1.2088420674453664 + ], + "population_cv": 0.11952987076282337 + }, + "bits_per_byte": 8.096463027059821, + "branch_output_rms_by_sublayer": [ + 0.0033055038657039404, + 0.0037518907338380814, + 0.0033751516602933407, + 0.003911525942385197, + 0.004229962360113859, + 0.0038461871445178986, + 0.004243654198944569, + 0.0038366704247891903, + 0.004372371360659599, + 0.003851204412057996, + 0.005078981164842844, + 0.003818925702944398, + 0.0054668826051056385, + 0.0038749484810978174, + 0.005823117680847645, + 0.0037566579412668943, + 0.0065947300754487514, + 0.003876061411574483, + 0.006763988174498081, + 0.003795720636844635, + 0.006863070651888847, + 0.0038706199266016483, + 0.007856340147554874, + 0.003839300014078617, + 0.00830968376249075, + 0.003927029203623533, + 0.009655521251261234, + 0.0038513424806296825, + 0.009004893712699413, + 0.0039484030567109585, + 0.008608299307525158, + 0.003791053779423237 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00809059897248305, + 0.007313812062277777, + 0.007090710224412361, + 0.0067033738367094624, + 0.006535407867454384, + 0.006785021795736036, + 0.006295993569919972, + 0.005509300326230647, + 0.006231736048910802, + 0.006047390980174007, + 0.005629405359532731, + 0.005784042737956458, + 0.0059591736113607996, + 0.0061983173180850636, + 0.006571690039252182, + 0.005060205437758869 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007299623773970663, + "first_to_last_ratio": 1.227374872012571, + "imbalance_abs_log_ratio": 0.2048776382299483, + "last_quartile_mean": 0.005947346601614228, + "mean": 0.006362886261765913, + "normalized": [ + 1.271529717747562, + 1.1494488132258316, + 1.1143858200043408, + 1.05351149791715, + 1.0271137340180259, + 1.066343404015675, + 0.9894870520870547, + 0.8658492545019393, + 0.9793882512652816, + 0.9504163254515969, + 0.8847251275509024, + 0.909028151691524, + 0.9365519618304368, + 0.9741361173356609, + 1.0328158902888074, + 0.7952688810682109 + ], + "population_cv": 0.11384977604868386 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02823694422841072, + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679 + ], + "loss_nats": 5.6120405197143555, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679, + 0.04121527820825577 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.69471885683015e-05, + 5.527245593839325e-05, + 5.393734318204224e-05, + 5.292302375892177e-05, + 5.23102717124857e-05, + 5.178990977583453e-05, + 5.1564093155320734e-05, + 5.129844430484809e-05, + 5.121947833686136e-05, + 5.116340616950765e-05, + 5.109804988023825e-05, + 5.104695082991384e-05, + 5.09646451973822e-05, + 5.100828275317326e-05, + 5.106439857627265e-05, + 5.11144389747642e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.477000286191469e-05, + "first_to_last_ratio": 1.0731232762518041, + "imbalance_abs_log_ratio": 0.07057334637995329, + "last_quartile_mean": 5.103794137539808e-05, + "mean": 5.2170148819641327e-05, + "normalized": [ + 1.0915665348238701, + 1.0594651767139285, + 1.033873669184083, + 1.0144311441756324, + 1.002685882559561, + 0.9927115591500164, + 0.988383094968431, + 0.9832911246274795, + 0.9817775010367221, + 0.9807027069519371, + 0.9794499543578176, + 0.9784704852268963, + 0.9768928467805085, + 0.9777292936141551, + 0.9788049244944386, + 0.9797641013345227 + ], + "population_cv": 0.032815404485150704 + }, + "activation_output_rms_by_block": [ + 0.028743742033839226, + 0.02942933700978756, + 0.030596865341067314, + 0.03249936178326607, + 0.03542664647102356, + 0.03894684836268425, + 0.04318666458129883, + 0.04936420917510986, + 0.05370106175541878, + 0.06048284471035004, + 0.0664696991443634, + 0.07231792062520981, + 0.07627613097429276, + 0.07997097074985504, + 0.08494995534420013, + 0.08972473442554474 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03031732654199004, + "first_to_last_ratio": 0.3664591129538783, + "imbalance_abs_log_ratio": 1.0038683247140607, + "last_quartile_mean": 0.08273044787347317, + "mean": 0.05450543703045696, + "normalized": [ + 0.5273555006590916, + 0.5399339701348108, + 0.5613543713807885, + 0.5962590808162097, + 0.6499653686150171, + 0.7145497859400932, + 0.7923368187501488, + 0.9056749539963465, + 0.9852422929002714, + 1.1096662646068314, + 1.21950584686113, + 1.326801958945847, + 1.399422427007981, + 1.4672108895332452, + 1.5585592919240543, + 1.6461611779281335 + ], + "population_cv": 0.3808757530834043 + }, + "bits_per_byte": 6.7509657011324835, + "branch_output_rms_by_sublayer": [ + 0.003436450148001313, + 0.003791899885982275, + 0.003964710980653763, + 0.003938332665711641, + 0.0056790695525705814, + 0.003919641952961683, + 0.006960175931453705, + 0.0038887569680809975, + 0.008646421134471893, + 0.003928068559616804, + 0.009785857982933521, + 0.0038193853106349707, + 0.010625048540532589, + 0.003996006678789854, + 0.012892307713627815, + 0.0039014811627566814, + 0.012757784686982632, + 0.003861474571749568, + 0.014726920053362846, + 0.003962590359151363, + 0.012820222415030003, + 0.004551138263195753, + 0.01401793584227562, + 0.004161422606557608, + 0.01284075528383255, + 0.004107494372874498, + 0.012963366694748402, + 0.0034911984112113714, + 0.014392351731657982, + 0.004168955609202385, + 0.013408888131380081, + 0.003909120801836252 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004507690637370199, + 0.0004530669153600251, + 0.0005174622333717066, + 0.0005579942111201043, + 0.0006494724560479704, + 0.0007477190266084469, + 0.0008433132451421535, + 0.0008983201732199723, + 0.001008715304343127, + 0.0011143118429534733, + 0.0010428854825857738, + 0.0010916116075341876, + 0.0011085001103672855, + 0.0012404769719650786, + 0.0013545740271648265, + 0.0014243271893498689 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.000494823105897214, + "first_to_last_ratio": 0.385986622193018, + "imbalance_abs_log_ratio": 0.9519525676489338, + "last_quartile_mean": 0.001281969574711765, + "mean": 0.0009064699913044387, + "normalized": [ + 0.49727963204644987, + 0.4998145771025995, + 0.5708542349284638, + 0.6155683215912455, + 0.7164853357289404, + 0.824869034585972, + 0.930326710461313, + 0.99100927977468, + 1.112795033503044, + 1.2292870736404011, + 1.1504909071341995, + 1.2042446170372658, + 1.2228756836970622, + 1.3684699812069823, + 1.494339625314625, + 1.571289952246756 + ], + "population_cv": 0.338812252534038 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028241552412509918, + 0.028533460572361946, + 0.028743742033839226, + 0.029236938804388046, + 0.02942933700978756, + 0.03035305254161358, + 0.030596865341067314, + 0.032195623964071274, + 0.03249936178326607, + 0.03506496921181679, + 0.03542664647102356, + 0.03852483630180359, + 0.03894684836268425, + 0.04251723736524582, + 0.04318666458129883, + 0.04861506074666977, + 0.04936420917510986, + 0.052915990352630615, + 0.05370106175541878, + 0.0598941408097744, + 0.06048284471035004, + 0.06538087129592896, + 0.0664696991443634, + 0.07142146676778793, + 0.07231792062520981, + 0.07521561533212662, + 0.07627613097429276, + 0.07935076206922531, + 0.07997097074985504, + 0.08368266373872757, + 0.08494995534420013, + 0.08860929310321808 + ], + "loss_nats": 4.679412841796875, + "loss_scale": 1.0, + "output_weights": null, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.028533460572361946, + 0.028743742033839226, + 0.029236938804388046, + 0.02942933700978756, + 0.03035305254161358, + 0.030596865341067314, + 0.032195623964071274, + 0.03249936178326607, + 0.03506496921181679, + 0.03542664647102356, + 0.03852483630180359, + 0.03894684836268425, + 0.04251723736524582, + 0.04318666458129883, + 0.04861506074666977, + 0.04936420917510986, + 0.052915990352630615, + 0.05370106175541878, + 0.0598941408097744, + 0.06048284471035004, + 0.06538087129592896, + 0.0664696991443634, + 0.07142146676778793, + 0.07231792062520981, + 0.07521561533212662, + 0.07627613097429276, + 0.07935076206922531, + 0.07997097074985504, + 0.08368266373872757, + 0.08494995534420013, + 0.08860929310321808, + 0.08972473442554474 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.076786148009306, + "cross_entropy_nats": 5.5984015464782715, + "step": 0 + }, + { + "bits_per_byte": 6.752401670275862, + "cross_entropy_nats": 4.680408179759979, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0", + "final_optimizer_state": "a7bce44db1478ce53933758aa5033bbb1e0aa21296e9615c33146920f30a2057", + "final_public_parameters": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 1648265728, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.088097790921855, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.6062421798706055, + "step": 1, + "unclipped_grad_norm": 19.475919723510742 + }, + { + "bits_per_byte": 7.474302716882146, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.180791854858398, + "step": 10, + "unclipped_grad_norm": 12.938376426696777 + }, + { + "bits_per_byte": 6.789615018295581, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.706202507019043, + "step": 20, + "unclipped_grad_norm": 4.482712745666504 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-block.json b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-block.json new file mode 100644 index 0000000..c5f7945 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-16-block.json @@ -0,0 +1,1450 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "a0d2c00339b4ed2c671f57d7a804a28f3934cfaeaa7e1412046d0ce71e823782", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027713447343558073, + 0.00024001447309274226, + 0.00020889485313091427, + 0.00019106797117274255, + 0.00017709653184283525, + 0.00016345064796041697, + 0.0001520135992905125, + 0.00014396758342627436, + 0.00013824697816744447, + 0.0001333381951553747, + 0.00012879272981081158, + 0.00012578140012919903, + 0.00012338913802523166, + 0.0001212825154652819, + 0.00011928759340662509, + 0.00011723516217898577 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022927794270799495, + "first_to_last_ratio": 1.9059069547229377, + "imbalance_abs_log_ratio": 0.6449579870010378, + "last_quartile_mean": 0.00012029860226903111, + "mean": 0.00016006211535568582, + "normalized": [ + 1.7314182860806244, + 1.4995083162520277, + 1.3050861702453052, + 1.1937113960299495, + 1.1064237870985023, + 1.021170110098828, + 0.9497162957812465, + 0.8994482117542517, + 0.8637083038683806, + 0.8330403159990363, + 0.8046421823465957, + 0.7858286756343993, + 0.7708828397715447, + 0.7577215582573746, + 0.7452581339534802, + 0.7324354168284537 + ], + "population_cv": 0.28697332073733334 + }, + "activation_output_rms_by_block": [ + 0.005004490725696087, + 0.007150472607463598, + 0.0056586493737995625, + 0.007886149920523167, + 0.005775847006589174, + 0.008285986259579659, + 0.006597128231078386, + 0.009296777658164501, + 0.007501539774239063, + 0.01081777922809124, + 0.007698386441916227, + 0.011321330443024635, + 0.009009703993797302, + 0.013792803511023521, + 0.00953624676913023, + 0.012954792007803917 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0064249406568706036, + "first_to_last_ratio": 0.567404514268178, + "imbalance_abs_log_ratio": 0.5666828005795043, + "last_quartile_mean": 0.011323386570438743, + "mean": 0.008643005246995017, + "normalized": [ + 0.5790220626599802, + 0.8273132322752738, + 0.6547085431617609, + 0.9124314628022479, + 0.6682683674867963, + 0.9586927258271091, + 0.7632910130850682, + 1.0756417926966766, + 0.8679318778438997, + 1.251622429808468, + 0.8907071350665657, + 1.3098835554867694, + 1.04242722714183, + 1.5958342170183197, + 1.103348487778111, + 1.4988758698611242 + ], + "population_cv": 0.28641618387108014 + }, + "bits_per_byte": 8.092078157393424, + "branch_output_rms_by_sublayer": [ + 0.0033052791841328144, + 0.003737553022801876, + 0.0033692142460495234, + 0.0038969451561570168, + 0.004222586750984192, + 0.003809630637988448, + 0.004224879201501608, + 0.003801452461630106, + 0.004351162351667881, + 0.0037875697016716003, + 0.005037518683820963, + 0.0037579576019197702, + 0.005422801710665226, + 0.003782435320317745, + 0.005750134587287903, + 0.003670723643153906, + 0.006515308283269405, + 0.003754077944904566, + 0.006654165219515562, + 0.0036852110642939806, + 0.006751408334821463, + 0.003720880253240466, + 0.007695567328482866, + 0.003703791182488203, + 0.008140962570905685, + 0.003749529365450144, + 0.009426429867744446, + 0.0036915375385433435, + 0.008798080496490002, + 0.003752112854272127, + 0.008380400948226452, + 0.0036098985001444817 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007836087849617155, + 0.00707663314616961, + 0.006860545956961424, + 0.006477463491545069, + 0.00630808320659222, + 0.0065403273634858875, + 0.006076989561376891, + 0.0053044433268254945, + 0.006002429844215364, + 0.005811298045810947, + 0.00541486468124986, + 0.005552686577658202, + 0.005724443119797102, + 0.005942467552368739, + 0.006312250195815329, + 0.004848514440357715 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007062682611073314, + "first_to_last_ratio": 1.237564932158175, + "imbalance_abs_log_ratio": 0.21314568451503188, + "last_quartile_mean": 0.005706918827084721, + "mean": 0.006130595522490438, + "normalized": [ + 1.2781935818257821, + 1.154314147819503, + 1.119066807097797, + 1.0565798163950184, + 1.0289511326347756, + 1.0668339379905794, + 0.9912559944760846, + 0.865241118479574, + 0.9790940899942118, + 0.947917380047643, + 0.883252640201938, + 0.9057336366895933, + 0.9337499267072272, + 0.9693132633801169, + 1.0296308364592772, + 0.7908716898008789 + ], + "population_cv": 0.1166547247084935 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02823692187666893, + 0.014202049933373928, + 0.014330258592963219, + 0.014431006275117397, + 0.0145475585013628, + 0.009805543348193169, + 0.009861846454441547, + 0.009975572116672993, + 0.010043247602880001, + 0.007605585735291243, + 0.007654537446796894, + 0.0077590313740074635, + 0.007819762453436852, + 0.006375181954354048, + 0.006421996746212244, + 0.0065260399132966995, + 0.006563509814441204, + 0.005602568853646517, + 0.0056375316344201565, + 0.0057973419316112995, + 0.005820456892251968, + 0.005056646186858416, + 0.00509054446592927, + 0.005272198468446732, + 0.005295110866427422, + 0.0046798414550721645, + 0.004703808110207319, + 0.004869809839874506, + 0.004889858886599541, + 0.004459173884242773, + 0.00446688337251544, + 0.00453129643574357 + ], + "loss_nats": 5.609001159667969, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0033052791841328144, + 0.005004490725696087, + 0.005999269895255566, + 0.007150472607463598, + 0.004222586750984192, + 0.0056586493737995625, + 0.006890070624649525, + 0.007886149920523167, + 0.004351162351667881, + 0.005775847006589174, + 0.007411441300064325, + 0.008285986259579659, + 0.005422801710665226, + 0.006597128231078386, + 0.008578632958233356, + 0.009296777658164501, + 0.006515308283269405, + 0.007501539774239063, + 0.010256913490593433, + 0.01081777922809124, + 0.006751408334821463, + 0.007698386441916227, + 0.01081965770572424, + 0.011321330443024635, + 0.008140962570905685, + 0.009009703993797302, + 0.013233783654868603, + 0.013792803511023521, + 0.008798080496490002, + 0.00953624676913023, + 0.01231932733207941, + 0.012954792007803917 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.571244764723815e-05, + 5.411167876445688e-05, + 5.283674909151159e-05, + 5.185072586755268e-05, + 5.127084659761749e-05, + 5.077067544334568e-05, + 5.0770395318977535e-05, + 5.0520964578026906e-05, + 5.0666069000726566e-05, + 5.061549381935038e-05, + 5.055486690253019e-05, + 5.0508093409007415e-05, + 4.999646989745088e-05, + 5.004305057809688e-05, + 5.0446498789824545e-05, + 5.0497859774623066e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.362790034268983e-05, + "first_to_last_ratio": 1.067307499464034, + "imbalance_abs_log_ratio": 0.06513912148859854, + "last_quartile_mean": 5.024596975999884e-05, + "mean": 5.132330534252105e-05, + "normalized": [ + 1.0855194784401525, + 1.0543295760732245, + 1.0294884310137495, + 1.010276433318387, + 0.9989778767257201, + 0.9892323790237741, + 0.989226920989334, + 0.9843669311799096, + 0.9871941930199502, + 0.9862087697110136, + 0.985027495114306, + 0.9841161451299155, + 0.9741475059679974, + 0.9750550991234875, + 0.9829160155051417, + 0.9839167496639366 + ], + "population_cv": 0.029989256005628663 + }, + "activation_output_rms_by_block": [ + 0.005180664826184511, + 0.007948367856442928, + 0.0071101076900959015, + 0.011628935113549232, + 0.009834222495555878, + 0.015592618845403194, + 0.011937727220356464, + 0.02099907398223877, + 0.014203419908881187, + 0.023388667032122612, + 0.014353072270751, + 0.022941188886761665, + 0.013975659385323524, + 0.02055426314473152, + 0.015514259226620197, + 0.022412775084376335 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.007967018871568143, + "first_to_last_ratio": 0.4398207829260811, + "imbalance_abs_log_ratio": 0.8213879465753375, + "last_quartile_mean": 0.018114239210262895, + "mean": 0.014848438935587183, + "normalized": [ + 0.3489029957060359, + 0.5352998985902224, + 0.47884546792694416, + 0.7831756027684648, + 0.6623068282273259, + 1.0501183937950838, + 0.8039718701839673, + 1.4142277227480384, + 0.9565598087782763, + 1.575159997194527, + 0.966638468394887, + 1.5450236207510426, + 0.9412207873130777, + 1.3842709818787224, + 1.04484109702854, + 1.5094364587148448 + ], + "population_cv": 0.38149080499888866 + }, + "bits_per_byte": 6.756301290892851, + "branch_output_rms_by_sublayer": [ + 0.00344275776296854, + 0.0037783386651426554, + 0.003971979953348637, + 0.003924910444766283, + 0.0057159834541380405, + 0.0038860586937516928, + 0.007017855532467365, + 0.003858900861814618, + 0.008758665062487125, + 0.0038810574915260077, + 0.009854576550424099, + 0.0037807892076671124, + 0.010700107552111149, + 0.003945234231650829, + 0.012973167933523655, + 0.003866987768560648, + 0.012833327986299992, + 0.0038173284847289324, + 0.01477400679141283, + 0.003928930964320898, + 0.012848040089011192, + 0.004507644101977348, + 0.014026274904608727, + 0.004126352723687887, + 0.012859554030001163, + 0.004076994024217129, + 0.01296316273510456, + 0.0034613758325576782, + 0.01440997514873743, + 0.0041273003444075584, + 0.01337564829736948, + 0.0038736232090741396 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004508021297651076, + 0.0004524930113583151, + 0.0005201601896034384, + 0.0005594057048027396, + 0.0006552041707250334, + 0.0007507040984739487, + 0.000850349803788574, + 0.0009063852334487553, + 0.0010217915468968724, + 0.0011277818818807989, + 0.0010544629528892386, + 0.0011012469919556404, + 0.0011079217036188914, + 0.001237384982851684, + 0.0013584092718518022, + 0.0014241647000383885 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004957152588824002, + "first_to_last_ratio": 0.3866823679479825, + "imbalance_abs_log_ratio": 0.950151677612245, + "last_quartile_mean": 0.0012819701645901916, + "mean": 0.0009111667733718268, + "normalized": [ + 0.494752599567352, + 0.4966083318466913, + 0.5708726490086493, + 0.61394436359068, + 0.7190825981290028, + 0.8238931888351498, + 0.9332537452411748, + 0.9947522889740842, + 1.1214100170879497, + 1.2377337660232879, + 1.1572666867418089, + 1.208611885484378, + 1.215937340997369, + 1.3580225036879552, + 1.490845925850575, + 1.5630121089338918 + ], + "population_cv": 0.3380559322236514 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931420564651489, + "mean_weights": [ + 0.5014879107475281, + 0.49851205945014954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.693139910697937, + "mean_weights": [ + 0.5018156170845032, + 0.4981843829154968 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931352019309998, + "mean_weights": [ + 0.5023940205574036, + 0.49760597944259644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931431889533997, + "mean_weights": [ + 0.49870234727859497, + 0.501297652721405 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986039638519287, + "mean_weights": [ + 0.33161261677742004, + 0.33409368991851807, + 0.3342936635017395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986011028289795, + "mean_weights": [ + 0.3315771222114563, + 0.33318889141082764, + 0.3352339565753937 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986087322235107, + "mean_weights": [ + 0.3331664800643921, + 0.3342593312263489, + 0.3325742185115814 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985949039459229, + "mean_weights": [ + 0.3315378725528717, + 0.3324793577194214, + 0.3359827995300293 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862826824188232, + "mean_weights": [ + 0.24934276938438416, + 0.24841846525669098, + 0.2508545219898224, + 0.2513842284679413 + ], + "sources": 4 + }, + { + "entropy_mean": 1.38628089427948, + "mean_weights": [ + 0.248581200838089, + 0.24957583844661713, + 0.24980047345161438, + 0.2520424723625183 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386273741722107, + "mean_weights": [ + 0.2486811876296997, + 0.2481946051120758, + 0.2515612244606018, + 0.25156301259994507 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862909078598022, + "mean_weights": [ + 0.24919962882995605, + 0.24966104328632355, + 0.25073325634002686, + 0.25040602684020996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094350814819336, + "mean_weights": [ + 0.20043906569480896, + 0.20025089383125305, + 0.1992931365966797, + 0.19971072673797607, + 0.20030616223812103 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094260215759277, + "mean_weights": [ + 0.1990065574645996, + 0.19869452714920044, + 0.20100653171539307, + 0.20066148042678833, + 0.20063093304634094 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609431266784668, + "mean_weights": [ + 0.19923549890518188, + 0.20021051168441772, + 0.20099328458309174, + 0.2002217173576355, + 0.19933897256851196 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609424114227295, + "mean_weights": [ + 0.19919060170650482, + 0.19957327842712402, + 0.1993737816810608, + 0.1998673677444458, + 0.20199497044086456 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917512655258179, + "mean_weights": [ + 0.16589663922786713, + 0.16579726338386536, + 0.16652584075927734, + 0.1674002707004547, + 0.1673392355442047, + 0.16704075038433075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917526960372925, + "mean_weights": [ + 0.1660098284482956, + 0.1660982221364975, + 0.16643531620502472, + 0.16734513640403748, + 0.16757234930992126, + 0.16653914749622345 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917439937591553, + "mean_weights": [ + 0.1664077788591385, + 0.1667102724313736, + 0.16575725376605988, + 0.16558866202831268, + 0.16724394261837006, + 0.1682921051979065 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917406558990479, + "mean_weights": [ + 0.1662176251411438, + 0.16636696457862854, + 0.165130153298378, + 0.16684658825397491, + 0.16694699227809906, + 0.1684916913509369 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459023475646973, + "mean_weights": [ + 0.14233610033988953, + 0.1422998011112213, + 0.14257268607616425, + 0.1434200406074524, + 0.14240685105323792, + 0.14367292821407318, + 0.14329159259796143 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458985328674316, + "mean_weights": [ + 0.1425248682498932, + 0.14280283451080322, + 0.1430899053812027, + 0.14388513565063477, + 0.14334818720817566, + 0.14153052866458893, + 0.14281854033470154 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14222130179405212, + 0.14271843433380127, + 0.14239171147346497, + 0.1434115171432495, + 0.1425933688879013, + 0.14311325550079346, + 0.14355038106441498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14283692836761475, + 0.14230498671531677, + 0.14244258403778076, + 0.1435498595237732, + 0.14231878519058228, + 0.14314766228199005, + 0.143399178981781 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432964324951, + "mean_weights": [ + 0.12469614297151566, + 0.12493360042572021, + 0.12421932816505432, + 0.12482892721891403, + 0.12584659457206726, + 0.12576377391815186, + 0.12501835823059082, + 0.12469328939914703 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794296264648438, + "mean_weights": [ + 0.12482310831546783, + 0.12494714558124542, + 0.12461379170417786, + 0.12449270486831665, + 0.12426981329917908, + 0.12538880109786987, + 0.12516307830810547, + 0.12630154192447662 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794286727905273, + "mean_weights": [ + 0.12459714710712433, + 0.12472857534885406, + 0.12479612231254578, + 0.12524385750293732, + 0.12565991282463074, + 0.1262103021144867, + 0.12457992881536484, + 0.12418416142463684 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794296264648438, + "mean_weights": [ + 0.12493155896663666, + 0.12483802437782288, + 0.12460187077522278, + 0.12459874153137207, + 0.12501779198646545, + 0.12420021742582321, + 0.12559065222740173, + 0.12622115015983582 + ], + "sources": 8 + }, + { + "entropy_mean": 2.197211265563965, + "mean_weights": [ + 0.11090581119060516, + 0.11095649003982544, + 0.11032625287771225, + 0.11037011444568634, + 0.11067523062229156, + 0.11147095263004303, + 0.11175340414047241, + 0.1119384914636612, + 0.1116032600402832 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972126960754395, + "mean_weights": [ + 0.11083438992500305, + 0.11066233366727829, + 0.1106841117143631, + 0.11084605753421783, + 0.11061468720436096, + 0.11108818650245667, + 0.1115877777338028, + 0.11133615672588348, + 0.11234630644321442 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11089816689491272, + 0.1108870804309845, + 0.11106622219085693, + 0.11082258075475693, + 0.11085513234138489, + 0.1119755208492279, + 0.11102905869483948, + 0.11071982979774475, + 0.11174643039703369 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0282417144626379, + 0.01428750716149807, + 0.014416527934372425, + 0.014703329652547836, + 0.014682094566524029, + 0.01008235290646553, + 0.010167798958718777, + 0.010738944634795189, + 0.01082152035087347, + 0.00878758542239666, + 0.008862113580107689, + 0.009672283194959164, + 0.00976790115237236, + 0.008560523390769958, + 0.008684799075126648, + 0.00977813359349966, + 0.009954164735972881, + 0.008888072334229946, + 0.00902888085693121, + 0.01007006410509348, + 0.010184217244386673, + 0.009424775838851929, + 0.009566618129611015, + 0.010299356654286385, + 0.010436818934977055, + 0.009499363601207733, + 0.009606627747416496, + 0.010012919083237648, + 0.010090752504765987, + 0.00939567293971777, + 0.009533073753118515, + 0.009944267570972443 + ], + "loss_nats": 4.683111190795898, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972169876098633, + "mean_weights": [ + 0.11062228679656982, + 0.11076737940311432, + 0.1108340471982956, + 0.11088211834430695, + 0.11121642589569092, + 0.11167430132627487, + 0.111720010638237, + 0.11070731282234192, + 0.1115761250257492 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.00344275776296854, + 0.005180664826184511, + 0.006782237906008959, + 0.007948367856442928, + 0.0057159834541380405, + 0.0071101076900959015, + 0.010661935433745384, + 0.011628935113549232, + 0.008758665062487125, + 0.009834222495555878, + 0.014840804971754551, + 0.015592618845403194, + 0.010700107552111149, + 0.011937727220356464, + 0.019919002428650856, + 0.02099907398223877, + 0.012833327986299992, + 0.014203419908881187, + 0.022593334317207336, + 0.023388667032122612, + 0.012848040089011192, + 0.014353072270751, + 0.0218671802431345, + 0.022941188886761665, + 0.012859554030001163, + 0.013975659385323524, + 0.020060954615473747, + 0.02055426314473152, + 0.01440997514873743, + 0.015514259226620197, + 0.021293850615620613, + 0.022412775084376335 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.073347956844582, + "cross_entropy_nats": 5.596018373966217, + "step": 0 + }, + { + "bits_per_byte": 6.7577811156122, + "cross_entropy_nats": 4.684136927127838, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "5dd01b8270ff98adc6b1340e38ee82b56d324c449cbdbb97581afacdec1223dc", + "final_model_state": "cf7d109dd2a8a9a3c903435afe8c733b4191e11d3150f7eee30962a8de6e0a3d", + "final_optimizer_state": "96110c28f2690589414ee6ed445333520dd0bc9ac87eeb7d4f316b17494d8f30", + "final_public_parameters": "b20e9b701d001f45efdba361481e6a18c3eb9f6fec33fa6434bd9773667dd881", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 3462317568, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.084436624250287, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.603704452514648, + "step": 1, + "unclipped_grad_norm": 18.78145980834961 + }, + { + "bits_per_byte": 7.484355446790379, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.187759876251221, + "step": 10, + "unclipped_grad_norm": 12.867269515991211 + }, + { + "bits_per_byte": 6.795359238834398, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.710184097290039, + "step": 20, + "unclipped_grad_norm": 4.549929141998291 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-baseline.json b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-baseline.json new file mode 100644 index 0000000..f2df857 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-baseline.json @@ -0,0 +1,1132 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "5958d579e71547e6f80ee5807def35862d1fe1f475d705f6d6c7f20874d1f5b2", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002768727717921138, + 0.0002554995589889586, + 0.00023783760843798518, + 0.00022115583124104887, + 0.00020655356638599187, + 0.00019376208365429193, + 0.00018488570640329272, + 0.00017590694187674671, + 0.0001684112212387845, + 0.0001647331955609843, + 0.00015843863366171718, + 0.0001538397918920964, + 0.00015132295084185898, + 0.00014730033581145108, + 0.00014253577683120966, + 0.0001394751889165491, + 0.0001364792260574177, + 0.0001337067224085331, + 0.00013106675760354847, + 0.00012915431580040604, + 0.00012709009752143174, + 0.00012574320135172457, + 0.0001242366706719622, + 0.0001228937617270276, + 0.00012132258416386321, + 0.00011988351616309956, + 0.00011879197700181976, + 0.00011764531518565491, + 0.00011639683361863717, + 0.00011549647024367005, + 0.00011481995898066089, + 0.00011415599146857858 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002190592585975537, + "first_to_last_ratio": 1.8672887091156776, + "imbalance_abs_log_ratio": 0.6244874905881238, + "last_quartile_mean": 0.00011731408085324802, + "mean": 0.0001546067051094724, + "normalized": [ + 1.7908199492128654, + 1.6525774791465027, + 1.538339545297078, + 1.4304413969915148, + 1.3359935860462033, + 1.2532579587482624, + 1.1958453307208163, + 1.1377704592578457, + 1.0892879524179597, + 1.065498391187767, + 1.0247850088360109, + 0.9950395863049215, + 0.978760596021445, + 0.9527422223192203, + 0.9219249367631522, + 0.9021289782858439, + 0.8827510162691898, + 0.8648183939620171, + 0.8477430361816715, + 0.835373315206192, + 0.8220219002238166, + 0.8133101424203405, + 0.80356586464988, + 0.7948798963150414, + 0.7847174809007043, + 0.775409553409819, + 0.7683494510649245, + 0.7609328140222236, + 0.7528576043077825, + 0.7470340316863389, + 0.7426583400723811, + 0.7383637817502684 + ], + "population_cv": 0.27888268733798954 + }, + "activation_output_rms_by_block": [ + 0.028270306065678596, + 0.0284538883715868, + 0.02867252007126808, + 0.02881420962512493, + 0.029061447829008102, + 0.02929086796939373, + 0.029533835127949715, + 0.029811719432473183, + 0.030115574598312378, + 0.030315518379211426, + 0.030812423676252365, + 0.03120259940624237, + 0.03179442882537842, + 0.03198441118001938, + 0.03240228444337845, + 0.0329107940196991, + 0.033366311341524124, + 0.033811621367931366, + 0.03420914337038994, + 0.03470747172832489, + 0.03500960022211075, + 0.03567802160978317, + 0.035978782922029495, + 0.036836784332990646, + 0.03769209608435631, + 0.038124579936265945, + 0.038542140275239944, + 0.03904740884900093, + 0.0400773324072361, + 0.04093807935714722, + 0.04163828864693642, + 0.042313966900110245 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.028988599311560392, + "first_to_last_ratio": 0.728416493900548, + "imbalance_abs_log_ratio": 0.31688228737002583, + "last_quartile_mean": 0.03979673655703664, + "mean": 0.03379432682413608, + "normalized": [ + 0.8365399971656723, + 0.8419723381282118, + 0.8484418174825138, + 0.8526345198433033, + 0.8599504875549783, + 0.8667392051281828, + 0.873928789339177, + 0.8821515986281313, + 0.8911429055838948, + 0.8970593951159853, + 0.9117632032322649, + 0.9233088017589188, + 0.9408214873116122, + 0.9464432106153376, + 0.9588084003566117, + 0.9738555879797565, + 0.9873346942272493, + 1.0005117587897308, + 1.0122747391422395, + 1.0270206567197142, + 1.0359608701276664, + 1.055739970659861, + 1.0646397281194921, + 1.0900286466627183, + 1.1153379761196018, + 1.1281355043603702, + 1.1404914344295491, + 1.1554427183059932, + 1.1859189447920195, + 1.2113891059344621, + 1.2321088348236646, + 1.2521026715611152 + ], + "population_cv": 0.12455242949918699 + }, + "bits_per_byte": 7.95915895791699, + "branch_output_rms_by_sublayer": [ + 0.0022265268489718437, + 0.002717922907322645, + 0.0021652730647474527, + 0.002733128145337105, + 0.0025650423485785723, + 0.002695153234526515, + 0.002537536434829235, + 0.002696225419640541, + 0.0027307753916829824, + 0.0027451792266219854, + 0.0029185698367655277, + 0.0026901187375187874, + 0.002936159959062934, + 0.0027115964330732822, + 0.0030124466866254807, + 0.0027617362793534994, + 0.0031113612931221724, + 0.002723218873143196, + 0.0034121612552553415, + 0.0027921765577048063, + 0.0034701526165008545, + 0.002744668396189809, + 0.0039346772246062756, + 0.0027058592531830072, + 0.004295050632208586, + 0.0026920263189822435, + 0.004176756367087364, + 0.0027182905469089746, + 0.004023758694529533, + 0.0027150253299623728, + 0.004447397775948048, + 0.0027387896552681923, + 0.004260339308530092, + 0.00275799329392612, + 0.005327676888555288, + 0.002721488941460848, + 0.00474865036085248, + 0.0027454239316284657, + 0.00498866755515337, + 0.002719792304560542, + 0.0052231852896511555, + 0.0027601351030170918, + 0.005184618756175041, + 0.002755985129624605, + 0.0052438946440815926, + 0.0027470553759485483, + 0.0059704589657485485, + 0.0027777766808867455, + 0.0060942270793020725, + 0.0026787330862134695, + 0.005698176100850105, + 0.002644060645252466, + 0.006003713235259056, + 0.002661966485902667, + 0.006405353546142578, + 0.002804658142849803, + 0.006667613051831722, + 0.0026854604948312044, + 0.006290186662226915, + 0.0027102255262434483, + 0.0072579155676066875, + 0.0027289679273962975, + 0.006407012697309256, + 0.0026884053368121386 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.005888019831616423, + 0.005779358340731243, + 0.0060348395662513914, + 0.005586875351280001, + 0.005559071516712478, + 0.0054082748429522395, + 0.005481135467826832, + 0.0053805923812947415, + 0.004971746645687392, + 0.005439263395635174, + 0.005139424262259116, + 0.005696496994849179, + 0.005782818048228185, + 0.0054294510739720785, + 0.005067071208887471, + 0.005218073376917496, + 0.005181036474299639, + 0.005605812513830226, + 0.005170579144406511, + 0.004834742615880465, + 0.005150211089068095, + 0.00476651480071292, + 0.004776402486441839, + 0.00519066249960807, + 0.0053994045182471796, + 0.004795508114414784, + 0.004895792813412353, + 0.00496559115560922, + 0.00502742614163479, + 0.004657478744424888, + 0.005287606581362069, + 0.004870555415551657 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005639770912333169, + "first_to_last_ratio": 1.130799174678946, + "imbalance_abs_log_ratio": 0.12292461698112994, + "last_quartile_mean": 0.0049874204355821175, + "mean": 0.005263682419187692, + "normalized": [ + 1.1186122875029152, + 1.0979686615711766, + 1.146505257280075, + 1.061400537941684, + 1.0561183357962487, + 1.0274698228824493, + 1.0413119621059315, + 1.0222106792919112, + 0.9445377303850805, + 1.0333570611721248, + 0.976393302818647, + 1.0822265747043835, + 1.0986259404914112, + 1.0314929058371969, + 0.962647592570647, + 0.9913351455049914, + 0.9842988352437859, + 1.0649982402804865, + 0.9823121405573801, + 0.9185095586801336, + 0.9784425956805525, + 0.9055475655859389, + 0.9074260386664719, + 0.9861275977985596, + 1.0257846291342996, + 0.9110557462459602, + 0.9301079403205877, + 0.9433683038148654, + 0.9551157804103686, + 0.884832779319472, + 1.0045451378463044, + 0.9253133125579595 + ], + "population_cv": 0.06750436074287791 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028027428314089775, + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346 + ], + "loss_nats": 5.516868591308594, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346, + 0.042313966900110245 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.0331513446290046e-05, + 3.985461080446839e-05, + 3.939039379474707e-05, + 3.8955986383371055e-05, + 3.865906546707265e-05, + 3.843139347736724e-05, + 3.827241380349733e-05, + 3.811378701357171e-05, + 3.791215567616746e-05, + 3.7825848266948014e-05, + 3.78328186343424e-05, + 3.776324956561439e-05, + 3.772067429963499e-05, + 3.7761601561214775e-05, + 3.77303222194314e-05, + 3.774304786929861e-05, + 3.785186345339753e-05, + 3.794040094362572e-05, + 3.797327008214779e-05, + 3.8049773138482124e-05, + 3.8102883991086856e-05, + 3.8117221265565604e-05, + 3.811223359662108e-05, + 3.81382014893461e-05, + 3.817927427007817e-05, + 3.815999662037939e-05, + 3.819023913820274e-05, + 3.816752723651007e-05, + 3.8173260691110045e-05, + 3.818520053755492e-05, + 3.816522075794637e-05, + 3.822705912170932e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.9001145523798186e-05, + "first_to_last_ratio": 1.0214812032742024, + "imbalance_abs_log_ratio": 0.021253734004134984, + "last_quartile_mean": 3.818097229668638e-05, + "mean": 3.825101589427504e-05, + "normalized": [ + 1.0543906482840992, + 1.0419229364946974, + 1.0297868664095418, + 1.0184301115307508, + 1.0106676793611298, + 1.0047156285624088, + 1.000559407605838, + 0.9964124121282784, + 0.9911411446157616, + 0.9888848016873021, + 0.9890670286747801, + 0.9872482777971484, + 0.9861352285098545, + 0.987205193859086, + 0.9863874550081799, + 0.9867201429007677, + 0.9895649192172892, + 0.991879563368778, + 0.9927388644292499, + 0.9947388912140491, + 0.9961273733592431, + 0.9965021941095827, + 0.9963718009990231, + 0.9970506821246067, + 0.99812445179508, + 0.9976204743385841, + 0.998411107400643, + 0.9978173479628427, + 0.9979672382197662, + 0.9982793827776487, + 0.9977570494711616, + 0.9993736957828273 + ], + "population_cv": 0.01543848086298159 + }, + "activation_output_rms_by_block": [ + 0.028325499966740608, + 0.028658077120780945, + 0.029205938801169395, + 0.029751326888799667, + 0.03091570734977722, + 0.032307714223861694, + 0.034374311566352844, + 0.03687922656536102, + 0.03944294899702072, + 0.04215770959854126, + 0.04655195400118828, + 0.050798118114471436, + 0.05539363995194435, + 0.05819777771830559, + 0.062257908284664154, + 0.0653507262468338, + 0.06877651810646057, + 0.07320939004421234, + 0.07733170688152313, + 0.08104265481233597, + 0.08443007618188858, + 0.08700668811798096, + 0.09079501777887344, + 0.093939870595932, + 0.09777236729860306, + 0.10033779591321945, + 0.10455679148435593, + 0.10687003284692764, + 0.11059616506099701, + 0.11359578371047974, + 0.11617089062929153, + 0.11852118372917175 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.031302225310355425, + "first_to_last_ratio": 0.2883599077004872, + "imbalance_abs_log_ratio": 1.2435458995301245, + "last_quartile_mean": 0.10855262633413076, + "mean": 0.06861004745587707, + "normalized": [ + 0.4128476953023048, + 0.41769504880769653, + 0.4256802011389317, + 0.43362930054716353, + 0.4506002910092582, + 0.4708889648362173, + 0.5010098788877659, + 0.5375193274582407, + 0.5748859016951763, + 0.614453876098144, + 0.6785005363992164, + 0.7403889080114624, + 0.8073692120322151, + 0.8482398697615304, + 0.9074167792217611, + 0.9524949868145868, + 1.002426330497593, + 1.0670359919411672, + 1.1271192740575633, + 1.1812068030481144, + 1.2305788920520035, + 1.2681333324238657, + 1.323348709782827, + 1.3691853318764207, + 1.4250444493786454, + 1.4624358914450017, + 1.5239282781664902, + 1.5576440595767767, + 1.61195290138986, + 1.6556727173747092, + 1.693205221931973, + 1.7274610370353176 + ], + "population_cv": 0.4388810528199641 + }, + "bits_per_byte": 6.713833958880599, + "branch_output_rms_by_sublayer": [ + 0.002437001559883356, + 0.0027646415401250124, + 0.0026093844790011644, + 0.002773481421172619, + 0.0033957704436033964, + 0.0027507112827152014, + 0.003547805128619075, + 0.0027288019191473722, + 0.004703273996710777, + 0.0028269628528505564, + 0.005306650884449482, + 0.002743088873103261, + 0.006675844080746174, + 0.002795359119772911, + 0.006955645978450775, + 0.002845237497240305, + 0.006907123140990734, + 0.002933910582214594, + 0.00843191146850586, + 0.0027949840296059847, + 0.00897424016147852, + 0.0029047494754195213, + 0.009036343544721603, + 0.0032153772190213203, + 0.010582618415355682, + 0.0028866741340607405, + 0.009517580270767212, + 0.0026635508984327316, + 0.008616920560598373, + 0.003158343257382512, + 0.009494570083916187, + 0.0030213629361242056, + 0.009422268718481064, + 0.002808293094858527, + 0.010874342173337936, + 0.0031132514122873545, + 0.01018539723008871, + 0.002876743907108903, + 0.008500457741320133, + 0.0031567441765218973, + 0.01006424892693758, + 0.0032528946176171303, + 0.009540545754134655, + 0.0029259754810482264, + 0.010095320641994476, + 0.002696031704545021, + 0.009177232161164284, + 0.002929815324023366, + 0.011962441727519035, + 0.0029209970962256193, + 0.008841389790177345, + 0.0027309327851980925, + 0.010780212469398975, + 0.0031468512024730444, + 0.01038750447332859, + 0.0032646870240569115, + 0.009689838625490665, + 0.0026016440242528915, + 0.009929094463586807, + 0.003191312775015831, + 0.010107175447046757, + 0.0028179220389574766, + 0.009526536799967289, + 0.003107903292402625 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00017190496538337113, + 0.0001735542784589828, + 0.00019824933045045772, + 0.0002107982497012391, + 0.0002450550613783958, + 0.00026804391005753116, + 0.0003238054491237347, + 0.00031010311411301124, + 0.00031952489285031816, + 0.00034636354958654015, + 0.00034728721218647997, + 0.00036779944805402854, + 0.00040306175531293955, + 0.00038015703078453524, + 0.0003565887583279226, + 0.00036832720297469803, + 0.00036821166426922845, + 0.000375260819686975, + 0.00038062964117533765, + 0.00035620909008044117, + 0.0003690236577410054, + 0.00035091389469732654, + 0.00035793360754103896, + 0.000359518412683121, + 0.00040508202264603015, + 0.0003592735021571915, + 0.00038012959666749183, + 0.00035476778007974936, + 0.0003492989830982136, + 0.0003659888269167611, + 0.0003625696460584541, + 0.0003779058324425446 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023768929483334046, + "first_to_last_ratio": 0.6434869511235988, + "imbalance_abs_log_ratio": 0.44085353012576206, + "last_quartile_mean": 0.0003693770237583045, + "mean": 0.00033322941208390925, + "normalized": [ + 0.5158757275005617, + 0.5208252097965493, + 0.5949334700399654, + 0.632591968346896, + 0.7353944534664587, + 0.8043825074781695, + 0.9717192942206389, + 0.9305994695177908, + 0.9588736205850275, + 1.0394147005826835, + 1.0421865525454603, + 1.1037424510457507, + 1.2095623636350745, + 1.140826760780676, + 1.0700998933375405, + 1.105326209566252, + 1.1049794853538033, + 1.1261335466764921, + 1.1422450341192953, + 1.0689605333839665, + 1.107416225456362, + 1.053069993140234, + 1.074135699194851, + 1.0788915973377284, + 1.2156250557619686, + 1.0781566366258335, + 1.1407444327626544, + 1.0646352549168638, + 1.0482237474591767, + 1.0983088936477277, + 1.0880481521455758, + 1.134071059571973 + ], + "population_cv": 0.19078746332772342 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028024883940815926, + 0.02821914665400982, + 0.028325499966740608, + 0.02856484428048134, + 0.028658077120780945, + 0.02912120893597603, + 0.029205938801169395, + 0.02964756451547146, + 0.029751326888799667, + 0.030729545280337334, + 0.03091570734977722, + 0.03217839077115059, + 0.032307714223861694, + 0.03411358222365379, + 0.034374311566352844, + 0.036541424691677094, + 0.03687922656536102, + 0.03906221315264702, + 0.03944294899702072, + 0.04181833937764168, + 0.04215770959854126, + 0.04607149586081505, + 0.04655195400118828, + 0.0500124990940094, + 0.050798118114471436, + 0.05468229204416275, + 0.05539363995194435, + 0.05778742954134941, + 0.05819777771830559, + 0.06149948015809059, + 0.062257908284664154, + 0.06454319506883621, + 0.0653507262468338, + 0.06803742796182632, + 0.06877651810646057, + 0.07261326909065247, + 0.07320939004421234, + 0.07679533213376999, + 0.07733170688152313, + 0.08014968782663345, + 0.08104265481233597, + 0.0835338607430458, + 0.08443007618188858, + 0.08647285401821136, + 0.08700668811798096, + 0.09021914005279541, + 0.09079501777887344, + 0.09300754219293594, + 0.093939870595932, + 0.0970776379108429, + 0.09777236729860306, + 0.0996294915676117, + 0.10033779591321945, + 0.10360711812973022, + 0.10455679148435593, + 0.10598249733448029, + 0.10687003284692764, + 0.10993104428052902, + 0.11059616506099701, + 0.11287258565425873, + 0.11359578371047974, + 0.11536551266908646, + 0.11617089062929153, + 0.11782468110322952 + ], + "loss_nats": 4.653675079345703, + "loss_scale": 1.0, + "output_weights": null, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.02821914665400982, + 0.028325499966740608, + 0.02856484428048134, + 0.028658077120780945, + 0.02912120893597603, + 0.029205938801169395, + 0.02964756451547146, + 0.029751326888799667, + 0.030729545280337334, + 0.03091570734977722, + 0.03217839077115059, + 0.032307714223861694, + 0.03411358222365379, + 0.034374311566352844, + 0.036541424691677094, + 0.03687922656536102, + 0.03906221315264702, + 0.03944294899702072, + 0.04181833937764168, + 0.04215770959854126, + 0.04607149586081505, + 0.04655195400118828, + 0.0500124990940094, + 0.050798118114471436, + 0.05468229204416275, + 0.05539363995194435, + 0.05778742954134941, + 0.05819777771830559, + 0.06149948015809059, + 0.062257908284664154, + 0.06454319506883621, + 0.0653507262468338, + 0.06803742796182632, + 0.06877651810646057, + 0.07261326909065247, + 0.07320939004421234, + 0.07679533213376999, + 0.07733170688152313, + 0.08014968782663345, + 0.08104265481233597, + 0.0835338607430458, + 0.08443007618188858, + 0.08647285401821136, + 0.08700668811798096, + 0.09021914005279541, + 0.09079501777887344, + 0.09300754219293594, + 0.093939870595932, + 0.0970776379108429, + 0.09777236729860306, + 0.0996294915676117, + 0.10033779591321945, + 0.10360711812973022, + 0.10455679148435593, + 0.10598249733448029, + 0.10687003284692764, + 0.10993104428052902, + 0.11059616506099701, + 0.11287258565425873, + 0.11359578371047974, + 0.11536551266908646, + 0.11617089062929153, + 0.11782468110322952, + 0.11852118372917175 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.95552135286859, + "cross_entropy_nats": 5.514347195625305, + "step": 0 + }, + { + "bits_per_byte": 6.717222361067899, + "cross_entropy_nats": 4.656023740768433, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858", + "final_optimizer_state": "88ff74de5c6dcffa477d815ee1b7fd6ced81b387b2327a5bf0fd7a221cea9a6d", + "final_public_parameters": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 3185052160, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958522622108798, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516427516937256, + "step": 1, + "unclipped_grad_norm": 22.09486961364746 + }, + { + "bits_per_byte": 7.238571600789322, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.017395496368408, + "step": 10, + "unclipped_grad_norm": 11.172341346740723 + }, + { + "bits_per_byte": 6.729949421197048, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6648454666137695, + "step": 20, + "unclipped_grad_norm": 3.462282180786133 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-block.json b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-block.json new file mode 100644 index 0000000..ff4c261 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-a-depth-32-block.json @@ -0,0 +1,2618 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "fc6cf8d746ae7004851f9d52a5fc29a99061d0d6a5ec28f2f4c70a6313eb712a", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.9337450289167464e-05, + 3.887022103299387e-05, + 3.8419901102315634e-05, + 3.799897967837751e-05, + 3.775028380914591e-05, + 3.7539852201007307e-05, + 3.739175735972822e-05, + 3.724253110704012e-05, + 3.720997119671665e-05, + 3.713165278895758e-05, + 3.7142541259527206e-05, + 3.707727591972798e-05, + 3.688051219796762e-05, + 3.692117388709448e-05, + 3.689077857416123e-05, + 3.6904009903082624e-05, + 3.717117215273902e-05, + 3.7258152588037774e-05, + 3.729091622517444e-05, + 3.7364865420386195e-05, + 3.74124720110558e-05, + 3.742674380191602e-05, + 3.742256740224548e-05, + 3.74479714082554e-05, + 3.765593282878399e-05, + 3.763797212741338e-05, + 3.7667807191610336e-05, + 3.764635039260611e-05, + 3.751127951545641e-05, + 3.7523430364672095e-05, + 3.7504440115299076e-05, + 3.756426303880289e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.8068872072472004e-05, + "first_to_last_ratio": 1.0127680561501509, + "imbalance_abs_log_ratio": 0.01268723177426717, + "last_quartile_mean": 3.7588934446830535e-05, + "mean": 3.750672590285831e-05, + "normalized": [ + 1.0488105624322075, + 1.0363533498942825, + 1.0243469718423952, + 1.013124413386392, + 1.0064937128054956, + 1.0008832095404647, + 0.9969347219635258, + 0.9929560688260969, + 0.9920879602525092, + 0.9899998438980743, + 0.9902901510445264, + 0.9885500540824973, + 0.9833039624276305, + 0.9843880796932157, + 0.9835776833655818, + 0.9839304555311837, + 0.9910535046170554, + 0.9933725669506762, + 0.9942461072650594, + 0.9962177321785024, + 0.997487013607996, + 0.9978675264498043, + 0.9977561757634937, + 0.9984334944416348, + 1.0039781378495187, + 1.0034992717011608, + 1.0042947307416068, + 1.0037226520413813, + 1.0001214078938774, + 1.0004453724341882, + 0.9999390565957383, + 1.001534048482227 + ], + "population_cv": 0.013988937740460563 + }, + "activation_output_rms_by_block": [ + 0.0037520017940551043, + 0.005621022544801235, + 0.0076507688499987125, + 0.009552493691444397, + 0.005728918593376875, + 0.009368892759084702, + 0.013378600589931011, + 0.01761116087436676, + 0.00800272449851036, + 0.013154285028576851, + 0.01923154480755329, + 0.024512210860848427, + 0.011547097004950047, + 0.016813727095723152, + 0.02113443799316883, + 0.025982216000556946, + 0.010269254446029663, + 0.01641162671148777, + 0.02165052480995655, + 0.026260115206241608, + 0.011169232428073883, + 0.016121258959174156, + 0.019987935200333595, + 0.02486182004213333, + 0.01274633314460516, + 0.016767995432019234, + 0.022238746285438538, + 0.02633264660835266, + 0.0102624436840415, + 0.016213588416576385, + 0.020620016381144524, + 0.02528378553688526 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.00908298246213235, + "first_to_last_ratio": 0.4829268696139592, + "imbalance_abs_log_ratio": 0.7278900454597648, + "last_quartile_mean": 0.018808194436132908, + "mean": 0.015944982071232516, + "normalized": [ + 0.2353092513552812, + 0.35252611258450545, + 0.47982298229121323, + 0.5990909020009959, + 0.3592928839796457, + 0.5875762492068142, + 0.8390477035448226, + 1.1044954955540724, + 0.5018961114386356, + 0.8249796061112817, + 1.2061189358281121, + 1.537299367998203, + 0.7241837559531027, + 1.0544839135352808, + 1.3254601290081713, + 1.62949170368986, + 0.6440430224476176, + 1.0292659244250366, + 1.3578268519359336, + 1.6469203344931544, + 0.7004857313841133, + 1.0110553205487558, + 1.2535564550050677, + 1.5592253369157636, + 0.7993946363603464, + 1.0516158222762493, + 1.394717547256516, + 1.6514691889093607, + 0.6436158810461574, + 1.016845822977028, + 1.2931978404884237, + 1.5856891794504773 + ], + "population_cv": 0.416991705473353 + }, + "bits_per_byte": 6.71703146032544, + "branch_output_rms_by_sublayer": [ + 0.002443866338580847, + 0.0027545092161744833, + 0.0026208641938865185, + 0.002763434313237667, + 0.0034182921517640352, + 0.002741520293056965, + 0.0035785960499197245, + 0.002719961805269122, + 0.0047746808268129826, + 0.0028042772319167852, + 0.005376013461500406, + 0.0027222761418670416, + 0.006767150945961475, + 0.0027788877487182617, + 0.007074786350131035, + 0.002833561971783638, + 0.006997238378971815, + 0.0029156480450183153, + 0.008518846705555916, + 0.0027741468511521816, + 0.009081361815333366, + 0.0028916175942867994, + 0.00911131501197815, + 0.0032094581983983517, + 0.010676512494683266, + 0.002878024475648999, + 0.009562043473124504, + 0.0026487084105610847, + 0.008648304268717766, + 0.003162843408063054, + 0.009535389021039009, + 0.0030166516080498695, + 0.009431489743292332, + 0.002794384490698576, + 0.010901342146098614, + 0.00310111534781754, + 0.010227455757558346, + 0.0028713408391922712, + 0.008522198535501957, + 0.0031550333369523287, + 0.010077033191919327, + 0.0032391855493187904, + 0.009554130956530571, + 0.00291044800542295, + 0.010129180736839771, + 0.0026873303577303886, + 0.00920273270457983, + 0.002927709138020873, + 0.011994888074696064, + 0.0029113248456269503, + 0.008809582330286503, + 0.0027162786573171616, + 0.01077704131603241, + 0.0031467911321669817, + 0.010385709814727306, + 0.003254092764109373, + 0.009710066020488739, + 0.0025947801768779755, + 0.009926802478730679, + 0.0031805087346583605, + 0.0101145189255476, + 0.002813765313476324, + 0.009518872946500778, + 0.0030965779442340136 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00016739224645496468, + 0.0001695381430192955, + 0.0001941383401676919, + 0.00020674849822850373, + 0.00024134787186321302, + 0.00026490993231531454, + 0.000322331199796065, + 0.0003097008723390916, + 0.00031886187327784883, + 0.00034582398876540076, + 0.00034801623779775657, + 0.00036856862737498307, + 0.00040270040252095, + 0.0003782002933862871, + 0.0003548760100734237, + 0.0003670443346435906, + 0.0003676146253339179, + 0.0003749331083729355, + 0.00038033249912268637, + 0.0003562102847256418, + 0.00036931756079357426, + 0.00035104688530666954, + 0.0003583944935512815, + 0.0003605180195562536, + 0.0004079256274935599, + 0.0003614699495467052, + 0.00038354947558429407, + 0.0003579111697278514, + 0.0003529832358531257, + 0.00036980562702627114, + 0.00036596233134640236, + 0.0003819940876588112 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002345133880230175, + "first_to_last_ratio": 0.6292279841951004, + "imbalance_abs_log_ratio": 0.46326163295248246, + "last_quartile_mean": 0.00037270018802962763, + "mean": 0.0003331302454070113, + "normalized": [ + 0.502482883985657, + 0.5089244983209419, + 0.582770081205019, + 0.6206236181764371, + 0.7244850180695525, + 0.7952142922107038, + 0.9675831127319218, + 0.9296690306840971, + 0.9571687881064967, + 1.0381044457337716, + 1.044685202247421, + 1.1063799593599613, + 1.2088377085933444, + 1.1352925728019987, + 1.0652770649505088, + 1.1018042933782364, + 1.1035162084570687, + 1.1254850423889022, + 1.141693089614257, + 1.069282329169577, + 1.1086281293442752, + 1.0537826875424299, + 1.0758389503582988, + 1.0822134121018658, + 1.224522939954507, + 1.0850709430633327, + 1.1513499025454192, + 1.07438809493435, + 1.0595952805842004, + 1.1100932206694432, + 1.0985563046047577, + 1.146680894111248 + ], + "population_cv": 0.19646899587719932 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931430697441101, + "mean_weights": [ + 0.49865636229515076, + 0.5013436079025269 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931370496749878, + "mean_weights": [ + 0.49779537320137024, + 0.5022045969963074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931354999542236, + "mean_weights": [ + 0.49762701988220215, + 0.5023729801177979 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931448578834534, + "mean_weights": [ + 0.49909353256225586, + 0.5009064674377441 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931440234184265, + "mean_weights": [ + 0.49918538331985474, + 0.5008145570755005 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931461691856384, + "mean_weights": [ + 0.49945560097694397, + 0.5005444288253784 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931438446044922, + "mean_weights": [ + 0.49891161918640137, + 0.5010883808135986 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931411623954773, + "mean_weights": [ + 0.49836465716362, + 0.5016353130340576 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986062288284302, + "mean_weights": [ + 0.3321005403995514, + 0.3337631821632385, + 0.3341362476348877 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985990762710571, + "mean_weights": [ + 0.33337411284446716, + 0.33537548780441284, + 0.3312504291534424 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986008644104004, + "mean_weights": [ + 0.3350442051887512, + 0.33362624049186707, + 0.3313295841217041 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986016988754272, + "mean_weights": [ + 0.3313179612159729, + 0.3338589072227478, + 0.3348231613636017 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986084938049316, + "mean_weights": [ + 0.33454394340515137, + 0.3326125741004944, + 0.33284348249435425 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985959768295288, + "mean_weights": [ + 0.3312351107597351, + 0.332991361618042, + 0.3357735574245453 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986058712005615, + "mean_weights": [ + 0.33314070105552673, + 0.3320331275463104, + 0.33482617139816284 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.33257031440734863, + 0.3324887454509735, + 0.33494094014167786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862926959991455, + "mean_weights": [ + 0.24962691962718964, + 0.2496621459722519, + 0.25036555528640747, + 0.2503453493118286 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862905502319336, + "mean_weights": [ + 0.24916216731071472, + 0.24970784783363342, + 0.25086167454719543, + 0.2502683103084564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386286973953247, + "mean_weights": [ + 0.25067827105522156, + 0.24852371215820312, + 0.24991214275360107, + 0.25088587403297424 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862855434417725, + "mean_weights": [ + 0.24860653281211853, + 0.24958111345767975, + 0.2513706088066101, + 0.2504417300224304 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386291265487671, + "mean_weights": [ + 0.25055181980133057, + 0.24973659217357635, + 0.2504120469093323, + 0.2492995262145996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862907886505127, + "mean_weights": [ + 0.24893252551555634, + 0.25016576051712036, + 0.2503091096878052, + 0.25059258937835693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862874507904053, + "mean_weights": [ + 0.24930882453918457, + 0.24942341446876526, + 0.2499566376209259, + 0.25131112337112427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862828016281128, + "mean_weights": [ + 0.24865929782390594, + 0.24900496006011963, + 0.25141197443008423, + 0.2509238123893738 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094212532043457, + "mean_weights": [ + 0.19921688735485077, + 0.1992446780204773, + 0.19889645278453827, + 0.20075249671936035, + 0.20188948512077332 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609428882598877, + "mean_weights": [ + 0.19964422285556793, + 0.19919568300247192, + 0.19919705390930176, + 0.20121362805366516, + 0.20074939727783203 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094245910644531, + "mean_weights": [ + 0.20065900683403015, + 0.19955331087112427, + 0.2014925479888916, + 0.19979336857795715, + 0.19850176572799683 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094355583190918, + "mean_weights": [ + 0.19937855005264282, + 0.20006650686264038, + 0.20067161321640015, + 0.1999610960483551, + 0.19992223381996155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094313859939575, + "mean_weights": [ + 0.1997283697128296, + 0.19996783137321472, + 0.19928213953971863, + 0.1997123807668686, + 0.20130927860736847 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094284057617188, + "mean_weights": [ + 0.19933226704597473, + 0.19993427395820618, + 0.19907569885253906, + 0.2001166194677353, + 0.20154115557670593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094276905059814, + "mean_weights": [ + 0.19932052493095398, + 0.19971224665641785, + 0.2011900395154953, + 0.20086491107940674, + 0.19891227781772614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094303131103516, + "mean_weights": [ + 0.19955268502235413, + 0.19991174340248108, + 0.20052853226661682, + 0.20107966661453247, + 0.1989273726940155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917479276657104, + "mean_weights": [ + 0.16630326211452484, + 0.1665954291820526, + 0.1656438112258911, + 0.16612562537193298, + 0.16804365813732147, + 0.16728821396827698 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917546033859253, + "mean_weights": [ + 0.16606619954109192, + 0.16676779091358185, + 0.16676853597164154, + 0.16595947742462158, + 0.16733971238136292, + 0.1670982539653778 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791755199432373, + "mean_weights": [ + 0.16713324189186096, + 0.16643813252449036, + 0.16707737743854523, + 0.16582445800304413, + 0.16707313060760498, + 0.16645365953445435 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791752815246582, + "mean_weights": [ + 0.16595935821533203, + 0.16658073663711548, + 0.16717244684696198, + 0.1671261042356491, + 0.16733530163764954, + 0.16582605242729187 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917518615722656, + "mean_weights": [ + 0.1666388213634491, + 0.16675034165382385, + 0.16638417541980743, + 0.16550619900226593, + 0.16747808456420898, + 0.1672423779964447 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917499542236328, + "mean_weights": [ + 0.16634854674339294, + 0.16704240441322327, + 0.16620638966560364, + 0.16553011536598206, + 0.16771462559700012, + 0.16715793311595917 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917507886886597, + "mean_weights": [ + 0.16621501743793488, + 0.1665443778038025, + 0.16585618257522583, + 0.16727086901664734, + 0.16779379546642303, + 0.16631974279880524 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917485237121582, + "mean_weights": [ + 0.16604092717170715, + 0.1667710244655609, + 0.16534343361854553, + 0.1669321358203888, + 0.16766314208507538, + 0.16724932193756104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9458963871002197, + "mean_weights": [ + 0.14258044958114624, + 0.14276695251464844, + 0.14234250783920288, + 0.14281943440437317, + 0.14429350197315216, + 0.14340519905090332, + 0.14179196953773499 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458930492401123, + "mean_weights": [ + 0.14285841584205627, + 0.14281482994556427, + 0.14142268896102905, + 0.14248794317245483, + 0.1441671997308731, + 0.14373424649238586, + 0.1425146758556366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459006786346436, + "mean_weights": [ + 0.1428324282169342, + 0.14242856204509735, + 0.14238804578781128, + 0.14283627271652222, + 0.14203999936580658, + 0.14380358159542084, + 0.14367109537124634 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.14253869652748108, + 0.14282220602035522, + 0.1414484828710556, + 0.14307048916816711, + 0.1438344419002533, + 0.14343075454235077, + 0.1428549587726593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14281800389289856, + 0.1431872844696045, + 0.1432121843099594, + 0.14210283756256104, + 0.14336752891540527, + 0.14217621088027954, + 0.14313597977161407 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458997249603271, + "mean_weights": [ + 0.14271901547908783, + 0.14267435669898987, + 0.14293994009494781, + 0.14278604090213776, + 0.14186438918113708, + 0.1428055316209793, + 0.14421072602272034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459013938903809, + "mean_weights": [ + 0.14281892776489258, + 0.1429084837436676, + 0.14260512590408325, + 0.14224481582641602, + 0.14402621984481812, + 0.14324231445789337, + 0.14215409755706787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459059238433838, + "mean_weights": [ + 0.14244785904884338, + 0.14288531243801117, + 0.14282266795635223, + 0.14290127158164978, + 0.1435883343219757, + 0.14224958419799805, + 0.14310497045516968 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432487487793, + "mean_weights": [ + 0.12496939301490784, + 0.12501314282417297, + 0.12449057400226593, + 0.12434226274490356, + 0.12589572370052338, + 0.12463213503360748, + 0.12490519136190414, + 0.1257515549659729 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794320106506348, + "mean_weights": [ + 0.12487496435642242, + 0.12489178031682968, + 0.12453664839267731, + 0.12434129416942596, + 0.1249510794878006, + 0.12465334683656693, + 0.12601137161254883, + 0.12573951482772827 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794363021850586, + "mean_weights": [ + 0.12491423636674881, + 0.12523731589317322, + 0.12523618340492249, + 0.12467047572135925, + 0.12580403685569763, + 0.12441615015268326, + 0.12498776614665985, + 0.1247338205575943 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079427480697632, + "mean_weights": [ + 0.12474467605352402, + 0.1245410293340683, + 0.12462934106588364, + 0.1246577650308609, + 0.12418574839830399, + 0.12514574825763702, + 0.12619128823280334, + 0.1259044110774994 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794358253479004, + "mean_weights": [ + 0.12475918978452682, + 0.12487104535102844, + 0.1250094473361969, + 0.1255180537700653, + 0.12490293383598328, + 0.12545335292816162, + 0.12531054019927979, + 0.12417542934417725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079436779022217, + "mean_weights": [ + 0.12494880706071854, + 0.12516388297080994, + 0.12451333552598953, + 0.124576136469841, + 0.12569281458854675, + 0.12472400069236755, + 0.1254362016916275, + 0.12494482100009918 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079434871673584, + "mean_weights": [ + 0.12489444017410278, + 0.12512432038784027, + 0.12540367245674133, + 0.1253792643547058, + 0.1248319000005722, + 0.12483960390090942, + 0.125506192445755, + 0.12402061372995377 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079428195953369, + "mean_weights": [ + 0.12488104403018951, + 0.12459895014762878, + 0.12424551695585251, + 0.12444087862968445, + 0.12461263686418533, + 0.12524595856666565, + 0.12623095512390137, + 0.1257440745830536 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11096074432134628, + 0.11101590096950531, + 0.1106940507888794, + 0.11131151020526886, + 0.11191274225711823, + 0.11117846518754959, + 0.11035685986280441, + 0.11126935482025146, + 0.11130036413669586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216749191284, + "mean_weights": [ + 0.11095131188631058, + 0.11080048978328705, + 0.11054704338312149, + 0.11075608432292938, + 0.1108987033367157, + 0.11123715341091156, + 0.11191022396087646, + 0.11176811158657074, + 0.11113087832927704 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216510772705, + "mean_weights": [ + 0.1109621673822403, + 0.1111164540052414, + 0.11043901741504669, + 0.11095462739467621, + 0.11190879344940186, + 0.11175167560577393, + 0.11079787462949753, + 0.1112256720662117, + 0.1108437031507492 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197213649749756, + "mean_weights": [ + 0.11101574450731277, + 0.11100780963897705, + 0.11082783341407776, + 0.11142586171627045, + 0.11015472561120987, + 0.11108498275279999, + 0.11191210150718689, + 0.11077474802732468, + 0.11179618537425995 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197218894958496, + "mean_weights": [ + 0.11101824045181274, + 0.11099033057689667, + 0.11046893894672394, + 0.11147398501634598, + 0.11092066764831543, + 0.11164902150630951, + 0.11152558028697968, + 0.11112039536237717, + 0.1108328253030777 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216033935547, + "mean_weights": [ + 0.11115090548992157, + 0.11169334501028061, + 0.1117531955242157, + 0.1108616292476654, + 0.11132021248340607, + 0.11068520694971085, + 0.1114465519785881, + 0.11033187806606293, + 0.11075706779956818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972179412841797, + "mean_weights": [ + 0.11092537641525269, + 0.11108530312776566, + 0.11051007360219955, + 0.11084415018558502, + 0.11164555698633194, + 0.11119014024734497, + 0.11111340671777725, + 0.11183696240186691, + 0.1108490377664566 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802533470094204, + 0.014066163450479507, + 0.014108670875430107, + 0.014229752123355865, + 0.01430919673293829, + 0.014553130604326725, + 0.01460581086575985, + 0.014812584035098553, + 0.014851128682494164, + 0.010233328677713871, + 0.01034309808164835, + 0.010804922319948673, + 0.010769715532660484, + 0.011443440802395344, + 0.011492080986499786, + 0.01227729581296444, + 0.012374433688819408, + 0.009845216758549213, + 0.00994168221950531, + 0.010567485354840755, + 0.01064346730709076, + 0.01164715364575386, + 0.01175782922655344, + 0.012674410827457905, + 0.012870704755187035, + 0.011088024824857712, + 0.011239404790103436, + 0.011724143289029598, + 0.011804761365056038, + 0.012471788562834263, + 0.012632876634597778, + 0.01310542318969965, + 0.013265094719827175, + 0.011508332565426826, + 0.011626233346760273, + 0.012281009927392006, + 0.012384294532239437, + 0.012981574982404709, + 0.013078294694423676, + 0.013564560562372208, + 0.01372351124882698, + 0.012140991166234016, + 0.012265652418136597, + 0.012530013918876648, + 0.012632926926016808, + 0.013094775378704071, + 0.01315103005617857, + 0.01351068913936615, + 0.013631501235067844, + 0.01232061255723238, + 0.012407775968313217, + 0.012646659277379513, + 0.012729056179523468, + 0.01315284613519907, + 0.013267731294035912, + 0.013451422564685345, + 0.01357149239629507, + 0.012415111064910889, + 0.012491659261286259, + 0.01274929754436016, + 0.012818088755011559, + 0.013031110167503357, + 0.013103967532515526, + 0.013315660879015923 + ], + "loss_nats": 4.655891418457031, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.197218418121338, + "mean_weights": [ + 0.11057738959789276, + 0.11068764328956604, + 0.11082598567008972, + 0.11127202212810516, + 0.11091771721839905, + 0.11130063980817795, + 0.11115016043186188, + 0.11173908412456512, + 0.11152935773134232 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.002443866338580847, + 0.0037520017940551043, + 0.004732752684503794, + 0.005621022544801235, + 0.007002863567322493, + 0.0076507688499987125, + 0.008998939767479897, + 0.009552493691444397, + 0.0047746808268129826, + 0.005728918593376875, + 0.008738124743103981, + 0.009368892759084702, + 0.012762236408889294, + 0.013378600589931011, + 0.01696598529815674, + 0.01761116087436676, + 0.006997238378971815, + 0.00800272449851036, + 0.012584065087139606, + 0.013154285028576851, + 0.018650829792022705, + 0.01923154480755329, + 0.023473192006349564, + 0.024512210860848427, + 0.010676512494683266, + 0.011547097004950047, + 0.01631898619234562, + 0.016813727095723152, + 0.02041666954755783, + 0.02113443799316883, + 0.02490287274122238, + 0.025982216000556946, + 0.009431489743292332, + 0.010269254446029663, + 0.015727587044239044, + 0.01641162671148777, + 0.02073165774345398, + 0.02165052480995655, + 0.02527376264333725, + 0.026260115206241608, + 0.010077033191919327, + 0.011169232428073883, + 0.015674244612455368, + 0.016121258959174156, + 0.019633492454886436, + 0.019987935200333595, + 0.02385960891842842, + 0.02486182004213333, + 0.011994888074696064, + 0.01274633314460516, + 0.015906881541013718, + 0.016767995432019234, + 0.02143402211368084, + 0.022238746285438538, + 0.02530583366751671, + 0.02633264660835266, + 0.009710066020488739, + 0.0102624436840415, + 0.015399535186588764, + 0.016213588416576385, + 0.01968187838792801, + 0.020620016381144524, + 0.024621067568659782, + 0.02528378553688526 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 6.720464749984615, + "cross_entropy_nats": 4.6582711935043335, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "0f212e3b0862c8a0fa1714765ff7cc8edeb23e1470b2107592e11f4eeae72674", + "final_model_state": "ac8be80504f01ca19f023c383b7139873a7635445148dec83aff2b39b674dd20", + "final_optimizer_state": "de50114df8c8cb4b59e160333f861411e6676277c7a3a849bd21ada1ea6abc54", + "final_public_parameters": "063f8f52a2b325c61dff3a9a7bdd1302f383a71c59990ca1765419e2e5dcbb96", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 6751356416, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-baseline.json b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-baseline.json new file mode 100644 index 0000000..cd8ae6f --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-baseline.json @@ -0,0 +1,700 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "0ae9ad1697ca15ec4c84270ad82b9fa98056f14f20fc7372c4b9886c29845415", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002865509013645351, + 0.00024785567075014114, + 0.00021551651298068464, + 0.00019683866412378848, + 0.0001823508064262569, + 0.0001679604029050097, + 0.00015584587526973337, + 0.00014737028686795384, + 0.00014130656199995428, + 0.00013612695329356939, + 0.00013129493163432926, + 0.00012807638267986476, + 0.00012550120300147682, + 0.00012324050476308912, + 0.00012108208466088399, + 0.00011888353037647903 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023669043730478734, + "first_to_last_ratio": 1.9372775995887173, + "imbalance_abs_log_ratio": 0.6612836883477485, + "last_quartile_mean": 0.00012217683070048224, + "mean": 0.00016411257956860936, + "normalized": [ + 1.7460629899168627, + 1.5102783187106137, + 1.313223602646409, + 1.1994124072706904, + 1.1111324123086055, + 1.0234462424910682, + 0.9496278449793059, + 0.8979828801383491, + 0.8610343117596252, + 0.8294729974472175, + 0.8000296624393728, + 0.7804178266926869, + 0.7647262832098098, + 0.7509509940495869, + 0.7377989242455608, + 0.7244023016942358 + ], + "population_cv": 0.29361872380036635 + }, + "activation_output_rms_by_block": [ + 0.02866268903017044, + 0.029099803417921066, + 0.02953805774450302, + 0.030088091269135475, + 0.030644793063402176, + 0.03131929785013199, + 0.03213750571012497, + 0.03286394104361534, + 0.033869802951812744, + 0.03499744459986687, + 0.03588006645441055, + 0.037365153431892395, + 0.03789033368229866, + 0.039445169270038605, + 0.04050002992153168, + 0.04121527820825577 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0293471603654325, + "first_to_last_ratio": 0.7380574840395957, + "imbalance_abs_log_ratio": 0.3037335657624931, + "last_quartile_mean": 0.03976270277053118, + "mean": 0.034094841103069484, + "normalized": [ + 0.8406752488894867, + 0.8534957922212247, + 0.8663497699023965, + 0.8824822259232266, + 0.8988102619619861, + 0.9185934539320196, + 0.942591449919727, + 0.9638977622528551, + 0.9933996421752943, + 1.0264733158329964, + 1.0523605710888722, + 1.0959180985456622, + 1.1113216092650298, + 1.1569248600043318, + 1.1878638706395246, + 1.2088420674453664 + ], + "population_cv": 0.11952987076282337 + }, + "bits_per_byte": 8.096463027059821, + "branch_output_rms_by_sublayer": [ + 0.0033055038657039404, + 0.0037518907338380814, + 0.0033751516602933407, + 0.003911525942385197, + 0.004229962360113859, + 0.0038461871445178986, + 0.004243654198944569, + 0.0038366704247891903, + 0.004372371360659599, + 0.003851204412057996, + 0.005078981164842844, + 0.003818925702944398, + 0.0054668826051056385, + 0.0038749484810978174, + 0.005823117680847645, + 0.0037566579412668943, + 0.0065947300754487514, + 0.003876061411574483, + 0.006763988174498081, + 0.003795720636844635, + 0.006863070651888847, + 0.0038706199266016483, + 0.007856340147554874, + 0.003839300014078617, + 0.00830968376249075, + 0.003927029203623533, + 0.009655521251261234, + 0.0038513424806296825, + 0.009004893712699413, + 0.0039484030567109585, + 0.008608299307525158, + 0.003791053779423237 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00809059897248305, + 0.007313812062277777, + 0.007090710224412361, + 0.0067033738367094624, + 0.006535407867454384, + 0.006785021795736036, + 0.006295993569919972, + 0.005509300326230647, + 0.006231736048910802, + 0.006047390980174007, + 0.005629405359532731, + 0.005784042737956458, + 0.0059591736113607996, + 0.0061983173180850636, + 0.006571690039252182, + 0.005060205437758869 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007299623773970663, + "first_to_last_ratio": 1.227374872012571, + "imbalance_abs_log_ratio": 0.2048776382299483, + "last_quartile_mean": 0.005947346601614228, + "mean": 0.006362886261765913, + "normalized": [ + 1.271529717747562, + 1.1494488132258316, + 1.1143858200043408, + 1.05351149791715, + 1.0271137340180259, + 1.066343404015675, + 0.9894870520870547, + 0.8658492545019393, + 0.9793882512652816, + 0.9504163254515969, + 0.8847251275509024, + 0.909028151691524, + 0.9365519618304368, + 0.9741361173356609, + 1.0328158902888074, + 0.7952688810682109 + ], + "population_cv": 0.11384977604868386 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02823694422841072, + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679 + ], + "loss_nats": 5.6120405197143555, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679, + 0.04121527820825577 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.69471885683015e-05, + 5.527245593839325e-05, + 5.393734318204224e-05, + 5.292302375892177e-05, + 5.23102717124857e-05, + 5.178990977583453e-05, + 5.1564093155320734e-05, + 5.129844430484809e-05, + 5.121947833686136e-05, + 5.116340616950765e-05, + 5.109804988023825e-05, + 5.104695082991384e-05, + 5.09646451973822e-05, + 5.100828275317326e-05, + 5.106439857627265e-05, + 5.11144389747642e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.477000286191469e-05, + "first_to_last_ratio": 1.0731232762518041, + "imbalance_abs_log_ratio": 0.07057334637995329, + "last_quartile_mean": 5.103794137539808e-05, + "mean": 5.2170148819641327e-05, + "normalized": [ + 1.0915665348238701, + 1.0594651767139285, + 1.033873669184083, + 1.0144311441756324, + 1.002685882559561, + 0.9927115591500164, + 0.988383094968431, + 0.9832911246274795, + 0.9817775010367221, + 0.9807027069519371, + 0.9794499543578176, + 0.9784704852268963, + 0.9768928467805085, + 0.9777292936141551, + 0.9788049244944386, + 0.9797641013345227 + ], + "population_cv": 0.032815404485150704 + }, + "activation_output_rms_by_block": [ + 0.028743742033839226, + 0.02942933700978756, + 0.030596865341067314, + 0.03249936178326607, + 0.03542664647102356, + 0.03894684836268425, + 0.04318666458129883, + 0.04936420917510986, + 0.05370106175541878, + 0.06048284471035004, + 0.0664696991443634, + 0.07231792062520981, + 0.07627613097429276, + 0.07997097074985504, + 0.08494995534420013, + 0.08972473442554474 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03031732654199004, + "first_to_last_ratio": 0.3664591129538783, + "imbalance_abs_log_ratio": 1.0038683247140607, + "last_quartile_mean": 0.08273044787347317, + "mean": 0.05450543703045696, + "normalized": [ + 0.5273555006590916, + 0.5399339701348108, + 0.5613543713807885, + 0.5962590808162097, + 0.6499653686150171, + 0.7145497859400932, + 0.7923368187501488, + 0.9056749539963465, + 0.9852422929002714, + 1.1096662646068314, + 1.21950584686113, + 1.326801958945847, + 1.399422427007981, + 1.4672108895332452, + 1.5585592919240543, + 1.6461611779281335 + ], + "population_cv": 0.3808757530834043 + }, + "bits_per_byte": 6.7509657011324835, + "branch_output_rms_by_sublayer": [ + 0.003436450148001313, + 0.003791899885982275, + 0.003964710980653763, + 0.003938332665711641, + 0.0056790695525705814, + 0.003919641952961683, + 0.006960175931453705, + 0.0038887569680809975, + 0.008646421134471893, + 0.003928068559616804, + 0.009785857982933521, + 0.0038193853106349707, + 0.010625048540532589, + 0.003996006678789854, + 0.012892307713627815, + 0.0039014811627566814, + 0.012757784686982632, + 0.003861474571749568, + 0.014726920053362846, + 0.003962590359151363, + 0.012820222415030003, + 0.004551138263195753, + 0.01401793584227562, + 0.004161422606557608, + 0.01284075528383255, + 0.004107494372874498, + 0.012963366694748402, + 0.0034911984112113714, + 0.014392351731657982, + 0.004168955609202385, + 0.013408888131380081, + 0.003909120801836252 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004507690637370199, + 0.0004530669153600251, + 0.0005174622333717066, + 0.0005579942111201043, + 0.0006494724560479704, + 0.0007477190266084469, + 0.0008433132451421535, + 0.0008983201732199723, + 0.001008715304343127, + 0.0011143118429534733, + 0.0010428854825857738, + 0.0010916116075341876, + 0.0011085001103672855, + 0.0012404769719650786, + 0.0013545740271648265, + 0.0014243271893498689 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.000494823105897214, + "first_to_last_ratio": 0.385986622193018, + "imbalance_abs_log_ratio": 0.9519525676489338, + "last_quartile_mean": 0.001281969574711765, + "mean": 0.0009064699913044387, + "normalized": [ + 0.49727963204644987, + 0.4998145771025995, + 0.5708542349284638, + 0.6155683215912455, + 0.7164853357289404, + 0.824869034585972, + 0.930326710461313, + 0.99100927977468, + 1.112795033503044, + 1.2292870736404011, + 1.1504909071341995, + 1.2042446170372658, + 1.2228756836970622, + 1.3684699812069823, + 1.494339625314625, + 1.571289952246756 + ], + "population_cv": 0.338812252534038 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028241552412509918, + 0.028533460572361946, + 0.028743742033839226, + 0.029236938804388046, + 0.02942933700978756, + 0.03035305254161358, + 0.030596865341067314, + 0.032195623964071274, + 0.03249936178326607, + 0.03506496921181679, + 0.03542664647102356, + 0.03852483630180359, + 0.03894684836268425, + 0.04251723736524582, + 0.04318666458129883, + 0.04861506074666977, + 0.04936420917510986, + 0.052915990352630615, + 0.05370106175541878, + 0.0598941408097744, + 0.06048284471035004, + 0.06538087129592896, + 0.0664696991443634, + 0.07142146676778793, + 0.07231792062520981, + 0.07521561533212662, + 0.07627613097429276, + 0.07935076206922531, + 0.07997097074985504, + 0.08368266373872757, + 0.08494995534420013, + 0.08860929310321808 + ], + "loss_nats": 4.679412841796875, + "loss_scale": 1.0, + "output_weights": null, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.028533460572361946, + 0.028743742033839226, + 0.029236938804388046, + 0.02942933700978756, + 0.03035305254161358, + 0.030596865341067314, + 0.032195623964071274, + 0.03249936178326607, + 0.03506496921181679, + 0.03542664647102356, + 0.03852483630180359, + 0.03894684836268425, + 0.04251723736524582, + 0.04318666458129883, + 0.04861506074666977, + 0.04936420917510986, + 0.052915990352630615, + 0.05370106175541878, + 0.0598941408097744, + 0.06048284471035004, + 0.06538087129592896, + 0.0664696991443634, + 0.07142146676778793, + 0.07231792062520981, + 0.07521561533212662, + 0.07627613097429276, + 0.07935076206922531, + 0.07997097074985504, + 0.08368266373872757, + 0.08494995534420013, + 0.08860929310321808, + 0.08972473442554474 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.076786148009306, + "cross_entropy_nats": 5.5984015464782715, + "step": 0 + }, + { + "bits_per_byte": 6.752401670275862, + "cross_entropy_nats": 4.680408179759979, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0", + "final_optimizer_state": "a7bce44db1478ce53933758aa5033bbb1e0aa21296e9615c33146920f30a2057", + "final_public_parameters": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 1648265728, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.088097790921855, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.6062421798706055, + "step": 1, + "unclipped_grad_norm": 19.475919723510742 + }, + { + "bits_per_byte": 7.474302716882146, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.180791854858398, + "step": 10, + "unclipped_grad_norm": 12.938376426696777 + }, + { + "bits_per_byte": 6.789615018295581, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.706202507019043, + "step": 20, + "unclipped_grad_norm": 4.482712745666504 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-block.json b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-block.json new file mode 100644 index 0000000..c5f7945 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-16-block.json @@ -0,0 +1,1450 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "a0d2c00339b4ed2c671f57d7a804a28f3934cfaeaa7e1412046d0ce71e823782", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027713447343558073, + 0.00024001447309274226, + 0.00020889485313091427, + 0.00019106797117274255, + 0.00017709653184283525, + 0.00016345064796041697, + 0.0001520135992905125, + 0.00014396758342627436, + 0.00013824697816744447, + 0.0001333381951553747, + 0.00012879272981081158, + 0.00012578140012919903, + 0.00012338913802523166, + 0.0001212825154652819, + 0.00011928759340662509, + 0.00011723516217898577 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022927794270799495, + "first_to_last_ratio": 1.9059069547229377, + "imbalance_abs_log_ratio": 0.6449579870010378, + "last_quartile_mean": 0.00012029860226903111, + "mean": 0.00016006211535568582, + "normalized": [ + 1.7314182860806244, + 1.4995083162520277, + 1.3050861702453052, + 1.1937113960299495, + 1.1064237870985023, + 1.021170110098828, + 0.9497162957812465, + 0.8994482117542517, + 0.8637083038683806, + 0.8330403159990363, + 0.8046421823465957, + 0.7858286756343993, + 0.7708828397715447, + 0.7577215582573746, + 0.7452581339534802, + 0.7324354168284537 + ], + "population_cv": 0.28697332073733334 + }, + "activation_output_rms_by_block": [ + 0.005004490725696087, + 0.007150472607463598, + 0.0056586493737995625, + 0.007886149920523167, + 0.005775847006589174, + 0.008285986259579659, + 0.006597128231078386, + 0.009296777658164501, + 0.007501539774239063, + 0.01081777922809124, + 0.007698386441916227, + 0.011321330443024635, + 0.009009703993797302, + 0.013792803511023521, + 0.00953624676913023, + 0.012954792007803917 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0064249406568706036, + "first_to_last_ratio": 0.567404514268178, + "imbalance_abs_log_ratio": 0.5666828005795043, + "last_quartile_mean": 0.011323386570438743, + "mean": 0.008643005246995017, + "normalized": [ + 0.5790220626599802, + 0.8273132322752738, + 0.6547085431617609, + 0.9124314628022479, + 0.6682683674867963, + 0.9586927258271091, + 0.7632910130850682, + 1.0756417926966766, + 0.8679318778438997, + 1.251622429808468, + 0.8907071350665657, + 1.3098835554867694, + 1.04242722714183, + 1.5958342170183197, + 1.103348487778111, + 1.4988758698611242 + ], + "population_cv": 0.28641618387108014 + }, + "bits_per_byte": 8.092078157393424, + "branch_output_rms_by_sublayer": [ + 0.0033052791841328144, + 0.003737553022801876, + 0.0033692142460495234, + 0.0038969451561570168, + 0.004222586750984192, + 0.003809630637988448, + 0.004224879201501608, + 0.003801452461630106, + 0.004351162351667881, + 0.0037875697016716003, + 0.005037518683820963, + 0.0037579576019197702, + 0.005422801710665226, + 0.003782435320317745, + 0.005750134587287903, + 0.003670723643153906, + 0.006515308283269405, + 0.003754077944904566, + 0.006654165219515562, + 0.0036852110642939806, + 0.006751408334821463, + 0.003720880253240466, + 0.007695567328482866, + 0.003703791182488203, + 0.008140962570905685, + 0.003749529365450144, + 0.009426429867744446, + 0.0036915375385433435, + 0.008798080496490002, + 0.003752112854272127, + 0.008380400948226452, + 0.0036098985001444817 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007836087849617155, + 0.00707663314616961, + 0.006860545956961424, + 0.006477463491545069, + 0.00630808320659222, + 0.0065403273634858875, + 0.006076989561376891, + 0.0053044433268254945, + 0.006002429844215364, + 0.005811298045810947, + 0.00541486468124986, + 0.005552686577658202, + 0.005724443119797102, + 0.005942467552368739, + 0.006312250195815329, + 0.004848514440357715 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007062682611073314, + "first_to_last_ratio": 1.237564932158175, + "imbalance_abs_log_ratio": 0.21314568451503188, + "last_quartile_mean": 0.005706918827084721, + "mean": 0.006130595522490438, + "normalized": [ + 1.2781935818257821, + 1.154314147819503, + 1.119066807097797, + 1.0565798163950184, + 1.0289511326347756, + 1.0668339379905794, + 0.9912559944760846, + 0.865241118479574, + 0.9790940899942118, + 0.947917380047643, + 0.883252640201938, + 0.9057336366895933, + 0.9337499267072272, + 0.9693132633801169, + 1.0296308364592772, + 0.7908716898008789 + ], + "population_cv": 0.1166547247084935 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02823692187666893, + 0.014202049933373928, + 0.014330258592963219, + 0.014431006275117397, + 0.0145475585013628, + 0.009805543348193169, + 0.009861846454441547, + 0.009975572116672993, + 0.010043247602880001, + 0.007605585735291243, + 0.007654537446796894, + 0.0077590313740074635, + 0.007819762453436852, + 0.006375181954354048, + 0.006421996746212244, + 0.0065260399132966995, + 0.006563509814441204, + 0.005602568853646517, + 0.0056375316344201565, + 0.0057973419316112995, + 0.005820456892251968, + 0.005056646186858416, + 0.00509054446592927, + 0.005272198468446732, + 0.005295110866427422, + 0.0046798414550721645, + 0.004703808110207319, + 0.004869809839874506, + 0.004889858886599541, + 0.004459173884242773, + 0.00446688337251544, + 0.00453129643574357 + ], + "loss_nats": 5.609001159667969, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0033052791841328144, + 0.005004490725696087, + 0.005999269895255566, + 0.007150472607463598, + 0.004222586750984192, + 0.0056586493737995625, + 0.006890070624649525, + 0.007886149920523167, + 0.004351162351667881, + 0.005775847006589174, + 0.007411441300064325, + 0.008285986259579659, + 0.005422801710665226, + 0.006597128231078386, + 0.008578632958233356, + 0.009296777658164501, + 0.006515308283269405, + 0.007501539774239063, + 0.010256913490593433, + 0.01081777922809124, + 0.006751408334821463, + 0.007698386441916227, + 0.01081965770572424, + 0.011321330443024635, + 0.008140962570905685, + 0.009009703993797302, + 0.013233783654868603, + 0.013792803511023521, + 0.008798080496490002, + 0.00953624676913023, + 0.01231932733207941, + 0.012954792007803917 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.571244764723815e-05, + 5.411167876445688e-05, + 5.283674909151159e-05, + 5.185072586755268e-05, + 5.127084659761749e-05, + 5.077067544334568e-05, + 5.0770395318977535e-05, + 5.0520964578026906e-05, + 5.0666069000726566e-05, + 5.061549381935038e-05, + 5.055486690253019e-05, + 5.0508093409007415e-05, + 4.999646989745088e-05, + 5.004305057809688e-05, + 5.0446498789824545e-05, + 5.0497859774623066e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.362790034268983e-05, + "first_to_last_ratio": 1.067307499464034, + "imbalance_abs_log_ratio": 0.06513912148859854, + "last_quartile_mean": 5.024596975999884e-05, + "mean": 5.132330534252105e-05, + "normalized": [ + 1.0855194784401525, + 1.0543295760732245, + 1.0294884310137495, + 1.010276433318387, + 0.9989778767257201, + 0.9892323790237741, + 0.989226920989334, + 0.9843669311799096, + 0.9871941930199502, + 0.9862087697110136, + 0.985027495114306, + 0.9841161451299155, + 0.9741475059679974, + 0.9750550991234875, + 0.9829160155051417, + 0.9839167496639366 + ], + "population_cv": 0.029989256005628663 + }, + "activation_output_rms_by_block": [ + 0.005180664826184511, + 0.007948367856442928, + 0.0071101076900959015, + 0.011628935113549232, + 0.009834222495555878, + 0.015592618845403194, + 0.011937727220356464, + 0.02099907398223877, + 0.014203419908881187, + 0.023388667032122612, + 0.014353072270751, + 0.022941188886761665, + 0.013975659385323524, + 0.02055426314473152, + 0.015514259226620197, + 0.022412775084376335 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.007967018871568143, + "first_to_last_ratio": 0.4398207829260811, + "imbalance_abs_log_ratio": 0.8213879465753375, + "last_quartile_mean": 0.018114239210262895, + "mean": 0.014848438935587183, + "normalized": [ + 0.3489029957060359, + 0.5352998985902224, + 0.47884546792694416, + 0.7831756027684648, + 0.6623068282273259, + 1.0501183937950838, + 0.8039718701839673, + 1.4142277227480384, + 0.9565598087782763, + 1.575159997194527, + 0.966638468394887, + 1.5450236207510426, + 0.9412207873130777, + 1.3842709818787224, + 1.04484109702854, + 1.5094364587148448 + ], + "population_cv": 0.38149080499888866 + }, + "bits_per_byte": 6.756301290892851, + "branch_output_rms_by_sublayer": [ + 0.00344275776296854, + 0.0037783386651426554, + 0.003971979953348637, + 0.003924910444766283, + 0.0057159834541380405, + 0.0038860586937516928, + 0.007017855532467365, + 0.003858900861814618, + 0.008758665062487125, + 0.0038810574915260077, + 0.009854576550424099, + 0.0037807892076671124, + 0.010700107552111149, + 0.003945234231650829, + 0.012973167933523655, + 0.003866987768560648, + 0.012833327986299992, + 0.0038173284847289324, + 0.01477400679141283, + 0.003928930964320898, + 0.012848040089011192, + 0.004507644101977348, + 0.014026274904608727, + 0.004126352723687887, + 0.012859554030001163, + 0.004076994024217129, + 0.01296316273510456, + 0.0034613758325576782, + 0.01440997514873743, + 0.0041273003444075584, + 0.01337564829736948, + 0.0038736232090741396 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004508021297651076, + 0.0004524930113583151, + 0.0005201601896034384, + 0.0005594057048027396, + 0.0006552041707250334, + 0.0007507040984739487, + 0.000850349803788574, + 0.0009063852334487553, + 0.0010217915468968724, + 0.0011277818818807989, + 0.0010544629528892386, + 0.0011012469919556404, + 0.0011079217036188914, + 0.001237384982851684, + 0.0013584092718518022, + 0.0014241647000383885 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004957152588824002, + "first_to_last_ratio": 0.3866823679479825, + "imbalance_abs_log_ratio": 0.950151677612245, + "last_quartile_mean": 0.0012819701645901916, + "mean": 0.0009111667733718268, + "normalized": [ + 0.494752599567352, + 0.4966083318466913, + 0.5708726490086493, + 0.61394436359068, + 0.7190825981290028, + 0.8238931888351498, + 0.9332537452411748, + 0.9947522889740842, + 1.1214100170879497, + 1.2377337660232879, + 1.1572666867418089, + 1.208611885484378, + 1.215937340997369, + 1.3580225036879552, + 1.490845925850575, + 1.5630121089338918 + ], + "population_cv": 0.3380559322236514 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931420564651489, + "mean_weights": [ + 0.5014879107475281, + 0.49851205945014954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.693139910697937, + "mean_weights": [ + 0.5018156170845032, + 0.4981843829154968 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931352019309998, + "mean_weights": [ + 0.5023940205574036, + 0.49760597944259644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931431889533997, + "mean_weights": [ + 0.49870234727859497, + 0.501297652721405 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986039638519287, + "mean_weights": [ + 0.33161261677742004, + 0.33409368991851807, + 0.3342936635017395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986011028289795, + "mean_weights": [ + 0.3315771222114563, + 0.33318889141082764, + 0.3352339565753937 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986087322235107, + "mean_weights": [ + 0.3331664800643921, + 0.3342593312263489, + 0.3325742185115814 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985949039459229, + "mean_weights": [ + 0.3315378725528717, + 0.3324793577194214, + 0.3359827995300293 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862826824188232, + "mean_weights": [ + 0.24934276938438416, + 0.24841846525669098, + 0.2508545219898224, + 0.2513842284679413 + ], + "sources": 4 + }, + { + "entropy_mean": 1.38628089427948, + "mean_weights": [ + 0.248581200838089, + 0.24957583844661713, + 0.24980047345161438, + 0.2520424723625183 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386273741722107, + "mean_weights": [ + 0.2486811876296997, + 0.2481946051120758, + 0.2515612244606018, + 0.25156301259994507 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862909078598022, + "mean_weights": [ + 0.24919962882995605, + 0.24966104328632355, + 0.25073325634002686, + 0.25040602684020996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094350814819336, + "mean_weights": [ + 0.20043906569480896, + 0.20025089383125305, + 0.1992931365966797, + 0.19971072673797607, + 0.20030616223812103 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094260215759277, + "mean_weights": [ + 0.1990065574645996, + 0.19869452714920044, + 0.20100653171539307, + 0.20066148042678833, + 0.20063093304634094 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609431266784668, + "mean_weights": [ + 0.19923549890518188, + 0.20021051168441772, + 0.20099328458309174, + 0.2002217173576355, + 0.19933897256851196 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609424114227295, + "mean_weights": [ + 0.19919060170650482, + 0.19957327842712402, + 0.1993737816810608, + 0.1998673677444458, + 0.20199497044086456 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917512655258179, + "mean_weights": [ + 0.16589663922786713, + 0.16579726338386536, + 0.16652584075927734, + 0.1674002707004547, + 0.1673392355442047, + 0.16704075038433075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917526960372925, + "mean_weights": [ + 0.1660098284482956, + 0.1660982221364975, + 0.16643531620502472, + 0.16734513640403748, + 0.16757234930992126, + 0.16653914749622345 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917439937591553, + "mean_weights": [ + 0.1664077788591385, + 0.1667102724313736, + 0.16575725376605988, + 0.16558866202831268, + 0.16724394261837006, + 0.1682921051979065 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917406558990479, + "mean_weights": [ + 0.1662176251411438, + 0.16636696457862854, + 0.165130153298378, + 0.16684658825397491, + 0.16694699227809906, + 0.1684916913509369 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459023475646973, + "mean_weights": [ + 0.14233610033988953, + 0.1422998011112213, + 0.14257268607616425, + 0.1434200406074524, + 0.14240685105323792, + 0.14367292821407318, + 0.14329159259796143 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458985328674316, + "mean_weights": [ + 0.1425248682498932, + 0.14280283451080322, + 0.1430899053812027, + 0.14388513565063477, + 0.14334818720817566, + 0.14153052866458893, + 0.14281854033470154 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14222130179405212, + 0.14271843433380127, + 0.14239171147346497, + 0.1434115171432495, + 0.1425933688879013, + 0.14311325550079346, + 0.14355038106441498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14283692836761475, + 0.14230498671531677, + 0.14244258403778076, + 0.1435498595237732, + 0.14231878519058228, + 0.14314766228199005, + 0.143399178981781 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432964324951, + "mean_weights": [ + 0.12469614297151566, + 0.12493360042572021, + 0.12421932816505432, + 0.12482892721891403, + 0.12584659457206726, + 0.12576377391815186, + 0.12501835823059082, + 0.12469328939914703 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794296264648438, + "mean_weights": [ + 0.12482310831546783, + 0.12494714558124542, + 0.12461379170417786, + 0.12449270486831665, + 0.12426981329917908, + 0.12538880109786987, + 0.12516307830810547, + 0.12630154192447662 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794286727905273, + "mean_weights": [ + 0.12459714710712433, + 0.12472857534885406, + 0.12479612231254578, + 0.12524385750293732, + 0.12565991282463074, + 0.1262103021144867, + 0.12457992881536484, + 0.12418416142463684 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794296264648438, + "mean_weights": [ + 0.12493155896663666, + 0.12483802437782288, + 0.12460187077522278, + 0.12459874153137207, + 0.12501779198646545, + 0.12420021742582321, + 0.12559065222740173, + 0.12622115015983582 + ], + "sources": 8 + }, + { + "entropy_mean": 2.197211265563965, + "mean_weights": [ + 0.11090581119060516, + 0.11095649003982544, + 0.11032625287771225, + 0.11037011444568634, + 0.11067523062229156, + 0.11147095263004303, + 0.11175340414047241, + 0.1119384914636612, + 0.1116032600402832 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972126960754395, + "mean_weights": [ + 0.11083438992500305, + 0.11066233366727829, + 0.1106841117143631, + 0.11084605753421783, + 0.11061468720436096, + 0.11108818650245667, + 0.1115877777338028, + 0.11133615672588348, + 0.11234630644321442 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11089816689491272, + 0.1108870804309845, + 0.11106622219085693, + 0.11082258075475693, + 0.11085513234138489, + 0.1119755208492279, + 0.11102905869483948, + 0.11071982979774475, + 0.11174643039703369 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0282417144626379, + 0.01428750716149807, + 0.014416527934372425, + 0.014703329652547836, + 0.014682094566524029, + 0.01008235290646553, + 0.010167798958718777, + 0.010738944634795189, + 0.01082152035087347, + 0.00878758542239666, + 0.008862113580107689, + 0.009672283194959164, + 0.00976790115237236, + 0.008560523390769958, + 0.008684799075126648, + 0.00977813359349966, + 0.009954164735972881, + 0.008888072334229946, + 0.00902888085693121, + 0.01007006410509348, + 0.010184217244386673, + 0.009424775838851929, + 0.009566618129611015, + 0.010299356654286385, + 0.010436818934977055, + 0.009499363601207733, + 0.009606627747416496, + 0.010012919083237648, + 0.010090752504765987, + 0.00939567293971777, + 0.009533073753118515, + 0.009944267570972443 + ], + "loss_nats": 4.683111190795898, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972169876098633, + "mean_weights": [ + 0.11062228679656982, + 0.11076737940311432, + 0.1108340471982956, + 0.11088211834430695, + 0.11121642589569092, + 0.11167430132627487, + 0.111720010638237, + 0.11070731282234192, + 0.1115761250257492 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.00344275776296854, + 0.005180664826184511, + 0.006782237906008959, + 0.007948367856442928, + 0.0057159834541380405, + 0.0071101076900959015, + 0.010661935433745384, + 0.011628935113549232, + 0.008758665062487125, + 0.009834222495555878, + 0.014840804971754551, + 0.015592618845403194, + 0.010700107552111149, + 0.011937727220356464, + 0.019919002428650856, + 0.02099907398223877, + 0.012833327986299992, + 0.014203419908881187, + 0.022593334317207336, + 0.023388667032122612, + 0.012848040089011192, + 0.014353072270751, + 0.0218671802431345, + 0.022941188886761665, + 0.012859554030001163, + 0.013975659385323524, + 0.020060954615473747, + 0.02055426314473152, + 0.01440997514873743, + 0.015514259226620197, + 0.021293850615620613, + 0.022412775084376335 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.073347956844582, + "cross_entropy_nats": 5.596018373966217, + "step": 0 + }, + { + "bits_per_byte": 6.7577811156122, + "cross_entropy_nats": 4.684136927127838, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "5dd01b8270ff98adc6b1340e38ee82b56d324c449cbdbb97581afacdec1223dc", + "final_model_state": "cf7d109dd2a8a9a3c903435afe8c733b4191e11d3150f7eee30962a8de6e0a3d", + "final_optimizer_state": "96110c28f2690589414ee6ed445333520dd0bc9ac87eeb7d4f316b17494d8f30", + "final_public_parameters": "b20e9b701d001f45efdba361481e6a18c3eb9f6fec33fa6434bd9773667dd881", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 3462317568, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.084436624250287, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.603704452514648, + "step": 1, + "unclipped_grad_norm": 18.78145980834961 + }, + { + "bits_per_byte": 7.484355446790379, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.187759876251221, + "step": 10, + "unclipped_grad_norm": 12.867269515991211 + }, + { + "bits_per_byte": 6.795359238834398, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.710184097290039, + "step": 20, + "unclipped_grad_norm": 4.549929141998291 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-baseline.json b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-baseline.json new file mode 100644 index 0000000..f2df857 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-baseline.json @@ -0,0 +1,1132 @@ +{ + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "5958d579e71547e6f80ee5807def35862d1fe1f475d705f6d6c7f20874d1f5b2", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002768727717921138, + 0.0002554995589889586, + 0.00023783760843798518, + 0.00022115583124104887, + 0.00020655356638599187, + 0.00019376208365429193, + 0.00018488570640329272, + 0.00017590694187674671, + 0.0001684112212387845, + 0.0001647331955609843, + 0.00015843863366171718, + 0.0001538397918920964, + 0.00015132295084185898, + 0.00014730033581145108, + 0.00014253577683120966, + 0.0001394751889165491, + 0.0001364792260574177, + 0.0001337067224085331, + 0.00013106675760354847, + 0.00012915431580040604, + 0.00012709009752143174, + 0.00012574320135172457, + 0.0001242366706719622, + 0.0001228937617270276, + 0.00012132258416386321, + 0.00011988351616309956, + 0.00011879197700181976, + 0.00011764531518565491, + 0.00011639683361863717, + 0.00011549647024367005, + 0.00011481995898066089, + 0.00011415599146857858 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002190592585975537, + "first_to_last_ratio": 1.8672887091156776, + "imbalance_abs_log_ratio": 0.6244874905881238, + "last_quartile_mean": 0.00011731408085324802, + "mean": 0.0001546067051094724, + "normalized": [ + 1.7908199492128654, + 1.6525774791465027, + 1.538339545297078, + 1.4304413969915148, + 1.3359935860462033, + 1.2532579587482624, + 1.1958453307208163, + 1.1377704592578457, + 1.0892879524179597, + 1.065498391187767, + 1.0247850088360109, + 0.9950395863049215, + 0.978760596021445, + 0.9527422223192203, + 0.9219249367631522, + 0.9021289782858439, + 0.8827510162691898, + 0.8648183939620171, + 0.8477430361816715, + 0.835373315206192, + 0.8220219002238166, + 0.8133101424203405, + 0.80356586464988, + 0.7948798963150414, + 0.7847174809007043, + 0.775409553409819, + 0.7683494510649245, + 0.7609328140222236, + 0.7528576043077825, + 0.7470340316863389, + 0.7426583400723811, + 0.7383637817502684 + ], + "population_cv": 0.27888268733798954 + }, + "activation_output_rms_by_block": [ + 0.028270306065678596, + 0.0284538883715868, + 0.02867252007126808, + 0.02881420962512493, + 0.029061447829008102, + 0.02929086796939373, + 0.029533835127949715, + 0.029811719432473183, + 0.030115574598312378, + 0.030315518379211426, + 0.030812423676252365, + 0.03120259940624237, + 0.03179442882537842, + 0.03198441118001938, + 0.03240228444337845, + 0.0329107940196991, + 0.033366311341524124, + 0.033811621367931366, + 0.03420914337038994, + 0.03470747172832489, + 0.03500960022211075, + 0.03567802160978317, + 0.035978782922029495, + 0.036836784332990646, + 0.03769209608435631, + 0.038124579936265945, + 0.038542140275239944, + 0.03904740884900093, + 0.0400773324072361, + 0.04093807935714722, + 0.04163828864693642, + 0.042313966900110245 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.028988599311560392, + "first_to_last_ratio": 0.728416493900548, + "imbalance_abs_log_ratio": 0.31688228737002583, + "last_quartile_mean": 0.03979673655703664, + "mean": 0.03379432682413608, + "normalized": [ + 0.8365399971656723, + 0.8419723381282118, + 0.8484418174825138, + 0.8526345198433033, + 0.8599504875549783, + 0.8667392051281828, + 0.873928789339177, + 0.8821515986281313, + 0.8911429055838948, + 0.8970593951159853, + 0.9117632032322649, + 0.9233088017589188, + 0.9408214873116122, + 0.9464432106153376, + 0.9588084003566117, + 0.9738555879797565, + 0.9873346942272493, + 1.0005117587897308, + 1.0122747391422395, + 1.0270206567197142, + 1.0359608701276664, + 1.055739970659861, + 1.0646397281194921, + 1.0900286466627183, + 1.1153379761196018, + 1.1281355043603702, + 1.1404914344295491, + 1.1554427183059932, + 1.1859189447920195, + 1.2113891059344621, + 1.2321088348236646, + 1.2521026715611152 + ], + "population_cv": 0.12455242949918699 + }, + "bits_per_byte": 7.95915895791699, + "branch_output_rms_by_sublayer": [ + 0.0022265268489718437, + 0.002717922907322645, + 0.0021652730647474527, + 0.002733128145337105, + 0.0025650423485785723, + 0.002695153234526515, + 0.002537536434829235, + 0.002696225419640541, + 0.0027307753916829824, + 0.0027451792266219854, + 0.0029185698367655277, + 0.0026901187375187874, + 0.002936159959062934, + 0.0027115964330732822, + 0.0030124466866254807, + 0.0027617362793534994, + 0.0031113612931221724, + 0.002723218873143196, + 0.0034121612552553415, + 0.0027921765577048063, + 0.0034701526165008545, + 0.002744668396189809, + 0.0039346772246062756, + 0.0027058592531830072, + 0.004295050632208586, + 0.0026920263189822435, + 0.004176756367087364, + 0.0027182905469089746, + 0.004023758694529533, + 0.0027150253299623728, + 0.004447397775948048, + 0.0027387896552681923, + 0.004260339308530092, + 0.00275799329392612, + 0.005327676888555288, + 0.002721488941460848, + 0.00474865036085248, + 0.0027454239316284657, + 0.00498866755515337, + 0.002719792304560542, + 0.0052231852896511555, + 0.0027601351030170918, + 0.005184618756175041, + 0.002755985129624605, + 0.0052438946440815926, + 0.0027470553759485483, + 0.0059704589657485485, + 0.0027777766808867455, + 0.0060942270793020725, + 0.0026787330862134695, + 0.005698176100850105, + 0.002644060645252466, + 0.006003713235259056, + 0.002661966485902667, + 0.006405353546142578, + 0.002804658142849803, + 0.006667613051831722, + 0.0026854604948312044, + 0.006290186662226915, + 0.0027102255262434483, + 0.0072579155676066875, + 0.0027289679273962975, + 0.006407012697309256, + 0.0026884053368121386 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.005888019831616423, + 0.005779358340731243, + 0.0060348395662513914, + 0.005586875351280001, + 0.005559071516712478, + 0.0054082748429522395, + 0.005481135467826832, + 0.0053805923812947415, + 0.004971746645687392, + 0.005439263395635174, + 0.005139424262259116, + 0.005696496994849179, + 0.005782818048228185, + 0.0054294510739720785, + 0.005067071208887471, + 0.005218073376917496, + 0.005181036474299639, + 0.005605812513830226, + 0.005170579144406511, + 0.004834742615880465, + 0.005150211089068095, + 0.00476651480071292, + 0.004776402486441839, + 0.00519066249960807, + 0.0053994045182471796, + 0.004795508114414784, + 0.004895792813412353, + 0.00496559115560922, + 0.00502742614163479, + 0.004657478744424888, + 0.005287606581362069, + 0.004870555415551657 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005639770912333169, + "first_to_last_ratio": 1.130799174678946, + "imbalance_abs_log_ratio": 0.12292461698112994, + "last_quartile_mean": 0.0049874204355821175, + "mean": 0.005263682419187692, + "normalized": [ + 1.1186122875029152, + 1.0979686615711766, + 1.146505257280075, + 1.061400537941684, + 1.0561183357962487, + 1.0274698228824493, + 1.0413119621059315, + 1.0222106792919112, + 0.9445377303850805, + 1.0333570611721248, + 0.976393302818647, + 1.0822265747043835, + 1.0986259404914112, + 1.0314929058371969, + 0.962647592570647, + 0.9913351455049914, + 0.9842988352437859, + 1.0649982402804865, + 0.9823121405573801, + 0.9185095586801336, + 0.9784425956805525, + 0.9055475655859389, + 0.9074260386664719, + 0.9861275977985596, + 1.0257846291342996, + 0.9110557462459602, + 0.9301079403205877, + 0.9433683038148654, + 0.9551157804103686, + 0.884832779319472, + 1.0045451378463044, + 0.9253133125579595 + ], + "population_cv": 0.06750436074287791 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028027428314089775, + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346 + ], + "loss_nats": 5.516868591308594, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346, + 0.042313966900110245 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.0331513446290046e-05, + 3.985461080446839e-05, + 3.939039379474707e-05, + 3.8955986383371055e-05, + 3.865906546707265e-05, + 3.843139347736724e-05, + 3.827241380349733e-05, + 3.811378701357171e-05, + 3.791215567616746e-05, + 3.7825848266948014e-05, + 3.78328186343424e-05, + 3.776324956561439e-05, + 3.772067429963499e-05, + 3.7761601561214775e-05, + 3.77303222194314e-05, + 3.774304786929861e-05, + 3.785186345339753e-05, + 3.794040094362572e-05, + 3.797327008214779e-05, + 3.8049773138482124e-05, + 3.8102883991086856e-05, + 3.8117221265565604e-05, + 3.811223359662108e-05, + 3.81382014893461e-05, + 3.817927427007817e-05, + 3.815999662037939e-05, + 3.819023913820274e-05, + 3.816752723651007e-05, + 3.8173260691110045e-05, + 3.818520053755492e-05, + 3.816522075794637e-05, + 3.822705912170932e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.9001145523798186e-05, + "first_to_last_ratio": 1.0214812032742024, + "imbalance_abs_log_ratio": 0.021253734004134984, + "last_quartile_mean": 3.818097229668638e-05, + "mean": 3.825101589427504e-05, + "normalized": [ + 1.0543906482840992, + 1.0419229364946974, + 1.0297868664095418, + 1.0184301115307508, + 1.0106676793611298, + 1.0047156285624088, + 1.000559407605838, + 0.9964124121282784, + 0.9911411446157616, + 0.9888848016873021, + 0.9890670286747801, + 0.9872482777971484, + 0.9861352285098545, + 0.987205193859086, + 0.9863874550081799, + 0.9867201429007677, + 0.9895649192172892, + 0.991879563368778, + 0.9927388644292499, + 0.9947388912140491, + 0.9961273733592431, + 0.9965021941095827, + 0.9963718009990231, + 0.9970506821246067, + 0.99812445179508, + 0.9976204743385841, + 0.998411107400643, + 0.9978173479628427, + 0.9979672382197662, + 0.9982793827776487, + 0.9977570494711616, + 0.9993736957828273 + ], + "population_cv": 0.01543848086298159 + }, + "activation_output_rms_by_block": [ + 0.028325499966740608, + 0.028658077120780945, + 0.029205938801169395, + 0.029751326888799667, + 0.03091570734977722, + 0.032307714223861694, + 0.034374311566352844, + 0.03687922656536102, + 0.03944294899702072, + 0.04215770959854126, + 0.04655195400118828, + 0.050798118114471436, + 0.05539363995194435, + 0.05819777771830559, + 0.062257908284664154, + 0.0653507262468338, + 0.06877651810646057, + 0.07320939004421234, + 0.07733170688152313, + 0.08104265481233597, + 0.08443007618188858, + 0.08700668811798096, + 0.09079501777887344, + 0.093939870595932, + 0.09777236729860306, + 0.10033779591321945, + 0.10455679148435593, + 0.10687003284692764, + 0.11059616506099701, + 0.11359578371047974, + 0.11617089062929153, + 0.11852118372917175 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.031302225310355425, + "first_to_last_ratio": 0.2883599077004872, + "imbalance_abs_log_ratio": 1.2435458995301245, + "last_quartile_mean": 0.10855262633413076, + "mean": 0.06861004745587707, + "normalized": [ + 0.4128476953023048, + 0.41769504880769653, + 0.4256802011389317, + 0.43362930054716353, + 0.4506002910092582, + 0.4708889648362173, + 0.5010098788877659, + 0.5375193274582407, + 0.5748859016951763, + 0.614453876098144, + 0.6785005363992164, + 0.7403889080114624, + 0.8073692120322151, + 0.8482398697615304, + 0.9074167792217611, + 0.9524949868145868, + 1.002426330497593, + 1.0670359919411672, + 1.1271192740575633, + 1.1812068030481144, + 1.2305788920520035, + 1.2681333324238657, + 1.323348709782827, + 1.3691853318764207, + 1.4250444493786454, + 1.4624358914450017, + 1.5239282781664902, + 1.5576440595767767, + 1.61195290138986, + 1.6556727173747092, + 1.693205221931973, + 1.7274610370353176 + ], + "population_cv": 0.4388810528199641 + }, + "bits_per_byte": 6.713833958880599, + "branch_output_rms_by_sublayer": [ + 0.002437001559883356, + 0.0027646415401250124, + 0.0026093844790011644, + 0.002773481421172619, + 0.0033957704436033964, + 0.0027507112827152014, + 0.003547805128619075, + 0.0027288019191473722, + 0.004703273996710777, + 0.0028269628528505564, + 0.005306650884449482, + 0.002743088873103261, + 0.006675844080746174, + 0.002795359119772911, + 0.006955645978450775, + 0.002845237497240305, + 0.006907123140990734, + 0.002933910582214594, + 0.00843191146850586, + 0.0027949840296059847, + 0.00897424016147852, + 0.0029047494754195213, + 0.009036343544721603, + 0.0032153772190213203, + 0.010582618415355682, + 0.0028866741340607405, + 0.009517580270767212, + 0.0026635508984327316, + 0.008616920560598373, + 0.003158343257382512, + 0.009494570083916187, + 0.0030213629361242056, + 0.009422268718481064, + 0.002808293094858527, + 0.010874342173337936, + 0.0031132514122873545, + 0.01018539723008871, + 0.002876743907108903, + 0.008500457741320133, + 0.0031567441765218973, + 0.01006424892693758, + 0.0032528946176171303, + 0.009540545754134655, + 0.0029259754810482264, + 0.010095320641994476, + 0.002696031704545021, + 0.009177232161164284, + 0.002929815324023366, + 0.011962441727519035, + 0.0029209970962256193, + 0.008841389790177345, + 0.0027309327851980925, + 0.010780212469398975, + 0.0031468512024730444, + 0.01038750447332859, + 0.0032646870240569115, + 0.009689838625490665, + 0.0026016440242528915, + 0.009929094463586807, + 0.003191312775015831, + 0.010107175447046757, + 0.0028179220389574766, + 0.009526536799967289, + 0.003107903292402625 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00017190496538337113, + 0.0001735542784589828, + 0.00019824933045045772, + 0.0002107982497012391, + 0.0002450550613783958, + 0.00026804391005753116, + 0.0003238054491237347, + 0.00031010311411301124, + 0.00031952489285031816, + 0.00034636354958654015, + 0.00034728721218647997, + 0.00036779944805402854, + 0.00040306175531293955, + 0.00038015703078453524, + 0.0003565887583279226, + 0.00036832720297469803, + 0.00036821166426922845, + 0.000375260819686975, + 0.00038062964117533765, + 0.00035620909008044117, + 0.0003690236577410054, + 0.00035091389469732654, + 0.00035793360754103896, + 0.000359518412683121, + 0.00040508202264603015, + 0.0003592735021571915, + 0.00038012959666749183, + 0.00035476778007974936, + 0.0003492989830982136, + 0.0003659888269167611, + 0.0003625696460584541, + 0.0003779058324425446 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023768929483334046, + "first_to_last_ratio": 0.6434869511235988, + "imbalance_abs_log_ratio": 0.44085353012576206, + "last_quartile_mean": 0.0003693770237583045, + "mean": 0.00033322941208390925, + "normalized": [ + 0.5158757275005617, + 0.5208252097965493, + 0.5949334700399654, + 0.632591968346896, + 0.7353944534664587, + 0.8043825074781695, + 0.9717192942206389, + 0.9305994695177908, + 0.9588736205850275, + 1.0394147005826835, + 1.0421865525454603, + 1.1037424510457507, + 1.2095623636350745, + 1.140826760780676, + 1.0700998933375405, + 1.105326209566252, + 1.1049794853538033, + 1.1261335466764921, + 1.1422450341192953, + 1.0689605333839665, + 1.107416225456362, + 1.053069993140234, + 1.074135699194851, + 1.0788915973377284, + 1.2156250557619686, + 1.0781566366258335, + 1.1407444327626544, + 1.0646352549168638, + 1.0482237474591767, + 1.0983088936477277, + 1.0880481521455758, + 1.134071059571973 + ], + "population_cv": 0.19078746332772342 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028024883940815926, + 0.02821914665400982, + 0.028325499966740608, + 0.02856484428048134, + 0.028658077120780945, + 0.02912120893597603, + 0.029205938801169395, + 0.02964756451547146, + 0.029751326888799667, + 0.030729545280337334, + 0.03091570734977722, + 0.03217839077115059, + 0.032307714223861694, + 0.03411358222365379, + 0.034374311566352844, + 0.036541424691677094, + 0.03687922656536102, + 0.03906221315264702, + 0.03944294899702072, + 0.04181833937764168, + 0.04215770959854126, + 0.04607149586081505, + 0.04655195400118828, + 0.0500124990940094, + 0.050798118114471436, + 0.05468229204416275, + 0.05539363995194435, + 0.05778742954134941, + 0.05819777771830559, + 0.06149948015809059, + 0.062257908284664154, + 0.06454319506883621, + 0.0653507262468338, + 0.06803742796182632, + 0.06877651810646057, + 0.07261326909065247, + 0.07320939004421234, + 0.07679533213376999, + 0.07733170688152313, + 0.08014968782663345, + 0.08104265481233597, + 0.0835338607430458, + 0.08443007618188858, + 0.08647285401821136, + 0.08700668811798096, + 0.09021914005279541, + 0.09079501777887344, + 0.09300754219293594, + 0.093939870595932, + 0.0970776379108429, + 0.09777236729860306, + 0.0996294915676117, + 0.10033779591321945, + 0.10360711812973022, + 0.10455679148435593, + 0.10598249733448029, + 0.10687003284692764, + 0.10993104428052902, + 0.11059616506099701, + 0.11287258565425873, + 0.11359578371047974, + 0.11536551266908646, + 0.11617089062929153, + 0.11782468110322952 + ], + "loss_nats": 4.653675079345703, + "loss_scale": 1.0, + "output_weights": null, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.02821914665400982, + 0.028325499966740608, + 0.02856484428048134, + 0.028658077120780945, + 0.02912120893597603, + 0.029205938801169395, + 0.02964756451547146, + 0.029751326888799667, + 0.030729545280337334, + 0.03091570734977722, + 0.03217839077115059, + 0.032307714223861694, + 0.03411358222365379, + 0.034374311566352844, + 0.036541424691677094, + 0.03687922656536102, + 0.03906221315264702, + 0.03944294899702072, + 0.04181833937764168, + 0.04215770959854126, + 0.04607149586081505, + 0.04655195400118828, + 0.0500124990940094, + 0.050798118114471436, + 0.05468229204416275, + 0.05539363995194435, + 0.05778742954134941, + 0.05819777771830559, + 0.06149948015809059, + 0.062257908284664154, + 0.06454319506883621, + 0.0653507262468338, + 0.06803742796182632, + 0.06877651810646057, + 0.07261326909065247, + 0.07320939004421234, + 0.07679533213376999, + 0.07733170688152313, + 0.08014968782663345, + 0.08104265481233597, + 0.0835338607430458, + 0.08443007618188858, + 0.08647285401821136, + 0.08700668811798096, + 0.09021914005279541, + 0.09079501777887344, + 0.09300754219293594, + 0.093939870595932, + 0.0970776379108429, + 0.09777236729860306, + 0.0996294915676117, + 0.10033779591321945, + 0.10360711812973022, + 0.10455679148435593, + 0.10598249733448029, + 0.10687003284692764, + 0.10993104428052902, + 0.11059616506099701, + 0.11287258565425873, + 0.11359578371047974, + 0.11536551266908646, + 0.11617089062929153, + 0.11782468110322952, + 0.11852118372917175 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.95552135286859, + "cross_entropy_nats": 5.514347195625305, + "step": 0 + }, + { + "bits_per_byte": 6.717222361067899, + "cross_entropy_nats": 4.656023740768433, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858", + "final_optimizer_state": "88ff74de5c6dcffa477d815ee1b7fd6ced81b387b2327a5bf0fd7a221cea9a6d", + "final_public_parameters": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 3185052160, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958522622108798, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516427516937256, + "step": 1, + "unclipped_grad_norm": 22.09486961364746 + }, + { + "bits_per_byte": 7.238571600789322, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.017395496368408, + "step": 10, + "unclipped_grad_norm": 11.172341346740723 + }, + { + "bits_per_byte": 6.729949421197048, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6648454666137695, + "step": 20, + "unclipped_grad_norm": 3.462282180786133 + } + ] +} diff --git a/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-block.json b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-block.json new file mode 100644 index 0000000..ff4c261 --- /dev/null +++ b/experiments/k3/attnres_gradient/results/raw/smoke-b-depth-32-block.json @@ -0,0 +1,2618 @@ +{ + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "fc6cf8d746ae7004851f9d52a5fc29a99061d0d6a5ec28f2f4c70a6313eb712a", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.9337450289167464e-05, + 3.887022103299387e-05, + 3.8419901102315634e-05, + 3.799897967837751e-05, + 3.775028380914591e-05, + 3.7539852201007307e-05, + 3.739175735972822e-05, + 3.724253110704012e-05, + 3.720997119671665e-05, + 3.713165278895758e-05, + 3.7142541259527206e-05, + 3.707727591972798e-05, + 3.688051219796762e-05, + 3.692117388709448e-05, + 3.689077857416123e-05, + 3.6904009903082624e-05, + 3.717117215273902e-05, + 3.7258152588037774e-05, + 3.729091622517444e-05, + 3.7364865420386195e-05, + 3.74124720110558e-05, + 3.742674380191602e-05, + 3.742256740224548e-05, + 3.74479714082554e-05, + 3.765593282878399e-05, + 3.763797212741338e-05, + 3.7667807191610336e-05, + 3.764635039260611e-05, + 3.751127951545641e-05, + 3.7523430364672095e-05, + 3.7504440115299076e-05, + 3.756426303880289e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.8068872072472004e-05, + "first_to_last_ratio": 1.0127680561501509, + "imbalance_abs_log_ratio": 0.01268723177426717, + "last_quartile_mean": 3.7588934446830535e-05, + "mean": 3.750672590285831e-05, + "normalized": [ + 1.0488105624322075, + 1.0363533498942825, + 1.0243469718423952, + 1.013124413386392, + 1.0064937128054956, + 1.0008832095404647, + 0.9969347219635258, + 0.9929560688260969, + 0.9920879602525092, + 0.9899998438980743, + 0.9902901510445264, + 0.9885500540824973, + 0.9833039624276305, + 0.9843880796932157, + 0.9835776833655818, + 0.9839304555311837, + 0.9910535046170554, + 0.9933725669506762, + 0.9942461072650594, + 0.9962177321785024, + 0.997487013607996, + 0.9978675264498043, + 0.9977561757634937, + 0.9984334944416348, + 1.0039781378495187, + 1.0034992717011608, + 1.0042947307416068, + 1.0037226520413813, + 1.0001214078938774, + 1.0004453724341882, + 0.9999390565957383, + 1.001534048482227 + ], + "population_cv": 0.013988937740460563 + }, + "activation_output_rms_by_block": [ + 0.0037520017940551043, + 0.005621022544801235, + 0.0076507688499987125, + 0.009552493691444397, + 0.005728918593376875, + 0.009368892759084702, + 0.013378600589931011, + 0.01761116087436676, + 0.00800272449851036, + 0.013154285028576851, + 0.01923154480755329, + 0.024512210860848427, + 0.011547097004950047, + 0.016813727095723152, + 0.02113443799316883, + 0.025982216000556946, + 0.010269254446029663, + 0.01641162671148777, + 0.02165052480995655, + 0.026260115206241608, + 0.011169232428073883, + 0.016121258959174156, + 0.019987935200333595, + 0.02486182004213333, + 0.01274633314460516, + 0.016767995432019234, + 0.022238746285438538, + 0.02633264660835266, + 0.0102624436840415, + 0.016213588416576385, + 0.020620016381144524, + 0.02528378553688526 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.00908298246213235, + "first_to_last_ratio": 0.4829268696139592, + "imbalance_abs_log_ratio": 0.7278900454597648, + "last_quartile_mean": 0.018808194436132908, + "mean": 0.015944982071232516, + "normalized": [ + 0.2353092513552812, + 0.35252611258450545, + 0.47982298229121323, + 0.5990909020009959, + 0.3592928839796457, + 0.5875762492068142, + 0.8390477035448226, + 1.1044954955540724, + 0.5018961114386356, + 0.8249796061112817, + 1.2061189358281121, + 1.537299367998203, + 0.7241837559531027, + 1.0544839135352808, + 1.3254601290081713, + 1.62949170368986, + 0.6440430224476176, + 1.0292659244250366, + 1.3578268519359336, + 1.6469203344931544, + 0.7004857313841133, + 1.0110553205487558, + 1.2535564550050677, + 1.5592253369157636, + 0.7993946363603464, + 1.0516158222762493, + 1.394717547256516, + 1.6514691889093607, + 0.6436158810461574, + 1.016845822977028, + 1.2931978404884237, + 1.5856891794504773 + ], + "population_cv": 0.416991705473353 + }, + "bits_per_byte": 6.71703146032544, + "branch_output_rms_by_sublayer": [ + 0.002443866338580847, + 0.0027545092161744833, + 0.0026208641938865185, + 0.002763434313237667, + 0.0034182921517640352, + 0.002741520293056965, + 0.0035785960499197245, + 0.002719961805269122, + 0.0047746808268129826, + 0.0028042772319167852, + 0.005376013461500406, + 0.0027222761418670416, + 0.006767150945961475, + 0.0027788877487182617, + 0.007074786350131035, + 0.002833561971783638, + 0.006997238378971815, + 0.0029156480450183153, + 0.008518846705555916, + 0.0027741468511521816, + 0.009081361815333366, + 0.0028916175942867994, + 0.00911131501197815, + 0.0032094581983983517, + 0.010676512494683266, + 0.002878024475648999, + 0.009562043473124504, + 0.0026487084105610847, + 0.008648304268717766, + 0.003162843408063054, + 0.009535389021039009, + 0.0030166516080498695, + 0.009431489743292332, + 0.002794384490698576, + 0.010901342146098614, + 0.00310111534781754, + 0.010227455757558346, + 0.0028713408391922712, + 0.008522198535501957, + 0.0031550333369523287, + 0.010077033191919327, + 0.0032391855493187904, + 0.009554130956530571, + 0.00291044800542295, + 0.010129180736839771, + 0.0026873303577303886, + 0.00920273270457983, + 0.002927709138020873, + 0.011994888074696064, + 0.0029113248456269503, + 0.008809582330286503, + 0.0027162786573171616, + 0.01077704131603241, + 0.0031467911321669817, + 0.010385709814727306, + 0.003254092764109373, + 0.009710066020488739, + 0.0025947801768779755, + 0.009926802478730679, + 0.0031805087346583605, + 0.0101145189255476, + 0.002813765313476324, + 0.009518872946500778, + 0.0030965779442340136 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00016739224645496468, + 0.0001695381430192955, + 0.0001941383401676919, + 0.00020674849822850373, + 0.00024134787186321302, + 0.00026490993231531454, + 0.000322331199796065, + 0.0003097008723390916, + 0.00031886187327784883, + 0.00034582398876540076, + 0.00034801623779775657, + 0.00036856862737498307, + 0.00040270040252095, + 0.0003782002933862871, + 0.0003548760100734237, + 0.0003670443346435906, + 0.0003676146253339179, + 0.0003749331083729355, + 0.00038033249912268637, + 0.0003562102847256418, + 0.00036931756079357426, + 0.00035104688530666954, + 0.0003583944935512815, + 0.0003605180195562536, + 0.0004079256274935599, + 0.0003614699495467052, + 0.00038354947558429407, + 0.0003579111697278514, + 0.0003529832358531257, + 0.00036980562702627114, + 0.00036596233134640236, + 0.0003819940876588112 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002345133880230175, + "first_to_last_ratio": 0.6292279841951004, + "imbalance_abs_log_ratio": 0.46326163295248246, + "last_quartile_mean": 0.00037270018802962763, + "mean": 0.0003331302454070113, + "normalized": [ + 0.502482883985657, + 0.5089244983209419, + 0.582770081205019, + 0.6206236181764371, + 0.7244850180695525, + 0.7952142922107038, + 0.9675831127319218, + 0.9296690306840971, + 0.9571687881064967, + 1.0381044457337716, + 1.044685202247421, + 1.1063799593599613, + 1.2088377085933444, + 1.1352925728019987, + 1.0652770649505088, + 1.1018042933782364, + 1.1035162084570687, + 1.1254850423889022, + 1.141693089614257, + 1.069282329169577, + 1.1086281293442752, + 1.0537826875424299, + 1.0758389503582988, + 1.0822134121018658, + 1.224522939954507, + 1.0850709430633327, + 1.1513499025454192, + 1.07438809493435, + 1.0595952805842004, + 1.1100932206694432, + 1.0985563046047577, + 1.146680894111248 + ], + "population_cv": 0.19646899587719932 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931430697441101, + "mean_weights": [ + 0.49865636229515076, + 0.5013436079025269 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931370496749878, + "mean_weights": [ + 0.49779537320137024, + 0.5022045969963074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931354999542236, + "mean_weights": [ + 0.49762701988220215, + 0.5023729801177979 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931448578834534, + "mean_weights": [ + 0.49909353256225586, + 0.5009064674377441 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931440234184265, + "mean_weights": [ + 0.49918538331985474, + 0.5008145570755005 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931461691856384, + "mean_weights": [ + 0.49945560097694397, + 0.5005444288253784 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931438446044922, + "mean_weights": [ + 0.49891161918640137, + 0.5010883808135986 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931411623954773, + "mean_weights": [ + 0.49836465716362, + 0.5016353130340576 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986062288284302, + "mean_weights": [ + 0.3321005403995514, + 0.3337631821632385, + 0.3341362476348877 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985990762710571, + "mean_weights": [ + 0.33337411284446716, + 0.33537548780441284, + 0.3312504291534424 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986008644104004, + "mean_weights": [ + 0.3350442051887512, + 0.33362624049186707, + 0.3313295841217041 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986016988754272, + "mean_weights": [ + 0.3313179612159729, + 0.3338589072227478, + 0.3348231613636017 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986084938049316, + "mean_weights": [ + 0.33454394340515137, + 0.3326125741004944, + 0.33284348249435425 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985959768295288, + "mean_weights": [ + 0.3312351107597351, + 0.332991361618042, + 0.3357735574245453 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986058712005615, + "mean_weights": [ + 0.33314070105552673, + 0.3320331275463104, + 0.33482617139816284 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986061096191406, + "mean_weights": [ + 0.33257031440734863, + 0.3324887454509735, + 0.33494094014167786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862926959991455, + "mean_weights": [ + 0.24962691962718964, + 0.2496621459722519, + 0.25036555528640747, + 0.2503453493118286 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862905502319336, + "mean_weights": [ + 0.24916216731071472, + 0.24970784783363342, + 0.25086167454719543, + 0.2502683103084564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386286973953247, + "mean_weights": [ + 0.25067827105522156, + 0.24852371215820312, + 0.24991214275360107, + 0.25088587403297424 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862855434417725, + "mean_weights": [ + 0.24860653281211853, + 0.24958111345767975, + 0.2513706088066101, + 0.2504417300224304 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386291265487671, + "mean_weights": [ + 0.25055181980133057, + 0.24973659217357635, + 0.2504120469093323, + 0.2492995262145996 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862907886505127, + "mean_weights": [ + 0.24893252551555634, + 0.25016576051712036, + 0.2503091096878052, + 0.25059258937835693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862874507904053, + "mean_weights": [ + 0.24930882453918457, + 0.24942341446876526, + 0.2499566376209259, + 0.25131112337112427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862828016281128, + "mean_weights": [ + 0.24865929782390594, + 0.24900496006011963, + 0.25141197443008423, + 0.2509238123893738 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094212532043457, + "mean_weights": [ + 0.19921688735485077, + 0.1992446780204773, + 0.19889645278453827, + 0.20075249671936035, + 0.20188948512077332 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609428882598877, + "mean_weights": [ + 0.19964422285556793, + 0.19919568300247192, + 0.19919705390930176, + 0.20121362805366516, + 0.20074939727783203 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094245910644531, + "mean_weights": [ + 0.20065900683403015, + 0.19955331087112427, + 0.2014925479888916, + 0.19979336857795715, + 0.19850176572799683 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094355583190918, + "mean_weights": [ + 0.19937855005264282, + 0.20006650686264038, + 0.20067161321640015, + 0.1999610960483551, + 0.19992223381996155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094313859939575, + "mean_weights": [ + 0.1997283697128296, + 0.19996783137321472, + 0.19928213953971863, + 0.1997123807668686, + 0.20130927860736847 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094284057617188, + "mean_weights": [ + 0.19933226704597473, + 0.19993427395820618, + 0.19907569885253906, + 0.2001166194677353, + 0.20154115557670593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094276905059814, + "mean_weights": [ + 0.19932052493095398, + 0.19971224665641785, + 0.2011900395154953, + 0.20086491107940674, + 0.19891227781772614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094303131103516, + "mean_weights": [ + 0.19955268502235413, + 0.19991174340248108, + 0.20052853226661682, + 0.20107966661453247, + 0.1989273726940155 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7917479276657104, + "mean_weights": [ + 0.16630326211452484, + 0.1665954291820526, + 0.1656438112258911, + 0.16612562537193298, + 0.16804365813732147, + 0.16728821396827698 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917546033859253, + "mean_weights": [ + 0.16606619954109192, + 0.16676779091358185, + 0.16676853597164154, + 0.16595947742462158, + 0.16733971238136292, + 0.1670982539653778 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791755199432373, + "mean_weights": [ + 0.16713324189186096, + 0.16643813252449036, + 0.16707737743854523, + 0.16582445800304413, + 0.16707313060760498, + 0.16645365953445435 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791752815246582, + "mean_weights": [ + 0.16595935821533203, + 0.16658073663711548, + 0.16717244684696198, + 0.1671261042356491, + 0.16733530163764954, + 0.16582605242729187 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917518615722656, + "mean_weights": [ + 0.1666388213634491, + 0.16675034165382385, + 0.16638417541980743, + 0.16550619900226593, + 0.16747808456420898, + 0.1672423779964447 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917499542236328, + "mean_weights": [ + 0.16634854674339294, + 0.16704240441322327, + 0.16620638966560364, + 0.16553011536598206, + 0.16771462559700012, + 0.16715793311595917 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917507886886597, + "mean_weights": [ + 0.16621501743793488, + 0.1665443778038025, + 0.16585618257522583, + 0.16727086901664734, + 0.16779379546642303, + 0.16631974279880524 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917485237121582, + "mean_weights": [ + 0.16604092717170715, + 0.1667710244655609, + 0.16534343361854553, + 0.1669321358203888, + 0.16766314208507538, + 0.16724932193756104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9458963871002197, + "mean_weights": [ + 0.14258044958114624, + 0.14276695251464844, + 0.14234250783920288, + 0.14281943440437317, + 0.14429350197315216, + 0.14340519905090332, + 0.14179196953773499 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458930492401123, + "mean_weights": [ + 0.14285841584205627, + 0.14281482994556427, + 0.14142268896102905, + 0.14248794317245483, + 0.1441671997308731, + 0.14373424649238586, + 0.1425146758556366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459006786346436, + "mean_weights": [ + 0.1428324282169342, + 0.14242856204509735, + 0.14238804578781128, + 0.14283627271652222, + 0.14203999936580658, + 0.14380358159542084, + 0.14367109537124634 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.14253869652748108, + 0.14282220602035522, + 0.1414484828710556, + 0.14307048916816711, + 0.1438344419002533, + 0.14343075454235077, + 0.1428549587726593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.94590425491333, + "mean_weights": [ + 0.14281800389289856, + 0.1431872844696045, + 0.1432121843099594, + 0.14210283756256104, + 0.14336752891540527, + 0.14217621088027954, + 0.14313597977161407 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458997249603271, + "mean_weights": [ + 0.14271901547908783, + 0.14267435669898987, + 0.14293994009494781, + 0.14278604090213776, + 0.14186438918113708, + 0.1428055316209793, + 0.14421072602272034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459013938903809, + "mean_weights": [ + 0.14281892776489258, + 0.1429084837436676, + 0.14260512590408325, + 0.14224481582641602, + 0.14402621984481812, + 0.14324231445789337, + 0.14215409755706787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459059238433838, + "mean_weights": [ + 0.14244785904884338, + 0.14288531243801117, + 0.14282266795635223, + 0.14290127158164978, + 0.1435883343219757, + 0.14224958419799805, + 0.14310497045516968 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079432487487793, + "mean_weights": [ + 0.12496939301490784, + 0.12501314282417297, + 0.12449057400226593, + 0.12434226274490356, + 0.12589572370052338, + 0.12463213503360748, + 0.12490519136190414, + 0.1257515549659729 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794320106506348, + "mean_weights": [ + 0.12487496435642242, + 0.12489178031682968, + 0.12453664839267731, + 0.12434129416942596, + 0.1249510794878006, + 0.12465334683656693, + 0.12601137161254883, + 0.12573951482772827 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794363021850586, + "mean_weights": [ + 0.12491423636674881, + 0.12523731589317322, + 0.12523618340492249, + 0.12467047572135925, + 0.12580403685569763, + 0.12441615015268326, + 0.12498776614665985, + 0.1247338205575943 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079427480697632, + "mean_weights": [ + 0.12474467605352402, + 0.1245410293340683, + 0.12462934106588364, + 0.1246577650308609, + 0.12418574839830399, + 0.12514574825763702, + 0.12619128823280334, + 0.1259044110774994 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794358253479004, + "mean_weights": [ + 0.12475918978452682, + 0.12487104535102844, + 0.1250094473361969, + 0.1255180537700653, + 0.12490293383598328, + 0.12545335292816162, + 0.12531054019927979, + 0.12417542934417725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079436779022217, + "mean_weights": [ + 0.12494880706071854, + 0.12516388297080994, + 0.12451333552598953, + 0.124576136469841, + 0.12569281458854675, + 0.12472400069236755, + 0.1254362016916275, + 0.12494482100009918 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079434871673584, + "mean_weights": [ + 0.12489444017410278, + 0.12512432038784027, + 0.12540367245674133, + 0.1253792643547058, + 0.1248319000005722, + 0.12483960390090942, + 0.125506192445755, + 0.12402061372995377 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079428195953369, + "mean_weights": [ + 0.12488104403018951, + 0.12459895014762878, + 0.12424551695585251, + 0.12444087862968445, + 0.12461263686418533, + 0.12524595856666565, + 0.12623095512390137, + 0.1257440745830536 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972174644470215, + "mean_weights": [ + 0.11096074432134628, + 0.11101590096950531, + 0.1106940507888794, + 0.11131151020526886, + 0.11191274225711823, + 0.11117846518754959, + 0.11035685986280441, + 0.11126935482025146, + 0.11130036413669586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216749191284, + "mean_weights": [ + 0.11095131188631058, + 0.11080048978328705, + 0.11054704338312149, + 0.11075608432292938, + 0.1108987033367157, + 0.11123715341091156, + 0.11191022396087646, + 0.11176811158657074, + 0.11113087832927704 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216510772705, + "mean_weights": [ + 0.1109621673822403, + 0.1111164540052414, + 0.11043901741504669, + 0.11095462739467621, + 0.11190879344940186, + 0.11175167560577393, + 0.11079787462949753, + 0.1112256720662117, + 0.1108437031507492 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197213649749756, + "mean_weights": [ + 0.11101574450731277, + 0.11100780963897705, + 0.11082783341407776, + 0.11142586171627045, + 0.11015472561120987, + 0.11108498275279999, + 0.11191210150718689, + 0.11077474802732468, + 0.11179618537425995 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197218894958496, + "mean_weights": [ + 0.11101824045181274, + 0.11099033057689667, + 0.11046893894672394, + 0.11147398501634598, + 0.11092066764831543, + 0.11164902150630951, + 0.11152558028697968, + 0.11112039536237717, + 0.1108328253030777 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197216033935547, + "mean_weights": [ + 0.11115090548992157, + 0.11169334501028061, + 0.1117531955242157, + 0.1108616292476654, + 0.11132021248340607, + 0.11068520694971085, + 0.1114465519785881, + 0.11033187806606293, + 0.11075706779956818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972179412841797, + "mean_weights": [ + 0.11092537641525269, + 0.11108530312776566, + 0.11051007360219955, + 0.11084415018558502, + 0.11164555698633194, + 0.11119014024734497, + 0.11111340671777725, + 0.11183696240186691, + 0.1108490377664566 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802533470094204, + 0.014066163450479507, + 0.014108670875430107, + 0.014229752123355865, + 0.01430919673293829, + 0.014553130604326725, + 0.01460581086575985, + 0.014812584035098553, + 0.014851128682494164, + 0.010233328677713871, + 0.01034309808164835, + 0.010804922319948673, + 0.010769715532660484, + 0.011443440802395344, + 0.011492080986499786, + 0.01227729581296444, + 0.012374433688819408, + 0.009845216758549213, + 0.00994168221950531, + 0.010567485354840755, + 0.01064346730709076, + 0.01164715364575386, + 0.01175782922655344, + 0.012674410827457905, + 0.012870704755187035, + 0.011088024824857712, + 0.011239404790103436, + 0.011724143289029598, + 0.011804761365056038, + 0.012471788562834263, + 0.012632876634597778, + 0.01310542318969965, + 0.013265094719827175, + 0.011508332565426826, + 0.011626233346760273, + 0.012281009927392006, + 0.012384294532239437, + 0.012981574982404709, + 0.013078294694423676, + 0.013564560562372208, + 0.01372351124882698, + 0.012140991166234016, + 0.012265652418136597, + 0.012530013918876648, + 0.012632926926016808, + 0.013094775378704071, + 0.01315103005617857, + 0.01351068913936615, + 0.013631501235067844, + 0.01232061255723238, + 0.012407775968313217, + 0.012646659277379513, + 0.012729056179523468, + 0.01315284613519907, + 0.013267731294035912, + 0.013451422564685345, + 0.01357149239629507, + 0.012415111064910889, + 0.012491659261286259, + 0.01274929754436016, + 0.012818088755011559, + 0.013031110167503357, + 0.013103967532515526, + 0.013315660879015923 + ], + "loss_nats": 4.655891418457031, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.197218418121338, + "mean_weights": [ + 0.11057738959789276, + 0.11068764328956604, + 0.11082598567008972, + 0.11127202212810516, + 0.11091771721839905, + 0.11130063980817795, + 0.11115016043186188, + 0.11173908412456512, + 0.11152935773134232 + ], + "sources": 9 + }, + "step": 20, + "stream_state_rms_by_sublayer": [ + 0.002443866338580847, + 0.0037520017940551043, + 0.004732752684503794, + 0.005621022544801235, + 0.007002863567322493, + 0.0076507688499987125, + 0.008998939767479897, + 0.009552493691444397, + 0.0047746808268129826, + 0.005728918593376875, + 0.008738124743103981, + 0.009368892759084702, + 0.012762236408889294, + 0.013378600589931011, + 0.01696598529815674, + 0.01761116087436676, + 0.006997238378971815, + 0.00800272449851036, + 0.012584065087139606, + 0.013154285028576851, + 0.018650829792022705, + 0.01923154480755329, + 0.023473192006349564, + 0.024512210860848427, + 0.010676512494683266, + 0.011547097004950047, + 0.01631898619234562, + 0.016813727095723152, + 0.02041666954755783, + 0.02113443799316883, + 0.02490287274122238, + 0.025982216000556946, + 0.009431489743292332, + 0.010269254446029663, + 0.015727587044239044, + 0.01641162671148777, + 0.02073165774345398, + 0.02165052480995655, + 0.02527376264333725, + 0.026260115206241608, + 0.010077033191919327, + 0.011169232428073883, + 0.015674244612455368, + 0.016121258959174156, + 0.019633492454886436, + 0.019987935200333595, + 0.02385960891842842, + 0.02486182004213333, + 0.011994888074696064, + 0.01274633314460516, + 0.015906881541013718, + 0.016767995432019234, + 0.02143402211368084, + 0.022238746285438538, + 0.02530583366751671, + 0.02633264660835266, + 0.009710066020488739, + 0.0102624436840415, + 0.015399535186588764, + 0.016213588416576385, + 0.01968187838792801, + 0.020620016381144524, + 0.024621067568659782, + 0.02528378553688526 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 6.720464749984615, + "cross_entropy_nats": 4.6582711935043335, + "step": 20 + } + ], + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + }, + "hashes": { + "final_mixer_parameters": "0f212e3b0862c8a0fa1714765ff7cc8edeb23e1470b2107592e11f4eeae72674", + "final_model_state": "ac8be80504f01ca19f023c383b7139873a7635445148dec83aff2b39b674dd20", + "final_optimizer_state": "de50114df8c8cb4b59e160333f861411e6676277c7a3a849bd21ada1ea6abc54", + "final_public_parameters": "063f8f52a2b325c61dff3a9a7bdd1302f383a71c59990ca1765419e2e5dcbb96", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "smoke", + "schema_version": 1, + "seed": 2026073001, + "steps": 20, + "target_bytes_seen": 163840, + "timing": { + "mean_ms": null, + "measured_steps": 0, + "median_ms": null, + "p95_ms": null, + "peak_allocated_bytes": 6751356416, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + } + ] +} diff --git a/research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md b/research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md new file mode 100644 index 0000000..5c38c9b --- /dev/null +++ b/research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md @@ -0,0 +1,545 @@ +# Kimi K3 第五轮：Attention Residuals 梯度定义与深度扩展实验审计 + +> 协议：`llm-atlas-k3-attnres-gradient-scale-v1` +> 前置定义审计：`research/K3_ATTNRES_GRADIENT_DEFINITION_AUDIT.md` +> 预注册协议：`research/K3_ATTNRES_GRADIENT_SCALE_PROTOCOL.md` +> 数据清单：`experiments/k3/attnres_gradient/manifest.json` +> 执行日期：2026-07-30 +> 设备：NVIDIA GeForce RTX 5090；PyTorch `2.11.0+cu128` + +## 0. 先说结论 + +这一轮本来想澄清一个看似矛盾的问题： + +```text +论文 Figure 5： +Block AttnRes 的梯度沿深度“明显更均匀” + +本站 Round 04： +Block AttnRes 的核心参数梯度 RMS 跨层 CV 反而更高 +``` + +一手工件审计先确认：论文没有公开 Figure 5(c) 的确切 gradient tensor、norm、reduction、 +diagnostic batch、AMP / clip 时点或统计代码。因此，Round 05 没有假装恢复作者的隐藏实现， +而是冻结一个与 Figure 5 的 output/gradient 并列叙述对齐、可复现的定义： + +```text +h_l： +第 l 个 Transformer block 完成 attention + MLP 后的 FP32 residual output + +m_l： +sqrt(mean((∂L / ∂h_l)² over batch × time × channel)) + +L： +固定 16 × 256 targets 的 token-mean cross entropy +``` + +结果不是一句“是”或“不是”，而是一个更有信息量的分解： + +1. **Block 确实大幅缓解了早层整体偏大。** + Baseline 首四分位梯度平均是末四分位的 `3.01–3.90×`；Block 把它改到 + `0.55–1.40×`。预注册的首尾失衡指标在 6 / 6 个 depth×seed 配对中都改善， + depth-16 平均改善 `61.0%`，depth-32 平均改善 `72.0%`。 + +2. **但 Block 没有让整条深度谱更平。** + 它在中后段形成了局部尖峰，所以 population CV 在 6 / 6 个配对中都恶化： + depth-16 平均相对恶化 `10.3%`，depth-32 平均相对恶化 `60.0%`。 + +3. **Block 的绝对 activation-gradient 平均尺度更小。** + 它只有 Baseline 的 `57.4%`（depth-16）和 `54.4%`（depth-32）。这意味着“首尾更接近” + 不能自动解释为所有层都获得更强更新信号。 + +4. **参数梯度仍与 Round 04 同方向。** + 核心参数梯度 CV 从 `0.416→0.683`（depth-16），从 `0.397→0.772` + （depth-32）；Block 更不均匀。 + +5. **验证 BPC 在 6 / 6 配对中都更低，但不是同算力优势。** + 平均改善 `0.00894 BPC`（depth-16）与 `0.00987 BPC`（depth-32）；Block 实际 step + time 是 Baseline 的约 `2.55–2.60×`，peak allocated memory 约 `2.15–2.16×`。 + +按看结果前冻结的联合判据，CV 与首尾失衡必须同时改善才算 support；必须同时恶化才算 +concern。这里二者方向相反，所以： + +| depth | 预注册判定 | +|---:|---| +| 16 | **mixed / inconclusive at this depth** | +| 32 | **mixed / inconclusive at this depth** | +| 总判定 | **depth-dependent or inconclusive** | + +最准确的中文总结是： + +> 在这套公开 operationalization 中，Block AttnRes 把“早层系统性偏大”变成了“首尾更接近、 +> 但中后段有局部尖峰”的另一种梯度分布。它修正了一类失衡，却没有降低全层离散度。 + +这不复现论文 Figure 5 的数值，也不反驳一个没有公开测量合同的隐藏实现。 + +--- + +## 1. 为什么要另开一轮，而不是改写 Round 04 + +Round 04 的梯度对象是： + +```text +每个 Transformer block 的 attention、MLP 与两个 input norm +所有核心参数梯度拼接后的 RMS +``` + +这回答“该层权重收到多大更新信号”。Round 05 的对象是 `∂L/∂h_l`，回答“损失对该深度 +表征有多敏感”。链式法则把两者联系起来，但不会保证跨层形状同方向。 + +因此： + +- Round 04 参数梯度反结果继续有效； +- Round 05 不把它改名为 activation gradient； +- 两种对象在网站并排展示； +- 任何方向冲突都保留，而不是选择更像论文的一种。 + +官方工件边界见前置定义审计。固定 revision 为： + +```text +MoonshotAI/Attention-Residuals +85e22310fe5ee860b4a023de312d791de8a5a5e6 + +Attention_Residuals.pdf SHA-256 +e5831b0db1347606453b5176b0142115a18887b6a9c2e1d05a266d4805a26b2f +``` + +官方仓库没有可执行训练代码或 Figure 5 原始数组。 + +## 2. 实验规模与配对合同 + +正式网格： + +```text +2 depths +× 2 residual graphs +× 3 seeds +× 8,000 steps +× 32 windows +× 256 target bytes += 12 independent runs += 786,432,000 formal target bytes +``` + +| 项 | depth-16 | depth-32 | +|---|---:|---:| +| Transformer blocks | 16 | 32 | +| residual sublayers | 32 | 64 | +| AttnRes aggregation groups | 8 | 8 | +| sublayers / group | 4 | 8 | +| Transformer blocks / group | 2 | 4 | +| width / heads / FFN | 192 / 6 / 768 | 192 / 6 / 768 | + +每个 depth / seed 的 Baseline 与 Block： + +- 使用逐 tensor exact 的公共主干初始化； +- 使用逐 step / row exact 的 byte windows； +- 使用相同 optimizer、LR schedule、batch、context 与 target-byte budget； +- 分支线性计算走 BF16 autocast； +- residual accumulator 与被测 `h_l` 都为 FP32； +- 每个结构在全新进程中从零训练。 + +二者**不匹配**： + +- mixer 参数； +- mixer FLOPs； +- step wall time； +- activation memory。 + +所以 BPC 只能叫“同 token / step 预算对比”，不能叫“同算力优势”。 + +## 3. 数据与日程 + +| 对象 | 固定值 | +|---|---| +| dataset | `Salesforce/wikitext` | +| revision | `b08601e04326c79dfdd32d625aee71d232d685c3` | +| variant | `wikitext-2-raw-v1` | +| train bytes | 10,951,563 | +| train SHA-256 | `0ca7d3e7…e9b4` | +| validation bytes | 1,148,008 | +| validation SHA-256 | `a42356f6…e719` | +| formal schedule cells | 768,000 | +| schedule SHA-256 | `5041e09b…f4e` | +| validation tensor SHA-256 | `f459316f…338` | +| diagnostic tensor SHA-256 | `21117e31…716` | + +固定诊断时点： + +```text +0, 100, 500, 2,000, 4,000, 8,000 +``` + +六个时点的全部 activation gradient、output RMS、parameter gradient、mixer 权重与验证 BPC +都进入 raw JSON；没有只挑“最好看”的 checkpoint。 + +## 4. 正式训练前的故障与修订 + +### 4.1 第一次 smoke 的标量序列化错误 + +首个 depth-16 Baseline 20-step smoke 已完成数值计算，但在写 JSON 前失败： + +```text +RuntimeError: +self.dim() cannot be 0 to view Float as Byte +``` + +原因是 optimizer 的 step 是 0 维 tensor，hash helper 直接把它 `view(torch.uint8)`。修复为： + +```text +tensor.reshape(-1).view(torch.uint8) +``` + +当时： + +- 没有 formal 运行； +- 没有输出 JSON； +- 没有可供选择的正式结果。 + +### 4.2 smoke 发现被测 residual dtype 不一致 + +第一版 smoke 通过了 finite / loss×2 / replay 闸门，但检查 capture metadata 时发现： + +```text +Baseline post-MLP h_l：FP32 +Block aggregation partial：BF16 +``` + +原因： + +- Baseline 把 BF16 branch 加到 FP32 embedding/residual stream； +- Block 每组第一个 partial 直接引用 BF16 branch output。 + +这会把数值精度差异混进结构比较。正式训练前，协议与实现补充为： + +```text +BF16 branch output → 显式转 FP32 → residual partial 累加 +``` + +随后 4 个 depth×architecture 格的 smoke 全部从头重跑两次。正式输出是在这次修订之后才开始。 + +### 4.3 一次未启动模型的 zsh 调度错误 + +首个正式 Baseline 完成后，批处理脚本用 Bash 式标量切分处理 zsh 字符串，第一行就退出： + +```text +argument --architecture: invalid choice: '' +``` + +runner 没有启动，也没有创建新结果文件。调度改为显式 `:` 分隔数组后继续。这是 orchestration +故障，不是模型运行失败，但仍在审计时间线中保留。 + +## 5. 运行前与复现闸门 + +### 5.1 activation-gradient 测量闸门 + +4 / 4 个 depth×architecture 格都通过： + +- 捕获数量严格等于 16 / 32； +- shape 严格为 `[16,256,192]`； +- dtype 全部为 FP32； +- gradient 全部 present、finite； +- capture storage 全部不别名； +- diagnostic loss×2 后每层 gradient RMS 精确×2； +- CV、归一化谱、首尾比不变。 + +### 5.2 两次独立 smoke + +每个格都在两个全新进程中训练 20 steps。排除 timing 后的冻结字段： + +| 格 | compare SHA-256 | +|---|---| +| depth-16 Baseline | `525cdcd9…bd2a` | +| depth-16 Block | `e4330a0d…90cc` | +| depth-32 Baseline | `8ee0dc37…8e56` | +| depth-32 Block | `289073b1…34c` | + +4 / 4 exact。 + +### 5.3 完整 formal replay + +预注册格： + +```text +depth-32 / Block / seed-2026073001 +``` + +从初始化重新训练完整 8,000 steps，不加载 formal checkpoint。排除 `run_kind`、timing、 +memory 与进程元数据后的全部冻结字段： + +```text +formal compare SHA-256 +46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817 + +replay compare SHA-256 +46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817 +``` + +最终状态： + +| 对象 | formal | replay | +|---|---|---| +| model state | `3f0b97ec…2f59` | `3f0b97ec…2f59` | +| optimizer state | `ed03e6fb…4637` | `ed03e6fb…4637` | + +字段级 exact。 + +## 6. 主结果：CV 与首尾失衡为什么方向相反 + +### 6.1 depth-16 + +| seed | Base CV | Block CV | 相对 CV 改善 | Base imbalance | Block imbalance | 相对 imbalance 改善 | +|---:|---:|---:|---:|---:|---:|---:| +| 2026073001 | 0.41719 | 0.46768 | −12.1% | 1.29458 | 0.38821 | +70.0% | +| 2026073002 | 0.43744 | 0.45175 | −3.3% | 1.35991 | 0.58958 | +56.6% | +| 2026073003 | 0.41927 | 0.48391 | −15.4% | 1.29826 | 0.56693 | +56.3% | +| 均值 | 0.42463 | 0.46778 | **−10.3%** | 1.31759 | 0.51491 | **+61.0%** | + +这里“相对 CV 改善”为负，表示恶化。 + +原始首/末四分位比： + +```text +Baseline：3.65×, 3.90×, 3.66× +Block： 0.68×, 0.55×, 0.57× +``` + +Block 不只是把早层优势降到 1；它在三个 seed 中都略微“过冲”，变成末四分位平均更大。 +但 `abs(log(first/last))` 仍比 Baseline 更接近 0，所以失衡改善。 + +三 seed 平均 normalized activation-gradient 的最高点： + +```text +Baseline：layer 3 = 1.50× mean；layer 4 = 1.42× +Block： layer 11 = 2.09× mean；layer 13 = 1.95× +``` + +Baseline 是宽而平滑的早层隆起；Block 是更局部的中后段尖峰。CV 对尖峰敏感，所以升高。 + +### 6.2 depth-32 + +| seed | Base CV | Block CV | 相对 CV 改善 | Base imbalance | Block imbalance | 相对 imbalance 改善 | +|---:|---:|---:|---:|---:|---:|---:| +| 2026073001 | 0.36109 | 0.64203 | −77.8% | 1.10030 | 0.20781 | +81.1% | +| 2026073002 | 0.37162 | 0.72997 | −96.4% | 1.14087 | 0.43810 | +61.6% | +| 2026073003 | 0.40320 | 0.42685 | −5.9% | 1.25169 | 0.33417 | +73.3% | +| 均值 | 0.37864 | 0.59962 | **−60.0%** | 1.16429 | 0.32669 | **+72.0%** | + +原始首/末四分位比： + +```text +Baseline：3.01×, 3.13×, 3.50× +Block： 0.81×, 0.65×, 1.40× +``` + +三 seed 平均 normalized spectrum 的 Block 峰值： + +```text +layer 21 = 3.04× mean +layer 22 = 2.41× +layer 23 = 1.91× +layer 25 = 1.77× +``` + +depth-32 每个 AttnRes aggregation group 含 4 个 Transformer blocks。21–24 是第 6 组， +25–28 是第 7 组。尖峰集中在这两个中后段组附近，是数据中直接可见的结构；但仅凭本实验 +不能断言 pseudo-query、某个 source 或组边界是唯一因果。 + +### 6.3 seed-3 的中期反例 + +depth-32 seed-3 在 step 2,000： + +```text +Baseline CV 0.39965 +Block CV 0.34914 +``` + +此时 Block 更平；到 step 8,000 才变成： + +```text +Baseline CV 0.40320 +Block CV 0.42685 +``` + +前两个 seed 在 step 2,000 已明显恶化，seed-3 没有。网站必须保留 seed switch，不能用最终 +均值倒写成“三个 seed 从头到尾都一样”。 + +## 7. 绝对梯度尺度：更平不等于更强 + +最终 activation-gradient mean： + +| depth | Baseline | Block | Block / Baseline | +|---:|---:|---:|---:| +| 16 | 约 `2.02×10⁻⁴` | 约 `1.16×10⁻⁴` | **0.574×** | +| 32 | 约 `1.44×10⁻⁴` | 约 `0.78×10⁻⁴` | **0.544×** | + +所以 Block 的首尾比更接近 1，并不是因为它把晚层全部抬高到 Baseline 早层的强度。更接近的 +描述是： + +> 整体尺度下降，早层系统性高值被削弱，同时某些中后段位置相对全层均值形成尖峰。 + +这也是只看 normalized curve 或只看 CV 都不够的原因。 + +## 8. 参数梯度没有翻转 Round 04 + +最终核心参数梯度 CV： + +| depth | Baseline mean | Block mean | Block / Baseline | +|---:|---:|---:|---:| +| 16 | 0.41596 | 0.68287 | 1.64× | +| 32 | 0.39661 | 0.77176 | 1.95× | + +6 / 6 个配对中 Block 都更高。Round 04 的反结果不是在把梯度对象改成 activation 后自动消失； +两种梯度对象在本轮 final endpoint 都显示更高的跨层 CV。 + +但 activation gradient 又显示首尾失衡大幅改善，这说明“均匀”至少要拆成： + +```text +首尾是否平衡 +全层是否有尖峰 +绝对尺度多大 +参数更新信号是否平衡 +``` + +一个标量不能代替全部。 + +## 9. Output RMS：最接近论文叙述的正向结果 + +三 seed 最终 post-MLP output RMS 的最后/第一层比： + +| depth | Baseline | Block | +|---:|---:|---:| +| 16 | 4.59× | 1.17× | +| 32 | 6.08× | 1.89× | + +Baseline output magnitude 随深度明显累积；Block 把增长限制在 aggregation group 内并产生 +周期性 reset。这个缩小实验的 output-RMS 方向与论文 Figure 5(b) 的叙述一致。 + +仍不能把数值直接叠到论文图上： + +- 模型宽度、深度和数据不同； +- 训练 Token 相差巨大； +- 论文图的确切 output norm / reduction 也未完整公开； +- 本实验的 Block group 是 8 组固定设计。 + +## 10. 验证 BPC 与真实成本 + +最终 BPC： + +### depth-16 + +| seed | Baseline | Block | Block − Base | +|---:|---:|---:|---:| +| 2026073001 | 1.74880 | 1.73739 | −0.01141 | +| 2026073002 | 1.73683 | 1.73118 | −0.00565 | +| 2026073003 | 1.73637 | 1.72661 | −0.00976 | +| 均值 | 1.74067 | 1.73173 | **−0.00894** | + +### depth-32 + +| seed | Baseline | Block | Block − Base | +|---:|---:|---:|---:| +| 2026073001 | 1.71790 | 1.71235 | −0.00554 | +| 2026073002 | 1.72698 | 1.70932 | −0.01765 | +| 2026073003 | 1.70951 | 1.70310 | −0.00642 | +| 均值 | 1.71813 | 1.70826 | **−0.00987** | + +6 / 6 为负。这是有价值的次要方向，但 Round 05 没有为 BPC 再预注册一个新的 support 阈值， +因此不追加事后显著性结论。 + +真实成本： + +| depth | Base mean ms | Block mean ms | time ratio | Base peak alloc | Block peak alloc | memory ratio | +|---:|---:|---:|---:|---:|---:|---:| +| 16 | 21.47 | 54.71 | 2.55× | 3.04 GB | 6.54 GB | 2.15× | +| 32 | 42.11 | 109.38 | 2.60× | 5.94 GB | 12.82 GB | 2.16× | + +这个简单 eager 实现没有论文训练系统的 kernel、并行或工程优化；成本数值不应外推到 K3。 +但它足以说明本站的 BPC 对比不是同 wall time / FLOPs。 + +## 11. 预注册判定为何是 mixed + +支持需要： + +```text +CV：3 / 3 seeds 改善，平均相对改善 ≥20% +AND +imbalance：3 / 3 seeds 改善，平均相对改善 ≥20% +``` + +concern 需要两项都以相同规则恶化。 + +实际： + +```text +depth-16： +CV 3 / 3 恶化，平均 10.3% +imbalance 3 / 3 改善，平均 61.0% + +depth-32： +CV 3 / 3 恶化，平均 60.0% +imbalance 3 / 3 改善，平均 72.0% +``` + +两个指标相反，所以两个 depth 都是 `mixed / inconclusive at this depth`。这不是“数据没规律”， +而是预注册的“更均匀”概念被实验拆成了两个方向相反的组成部分。 + +## 12. 开放工件与校验哈希 + +| 工件 | SHA-256 | +|---|---| +| manifest | `080afb17…6371` | +| protocol | `f772629b…ce22` | +| definition audit | `79221c56…cc6` | +| runner | `04ae69e1…800f` | +| analyzer | `017d38d9…2fc8` | +| aggregate canonical | `69be133c…7b51` | +| compact canonical | `8cdb7180…044f` | +| reproduction canonical | `addb2e59…f68a` | + +公开目录包含： + +- 12 个完整 formal raw JSON； +- 8 个两套 smoke raw JSON； +- 1 个完整 replay raw JSON； +- 完整 aggregate； +- 网站 compact payload； +- manifest、runner、analyzer 与 reproduction 清单； +- 前置定义审计、本协议和本结果审计。 + +`experiments/k3/attnres_gradient/reproduction.json` 记录每个 raw 文件 SHA-256。 + +## 13. 允许和禁止的结论 + +允许： + +> 在本轮公开定义下，Block AttnRes 一致缓解了首/末深度四分位失衡，但在中后段形成局部 +> 梯度尖峰，导致全层 CV 一致升高；因此“梯度更均匀”必须拆成多个指标解释。 + +> 同 token / step 预算下，Block 的最终验证 BPC 在 6 / 6 个配对中更低，但实际运行成本 +> 约为 Baseline 的 2.6× step time 与 2.2× peak allocated memory。 + +禁止： + +- “复现了论文 Figure 5(c)”； +- “论文的梯度结论是错的”； +- “已测到 Kimi K3 checkpoint 的真实梯度”； +- “Block 解决了梯度消失 / 爆炸”； +- “CV 更高就代表训练一定更不稳定”； +- “BPC 改善是同 FLOPs / wall time 优势”； +- 从 3 seeds 推导总体显著性； +- 从 depth 16 / 32 外推到 48B、1T+400B Token 或 K3 2.8T 参数。 + +## 14. 下一步最值得问什么 + +这轮已经把“梯度”从一个模糊词拆成了可复查对象。下一个有价值的问题不是再换一个漂亮 +汇总指标，而是追踪局部尖峰从哪里来： + +1. 分开捕获 pre-attention 与 pre-MLP residual positions； +2. 把 layer 21–25 的 activation-gradient 与 mixer source weights 同步对齐； +3. 比较 aggregation-group boundary 前后； +4. 在不改变 formal 结果的前提下，对相同 raw gradient tensor 做多种公开 reduction + sensitivity analysis； +5. 若官方之后发布 Figure 5 telemetry 代码，再按其定义单独开新 protocol。 + +这些属于后续轮次，不能倒写进本轮预注册结论。 diff --git a/src/data/k3-attnres-gradient-compact.json b/src/data/k3-attnres-gradient-compact.json new file mode 100644 index 0000000..6713b77 --- /dev/null +++ b/src/data/k3-attnres-gradient-compact.json @@ -0,0 +1,34530 @@ +{ + "canonical_sha256_without_self": "8cdb71808a429e5f9513c1c6c9426bdc93e30f237001a1b47a2a2fba574a044f", + "cells": [ + { + "architecture": "baseline", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002865509013645351, + 0.00024785567075014114, + 0.00021551651298068464, + 0.00019683866412378848, + 0.0001823508064262569, + 0.0001679604029050097, + 0.00015584587526973337, + 0.00014737028686795384, + 0.00014130656199995428, + 0.00013612695329356939, + 0.00013129493163432926, + 0.00012807638267986476, + 0.00012550120300147682, + 0.00012324050476308912, + 0.00012108208466088399, + 0.00011888353037647903 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023669043730478734, + "first_to_last_ratio": 1.9372775995887173, + "imbalance_abs_log_ratio": 0.6612836883477485, + "last_quartile_mean": 0.00012217683070048224, + "mean": 0.00016411257956860936, + "normalized": [ + 1.7460629899168627, + 1.5102783187106137, + 1.313223602646409, + 1.1994124072706904, + 1.1111324123086055, + 1.0234462424910682, + 0.9496278449793059, + 0.8979828801383491, + 0.8610343117596252, + 0.8294729974472175, + 0.8000296624393728, + 0.7804178266926869, + 0.7647262832098098, + 0.7509509940495869, + 0.7377989242455608, + 0.7244023016942358 + ], + "population_cv": 0.29361872380036635 + }, + "activation_output_rms_by_block": [ + 0.02866268903017044, + 0.029099803417921066, + 0.02953805774450302, + 0.030088091269135475, + 0.030644793063402176, + 0.03131929785013199, + 0.03213750571012497, + 0.03286394104361534, + 0.033869802951812744, + 0.03499744459986687, + 0.03588006645441055, + 0.037365153431892395, + 0.03789033368229866, + 0.039445169270038605, + 0.04050002992153168, + 0.04121527820825577 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0293471603654325, + "first_to_last_ratio": 0.7380574840395957, + "imbalance_abs_log_ratio": 0.3037335657624931, + "last_quartile_mean": 0.03976270277053118, + "mean": 0.034094841103069484, + "normalized": [ + 0.8406752488894867, + 0.8534957922212247, + 0.8663497699023965, + 0.8824822259232266, + 0.8988102619619861, + 0.9185934539320196, + 0.942591449919727, + 0.9638977622528551, + 0.9933996421752943, + 1.0264733158329964, + 1.0523605710888722, + 1.0959180985456622, + 1.1113216092650298, + 1.1569248600043318, + 1.1878638706395246, + 1.2088420674453664 + ], + "population_cv": 0.11952987076282337 + }, + "bits_per_byte": 8.096463027059821, + "core_parameter_grad_rms_by_block": [ + 0.00809059897248305, + 0.007313812062277777, + 0.007090710224412361, + 0.0067033738367094624, + 0.006535407867454384, + 0.006785021795736036, + 0.006295993569919972, + 0.005509300326230647, + 0.006231736048910802, + 0.006047390980174007, + 0.005629405359532731, + 0.005784042737956458, + 0.0059591736113607996, + 0.0061983173180850636, + 0.006571690039252182, + 0.005060205437758869 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007299623773970663, + "first_to_last_ratio": 1.227374872012571, + "imbalance_abs_log_ratio": 0.2048776382299483, + "last_quartile_mean": 0.005947346601614228, + "mean": 0.006362886261765913, + "normalized": [ + 1.271529717747562, + 1.1494488132258316, + 1.1143858200043408, + 1.05351149791715, + 1.0271137340180259, + 1.066343404015675, + 0.9894870520870547, + 0.8658492545019393, + 0.9793882512652816, + 0.9504163254515969, + 0.8847251275509024, + 0.909028151691524, + 0.9365519618304368, + 0.9741361173356609, + 1.0328158902888074, + 0.7952688810682109 + ], + "population_cv": 0.11384977604868386 + }, + "loss_nats": 5.6120405197143555, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 8.509291365044191e-05, + 7.616560469614342e-05, + 6.756903894711286e-05, + 5.824641266372055e-05, + 5.0209364417241886e-05, + 4.2265295633114874e-05, + 3.587255923775956e-05, + 3.076014036196284e-05, + 2.70738746621646e-05, + 2.470596882631071e-05, + 2.293860052304808e-05, + 2.172161293856334e-05, + 2.083007166220341e-05, + 2.0244739062036388e-05, + 1.9782984963967465e-05, + 1.9472558051347733e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.176849248935468e-05, + "first_to_last_ratio": 3.5736674444153773, + "imbalance_abs_log_ratio": 1.2735923638392248, + "last_quartile_mean": 2.008258843488875e-05, + "mean": 3.8934483768571226e-05, + "normalized": [ + 2.185541078603713, + 1.9562505348440236, + 1.7354548566444863, + 1.496010914384804, + 1.28958598027623, + 1.0855491467240759, + 0.921357001957136, + 0.7900487533057547, + 0.6953700689366589, + 0.6345523668212576, + 0.5891589743271394, + 0.5579016551928063, + 0.5350031551983206, + 0.519969371685323, + 0.5081095997460412, + 0.5001365413522295 + ], + "population_cv": 0.5464287537448471 + }, + "activation_output_rms_by_block": [ + 0.030502017587423325, + 0.03408988192677498, + 0.038947492837905884, + 0.04481571167707443, + 0.0512830950319767, + 0.05888458341360092, + 0.06758055835962296, + 0.07802816480398178, + 0.08916938304901123, + 0.10193294286727905, + 0.11700897663831711, + 0.13432858884334564, + 0.15294352173805237, + 0.17351281642913818, + 0.19949108362197876, + 0.2260059416294098 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037088776007294655, + "first_to_last_ratio": 0.19729295890734105, + "imbalance_abs_log_ratio": 1.623065553831421, + "last_quartile_mean": 0.18798834085464478, + "mean": 0.09990779752843082, + "normalized": [ + 0.3053016715611547, + 0.3412134264802905, + 0.38983436529889043, + 0.4485707100521478, + 0.5133042295060407, + 0.5893892656060614, + 0.676429268099835, + 0.7810017509571989, + 0.8925167529955432, + 1.0202701429612833, + 1.1711696137132819, + 1.344525574243789, + 1.530846695870051, + 1.7367294717888415, + 1.9967518908142226, + 2.262145170051368 + ], + "population_cv": 0.5980263271119333 + }, + "bits_per_byte": 5.097173307164379, + "core_parameter_grad_rms_by_block": [ + 0.0005223696653307753, + 0.0004106838611462913, + 0.0004143731432945227, + 0.00042601318321813184, + 0.0003557105812490162, + 0.00034956609676611394, + 0.00031105173709675404, + 0.00028609388990673116, + 0.0002585596876336936, + 0.00022586454081316873, + 0.00020762781370074637, + 0.0002072362198298983, + 0.00019288470561650055, + 0.00019339001384203696, + 0.00020597582961175812, + 0.0001909336964292271 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004433599632474303, + "first_to_last_ratio": 2.2643967408442993, + "imbalance_abs_log_ratio": 0.8173083838772207, + "last_quartile_mean": 0.0001957960613748807, + "mean": 0.00029739591659283537, + "normalized": [ + 1.7564789433405417, + 1.3809330869480578, + 1.3933383754620974, + 1.4324782535645448, + 1.1960842816010129, + 1.175423323800053, + 1.0459179825344234, + 0.9619966984900542, + 0.8694123665043039, + 0.7594742503556288, + 0.6981528733799311, + 0.696836130785297, + 0.6485788635779385, + 0.6502779730725327, + 0.6925980422715741, + 0.6420185543120094 + ], + "population_cv": 0.3407962002760142 + }, + "loss_nats": 3.5330913066864014, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00014349650882650167, + 0.00014052700134925544, + 0.00012983661144971848, + 0.0001149297459051013, + 0.0001033403241308406, + 9.356468945043162e-05, + 8.457523654215038e-05, + 7.733132952125743e-05, + 7.047204417176545e-05, + 6.370909250108525e-05, + 5.659156886395067e-05, + 4.938157871947624e-05, + 4.27361810579896e-05, + 3.8237969420151785e-05, + 3.594611553126015e-05, + 3.5643912269733846e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013219746688264422, + "first_to_last_ratio": 3.4660158989817993, + "imbalance_abs_log_ratio": 1.243005778409783, + "last_quartile_mean": 3.8141044569783844e-05, + "mean": 8.001999435691687e-05, + "normalized": [ + 1.7932581722820122, + 1.7561486035909524, + 1.622552119544051, + 1.4362628594107976, + 1.2914312849099563, + 1.1692663840127346, + 1.0569262997559778, + 0.9664000871623777, + 0.8806804441579406, + 0.7961646712560445, + 0.7072178562214424, + 0.6171154986492164, + 0.5340687837013764, + 0.47785518766218865, + 0.44921417228459215, + 0.44543757539833934 + ], + "population_cv": 0.4531561582313658 + }, + "activation_output_rms_by_block": [ + 0.05229421332478523, + 0.06407254934310913, + 0.07844720035791397, + 0.08364550769329071, + 0.0890495702624321, + 0.09326893091201782, + 0.09662611037492752, + 0.09771094471216202, + 0.09944888204336166, + 0.10062506794929504, + 0.10494095832109451, + 0.11238908767700195, + 0.12405165284872055, + 0.14480136334896088, + 0.17238570749759674, + 0.2138574868440628 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.06961486767977476, + "first_to_last_ratio": 0.4250665264722616, + "imbalance_abs_log_ratio": 0.8555095894318028, + "last_quartile_mean": 0.16377405263483524, + "mean": 0.10797595209442079, + "normalized": [ + 0.48431351898667185, + 0.5933964748658124, + 0.7265247384835739, + 0.7746679336538376, + 0.8247166941816979, + 0.8637935494234656, + 0.8948854675569957, + 0.9049324670619027, + 0.9210280633265217, + 0.9319210990730817, + 0.9718919471006628, + 1.040871467183008, + 1.1488822320385021, + 1.3410519707419453, + 1.596519448579181, + 1.9806029277431396 + ], + "population_cv": 0.361888909970992 + }, + "bits_per_byte": 3.1304997557876018, + "core_parameter_grad_rms_by_block": [ + 0.0006452065099731355, + 0.00042477980025779674, + 0.0003406169302843733, + 0.00031739507901072504, + 0.0002899614562735046, + 0.0003022750042151422, + 0.0002946052417003999, + 0.0002490739832611916, + 0.00022953214745651343, + 0.00020477988838642542, + 0.00019411347501299613, + 0.00019138854173966402, + 0.00017481673923870362, + 0.00016715970609089626, + 0.0001418296786517743, + 0.00014407196058940572 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00043199957988150763, + "first_to_last_ratio": 2.752123958438998, + "imbalance_abs_log_ratio": 1.0123729620941166, + "last_quartile_mean": 0.00015696952114269496, + "mean": 0.00026947538388391546, + "normalized": [ + 2.394305931301975, + 1.576321347558719, + 1.2640001671956211, + 1.177825872018991, + 1.0760220547581225, + 1.121716573360022, + 1.0932547435475957, + 0.9242921548948887, + 0.8517740809876394, + 0.7599205739498657, + 0.720338430231595, + 0.7102264369427908, + 0.6487299014815064, + 0.6203153092562447, + 0.526317753434843, + 0.5346386690795809 + ], + "population_cv": 0.4584519745606109 + }, + "loss_nats": 2.1698970794677734, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.00016704612062312663, + 0.00018574431305751204, + 0.0002023741544689983, + 0.00018453448137734085, + 0.00017654163821134716, + 0.00017114127695094794, + 0.00016412502736784518, + 0.00015690893633291125, + 0.0001434983714716509, + 0.00012688931019511074, + 0.00010475149611011147, + 9.039731958182529e-05, + 7.724175520706922e-05, + 6.240545917535201e-05, + 4.393732160679065e-05, + 2.913598837039899e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018492476738174446, + "first_to_last_ratio": 3.4773281598182453, + "imbalance_abs_log_ratio": 1.246264228732751, + "last_quartile_mean": 5.3180131089902716e-05, + "mean": 0.00013041706063177116, + "normalized": [ + 1.2808609534206308, + 1.4242332418605552, + 1.551746017650221, + 1.4149566052433022, + 1.3536698140269194, + 1.312261418267664, + 1.2584628619352762, + 1.2031319796106978, + 1.100303677881642, + 0.9729502381086403, + 0.8032039336162798, + 0.6931402927187534, + 0.5922672603790627, + 0.47850686768314793, + 0.33689857288569336, + 0.22340626471151362 + ], + "population_cv": 0.41009229518876916 + }, + "activation_output_rms_by_block": [ + 0.08545628190040588, + 0.1388079822063446, + 0.16856427490711212, + 0.17068639397621155, + 0.17051894962787628, + 0.1691380739212036, + 0.16795755922794342, + 0.15964734554290771, + 0.14745007455348969, + 0.13513104617595673, + 0.13549363613128662, + 0.14635984599590302, + 0.16386321187019348, + 0.19344064593315125, + 0.2505952715873718, + 0.40341031551361084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14087873324751854, + "first_to_last_ratio": 0.5572131614407935, + "imbalance_abs_log_ratio": 0.5848074166792235, + "last_quartile_mean": 0.25282736122608185, + "mean": 0.17540755681693554, + "normalized": [ + 0.4871870029498929, + 0.7913455082850954, + 0.9609863905865502, + 0.9730846097716802, + 0.9721300080921755, + 0.9642576237335369, + 0.9575274992469833, + 0.9101508990831222, + 0.8406141515748735, + 0.77038326414287, + 0.7724503926173194, + 0.8343987491294447, + 0.9341855895137383, + 1.1028067971725755, + 1.4286458128420665, + 2.2998457012580755 + ], + "population_cv": 0.3845661956400748 + }, + "bits_per_byte": 2.092252187349085, + "core_parameter_grad_rms_by_block": [ + 0.00040658806478613896, + 0.0003235049151816101, + 0.00024397059654808256, + 0.0002149925859972515, + 0.00020566351114887415, + 0.00024487046739718384, + 0.00031061648932541417, + 0.0003911240721720923, + 0.000499392430316199, + 0.0005175204124242609, + 0.0005752549257371158, + 0.00046811732551685614, + 0.00041118767134459974, + 0.0003618888130405475, + 0.0003117946917671641, + 0.00025214502653225866 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029726404062827076, + "first_to_last_ratio": 0.8893356416516115, + "imbalance_abs_log_ratio": 0.11728056505781807, + "last_quartile_mean": 0.0003342540506711425, + "mean": 0.0003586644999522281, + "normalized": [ + 1.1336166942652368, + 0.9019708257290558, + 0.6802195271084202, + 0.5994253293144072, + 0.5734147404503854, + 0.6827284758591918, + 0.8660363358146301, + 1.0905012127606375, + 1.3923664884110778, + 1.4429094947874448, + 1.6038802998728234, + 1.3051677140592575, + 1.1464409535913576, + 1.0089897748139245, + 0.8693213067049939, + 0.7030108264571566 + ], + "population_cv": 0.30960458504189897 + }, + "loss_nats": 1.4502387046813965, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.00017181786824949086, + 0.0001916729233926162, + 0.00020346585370134562, + 0.00018768427253235132, + 0.00018128244846593589, + 0.00017597789701540023, + 0.0001705518807284534, + 0.00016394301201216877, + 0.0001481794024584815, + 0.0001306164776906371, + 0.00010858976020244882, + 9.323586709797382e-05, + 7.920334610389546e-05, + 6.374791701091453e-05, + 4.397464726935141e-05, + 2.655091702763457e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000188660229468951, + "first_to_last_ratio": 3.535001559771659, + "imbalance_abs_log_ratio": 1.2627137405851174, + "last_quartile_mean": 5.336920685294899e-05, + "mean": 0.00013378090568494372, + "normalized": [ + 1.2843228065305883, + 1.4327375226776322, + 1.5208885951221702, + 1.4029227233245896, + 1.3550696755847886, + 1.315418640010041, + 1.2748596659235207, + 1.2254589784154797, + 1.107627442794015, + 0.9763461909746755, + 0.8116984979768304, + 0.6969295552352281, + 0.5920377478264399, + 0.476509832883341, + 0.3287064551118561, + 0.19846566960880369 + ], + "population_cv": 0.41282075642614385 + }, + "activation_output_rms_by_block": [ + 0.10104851424694061, + 0.1665971875190735, + 0.18799307942390442, + 0.18892255425453186, + 0.18902969360351562, + 0.1888171136379242, + 0.18645423650741577, + 0.1778823733329773, + 0.16393223404884338, + 0.15616974234580994, + 0.16218091547489166, + 0.17675508558750153, + 0.19963829219341278, + 0.23325374722480774, + 0.2989467680454254, + 0.5114414095878601 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1611403338611126, + "first_to_last_ratio": 0.5184360907576058, + "imbalance_abs_log_ratio": 0.6569385168504118, + "last_quartile_mean": 0.3108200542628765, + "mean": 0.20556643418967724, + "normalized": [ + 0.49156135166358245, + 0.8104299136957027, + 0.9145125281029207, + 0.9190340582558922, + 0.9195552491273791, + 0.9185211310505179, + 0.907026661441106, + 0.8653279122837938, + 0.797465961284235, + 0.7597044865880744, + 0.7889464839636536, + 0.8598440999585013, + 0.9711619164887857, + 1.1346879082875139, + 1.4542586644743063, + 2.4879616733340346 + ], + "population_cv": 0.4294257518805603 + }, + "bits_per_byte": 1.9625077876075785, + "core_parameter_grad_rms_by_block": [ + 0.00044260282991257096, + 0.0003095538604202773, + 0.00019879384700301542, + 0.00020176150294426366, + 0.00022494965081173505, + 0.00026863158839826584, + 0.00037546716364749414, + 0.0004673111939829261, + 0.0006383031641084419, + 0.0006422551721113508, + 0.0007081700332195422, + 0.0005856936655792496, + 0.0005168621280586129, + 0.00045203145589075384, + 0.0003829553066507503, + 0.0003084257693651058 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002881780100700318, + "first_to_last_ratio": 0.6942899678444003, + "imbalance_abs_log_ratio": 0.36486558463682156, + "last_quartile_mean": 0.0004150686649913057, + "mean": 0.00042023552075652225, + "normalized": [ + 1.0532256509772957, + 0.7366199312780793, + 0.47305341215597385, + 0.48011529958497023, + 0.5352942331166415, + 0.6392405570920472, + 0.8934684125976915, + 1.1120221183151215, + 1.5189176844435874, + 1.5283219537347563, + 1.6851741422159423, + 1.3937271759532939, + 1.2299344118463382, + 1.075662178858635, + 0.9112873323067531, + 0.7339355055228726 + ], + "population_cv": 0.3791677394455083 + }, + "loss_nats": 1.360306739807129, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.0002529843186493963, + 0.0002798262285068631, + 0.0002946711319964379, + 0.00027859603869728744, + 0.0002704691141843796, + 0.0002655670396052301, + 0.0002578931744210422, + 0.00024818466044962406, + 0.00022750988136976957, + 0.00020225343178026378, + 0.00016641257388982922, + 0.0001406865194439888, + 0.000117213734483812, + 9.274082549382001e-05, + 6.181560456752777e-05, + 3.130940967821516e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002765194294624962, + "first_to_last_ratio": 3.6494630846841103, + "imbalance_abs_log_ratio": 1.2945800566875763, + "last_quartile_mean": 7.576989355584374e-05, + "mean": 0.00019925835545109294, + "normalized": [ + 1.269629662839861, + 1.4043387434036372, + 1.478839526349316, + 1.3981649003706025, + 1.3573790347314445, + 1.3327774336188996, + 1.2942652961137227, + 1.2455420496057434, + 1.1417833939998108, + 1.0150311203883542, + 0.8351598281190995, + 0.7060507908212645, + 0.5882500345767512, + 0.4654300457507431, + 0.31022841891666686, + 0.15712972039408363 + ], + "population_cv": 0.41719159337744666 + }, + "activation_output_rms_by_block": [ + 0.10425227880477905, + 0.15637855231761932, + 0.1697961539030075, + 0.17037443816661835, + 0.16992402076721191, + 0.16921496391296387, + 0.1664763242006302, + 0.1570158749818802, + 0.1424521654844284, + 0.13391907513141632, + 0.13948261737823486, + 0.15248820185661316, + 0.17248840630054474, + 0.20097237825393677, + 0.2573379576206207, + 0.4688045382499695 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15020035579800606, + "first_to_last_ratio": 0.5463801662721245, + "imbalance_abs_log_ratio": 0.6044402703006951, + "last_quartile_mean": 0.27490082010626793, + "mean": 0.18321112170815468, + "normalized": [ + 0.5690281126647281, + 0.853542900996599, + 0.9267786383267906, + 0.9299350202004414, + 0.927476559189958, + 0.9236063964638243, + 0.9086583972004597, + 0.8570215253198324, + 0.7775301202038757, + 0.7309549435800197, + 0.7613217804562272, + 0.8323086526347376, + 0.9414734470940544, + 1.0969442050252538, + 1.4045979044358783, + 2.5588213962073194 + ], + "population_cv": 0.43808900348348184 + }, + "bits_per_byte": 1.8689156889212923, + "core_parameter_grad_rms_by_block": [ + 0.0005010958566464473, + 0.000367257410216879, + 0.0002469638962710481, + 0.00025453790022962814, + 0.00027094925891000524, + 0.0003539210655471446, + 0.0005067460397887177, + 0.0006525825825832493, + 0.0009467684204760593, + 0.0009621097966915796, + 0.0010586988551551607, + 0.0008532929973946807, + 0.0007592856290288629, + 0.0006522457278840908, + 0.0005350698764559804, + 0.0004007556744895193 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034246376584100063, + "first_to_last_ratio": 0.5835734049551724, + "imbalance_abs_log_ratio": 0.5385850340449921, + "last_quartile_mean": 0.0005868392269646134, + "mean": 0.0005826425617355658, + "normalized": [ + 0.8600399104963966, + 0.6303305565643863, + 0.4238686160094385, + 0.43686801642402323, + 0.4650351290899624, + 0.6074411462370523, + 0.8697374223387168, + 1.1200393267518036, + 1.6249558179475279, + 1.6512865002955899, + 1.8170640538197664, + 1.4645222533226991, + 1.3031757013547307, + 1.1194611769198464, + 0.9183501371100032, + 0.6878242353180568 + ], + "population_cv": 0.44619098566843096 + }, + "loss_nats": 1.2954336404800415, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.076786148009306, + "cross_entropy_nats": 5.5984015464782715, + "step": 0 + }, + { + "bits_per_byte": 5.114718375266125, + "cross_entropy_nats": 3.5452526211738586, + "step": 100 + }, + { + "bits_per_byte": 3.1628078576350735, + "cross_entropy_nats": 2.192291349172592, + "step": 500 + }, + { + "bits_per_byte": 1.9765894690862489, + "cross_entropy_nats": 1.3700674176216125, + "step": 2000 + }, + { + "bits_per_byte": 1.840425322451167, + "cross_entropy_nats": 1.2756856232881546, + "step": 4000 + }, + { + "bits_per_byte": 1.7488000388579794, + "cross_entropy_nats": 1.2121758162975311, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "final_optimizer_state": "e9f9db84e99e09e299da28a6a358f3b99d039dde4de0eceec7d22aff7014508e", + "final_public_parameters": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "seed": 2026073001, + "timing": { + "mean_ms": 21.257796359896474, + "measured_steps": 7980, + "median_ms": 21.201465002377518, + "p95_ms": 21.46604766312521, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "baseline", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027575084823183715, + 0.00024360731185879558, + 0.00021477397240232676, + 0.00019141066877637058, + 0.00017414781905245036, + 0.00016276225505862385, + 0.00015418927068822086, + 0.00014831383305136114, + 0.0001412670244462788, + 0.00013702295836992562, + 0.0001326367782894522, + 0.00012906579650007188, + 0.00012653179874178022, + 0.00012431222421582788, + 0.000122369994642213, + 0.000120684111607261 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023138570031733252, + "first_to_last_ratio": 1.8739548634355478, + "imbalance_abs_log_ratio": 0.6280510978460953, + "last_quartile_mean": 0.00012347453230177052, + "mean": 0.0001624279166207998, + "normalized": [ + 1.6976813713346965, + 1.4997872097782008, + 1.3222725309204868, + 1.1784345496669344, + 1.072154483511671, + 1.0020583803866954, + 0.9492781407040226, + 0.9131055556023103, + 0.8697213347633911, + 0.8435924145343563, + 0.8165885584748509, + 0.79460353358706, + 0.77900278089005, + 0.7653377990807093, + 0.7533803128676146, + 0.74300104389695 + ], + "population_cv": 0.2778958958835334 + }, + "activation_output_rms_by_block": [ + 0.028587408363819122, + 0.028957132250070572, + 0.029405318200588226, + 0.029973436146974564, + 0.030426617711782455, + 0.030960483476519585, + 0.03149084374308586, + 0.03222988545894623, + 0.03307616710662842, + 0.03391994163393974, + 0.034928515553474426, + 0.035913772881031036, + 0.03729887306690216, + 0.038532089442014694, + 0.03956005349755287, + 0.0403735376894474 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02923082374036312, + "first_to_last_ratio": 0.7506412221981493, + "imbalance_abs_log_ratio": 0.2868274747935402, + "last_quartile_mean": 0.03894113842397928, + "mean": 0.033477129763923585, + "normalized": [ + 0.8539384518748725, + 0.8649825255113729, + 0.8783703503840009, + 0.8953406805883115, + 0.9088777301503161, + 0.9248249086719481, + 0.9406673739700877, + 0.962743391868635, + 0.9880227886881968, + 1.0132272949664087, + 1.0433545468140735, + 1.0727853054993168, + 1.114159825825243, + 1.1509974037122668, + 1.1817038610097486, + 1.206003560465201 + ], + "population_cv": 0.11262664842931464 + }, + "bits_per_byte": 8.031398550585404, + "core_parameter_grad_rms_by_block": [ + 0.007112265943118693, + 0.0061890396223896905, + 0.005352992918648162, + 0.005122911821075447, + 0.005146431820875539, + 0.004279647792521989, + 0.004250462252416264, + 0.004421502222176574, + 0.004575182382709001, + 0.004269268868925463, + 0.0040238830908244155, + 0.003973260546139901, + 0.0038964174441780434, + 0.004243476176619415, + 0.00401474940360252, + 0.003997243047861786 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005944302576307998, + "first_to_last_ratio": 1.472101165080992, + "imbalance_abs_log_ratio": 0.38669074422298993, + "last_quartile_mean": 0.004037971518065441, + "mean": 0.004679295959630182, + "normalized": [ + 1.5199435993103534, + 1.3226433369003716, + 1.1439740005398642, + 1.0948039759127195, + 1.0998303730466061, + 0.9145922440990935, + 0.9083550792867974, + 0.944907580183498, + 0.9777501620287747, + 0.9123741917070097, + 0.8599334441633468, + 0.8491150336329483, + 0.8326930969517022, + 0.9068621034508766, + 0.857981507953136, + 0.854240270832901 + ], + "population_cv": 0.1873316910212935 + }, + "loss_nats": 5.566941261291504, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 9.082402539206669e-05, + 8.28480115160346e-05, + 7.297925185412169e-05, + 6.382335413945839e-05, + 5.4197575082071126e-05, + 4.6132205170579255e-05, + 3.93990776501596e-05, + 3.41209233738482e-05, + 2.9922650355729274e-05, + 2.6855812393478118e-05, + 2.4530303562642075e-05, + 2.2887908926350065e-05, + 2.182140451623127e-05, + 2.1062218365841545e-05, + 2.0474079065024853e-05, + 2.0058148948010057e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.761866072542034e-05, + "first_to_last_ratio": 3.722010140399952, + "imbalance_abs_log_ratio": 1.3142638826655557, + "last_quartile_mean": 2.085396272377693e-05, + "mean": 4.1996059394477925e-05, + "normalized": [ + 2.1626797061823653, + 1.972756794579834, + 1.7377642785150873, + 1.5197462585704062, + 1.2905395378404856, + 1.098488901952672, + 0.938161299375155, + 0.8124791674700501, + 0.7125109066700628, + 0.6394841035254226, + 0.584109650198932, + 0.5450013467063437, + 0.5196060018693224, + 0.5015284450381319, + 0.4875238143823798, + 0.47761978712334874 + ], + "population_cv": 0.55073751796069 + }, + "activation_output_rms_by_block": [ + 0.03037329390645027, + 0.03339420258998871, + 0.03790459781885147, + 0.04353494197130203, + 0.04983566701412201, + 0.057033151388168335, + 0.06467092037200928, + 0.07435981929302216, + 0.08538646996021271, + 0.09887228161096573, + 0.11526436358690262, + 0.13244163990020752, + 0.15128590166568756, + 0.17302390933036804, + 0.19715538620948792, + 0.22326189279556274 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03630175907164812, + "first_to_last_ratio": 0.194980199104046, + "imbalance_abs_log_ratio": 1.63485726863041, + "last_quartile_mean": 0.18618177250027657, + "mean": 0.09798740246333182, + "normalized": [ + 0.30997141615031953, + 0.34080097798781084, + 0.38683133613053855, + 0.4442912137363105, + 0.5085925921219432, + 0.5820457523558795, + 0.65999219028395, + 0.7588712163366979, + 0.8714025253620276, + 1.0090305398999126, + 1.1763181867182986, + 1.351619050722045, + 1.543932157220932, + 1.7657770793048206, + 2.012048296547774, + 2.2784754691207403 + ], + "population_cv": 0.6071319453340859 + }, + "bits_per_byte": 5.218263195949167, + "core_parameter_grad_rms_by_block": [ + 0.0003576967965622096, + 0.00031529422707157425, + 0.00030800051934586363, + 0.00027582768007438345, + 0.00025927860434436177, + 0.0002499710324331713, + 0.00022865411268768897, + 0.00021433881461782036, + 0.00020517591210118428, + 0.00017936399546700514, + 0.000174300235585937, + 0.00015047843844280973, + 0.00014083116496318749, + 0.00013997136590409212, + 0.0001362051425885804, + 0.00013592122810918616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031420480576350773, + "first_to_last_ratio": 2.2730213947880955, + "imbalance_abs_log_ratio": 0.8211099574033609, + "last_quartile_mean": 0.00013823222539126155, + "mean": 0.000216956829393691, + "normalized": [ + 1.6487003315905355, + 1.4532579036700417, + 1.4196396592197809, + 1.271348225567332, + 1.19507002876537, + 1.1521694575448123, + 1.0539152573656578, + 0.9879330151386018, + 0.9456992558131047, + 0.8267266624805357, + 0.8033867201739516, + 0.6935870092835418, + 0.6491206815510496, + 0.6451576854955755, + 0.6277983641686699, + 0.6264897421714382 + ], + "population_cv": 0.3194559569879116 + }, + "loss_nats": 3.6170244216918945, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00015646869724150747, + 0.0001555694470880553, + 0.00014404315152205527, + 0.00012938148574903607, + 0.00011473113409010693, + 0.00010209716856479645, + 9.211692668031901e-05, + 8.257795707322657e-05, + 7.45681973057799e-05, + 6.577855674549937e-05, + 5.7623117754701525e-05, + 5.006180435884744e-05, + 4.3452899262774736e-05, + 3.91966023016721e-05, + 3.658098648884334e-05, + 3.536837175488472e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014636569540016353, + "first_to_last_ratio": 3.786979944917394, + "imbalance_abs_log_ratio": 1.3315688531352086, + "last_quartile_mean": 3.864971495204372e-05, + "mean": 8.622603149888164e-05, + "normalized": [ + 1.8146341020407148, + 1.8042051151347847, + 1.670529757871595, + 1.5004921773619393, + 1.330585811450629, + 1.1840643340534658, + 1.068319219602653, + 0.9576917276344513, + 0.8647991332727294, + 0.7628619293044065, + 0.6682798309632156, + 0.5805880601091649, + 0.5039417738173225, + 0.4545796857435157, + 0.42424527550381114, + 0.41018206613560143 + ], + "population_cv": 0.48370287188714023 + }, + "activation_output_rms_by_block": [ + 0.05259466543793678, + 0.067360520362854, + 0.0797014907002449, + 0.08837522566318512, + 0.09281065315008163, + 0.09652531147003174, + 0.09793278574943542, + 0.09766053408384323, + 0.09857666492462158, + 0.10091027617454529, + 0.10140916705131531, + 0.11094733327627182, + 0.12530675530433655, + 0.14505283534526825, + 0.17291401326656342, + 0.21681295335292816 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0720079755410552, + "first_to_last_ratio": 0.43635474619550435, + "imbalance_abs_log_ratio": 0.829299728394033, + "last_quartile_mean": 0.1650216393172741, + "mean": 0.10905569908209145, + "normalized": [ + 0.4822734243200464, + 0.6176707951057974, + 0.7308328805471209, + 0.8103677882680927, + 0.8510390005406192, + 0.8851010289464333, + 0.8980070420319504, + 0.8955105960150644, + 0.9039111734125715, + 0.9253095163654427, + 0.9298841592403141, + 1.0173455785447438, + 1.1490161115744364, + 1.33008028526854, + 1.5855568734321968, + 1.98809374638663 + ], + "population_cv": 0.3579222353559594 + }, + "bits_per_byte": 3.109639980064452, + "core_parameter_grad_rms_by_block": [ + 0.0010353898057018447, + 0.0006549276400495494, + 0.000583207798204784, + 0.0004740072073606198, + 0.00043399488807089015, + 0.0004525702167816539, + 0.0003859790612774441, + 0.0004183888521327983, + 0.00036657723102634496, + 0.00038828410964056754, + 0.0003961803157723344, + 0.00030130391836939404, + 0.00026972366751228246, + 0.0002437005745447571, + 0.00023517814034756533, + 0.0002654411672498285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006868831128291994, + "first_to_last_ratio": 2.7094817103793067, + "imbalance_abs_log_ratio": 0.9967573658183029, + "last_quartile_mean": 0.00025351088741360834, + "mean": 0.0004315534121276662, + "normalized": [ + 2.3992158944987, + 1.5176050557000407, + 1.3514151013878533, + 1.0983743704484823, + 1.0056574131373146, + 1.0487003556532426, + 0.8943946460172122, + 0.9694949463382451, + 0.8494365256412354, + 0.8997359277643753, + 0.9180330977319039, + 0.6981845349892853, + 0.6250064532741785, + 0.5647054749103996, + 0.5449572028363264, + 0.6150829996712046 + ], + "population_cv": 0.44646086778633726 + }, + "loss_nats": 2.155438184738159, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.0001656424137763679, + 0.00018305814592167735, + 0.00020359577320050448, + 0.00018629668920766562, + 0.0001775996934156865, + 0.00016671977937221527, + 0.00015271134907379746, + 0.0001390485413139686, + 0.00012298233923502266, + 0.00011299804464215413, + 9.434259118279442e-05, + 8.304810035042465e-05, + 7.153192564146593e-05, + 5.851028618053533e-05, + 4.2740048229461536e-05, + 2.919154394476209e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018464825552655384, + "first_to_last_ratio": 3.6568753347836163, + "imbalance_abs_log_ratio": 1.2966090491094597, + "last_quartile_mean": 5.049345099905622e-05, + "mean": 0.0001243760790430315, + "normalized": [ + 1.3317867475068026, + 1.4718115197886492, + 1.6369367386960691, + 1.4978498328702814, + 1.4279248452127258, + 1.3404489083028075, + 1.2278192900819984, + 1.1179685224347742, + 0.9887941490137615, + 0.9085191100377042, + 0.7585268156761388, + 0.6677176269698272, + 0.5751260707994935, + 0.4704303804294394, + 0.34363559744212857, + 0.23470384473739866 + ], + "population_cv": 0.4334270573824917 + }, + "activation_output_rms_by_block": [ + 0.08618219196796417, + 0.145728200674057, + 0.16391748189926147, + 0.16589416563510895, + 0.16637063026428223, + 0.1656494140625, + 0.16352342069149017, + 0.1510169357061386, + 0.14240477979183197, + 0.14064612984657288, + 0.14404520392417908, + 0.15505897998809814, + 0.1711372584104538, + 0.19663043320178986, + 0.24908031523227692, + 0.4027986526489258 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1404305100440979, + "first_to_last_ratio": 0.5508987205974385, + "imbalance_abs_log_ratio": 0.5962042968923701, + "last_quartile_mean": 0.2549116648733616, + "mean": 0.1756302621215582, + "normalized": [ + 0.49070240473885574, + 0.8297442531469593, + 0.9333100111517798, + 0.9445648126419658, + 0.9472776972178796, + 0.9431712511358084, + 0.9310663134939208, + 0.859857144673712, + 0.8108214271938511, + 0.8008080620481458, + 0.820161640620235, + 0.8828716538655387, + 0.9744178272193509, + 1.1195703452614385, + 1.4182084125108352, + 2.293446743079724 + ], + "population_cv": 0.38002251655845937 + }, + "bits_per_byte": 2.081392170877702, + "core_parameter_grad_rms_by_block": [ + 0.00041094985323999886, + 0.0003483840396349268, + 0.0002459421498427246, + 0.00023334166050981232, + 0.000254581953755271, + 0.0003388594313501643, + 0.0004326353023247584, + 0.0004560987158898983, + 0.0004819816052452772, + 0.0004606041111206281, + 0.00053396017856909, + 0.0004147477851776299, + 0.00037952749197896316, + 0.00033346538787636917, + 0.0002981284512971557, + 0.0002588526279031963 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030965442580686565, + "first_to_last_ratio": 0.9753095285106969, + "imbalance_abs_log_ratio": 0.0250003932265083, + "last_quartile_mean": 0.0003174934897639211, + "mean": 0.0003676287966072415, + "normalized": [ + 1.117839127490983, + 0.9476516607242959, + 0.6689958787572303, + 0.6347208452201429, + 0.6924973127914547, + 0.9217434392447414, + 1.176826479093712, + 1.2406501343178897, + 1.3110550906059877, + 1.2529054181049841, + 1.4524438332820528, + 1.1281700155298995, + 1.0323660591377277, + 0.9070709121642312, + 0.8109496700163646, + 0.7041141235183029 + ], + "population_cv": 0.2454972418327454 + }, + "loss_nats": 1.4427111148834229, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.0001670715573709458, + 0.0001843688660301268, + 0.00019890567637048662, + 0.00018651749996934086, + 0.00017948838649317622, + 0.0001703005691524595, + 0.00015747325960546732, + 0.0001445972011424601, + 0.00012834250810556114, + 0.00011769140110118315, + 9.774974023457617e-05, + 8.55058096931316e-05, + 7.324934267671779e-05, + 5.983562004985288e-05, + 4.2463150748517364e-05, + 2.6351406631874852e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018421589993522502, + "first_to_last_ratio": 3.6496550331101454, + "imbalance_abs_log_ratio": 1.2946326516512365, + "last_quartile_mean": 5.047488002674072e-05, + "mean": 0.00012624449971099239, + "normalized": [ + 1.323396724240799, + 1.4604110789158873, + 1.5755591477318631, + 1.477430703090666, + 1.42175213101619, + 1.348974169506975, + 1.2473672909787434, + 1.145374265599547, + 1.016618612291004, + 0.9322497326268504, + 0.7742891013735379, + 0.6773032479741881, + 0.5802180914368962, + 0.47396615446084944, + 0.336356442028975, + 0.2087331067270282 + ], + "population_cv": 0.42845315545725604 + }, + "activation_output_rms_by_block": [ + 0.10210617631673813, + 0.1711120903491974, + 0.18304166197776794, + 0.1841333508491516, + 0.18528582155704498, + 0.18522554636001587, + 0.18429037928581238, + 0.1677425056695938, + 0.15884697437286377, + 0.1616142988204956, + 0.1705332249403, + 0.18593955039978027, + 0.20567089319229126, + 0.23390550911426544, + 0.2939614951610565, + 0.5084910988807678 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16009831987321377, + "first_to_last_ratio": 0.5156025192452343, + "imbalance_abs_log_ratio": 0.6624191218832031, + "last_quartile_mean": 0.31050724908709526, + "mean": 0.20511878607794642, + "normalized": [ + 0.4977904670220559, + 0.8342097455869972, + 0.8923690778289365, + 0.8976913054622885, + 0.9033098581552409, + 0.9030160030765246, + 0.8984568542433792, + 0.8177822659590556, + 0.7744145595347903, + 0.7879058857099571, + 0.8313876471338725, + 0.9064969326072458, + 1.002691645777072, + 1.1403417189948652, + 1.4331280950997307, + 2.479007937807988 + ], + "population_cv": 0.4251348773789681 + }, + "bits_per_byte": 1.9581638498120857, + "core_parameter_grad_rms_by_block": [ + 0.00042380566817482924, + 0.00033628063183600165, + 0.000202638705536708, + 0.00018804477448005897, + 0.00023208100123381636, + 0.0003223172253228011, + 0.0004741333361060697, + 0.0004905991387745841, + 0.0005898184863404294, + 0.0005622263144383919, + 0.000639932357888532, + 0.0005200270037733745, + 0.0004834555123403282, + 0.0004217229809514017, + 0.0003667204712485774, + 0.0003029597160326848 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002876924450068995, + "first_to_last_ratio": 0.7307130437944471, + "imbalance_abs_log_ratio": 0.3137344492502099, + "last_quartile_mean": 0.000393714670143248, + "mean": 0.0004097977077799118, + "normalized": [ + 1.0341826226183788, + 0.8206015442541381, + 0.4944847218265511, + 0.4588721969646822, + 0.5663306476044494, + 0.7865276432827353, + 1.156993626622993, + 1.1971739457314583, + 1.4392918143339164, + 1.371960613162688, + 1.561581106335074, + 1.2689846573096573, + 1.1797418657109118, + 1.029100390131744, + 0.894881704525127, + 0.739290899585495 + ], + "population_cv": 0.3269281757976474 + }, + "loss_nats": 1.3572957515716553, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.000266833056230098, + 0.0002936421660706401, + 0.00031041845795698464, + 0.0002957055112347007, + 0.0002881943073589355, + 0.0002774772292468697, + 0.00026020428049378097, + 0.00024136141291819513, + 0.00021430903871078044, + 0.00019666840671561658, + 0.00016092424630187452, + 0.0001381110487272963, + 0.00011530883784871548, + 9.161087655229494e-05, + 6.125834624981508e-05, + 3.126815863652155e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029164979787310585, + "first_to_last_ratio": 3.8958554703706607, + "imbalance_abs_log_ratio": 1.3599132881647322, + "last_quartile_mean": 7.486155482183676e-05, + "mean": 0.00020270596132831997, + "normalized": [ + 1.3163552491577308, + 1.4486113982362465, + 1.531373107741041, + 1.4587904040757373, + 1.4217357273087359, + 1.3688656585557626, + 1.2836538145631138, + 1.1906971622174716, + 1.05724092822148, + 0.9702152093942396, + 0.7938801860949112, + 0.6813368872936094, + 0.5688477886545793, + 0.4519397256596421, + 0.3022029833182646, + 0.15425376950743425 + ], + "population_cv": 0.4374351028998013 + }, + "activation_output_rms_by_block": [ + 0.10593125969171524, + 0.16277404129505157, + 0.16965323686599731, + 0.16991853713989258, + 0.16967523097991943, + 0.1679684966802597, + 0.16474221646785736, + 0.14505790174007416, + 0.1350426822900772, + 0.13761113584041595, + 0.1448632776737213, + 0.15946830809116364, + 0.17768383026123047, + 0.2030501663684845, + 0.25671765208244324, + 0.47381165623664856 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15206926874816418, + "first_to_last_ratio": 0.5473743911850654, + "imbalance_abs_log_ratio": 0.6026222661058785, + "last_quartile_mean": 0.2778158262372017, + "mean": 0.18399810185655951, + "normalized": [ + 0.5757193070083771, + 0.8846506548309193, + 0.9220379729691717, + 0.9234798371580865, + 0.9221575074301264, + 0.9128816818512828, + 0.8953473693782256, + 0.7883662944151337, + 0.7339351924149371, + 0.7478943231039105, + 0.7873085440123357, + 0.8666845281669334, + 0.9656829525325674, + 1.1035448970380004, + 1.3952190239580522, + 2.57508991373194 + ], + "population_cv": 0.4416592993774559 + }, + "bits_per_byte": 1.8703918160136477, + "core_parameter_grad_rms_by_block": [ + 0.000552485054575078, + 0.0004878954173704819, + 0.00026717869748665266, + 0.00024508760509843496, + 0.0003479630309338483, + 0.0004644527429795662, + 0.0007202858637767138, + 0.0008028534351930377, + 0.0009144494502519988, + 0.0008971273720051169, + 0.001000538678877296, + 0.0008212401598758333, + 0.0007501935670996717, + 0.0006492653356624181, + 0.0005392275192896546, + 0.00040293680800761964 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00038816169363266187, + "first_to_last_ratio": 0.6630643028303428, + "imbalance_abs_log_ratio": 0.4108833058215335, + "last_quartile_mean": 0.000585405807514841, + "mean": 0.0006164487961552139, + "normalized": [ + 0.8962383542979121, + 0.7914613840005554, + 0.4334158800423393, + 0.3975798259758869, + 0.5644638015421408, + 0.7534328007068132, + 1.1684439458218279, + 1.3023846265909318, + 1.4834150962016837, + 1.4553153118320499, + 1.6230685908021032, + 1.3322114748181866, + 1.2169600650996852, + 1.0532348180609332, + 0.8747320501764496, + 0.6536419740305005 + ], + "population_cv": 0.37363268494405877 + }, + "loss_nats": 1.2964568138122559, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.032388224749795, + "cross_entropy_nats": 5.567627251148224, + "step": 0 + }, + { + "bits_per_byte": 5.232053968773903, + "cross_entropy_nats": 3.626583456993103, + "step": 100 + }, + { + "bits_per_byte": 3.1492899782815944, + "cross_entropy_nats": 2.1829214692115784, + "step": 500 + }, + { + "bits_per_byte": 1.9917407321684666, + "cross_entropy_nats": 1.3805694729089737, + "step": 2000 + }, + { + "bits_per_byte": 1.8310010171491868, + "cross_entropy_nats": 1.269153192639351, + "step": 4000 + }, + { + "bits_per_byte": 1.7368251448523782, + "cross_entropy_nats": 1.2038754522800446, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "final_optimizer_state": "34afebd4020a97001cb564ade8591d46cc164cbd65eb2e1cad4d8d953e6b7cce", + "final_public_parameters": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "seed": 2026073002, + "timing": { + "mean_ms": 21.682085667085165, + "measured_steps": 7980, + "median_ms": 21.630175004247576, + "p95_ms": 22.00388514320366, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "baseline", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00031658337684348226, + 0.0002727926184888929, + 0.00024126221251208335, + 0.00021358189405873418, + 0.0001949657453224063, + 0.00017900438979268074, + 0.00016492439317516983, + 0.00015548785449936986, + 0.00014800082135479897, + 0.0001436655584257096, + 0.00013764512550551444, + 0.0001334012922598049, + 0.00013005948858335614, + 0.00012698007049039006, + 0.00012402374704834074, + 0.000122041383292526 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002610550254757982, + "first_to_last_ratio": 2.075552313213765, + "imbalance_abs_log_ratio": 0.7302272933071482, + "last_quartile_mean": 0.00012577617235365324, + "mean": 0.00017527624822832877, + "normalized": [ + 1.8061966754963605, + 1.5563581560322512, + 1.3764683746413606, + 1.2185444189820027, + 1.1123340857252286, + 1.021270089941082, + 0.9409397727427746, + 0.887101681323182, + 0.8443860640033859, + 0.8196521769370567, + 0.7853039239304516, + 0.7610916687697798, + 0.7420257444918054, + 0.7244568033255466, + 0.7075901515576575, + 0.6962802121000742 + ], + "population_cv": 0.32194911387722014 + }, + "activation_output_rms_by_block": [ + 0.0284949392080307, + 0.028932223096489906, + 0.029410898685455322, + 0.029886392876505852, + 0.030463071539998055, + 0.031026815995573997, + 0.031805604696273804, + 0.03248070180416107, + 0.03315235301852226, + 0.034095361828804016, + 0.034913208335638046, + 0.03547592833638191, + 0.03669917955994606, + 0.03757266327738762, + 0.038674499839544296, + 0.039735738188028336 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029181113466620445, + "first_to_last_ratio": 0.7644934703880544, + "imbalance_abs_log_ratio": 0.2685417946723145, + "last_quartile_mean": 0.03817052021622658, + "mean": 0.03330122376792133, + "normalized": [ + 0.8556724343409726, + 0.8688036000755015, + 0.8831777141411389, + 0.8974562942427076, + 0.9147733354274736, + 0.9317019762337311, + 0.9550881648653333, + 0.9753606062804618, + 0.9955295712122606, + 1.0238471134399472, + 1.0484061660601651, + 1.065304043587595, + 1.102036964638457, + 1.1282667429651916, + 1.1613537120758601, + 1.193221560413203 + ], + "population_cv": 0.1043395636179365 + }, + "bits_per_byte": 8.022455452740537, + "core_parameter_grad_rms_by_block": [ + 0.007247792155837073, + 0.0070599825250150985, + 0.007115739546591776, + 0.006247289691276946, + 0.006127370040903895, + 0.0065579111453454445, + 0.0059278656112939756, + 0.005972691938247438, + 0.0055663256066952145, + 0.006567633669083912, + 0.005568350992310226, + 0.005243662280059585, + 0.00519845242839581, + 0.005229357864659918, + 0.005505343111599652, + 0.004934663781812996 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006917700979680223, + "first_to_last_ratio": 1.3260037536012093, + "imbalance_abs_log_ratio": 0.2821697225298069, + "last_quartile_mean": 0.005216954296617094, + "mean": 0.00600440202432056, + "normalized": [ + 1.2070797602292813, + 1.1758011033270186, + 1.1850871273725165, + 1.0404515996718042, + 1.0204796441153106, + 1.092183887551653, + 0.9872532830552356, + 0.9947188602720667, + 0.927040791763953, + 1.0938031201911544, + 0.9273781085536696, + 0.8733029965049588, + 0.8657735453655023, + 0.8709206751111334, + 0.9168844939596827, + 0.821841002955059 + ], + "population_cv": 0.119259291012628 + }, + "loss_nats": 5.560742378234863, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.706054748268798e-05, + 6.966097134863958e-05, + 6.121626211097464e-05, + 5.34596256329678e-05, + 4.684355008066632e-05, + 4.049177005072124e-05, + 3.4754324587993324e-05, + 3.020181247848086e-05, + 2.6929572413791902e-05, + 2.4378721718676388e-05, + 2.2718366381013766e-05, + 2.1439738702611066e-05, + 2.0652638340834528e-05, + 2.0034611225128174e-05, + 1.9586463167797774e-05, + 1.9270755728939548e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.53493516438175e-05, + "first_to_last_ratio": 3.2861795625404726, + "imbalance_abs_log_ratio": 1.1897256628392332, + "last_quartile_mean": 1.9886117115675006e-05, + "mean": 3.6793733215745306e-05, + "normalized": [ + 2.0943932769972666, + 1.893283590990143, + 1.663768711699472, + 1.45295464636599, + 1.2731393633256098, + 1.1005072470709065, + 0.9445718482603106, + 0.8208412095991516, + 0.7319064976605258, + 0.6625780965396546, + 0.6174520603223757, + 0.5827008250806217, + 0.5613085853434561, + 0.5445115098175108, + 0.5323314993058669, + 0.5237510316211383 + ], + "population_cv": 0.5070097402008534 + }, + "activation_output_rms_by_block": [ + 0.03020417131483555, + 0.03345569595694542, + 0.03829808160662651, + 0.04440607130527496, + 0.05133094638586044, + 0.059150952845811844, + 0.06815594434738159, + 0.07766561955213547, + 0.08808834105730057, + 0.10039259493350983, + 0.11500746011734009, + 0.132473424077034, + 0.1519528478384018, + 0.17349907755851746, + 0.19798777997493744, + 0.22741958498954773 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03659100504592061, + "first_to_last_ratio": 0.19492869311536967, + "imbalance_abs_log_ratio": 1.6351214636257212, + "last_quartile_mean": 0.1877148225903511, + "mean": 0.09934303711634129, + "normalized": [ + 0.30403913743308697, + 0.33676940959400337, + 0.3855134966507554, + 0.44699731953303096, + 0.5167040174717673, + 0.5954212249071846, + 0.6860666466997954, + 0.7817922806324186, + 0.8867087579992116, + 1.010564986209747, + 1.1576801299385893, + 1.3334948067059142, + 1.5295772330822623, + 1.746464398459366, + 1.9929708787045903, + 2.2892352759782764 + ], + "population_cv": 0.6013462998472742 + }, + "bits_per_byte": 5.091903414776641, + "core_parameter_grad_rms_by_block": [ + 0.00027208547857035954, + 0.0002379277587721014, + 0.0002494129668009364, + 0.00019949675850635897, + 0.0002037090250478814, + 0.0001757519905869641, + 0.00017904243730119992, + 0.00016456038574532694, + 0.0001493779474782176, + 0.00013508785060359837, + 0.0001355896704444294, + 0.00013648423482290936, + 0.00013660889397982336, + 0.00013630230473358484, + 0.0001407085634636149, + 0.00014945333748469326 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023973074066243908, + "first_to_last_ratio": 1.703016825392403, + "imbalance_abs_log_ratio": 0.532401281486357, + "last_quartile_mean": 0.0001407682749154291, + "mean": 0.00017509997527137498, + "normalized": [ + 1.5538864477203589, + 1.358810921608379, + 1.4244032097342618, + 1.1393305921212904, + 1.163386943557058, + 1.0037236745155367, + 1.0225154916425019, + 0.9398081609679643, + 0.8531009056209594, + 0.7714898325612858, + 0.774355737254038, + 0.7794646150656627, + 0.7801765463878734, + 0.7784256081266694, + 0.8035898534282527, + 0.8535314596879079 + ], + "population_cv": 0.2495872477003493 + }, + "loss_nats": 3.5294384956359863, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00015226140385493636, + 0.00015351864567492157, + 0.00014040769019629806, + 0.00012548675294965506, + 0.00010927267430815846, + 9.748537559062243e-05, + 8.759690535953268e-05, + 7.801534957252443e-05, + 7.03658297425136e-05, + 6.262787064770237e-05, + 5.5923766922205687e-05, + 4.834980791201815e-05, + 4.202092532068491e-05, + 3.7704347050748765e-05, + 3.55200027115643e-05, + 3.515387652441859e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014291862316895276, + "first_to_last_ratio": 3.801048653306501, + "imbalance_abs_log_ratio": 1.3352769900584454, + "last_quartile_mean": 3.759978790185414e-05, + "mean": 8.323195152115659e-05, + "normalized": [ + 1.8293624151805847, + 1.8444676938266782, + 1.686944588348479, + 1.5076752455787101, + 1.3128693045288338, + 1.1712494277614383, + 1.0524432475581997, + 0.9373245267797649, + 0.845418477597842, + 0.752449864542502, + 0.671902627538302, + 0.5809044126488876, + 0.504865313773205, + 0.4530032801305249, + 0.4267592200158317, + 0.4223603541902161 + ], + "population_cv": 0.48987170747402153 + }, + "activation_output_rms_by_block": [ + 0.05162646621465683, + 0.06355457007884979, + 0.07474122941493988, + 0.08428152650594711, + 0.08889172971248627, + 0.09293187409639359, + 0.0973934531211853, + 0.09854210913181305, + 0.10090499371290207, + 0.10472240298986435, + 0.11040307581424713, + 0.1186341941356659, + 0.130356103181839, + 0.149788036942482, + 0.17963695526123047, + 0.2188950628042221 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0685509480535984, + "first_to_last_ratio": 0.40402744210990815, + "imbalance_abs_log_ratio": 0.9062724773132301, + "last_quartile_mean": 0.1696690395474434, + "mean": 0.1103314864449203, + "normalized": [ + 0.4679214236856113, + 0.576032936079199, + 0.6774242949427878, + 0.7638936918340364, + 0.8056787103730612, + 0.8422969461468014, + 0.8827348951725225, + 0.8931458489957648, + 0.9145620798218456, + 0.9491615346100125, + 1.0006488571090046, + 1.0752523867689434, + 1.1814950326706182, + 1.3576182263914227, + 1.6281567578708265, + 1.9839763775275423 + ], + "population_cv": 0.3749688391491054 + }, + "bits_per_byte": 3.129900568231996, + "core_parameter_grad_rms_by_block": [ + 0.0009002454290164998, + 0.0005806619682142476, + 0.0004904299887450306, + 0.00047545530221616405, + 0.00040490888378542537, + 0.0003357588676961225, + 0.0003168596431849841, + 0.00029935138955839515, + 0.00029222591787988014, + 0.00027100955094896284, + 0.0002328601817845373, + 0.0002299739939536432, + 0.0002128774552133389, + 0.00018778710115789697, + 0.00017922956471394958, + 0.0001849955243427181 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006116981720479855, + "first_to_last_ratio": 3.1988832674328185, + "imbalance_abs_log_ratio": 1.1628017699711202, + "last_quartile_mean": 0.0001912224113569759, + "mean": 0.0003496644226507373, + "normalized": [ + 2.5745983025436674, + 1.6606263909046375, + 1.4025733159444054, + 1.359747436161308, + 1.157992799827591, + 0.9602317134548619, + 0.9061821067837941, + 0.8561105167322174, + 0.8357324879224854, + 0.7750561206498867, + 0.6659533160945287, + 0.6576991511182508, + 0.6088050182502319, + 0.537049493724068, + 0.512575924525708, + 0.5290659053623568 + ], + "population_cv": 0.5241885524055848 + }, + "loss_nats": 2.1694817543029785, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.0001594285131432116, + 0.00017026996647473425, + 0.00018808410095516592, + 0.00017561657296027988, + 0.0001699679996818304, + 0.00016689945186953992, + 0.00015736353816464543, + 0.00014705183275509626, + 0.0001390052493661642, + 0.00012381834676489234, + 0.000104674712929409, + 9.053898247657344e-05, + 7.686029857723042e-05, + 6.236408080440015e-05, + 4.532423918135464e-05, + 3.028271021321416e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00017334978838334791, + "first_to_last_ratio": 3.227644485017083, + "imbalance_abs_log_ratio": 1.171752609589353, + "last_quartile_mean": 5.370783219404984e-05, + "mean": 0.00012547191226985888, + "normalized": [ + 1.2706310939162266, + 1.3570365143437513, + 1.499013584415959, + 1.3996484933024078, + 1.3546298658162754, + 1.3301738122120965, + 1.2541734266884814, + 1.1719900501621776, + 1.1078594950174865, + 0.9868212297473388, + 0.834248167863097, + 0.7215876512812414, + 0.6125697551490491, + 0.4970361866349061, + 0.3612301618857412, + 0.24135051156376403 + ], + "population_cv": 0.3906610370884368 + }, + "activation_output_rms_by_block": [ + 0.08273108303546906, + 0.13354359567165375, + 0.1704990118741989, + 0.17198410630226135, + 0.17284893989562988, + 0.17146942019462585, + 0.16773979365825653, + 0.15899653732776642, + 0.14656992256641388, + 0.13979844748973846, + 0.1417049616575241, + 0.149912029504776, + 0.16512760519981384, + 0.19090111553668976, + 0.2512649893760681, + 0.390801340341568 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.13968944922089577, + "first_to_last_ratio": 0.5598242338035286, + "imbalance_abs_log_ratio": 0.5801324127278619, + "last_quartile_mean": 0.24952376261353493, + "mean": 0.17536830622702837, + "normalized": [ + 0.4717561845432151, + 0.7615035951751215, + 0.9722338975748231, + 0.9807023287298795, + 0.9856338560507224, + 0.9777674420407808, + 0.9565000499069877, + 0.906643513577263, + 0.8357834190213785, + 0.7971705406606262, + 0.8080420271270434, + 0.8548410641014165, + 0.9416046077678534, + 1.0885725000363118, + 1.4327844910059484, + 2.228460482680629 + ], + "population_cv": 0.3692959780115816 + }, + "bits_per_byte": 2.102960171157105, + "core_parameter_grad_rms_by_block": [ + 0.0004582000765955337, + 0.0003576182203712263, + 0.0003389913908322493, + 0.00022415790052185902, + 0.00024709072048410314, + 0.00027475536769928975, + 0.00033936291802973487, + 0.00039410580789665003, + 0.0004814106753566502, + 0.0005191302840338224, + 0.0005153457080868455, + 0.0004830357094583275, + 0.00041821493179685683, + 0.0003600919520715316, + 0.0003136989944772832, + 0.0002645229655850915 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034474189708021705, + "first_to_last_ratio": 1.0165412954471984, + "imbalance_abs_log_ratio": 0.016405978393597878, + "last_quartile_mean": 0.00033913221098269076, + "mean": 0.00037435835145606595, + "normalized": [ + 1.2239611452859689, + 0.9552831370804766, + 0.9055264548359695, + 0.5987789497683099, + 0.6600379543371862, + 0.7339367924627015, + 0.9065188921518208, + 1.052750142647483, + 1.2859621629494962, + 1.3867201893978494, + 1.3766106888824827, + 1.2903030146904997, + 1.1171513341967951, + 0.9618910615215468, + 0.8379644617440793, + 0.7066036180473336 + ], + "population_cv": 0.2506116126131003 + }, + "loss_nats": 1.4576609134674072, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.00016188873269129544, + 0.00017867473070509732, + 0.00019946519751101732, + 0.00018539158918429166, + 0.00018016427929978818, + 0.0001743910543154925, + 0.00016342199523933232, + 0.00015197545872069895, + 0.0001410306867910549, + 0.00012255505134817213, + 0.00010388438386144117, + 8.946077286964282e-05, + 7.552531315013766e-05, + 6.115865835454315e-05, + 4.3820087739732116e-05, + 2.6790259653353132e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018135506252292544, + "first_to_last_ratio": 3.499469999703496, + "imbalance_abs_log_ratio": 1.2526115283727475, + "last_quartile_mean": 5.1823579724441515e-05, + "mean": 0.00012872489071469317, + "normalized": [ + 1.257633483256217, + 1.3880355983452597, + 1.5495464506014887, + 1.4402155492615254, + 1.3996071645467982, + 1.3547578354679994, + 1.269544641537448, + 1.1806221615486823, + 1.0955976424453635, + 0.9520695699777604, + 0.8070263900374272, + 0.6949764911273015, + 0.58671879797927, + 0.475111363583098, + 0.3404165852962757, + 0.208120274988085 + ], + "population_cv": 0.4148571258695616 + }, + "activation_output_rms_by_block": [ + 0.09483804553747177, + 0.15792693197727203, + 0.18785393238067627, + 0.19029200077056885, + 0.19116424024105072, + 0.18977893888950348, + 0.18693965673446655, + 0.177143394947052, + 0.16181440651416779, + 0.1581212431192398, + 0.165044903755188, + 0.18009524047374725, + 0.1996685266494751, + 0.2291477918624878, + 0.29672855138778687, + 0.4953230619430542 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15772772766649723, + "first_to_last_ratio": 0.516772448690399, + "imbalance_abs_log_ratio": 0.6601526392924331, + "last_quartile_mean": 0.305216982960701, + "mean": 0.20386755419895053, + "normalized": [ + 0.46519440481893015, + 0.774654567264559, + 0.9214508562620668, + 0.9334099362611985, + 0.9376883977059789, + 0.9308932931245242, + 0.9169662012623924, + 0.8689141126114707, + 0.7937231951890498, + 0.7756076916728606, + 0.8095692539388771, + 0.8833933441806815, + 0.9794031592424085, + 1.1240032420208796, + 1.4554966951642292, + 2.429631649279893 + ], + "population_cv": 0.4175842970930077 + }, + "bits_per_byte": 1.9507894057257391, + "core_parameter_grad_rms_by_block": [ + 0.00041699002778082224, + 0.00029759414151732746, + 0.000265991514011251, + 0.0001953867836934275, + 0.00023560860135608575, + 0.00025046058190266666, + 0.00033611798182792255, + 0.0004636386291792892, + 0.0005491632897177901, + 0.0005951115480573408, + 0.0005882338016339408, + 0.0005513321353847482, + 0.0004945066783331371, + 0.00043544338635105514, + 0.0003756714670850321, + 0.0003149065782712711 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002939906167507071, + "first_to_last_ratio": 0.7256661946909654, + "imbalance_abs_log_ratio": 0.3206651568188065, + "last_quartile_mean": 0.00040513202751012384, + "mean": 0.00039788482163144424, + "normalized": [ + 1.0480169262829406, + 0.7479404222988562, + 0.6685138501152368, + 0.4910636774036378, + 0.5921527752428994, + 0.6294801115451073, + 0.8447620103972311, + 1.165258396332474, + 1.380206682592364, + 1.4956879873356546, + 1.4784022150480887, + 1.3856576210274245, + 1.2428387599846482, + 1.094395570470896, + 0.9441713949898092, + 0.7914515989327312 + ], + "population_cv": 0.32327688011764555 + }, + "loss_nats": 1.3521841764450073, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.00025817437563091516, + 0.00027957011479884386, + 0.0003037556598428637, + 0.0002888414019253105, + 0.00028409145306795835, + 0.00028240776737220585, + 0.00026678573340177536, + 0.0002519928675610572, + 0.00023668054200243205, + 0.00020713888807222247, + 0.00017311860574409366, + 0.0001455697201890871, + 0.00011939310206798837, + 9.376799425808713e-05, + 6.36211916571483e-05, + 3.1806663173483685e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002825853880494833, + "first_to_last_ratio": 3.662935915109688, + "imbalance_abs_log_ratio": 1.2982649884510844, + "last_quartile_mean": 7.714723778917687e-05, + "mean": 0.00020541975504784205, + "normalized": [ + 1.2568137644346165, + 1.3609699550743417, + 1.478707146603886, + 1.4061033314835745, + 1.3829801958521164, + 1.3747838775605266, + 1.2987345513075956, + 1.226721682646191, + 1.1521800420183999, + 1.0083688787574498, + 0.8427553898298366, + 0.7086451843698482, + 0.5812152878878755, + 0.4564701882554999, + 0.30971311226776116, + 0.15483741165048096 + ], + "population_cv": 0.41926708997335715 + }, + "activation_output_rms_by_block": [ + 0.0951872169971466, + 0.14984266459941864, + 0.17071209847927094, + 0.17147330939769745, + 0.17152385413646698, + 0.17004960775375366, + 0.1669105589389801, + 0.15593038499355316, + 0.1393769234418869, + 0.1344291716814041, + 0.13867342472076416, + 0.1521374136209488, + 0.16997921466827393, + 0.1961352825164795, + 0.25573307275772095, + 0.4589640200138092 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1468038223683834, + "first_to_last_ratio": 0.5433095785892573, + "imbalance_abs_log_ratio": 0.6100759951930727, + "last_quartile_mean": 0.2702028974890709, + "mean": 0.18106613866984844, + "normalized": [ + 0.5257041305260761, + 0.8275576300472066, + 0.9428162534052995, + 0.9470203024009789, + 0.94729945309774, + 0.9391574206142317, + 0.9218209443529402, + 0.861179158836942, + 0.7697569764605372, + 0.7424313163629063, + 0.7658716629154935, + 0.8402311704363035, + 0.9387686505990491, + 1.0832245275529278, + 1.4123738134385158, + 2.534786588952852 + ], + "population_cv": 0.4350736575110673 + }, + "bits_per_byte": 1.8670605120664352, + "core_parameter_grad_rms_by_block": [ + 0.0005002075550754018, + 0.0003412904728571966, + 0.0002592425753327288, + 0.0002585649119364213, + 0.00030071075691208866, + 0.0003425364164715924, + 0.0005240790570344413, + 0.0006975962934822548, + 0.0008810870404936847, + 0.000990036099264455, + 0.0009754970927650687, + 0.0008971247584696028, + 0.0007874986100445877, + 0.000666994667011454, + 0.0005527789362036235, + 0.0004234912235697502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00033982637880043714, + "first_to_last_ratio": 0.5592092980363276, + "imbalance_abs_log_ratio": 0.5812314608110569, + "last_quartile_mean": 0.0006076908592073538, + "mean": 0.0005874210291827721, + "normalized": [ + 0.8515315765445054, + 0.580998050635001, + 0.44132327998776366, + 0.4401696553086297, + 0.5119169079296352, + 0.5831190908301898, + 0.8921693827739654, + 1.1875575759566523, + 1.4999242395517653, + 1.6853943765714456, + 1.6606438045335101, + 1.5272261527948612, + 1.3406033678095697, + 1.135462698602033, + 0.9410268082718403, + 0.7209330318986312 + ], + "population_cv": 0.42806100897327076 + }, + "loss_nats": 1.2941477298736572, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.018429854831792, + "cross_entropy_nats": 5.557952046394348, + "step": 0 + }, + { + "bits_per_byte": 5.108882101013433, + "cross_entropy_nats": 3.5412072241306305, + "step": 100 + }, + { + "bits_per_byte": 3.154257052216826, + "cross_entropy_nats": 2.186364382505417, + "step": 500 + }, + { + "bits_per_byte": 1.977161870343983, + "cross_entropy_nats": 1.37046417593956, + "step": 2000 + }, + { + "bits_per_byte": 1.8329381437371801, + "cross_entropy_nats": 1.2704959064722061, + "step": 4000 + }, + { + "bits_per_byte": 1.7363699927668699, + "cross_entropy_nats": 1.2035599648952484, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "final_optimizer_state": "de12f865acf312961efff64f13ff04f15f5b52fc60a03097833a902e65a3e5f3", + "final_public_parameters": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "seed": 2026073003, + "timing": { + "mean_ms": 21.482708441361407, + "measured_steps": 7980, + "median_ms": 21.420384509838186, + "p95_ms": 21.788083588762674, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027713447343558073, + 0.00024001447309274226, + 0.00020889485313091427, + 0.00019106797117274255, + 0.00017709653184283525, + 0.00016345064796041697, + 0.0001520135992905125, + 0.00014396758342627436, + 0.00013824697816744447, + 0.0001333381951553747, + 0.00012879272981081158, + 0.00012578140012919903, + 0.00012338913802523166, + 0.0001212825154652819, + 0.00011928759340662509, + 0.00011723516217898577 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022927794270799495, + "first_to_last_ratio": 1.9059069547229377, + "imbalance_abs_log_ratio": 0.6449579870010378, + "last_quartile_mean": 0.00012029860226903111, + "mean": 0.00016006211535568582, + "normalized": [ + 1.7314182860806244, + 1.4995083162520277, + 1.3050861702453052, + 1.1937113960299495, + 1.1064237870985023, + 1.021170110098828, + 0.9497162957812465, + 0.8994482117542517, + 0.8637083038683806, + 0.8330403159990363, + 0.8046421823465957, + 0.7858286756343993, + 0.7708828397715447, + 0.7577215582573746, + 0.7452581339534802, + 0.7324354168284537 + ], + "population_cv": 0.28697332073733334 + }, + "activation_output_rms_by_block": [ + 0.005004490725696087, + 0.007150472607463598, + 0.0056586493737995625, + 0.007886149920523167, + 0.005775847006589174, + 0.008285986259579659, + 0.006597128231078386, + 0.009296777658164501, + 0.007501539774239063, + 0.01081777922809124, + 0.007698386441916227, + 0.011321330443024635, + 0.009009703993797302, + 0.013792803511023521, + 0.00953624676913023, + 0.012954792007803917 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0064249406568706036, + "first_to_last_ratio": 0.567404514268178, + "imbalance_abs_log_ratio": 0.5666828005795043, + "last_quartile_mean": 0.011323386570438743, + "mean": 0.008643005246995017, + "normalized": [ + 0.5790220626599802, + 0.8273132322752738, + 0.6547085431617609, + 0.9124314628022479, + 0.6682683674867963, + 0.9586927258271091, + 0.7632910130850682, + 1.0756417926966766, + 0.8679318778438997, + 1.251622429808468, + 0.8907071350665657, + 1.3098835554867694, + 1.04242722714183, + 1.5958342170183197, + 1.103348487778111, + 1.4988758698611242 + ], + "population_cv": 0.28641618387108014 + }, + "bits_per_byte": 8.092078157393424, + "core_parameter_grad_rms_by_block": [ + 0.007836087849617155, + 0.00707663314616961, + 0.006860545956961424, + 0.006477463491545069, + 0.00630808320659222, + 0.0065403273634858875, + 0.006076989561376891, + 0.0053044433268254945, + 0.006002429844215364, + 0.005811298045810947, + 0.00541486468124986, + 0.005552686577658202, + 0.005724443119797102, + 0.005942467552368739, + 0.006312250195815329, + 0.004848514440357715 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007062682611073314, + "first_to_last_ratio": 1.237564932158175, + "imbalance_abs_log_ratio": 0.21314568451503188, + "last_quartile_mean": 0.005706918827084721, + "mean": 0.006130595522490438, + "normalized": [ + 1.2781935818257821, + 1.154314147819503, + 1.119066807097797, + 1.0565798163950184, + 1.0289511326347756, + 1.0668339379905794, + 0.9912559944760846, + 0.865241118479574, + 0.9790940899942118, + 0.947917380047643, + 0.883252640201938, + 0.9057336366895933, + 0.9337499267072272, + 0.9693132633801169, + 1.0296308364592772, + 0.7908716898008789 + ], + "population_cv": 0.1166547247084935 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.609001159667969, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 8.295638690469787e-05, + 7.495813042623922e-05, + 6.424964522011578e-05, + 5.6337856221944094e-05, + 4.736588743980974e-05, + 4.000150511274114e-05, + 3.3406991860829294e-05, + 2.9257240385049954e-05, + 2.6081905161845498e-05, + 2.4221646526711993e-05, + 2.3017932107904926e-05, + 2.205361488449853e-05, + 2.1097952412674204e-05, + 2.0617559130187146e-05, + 2.0049723389092833e-05, + 1.9771381630562246e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.962550469324924e-05, + "first_to_last_ratio": 3.415668082811133, + "imbalance_abs_log_ratio": 1.2283731057520397, + "last_quartile_mean": 2.0384154140629107e-05, + "mean": 3.784033492593153e-05, + "normalized": [ + 2.1922741187961536, + 1.980905575306399, + 1.6979142850050797, + 1.4888308026929338, + 1.2517301322127163, + 1.0571128715176512, + 0.8828408079954885, + 0.7731760419752606, + 0.6892620060815551, + 0.640101272203941, + 0.6082909190143296, + 0.5828070741885916, + 0.557551946989138, + 0.5448566766267753, + 0.5298505795030106, + 0.5224948898909759 + ], + "population_cv": 0.5389270020989329 + }, + "activation_output_rms_by_block": [ + 0.00972363818436861, + 0.018413539975881577, + 0.011301152408123016, + 0.020593363791704178, + 0.012286243960261345, + 0.022828614339232445, + 0.014672646299004555, + 0.02904798462986946, + 0.01802690140902996, + 0.03463379293680191, + 0.0215163491666317, + 0.04299422353506088, + 0.024622667580842972, + 0.048123423010110855, + 0.032037004828453064, + 0.06016550958156586 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.015007923590019345, + "first_to_last_ratio": 0.36394181302547757, + "imbalance_abs_log_ratio": 1.010761278449072, + "last_quartile_mean": 0.04123715125024319, + "mean": 0.0263116909773089, + "normalized": [ + 0.3695558066851058, + 0.6998235116002747, + 0.4295106847320868, + 0.7826697193070493, + 0.4669499946186262, + 0.8676224708978132, + 0.5576474089657784, + 1.1039953553316026, + 0.6851289574879962, + 1.3162891342358027, + 0.8177486268437599, + 1.6340349836176982, + 0.9358071133503911, + 1.8289749241739084, + 1.2175958153385753, + 2.2866454928135314 + ], + "population_cv": 0.5242218345998277 + }, + "bits_per_byte": 5.094476963164152, + "core_parameter_grad_rms_by_block": [ + 0.00047768131303286524, + 0.00037801938737084786, + 0.0003757481810571057, + 0.0003849183374966076, + 0.0003205908784645981, + 0.0003135875371373823, + 0.0002702256708410831, + 0.00025052984822069896, + 0.00022846424191684566, + 0.00020301436013832287, + 0.00018877625003321395, + 0.00018901092530252878, + 0.00017529332745812984, + 0.0001779656532508539, + 0.00018745944157007413, + 0.00017444725355961893 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004040918047393566, + "first_to_last_ratio": 2.260129748343848, + "imbalance_abs_log_ratio": 0.8154222224078733, + "last_quartile_mean": 0.0001787914189596692, + "mean": 0.00026848328792817357, + "normalized": [ + 1.779184532188304, + 1.407981071328276, + 1.39952167584311, + 1.4336770845848084, + 1.1940813185749002, + 1.167996487071014, + 1.0064897257715932, + 0.9331301406280542, + 0.8509439961043904, + 0.7561526890740202, + 0.703121045223927, + 0.7039951229779867, + 0.6529021929477618, + 0.6628556087202881, + 0.6982164253747687, + 0.649750883586796 + ], + "population_cv": 0.3443291255737099 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6925109028816223, + "mean_weights": [ + 0.516845703125, + 0.4831542670726776 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6916897296905518, + "mean_weights": [ + 0.5256155133247375, + 0.47438448667526245 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6921632289886475, + "mean_weights": [ + 0.5142087936401367, + 0.4857912063598633 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927341222763062, + "mean_weights": [ + 0.5126981735229492, + 0.4873018264770508 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0980398654937744, + "mean_weights": [ + 0.3447725176811218, + 0.324658066034317, + 0.33056944608688354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098333477973938, + "mean_weights": [ + 0.3432988226413727, + 0.3276415765285492, + 0.3290596008300781 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0971107482910156, + "mean_weights": [ + 0.354239284992218, + 0.3338475227355957, + 0.3119131922721863 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098445177078247, + "mean_weights": [ + 0.33804112672805786, + 0.3361448645591736, + 0.3258139491081238 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3858705759048462, + "mean_weights": [ + 0.25839337706565857, + 0.24941343069076538, + 0.24149224162101746, + 0.2507009506225586 + ], + "sources": 4 + }, + { + "entropy_mean": 1.38621187210083, + "mean_weights": [ + 0.2517760992050171, + 0.2535240054130554, + 0.24683426320552826, + 0.24786563217639923 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385884165763855, + "mean_weights": [ + 0.2573123574256897, + 0.2553749680519104, + 0.24531587958335876, + 0.24199676513671875 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861134052276611, + "mean_weights": [ + 0.2514210641384125, + 0.2551700472831726, + 0.24895039200782776, + 0.24445852637290955 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6088017225265503, + "mean_weights": [ + 0.20787756145000458, + 0.20764237642288208, + 0.1969079077243805, + 0.1933688223361969, + 0.19420333206653595 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093868017196655, + "mean_weights": [ + 0.1996297687292099, + 0.200723797082901, + 0.20193718373775482, + 0.19867070019245148, + 0.199038565158844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.607616901397705, + "mean_weights": [ + 0.20617185533046722, + 0.2148669958114624, + 0.20431196689605713, + 0.19360044598579407, + 0.18104872107505798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094070672988892, + "mean_weights": [ + 0.1992160826921463, + 0.20051369071006775, + 0.198721244931221, + 0.19902712106704712, + 0.20252184569835663 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7898081541061401, + "mean_weights": [ + 0.17206060886383057, + 0.18212765455245972, + 0.17207278311252594, + 0.1640574187040329, + 0.15447911620140076, + 0.15520241856575012 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917404174804688, + "mean_weights": [ + 0.16550754010677338, + 0.16615179181098938, + 0.16667258739471436, + 0.16704785823822021, + 0.16769331693649292, + 0.16692689061164856 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7875205278396606, + "mean_weights": [ + 0.1741587072610855, + 0.18743160367012024, + 0.17662332952022552, + 0.16453337669372559, + 0.15226861834526062, + 0.14498436450958252 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916977405548096, + "mean_weights": [ + 0.1655852198600769, + 0.16538481414318085, + 0.16465555131435394, + 0.16688036918640137, + 0.16788282990455627, + 0.16961121559143066 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941636562347412, + "mean_weights": [ + 0.15022630989551544, + 0.1620834320783615, + 0.15417584776878357, + 0.14502081274986267, + 0.1326843798160553, + 0.12827157974243164, + 0.12753766775131226 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458396434783936, + "mean_weights": [ + 0.14176656305789948, + 0.14105038344860077, + 0.14134296774864197, + 0.14280946552753448, + 0.145027756690979, + 0.14322641491889954, + 0.14477643370628357 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395363330841064, + "mean_weights": [ + 0.1491325944662094, + 0.16522708535194397, + 0.15644297003746033, + 0.1476437896490097, + 0.13585588335990906, + 0.12666445970535278, + 0.11903323233127594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456722736358643, + "mean_weights": [ + 0.14162756502628326, + 0.1389600932598114, + 0.139722540974617, + 0.1427886188030243, + 0.14271582663059235, + 0.14634007215499878, + 0.14784526824951172 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0722482204437256, + "mean_weights": [ + 0.13311916589736938, + 0.1474711298942566, + 0.1391790360212326, + 0.13093729317188263, + 0.12214824557304382, + 0.11368396133184433, + 0.1066313236951828, + 0.10682985186576843 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079228401184082, + "mean_weights": [ + 0.12360984086990356, + 0.12237860262393951, + 0.12229418754577637, + 0.12343960255384445, + 0.1245395839214325, + 0.12723875045776367, + 0.12765389680862427, + 0.12884552776813507 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0708298683166504, + "mean_weights": [ + 0.13347145915031433, + 0.14857617020606995, + 0.14165782928466797, + 0.1326693594455719, + 0.12101326882839203, + 0.11364143341779709, + 0.10599445551633835, + 0.10297602415084839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079227924346924, + "mean_weights": [ + 0.12400860339403152, + 0.12165012210607529, + 0.12233424186706543, + 0.12391965091228485, + 0.12502582371234894, + 0.12579283118247986, + 0.12804390490055084, + 0.12922482192516327 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1904914379119873, + "mean_weights": [ + 0.1202651709318161, + 0.13243305683135986, + 0.1248447597026825, + 0.11750722676515579, + 0.10865370184183121, + 0.1034422218799591, + 0.09818274527788162, + 0.09678354859352112, + 0.09788758307695389 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969027519226074, + "mean_weights": [ + 0.10986955463886261, + 0.108255535364151, + 0.10776711255311966, + 0.10903829336166382, + 0.11005579680204391, + 0.11137478053569794, + 0.11346087604761124, + 0.11412093043327332, + 0.1160571351647377 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1876320838928223, + "mean_weights": [ + 0.12014202773571014, + 0.1339646577835083, + 0.12840712070465088, + 0.12060266733169556, + 0.11135218292474747, + 0.10444460809230804, + 0.09650979936122894, + 0.09281164407730103, + 0.09176531434059143 + ], + "sources": 9 + } + ], + "loss_nats": 3.531222343444824, + "output_weights": { + "entropy_mean": 2.1935343742370605, + "mean_weights": [ + 0.0999753400683403, + 0.09610259532928467, + 0.10262835025787354, + 0.1083034798502922, + 0.11600279062986374, + 0.11817552894353867, + 0.12095586210489273, + 0.11944535374641418, + 0.11841070652008057 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00015777233056724072, + 0.00015884912863839418, + 0.00014271712279878557, + 0.0001244439627043903, + 0.00010494918387848884, + 9.392325591761619e-05, + 8.538185647921637e-05, + 7.606096187373623e-05, + 7.664377335458994e-05, + 6.531321560032666e-05, + 6.149107503006235e-05, + 5.037344453739934e-05, + 4.3381573050282896e-05, + 3.9420847315341234e-05, + 3.5541354009183124e-05, + 3.5066059353994206e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001459456361772027, + "first_to_last_ratio": 3.805378902507801, + "imbalance_abs_log_ratio": 1.3364155665156117, + "last_quartile_mean": 3.8352458432200365e-05, + "mean": 8.445807156931551e-05, + "normalized": [ + 1.8680550909542795, + 1.8808045896243943, + 1.6897985017531016, + 1.4734407309105797, + 1.2426187566022755, + 1.1120696242813515, + 1.0109377930698158, + 0.9005765874172524, + 0.907477188745515, + 0.7733211804004252, + 0.7280662923921495, + 0.5964313842526867, + 0.513646265468887, + 0.46675050214695213, + 0.42081654658683526, + 0.4151889653934991 + ], + "population_cv": 0.4877143313799778 + }, + "activation_output_rms_by_block": [ + 0.025262881070375443, + 0.04275903478264809, + 0.027615655213594437, + 0.03926205262541771, + 0.023017436265945435, + 0.03517371043562889, + 0.029550904408097267, + 0.04735429957509041, + 0.028369002044200897, + 0.04305245727300644, + 0.030713967978954315, + 0.05133935809135437, + 0.032636724412441254, + 0.06983363628387451, + 0.0380333848297596, + 0.08374716341495514 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03372490592300892, + "first_to_last_ratio": 0.601556641750376, + "imbalance_abs_log_ratio": 0.508234580502337, + "last_quartile_mean": 0.056062727235257626, + "mean": 0.04048260429408401, + "normalized": [ + 0.6240428823910242, + 1.056232313317272, + 0.6821610342304507, + 0.9698499716125064, + 0.5685759764548824, + 0.868859873246076, + 0.7299654981044719, + 1.1697443976451536, + 0.7007701836106094, + 1.0634804263148154, + 0.7586954573335775, + 1.2681831860026092, + 0.8061913254234652, + 1.7250282560027828, + 0.939499458914941, + 2.0687197593953623 + ], + "population_cv": 0.3940384688106746 + }, + "bits_per_byte": 3.0824777281177833, + "core_parameter_grad_rms_by_block": [ + 0.0006504433508330581, + 0.00048523602227392907, + 0.0003980638932136608, + 0.0004387300445530515, + 0.000365155189968218, + 0.0002788779263831531, + 0.00029885208568229324, + 0.00026779671325771547, + 0.0002902285411340871, + 0.00023612702925164195, + 0.00026277969610552434, + 0.0002548116213942568, + 0.0001837802586554031, + 0.00016271499692659642, + 0.00013374164438063855, + 0.00014095740133328614 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004931183277184248, + "first_to_last_ratio": 3.1752920249892207, + "imbalance_abs_log_ratio": 1.1553996044896453, + "last_quartile_mean": 0.00015529857532398106, + "mean": 0.0003030185259591571, + "normalized": [ + 2.1465464818501867, + 1.6013411085609086, + 1.3136619022010376, + 1.4478654173513685, + 1.20505896071, + 0.9203329251913204, + 0.9862502127100129, + 0.8837635006310174, + 0.9577914096684838, + 0.7792494815431475, + 0.8672067005597657, + 0.8409110320489186, + 0.6064984247206531, + 0.536980359242221, + 0.4413645797968972, + 0.46517750321406204 + ], + "population_cv": 0.440031122144889 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6679806113243103, + "mean_weights": [ + 0.4191037714481354, + 0.580896258354187 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6774038076400757, + "mean_weights": [ + 0.4775225520133972, + 0.522477388381958 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686615526676178, + "mean_weights": [ + 0.49171629548072815, + 0.5082836747169495 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6786561012268066, + "mean_weights": [ + 0.48390668630599976, + 0.516093373298645 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0939385890960693, + "mean_weights": [ + 0.32296979427337646, + 0.3461979627609253, + 0.33083218336105347 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0876491069793701, + "mean_weights": [ + 0.3022151589393616, + 0.3410899341106415, + 0.35669487714767456 + ], + "sources": 3 + }, + { + "entropy_mean": 1.094268560409546, + "mean_weights": [ + 0.3278208076953888, + 0.3347434997558594, + 0.33743569254875183 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0785255432128906, + "mean_weights": [ + 0.30184268951416016, + 0.3319981098175049, + 0.3661591410636902 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3751009702682495, + "mean_weights": [ + 0.2335343211889267, + 0.2411448359489441, + 0.23957431316375732, + 0.2857465147972107 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3803684711456299, + "mean_weights": [ + 0.23392163217067719, + 0.24959179759025574, + 0.25211477279663086, + 0.264371782541275 + ], + "sources": 4 + }, + { + "entropy_mean": 1.377173662185669, + "mean_weights": [ + 0.2529916763305664, + 0.22492024302482605, + 0.24044713377952576, + 0.2816409468650818 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786447048187256, + "mean_weights": [ + 0.23216357827186584, + 0.24167919158935547, + 0.2545858919620514, + 0.2715713381767273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6062697172164917, + "mean_weights": [ + 0.20799711346626282, + 0.1897224485874176, + 0.18646025657653809, + 0.19469907879829407, + 0.22112110257148743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6025722026824951, + "mean_weights": [ + 0.20026245713233948, + 0.18544277548789978, + 0.1946466565132141, + 0.19751906394958496, + 0.22212904691696167 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6060729026794434, + "mean_weights": [ + 0.20172038674354553, + 0.20138651132583618, + 0.18545961380004883, + 0.19016142189502716, + 0.2212720513343811 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601726770401001, + "mean_weights": [ + 0.20073023438453674, + 0.1886606216430664, + 0.19257165491580963, + 0.19004762172698975, + 0.22798986732959747 + ], + "sources": 5 + }, + { + "entropy_mean": 1.788693904876709, + "mean_weights": [ + 0.16298983991146088, + 0.17133885622024536, + 0.15878048539161682, + 0.15357322990894318, + 0.17383432388305664, + 0.17948326468467712 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7880157232284546, + "mean_weights": [ + 0.16720382869243622, + 0.15951547026634216, + 0.1605137139558792, + 0.15542668104171753, + 0.17394030094146729, + 0.1834000051021576 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7797719240188599, + "mean_weights": [ + 0.15782463550567627, + 0.17797376215457916, + 0.15211433172225952, + 0.14025422930717468, + 0.16258476674556732, + 0.20924828946590424 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7884608507156372, + "mean_weights": [ + 0.16717270016670227, + 0.17295360565185547, + 0.1665436029434204, + 0.1547294557094574, + 0.1560850888490677, + 0.18251556158065796 + ], + "sources": 6 + }, + { + "entropy_mean": 1.927689790725708, + "mean_weights": [ + 0.1226624995470047, + 0.15493766963481903, + 0.12924720346927643, + 0.12375974655151367, + 0.1207568496465683, + 0.16289259493350983, + 0.18574345111846924 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9359079599380493, + "mean_weights": [ + 0.1374971866607666, + 0.15181128680706024, + 0.14001096785068512, + 0.13405972719192505, + 0.11918757855892181, + 0.14624495804309845, + 0.17118829488754272 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9246108531951904, + "mean_weights": [ + 0.12061166763305664, + 0.16360490024089813, + 0.1358117014169693, + 0.1262153834104538, + 0.1155187338590622, + 0.15234649181365967, + 0.18589113652706146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937231183052063, + "mean_weights": [ + 0.1359395533800125, + 0.16017206013202667, + 0.1499013900756836, + 0.1417488306760788, + 0.12114370614290237, + 0.13951142132282257, + 0.15158303081989288 + ], + "sources": 7 + }, + { + "entropy_mean": 2.052294969558716, + "mean_weights": [ + 0.0988478884100914, + 0.15235479176044464, + 0.12090037763118744, + 0.11009317636489868, + 0.09369668364524841, + 0.12056069076061249, + 0.13413047790527344, + 0.1694159209728241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.061406135559082, + "mean_weights": [ + 0.1149308905005455, + 0.14702443778514862, + 0.13390076160430908, + 0.12344712018966675, + 0.0966324657201767, + 0.11953848600387573, + 0.1299745738506317, + 0.1345512568950653 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0511577129364014, + "mean_weights": [ + 0.09969542920589447, + 0.16962948441505432, + 0.1319108009338379, + 0.11863738298416138, + 0.09143088757991791, + 0.12074249982833862, + 0.14481820166110992, + 0.12313534319400787 + ], + "sources": 8 + }, + { + "entropy_mean": 2.069282054901123, + "mean_weights": [ + 0.12384641170501709, + 0.13645249605178833, + 0.13362589478492737, + 0.12747827172279358, + 0.10400287806987762, + 0.11596200615167618, + 0.12676866352558136, + 0.13186340034008026 + ], + "sources": 8 + }, + { + "entropy_mean": 2.165989875793457, + "mean_weights": [ + 0.08691617846488953, + 0.1522500216960907, + 0.1198691874742508, + 0.10617777705192566, + 0.08211278915405273, + 0.11005747318267822, + 0.12842655181884766, + 0.08939606696367264, + 0.12479399144649506 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907525062561035, + "mean_weights": [ + 0.1032973974943161, + 0.12799270451068878, + 0.11672255396842957, + 0.11367214471101761, + 0.10446079820394516, + 0.11008220911026001, + 0.12035597860813141, + 0.1041758805513382, + 0.09924033284187317 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1538312435150146, + "mean_weights": [ + 0.09009353071451187, + 0.16798722743988037, + 0.13133394718170166, + 0.11392836272716522, + 0.08597822487354279, + 0.10755141079425812, + 0.13270318508148193, + 0.09302861988544464, + 0.07739552855491638 + ], + "sources": 9 + } + ], + "loss_nats": 2.136610746383667, + "output_weights": { + "entropy_mean": 2.1821343898773193, + "mean_weights": [ + 0.12100096046924591, + 0.09246665984392166, + 0.10230658948421478, + 0.10203823447227478, + 0.09980043023824692, + 0.10532397031784058, + 0.11232466995716095, + 0.1358221024274826, + 0.12891638278961182 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 7.445879600709304e-05, + 7.514825847465545e-05, + 8.149984205374494e-05, + 7.616886432515457e-05, + 7.93620856711641e-05, + 7.107412966433913e-05, + 6.609321280848235e-05, + 6.663892418146133e-05, + 0.00013160602247808129, + 0.00010466711682965979, + 0.00017213824321515858, + 0.0001231937640113756, + 0.00011169029312441126, + 7.821891631465405e-05, + 5.0732542149489745e-05, + 4.2012750782305375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.6818940215162e-05, + "first_to_last_ratio": 1.087107257387618, + "imbalance_abs_log_ratio": 0.08352027612092149, + "last_quartile_mean": 7.066362559271511e-05, + "mean": 8.779398513070191e-05, + "normalized": [ + 0.8481081693266763, + 0.8559613550151489, + 0.9283078098392884, + 0.8675863638238926, + 0.9039581191469443, + 0.8095557976839601, + 0.752821650709751, + 0.7590374680253287, + 1.499032334415107, + 1.1921900648869996, + 1.9607065672994624, + 1.403214170401, + 1.2721861635297007, + 0.8909370749967309, + 0.5778589737550069, + 0.47853791714500205 + ], + "population_cv": 0.365612445298668 + }, + "activation_output_rms_by_block": [ + 0.10262583196163177, + 0.17122715711593628, + 0.1563396155834198, + 0.23706908524036407, + 0.04811444878578186, + 0.07098451256752014, + 0.05486894026398659, + 0.08282698690891266, + 0.05263432487845421, + 0.09547916799783707, + 0.05760422348976135, + 0.09091533720493317, + 0.05383981764316559, + 0.1016460508108139, + 0.07459317892789841, + 0.18281812965869904 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16681542247533798, + "first_to_last_ratio": 1.6160480792916094, + "imbalance_abs_log_ratio": 0.4799837116932076, + "last_quartile_mean": 0.10322429426014423, + "mean": 0.10209917556494474, + "normalized": [ + 1.005158282559804, + 1.6770669906831872, + 1.531252477978855, + 2.3219490650006835, + 0.4712520793586283, + 0.6952505950683928, + 0.5374082597668455, + 0.8112405066016116, + 0.5155215464494499, + 0.9351610085930936, + 0.5641987133688422, + 0.8904610316574242, + 0.5273286228341615, + 0.9955619156411052, + 0.7305953110311854, + 1.7905935934067303 + ], + "population_cv": 0.5286889875537891 + }, + "bits_per_byte": 2.0839361382492645, + "core_parameter_grad_rms_by_block": [ + 0.0003583741677462985, + 0.0002652231972340971, + 0.00033417461006948805, + 0.0002275253539728081, + 0.0001893834795064905, + 0.00020636019253143534, + 0.0002096112726133392, + 0.0001923796657115476, + 0.00047635112247231166, + 0.000565001794959564, + 0.0009549288128880833, + 0.0007263576654831343, + 0.0005471283815528999, + 0.0004183078461688176, + 0.0002607600761312772, + 0.0002482750049477455 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029632433225567294, + "first_to_last_ratio": 0.8038795478406102, + "imbalance_abs_log_ratio": 0.21830583714537038, + "last_quartile_mean": 0.00036861782720018504, + "mean": 0.0003862589152493336, + "normalized": [ + 0.9278081452565691, + 0.6866461504529756, + 0.8651570148323317, + 0.5890488089470723, + 0.49030189862217616, + 0.5342535392308757, + 0.5426703807678671, + 0.49805883596852335, + 1.2332430493282536, + 1.462754056032209, + 2.472250542804088, + 1.8804942405387945, + 1.4164809146210218, + 1.0829726632945058, + 0.6750914110628468, + 0.6427683482398892 + ], + "population_cv": 0.550745478202196 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4576239287853241, + "mean_weights": [ + 0.8160475492477417, + 0.18395249545574188 + ], + "sources": 2 + }, + { + "entropy_mean": 0.622429370880127, + "mean_weights": [ + 0.626032829284668, + 0.37396717071533203 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5034195184707642, + "mean_weights": [ + 0.7919335961341858, + 0.20806637406349182 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6750276684761047, + "mean_weights": [ + 0.4605996012687683, + 0.5394004583358765 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9881983399391174, + "mean_weights": [ + 0.5066143870353699, + 0.2128569334745407, + 0.2805286943912506 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0713173151016235, + "mean_weights": [ + 0.2439693659543991, + 0.33285707235336304, + 0.42317354679107666 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0050503015518188, + "mean_weights": [ + 0.46526122093200684, + 0.19465598464012146, + 0.3400827646255493 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091313123703003, + "mean_weights": [ + 0.29163461923599243, + 0.3559022545814514, + 0.35246312618255615 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3032760620117188, + "mean_weights": [ + 0.25388818979263306, + 0.13041767477989197, + 0.27188149094581604, + 0.34381261467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3785908222198486, + "mean_weights": [ + 0.2572174668312073, + 0.24623332917690277, + 0.2131483256816864, + 0.28340083360671997 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2377310991287231, + "mean_weights": [ + 0.36895960569381714, + 0.12289825826883316, + 0.12992563843727112, + 0.3782165050506592 + ], + "sources": 4 + }, + { + "entropy_mean": 1.370632529258728, + "mean_weights": [ + 0.2267947793006897, + 0.24114219844341278, + 0.21443253755569458, + 0.31763046979904175 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5590848922729492, + "mean_weights": [ + 0.22265344858169556, + 0.11684703826904297, + 0.1529940664768219, + 0.24088799953460693, + 0.26661744713783264 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5987765789031982, + "mean_weights": [ + 0.19780412316322327, + 0.19732677936553955, + 0.1696196049451828, + 0.187029629945755, + 0.24821987748146057 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5600793361663818, + "mean_weights": [ + 0.19789859652519226, + 0.13498559594154358, + 0.15099790692329407, + 0.2153477668762207, + 0.300770103931427 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6049537658691406, + "mean_weights": [ + 0.21525533497333527, + 0.2069544643163681, + 0.18156641721725464, + 0.19566501677036285, + 0.20055876672267914 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7536540031433105, + "mean_weights": [ + 0.1706119179725647, + 0.12355935573577881, + 0.11797749251127243, + 0.17735670506954193, + 0.15967294573783875, + 0.250821590423584 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7486846446990967, + "mean_weights": [ + 0.1656067669391632, + 0.16547001898288727, + 0.14594469964504242, + 0.13893218338489532, + 0.11710066348314285, + 0.26694566011428833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.502744197845459, + "mean_weights": [ + 0.14790554344654083, + 0.12157414853572845, + 0.05193060636520386, + 0.11843410134315491, + 0.09838350117206573, + 0.4617721140384674 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7703553438186646, + "mean_weights": [ + 0.17112275958061218, + 0.18443143367767334, + 0.1734921634197235, + 0.14492332935333252, + 0.11953528970479965, + 0.2064950168132782 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6120144128799438, + "mean_weights": [ + 0.08869323134422302, + 0.07363369315862656, + 0.0430244579911232, + 0.08643164485692978, + 0.0696420818567276, + 0.21115350723266602, + 0.4274213910102844 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8458226919174194, + "mean_weights": [ + 0.12519317865371704, + 0.13179585337638855, + 0.1208491250872612, + 0.10168728232383728, + 0.08638136088848114, + 0.13217970728874207, + 0.3019134998321533 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5814783573150635, + "mean_weights": [ + 0.09110043942928314, + 0.07703252136707306, + 0.04712430387735367, + 0.08346284925937653, + 0.06837025284767151, + 0.1662226915359497, + 0.4666869342327118 + ], + "sources": 7 + }, + { + "entropy_mean": 1.864192008972168, + "mean_weights": [ + 0.13281965255737305, + 0.1363598257303238, + 0.09730516374111176, + 0.10794492065906525, + 0.09621791541576385, + 0.14952389895915985, + 0.27982860803604126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7858508825302124, + "mean_weights": [ + 0.0760645866394043, + 0.06870019435882568, + 0.041292984038591385, + 0.06624549627304077, + 0.05510660633444786, + 0.12923449277877808, + 0.3349960446357727, + 0.22835959494113922 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9164016246795654, + "mean_weights": [ + 0.10010629147291183, + 0.09289439767599106, + 0.06040576845407486, + 0.08482401818037033, + 0.06471984833478928, + 0.12172460556030273, + 0.20661231875419617, + 0.26871275901794434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7764818668365479, + "mean_weights": [ + 0.07597766816616058, + 0.07146216928958893, + 0.04444725811481476, + 0.06381020694971085, + 0.052670422941446304, + 0.10254879295825958, + 0.2695360779762268, + 0.3195474147796631 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9026113748550415, + "mean_weights": [ + 0.09874680638313293, + 0.09077301621437073, + 0.052286967635154724, + 0.08586552739143372, + 0.061087287962436676, + 0.1283089518547058, + 0.24689394235610962, + 0.2360374927520752 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0007686614990234, + "mean_weights": [ + 0.0738106444478035, + 0.07652983069419861, + 0.04215084761381149, + 0.06585545837879181, + 0.053016483783721924, + 0.10676231980323792, + 0.23121783137321472, + 0.19090119004249573, + 0.1597554087638855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.05625581741333, + "mean_weights": [ + 0.08243454247713089, + 0.09636277705430984, + 0.04735179618000984, + 0.07971908152103424, + 0.057041361927986145, + 0.1233949214220047, + 0.21250130236148834, + 0.17240752279758453, + 0.12878668308258057 + ], + "sources": 9 + }, + { + "entropy_mean": 2.023190498352051, + "mean_weights": [ + 0.08014722913503647, + 0.08487560600042343, + 0.04445280879735947, + 0.06905653327703476, + 0.057849444448947906, + 0.1067991554737091, + 0.20276135206222534, + 0.2105080783367157, + 0.14354978501796722 + ], + "sources": 9 + } + ], + "loss_nats": 1.444474458694458, + "output_weights": { + "entropy_mean": 1.9153306484222412, + "mean_weights": [ + 0.0914863795042038, + 0.05659400671720505, + 0.05275696888566017, + 0.06721317768096924, + 0.05635443329811096, + 0.0678911805152893, + 0.08835235983133316, + 0.2647843658924103, + 0.25456711649894714 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 5.599572978098877e-05, + 5.6567834690213203e-05, + 7.252454088302329e-05, + 6.816006498411298e-05, + 6.961339386180043e-05, + 6.354664219543338e-05, + 5.091843559057452e-05, + 5.004410559195094e-05, + 0.00011432960309321061, + 8.808868005871773e-05, + 0.0001649916812311858, + 0.00011419985094107687, + 0.0001274511159863323, + 8.248462108895183e-05, + 5.212613905314356e-05, + 4.046529647894204e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.331204258458456e-05, + "first_to_last_ratio": 0.8371088393670095, + "imbalance_abs_log_ratio": 0.1778011818604578, + "last_quartile_mean": 7.563179315184243e-05, + "mean": 7.946923346935364e-05, + "normalized": [ + 0.7046214910652542, + 0.711820565276094, + 0.9126115569114117, + 0.8576912348147676, + 0.8759791786420843, + 0.7996382929745935, + 0.6407314298584572, + 0.6297293104160852, + 1.438664978910366, + 1.1084626869175487, + 2.076170538310438, + 1.4370322444989567, + 1.6037793548805563, + 1.0379440883969395, + 0.6559285496725645, + 0.5091944984538828 + ], + "population_cv": 0.418216483754139 + }, + "activation_output_rms_by_block": [ + 0.16897788643836975, + 0.2546124756336212, + 0.24958282709121704, + 0.3488057255744934, + 0.05895345285534859, + 0.09218393266201019, + 0.05982525274157524, + 0.10435724258422852, + 0.07404384016990662, + 0.13799543678760529, + 0.072679303586483, + 0.12079861760139465, + 0.06610427051782608, + 0.11929895728826523, + 0.08813745528459549, + 0.2320432960987091 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.25549472868442535, + "first_to_last_ratio": 2.0213831070680737, + "imbalance_abs_log_ratio": 0.7037819835779293, + "last_quartile_mean": 0.12639599479734898, + "mean": 0.1405249983072281, + "normalized": [ + 1.2024756340429583, + 1.8118660644063134, + 1.776074222364032, + 2.4821613931772033, + 0.419522886073689, + 0.6559966822448885, + 0.425726763652258, + 0.7426240444143152, + 0.5269086714950565, + 0.9819992061904, + 0.5171983950327836, + 0.8596236901372816, + 0.47040933153618064, + 0.8489518500291555, + 0.6272012549105437, + 1.6512599102929408 + ], + "population_cv": 0.5965464726510664 + }, + "bits_per_byte": 1.9646943750306483, + "core_parameter_grad_rms_by_block": [ + 0.00033760869044818646, + 0.0001988249533593431, + 0.00022180290878374173, + 0.00015787488110689858, + 0.0001349156990162873, + 0.00018159413720443885, + 0.00013882426062051804, + 0.00016817749051027943, + 0.0004737077931871952, + 0.0005939057773758512, + 0.0010653831085878915, + 0.0008603315741687424, + 0.000762827758322017, + 0.0005453805470031765, + 0.00031779802289983946, + 0.00029451637801964313 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022902785842454247, + "first_to_last_ratio": 0.47701150875195986, + "imbalance_abs_log_ratio": 0.7402146610222566, + "last_quartile_mean": 0.000480130676561169, + "mean": 0.00040334212378837815, + "normalized": [ + 0.8370280973313858, + 0.49294368634717844, + 0.5499125821534951, + 0.3914167943185893, + 0.33449444295353004, + 0.45022358562210574, + 0.34418488036066147, + 0.41695989729681016, + 1.1744565351565805, + 1.4724615713271154, + 2.6413881559935133, + 2.1330070018179734, + 1.89126727245144, + 1.3521537048516206, + 0.7879118102392371, + 0.7301899817787623 + ], + "population_cv": 0.6909894412084847 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.27112114429473877, + "mean_weights": [ + 0.9155579805374146, + 0.08444204181432724 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5555346012115479, + "mean_weights": [ + 0.7120839953422546, + 0.28791600465774536 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4612463414669037, + "mean_weights": [ + 0.8216869831085205, + 0.1783129870891571 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6770275831222534, + "mean_weights": [ + 0.48782873153686523, + 0.5121712684631348 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9962381720542908, + "mean_weights": [ + 0.465767502784729, + 0.19465303421020508, + 0.3395794630050659 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0414454936981201, + "mean_weights": [ + 0.22931429743766785, + 0.28697463870048523, + 0.4837110638618469 + ], + "sources": 3 + }, + { + "entropy_mean": 0.989799976348877, + "mean_weights": [ + 0.40070968866348267, + 0.17750780284404755, + 0.4217824935913086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.085202932357788, + "mean_weights": [ + 0.2848654091358185, + 0.3230235278606415, + 0.39211106300354004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2675609588623047, + "mean_weights": [ + 0.23358023166656494, + 0.10898500680923462, + 0.35628458857536316, + 0.3011501431465149 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3760833740234375, + "mean_weights": [ + 0.2731572389602661, + 0.2245836853981018, + 0.22189702093601227, + 0.280362069606781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2111725807189941, + "mean_weights": [ + 0.41297993063926697, + 0.11292652785778046, + 0.1222275048494339, + 0.3518660068511963 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647139072418213, + "mean_weights": [ + 0.23418807983398438, + 0.22505050897598267, + 0.2112433910369873, + 0.32951802015304565 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5535553693771362, + "mean_weights": [ + 0.21889275312423706, + 0.10524161159992218, + 0.1673421710729599, + 0.2413085401058197, + 0.26721489429473877 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5950126647949219, + "mean_weights": [ + 0.2038668543100357, + 0.17965060472488403, + 0.17214898765087128, + 0.19203421473503113, + 0.25229936838150024 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5395398139953613, + "mean_weights": [ + 0.2043805718421936, + 0.12131387740373611, + 0.1313788890838623, + 0.23064203560352325, + 0.3122846484184265 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6000796556472778, + "mean_weights": [ + 0.22543779015541077, + 0.20209266245365143, + 0.18844717741012573, + 0.20001280307769775, + 0.18400955200195312 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7217694520950317, + "mean_weights": [ + 0.1735105812549591, + 0.1090376153588295, + 0.10670766234397888, + 0.18005813658237457, + 0.14383366703987122, + 0.28685232996940613 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7343032360076904, + "mean_weights": [ + 0.16843369603157043, + 0.15050029754638672, + 0.15599164366722107, + 0.14163106679916382, + 0.10262320935726166, + 0.2808200716972351 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4446122646331787, + "mean_weights": [ + 0.1531161069869995, + 0.1061544418334961, + 0.04045919328927994, + 0.12559477984905243, + 0.08803897351026535, + 0.4866364896297455 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7605862617492676, + "mean_weights": [ + 0.1779545247554779, + 0.16784369945526123, + 0.1857004165649414, + 0.14843404293060303, + 0.10648007690906525, + 0.21358723938465118 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5263525247573853, + "mean_weights": [ + 0.08794503659009933, + 0.0601879358291626, + 0.033082835376262665, + 0.08346094191074371, + 0.05888189375400543, + 0.20315150916576385, + 0.4732898473739624 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7944886684417725, + "mean_weights": [ + 0.12276870757341385, + 0.11536462604999542, + 0.1304849088191986, + 0.09876619279384613, + 0.07423829287290573, + 0.11369137465953827, + 0.3446859121322632 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4918596744537354, + "mean_weights": [ + 0.09284307807683945, + 0.06384406983852386, + 0.03825954347848892, + 0.08267752081155777, + 0.05671031028032303, + 0.1521964967250824, + 0.5134689807891846 + ], + "sources": 7 + }, + { + "entropy_mean": 1.830867052078247, + "mean_weights": [ + 0.13634362816810608, + 0.12310001999139786, + 0.09669966250658035, + 0.1085672378540039, + 0.08431269228458405, + 0.13993649184703827, + 0.3110402822494507 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6549372673034668, + "mean_weights": [ + 0.06966839730739594, + 0.049915459007024765, + 0.029968995600938797, + 0.05769501253962517, + 0.040842454880476, + 0.10442843288183212, + 0.30951738357543945, + 0.33796387910842896 + ], + "sources": 8 + }, + { + "entropy_mean": 1.859694480895996, + "mean_weights": [ + 0.09671731293201447, + 0.07844845205545425, + 0.05707312002778053, + 0.0829043835401535, + 0.06170058995485306, + 0.10576222091913223, + 0.19003936648368835, + 0.3273545503616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.677255392074585, + "mean_weights": [ + 0.07509785890579224, + 0.05865459889173508, + 0.03770594298839569, + 0.059113647788763046, + 0.042673639953136444, + 0.08797852694988251, + 0.24025270342826843, + 0.39852309226989746 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8695217370986938, + "mean_weights": [ + 0.0984041839838028, + 0.08118931949138641, + 0.05154065042734146, + 0.08640529215335846, + 0.05755901336669922, + 0.11663993448019028, + 0.23645231127738953, + 0.27180927991867065 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8854753971099854, + "mean_weights": [ + 0.06882007420063019, + 0.0593036487698555, + 0.03391250967979431, + 0.060251303017139435, + 0.038275185972452164, + 0.09127645194530487, + 0.20795494318008423, + 0.23203802108764648, + 0.20816785097122192 + ], + "sources": 9 + }, + { + "entropy_mean": 2.032858371734619, + "mean_weights": [ + 0.08241330087184906, + 0.08549442887306213, + 0.041513457894325256, + 0.08080939948558807, + 0.054553937166929245, + 0.12166906148195267, + 0.2086121290922165, + 0.17801183462142944, + 0.14692245423793793 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9540549516677856, + "mean_weights": [ + 0.07821957767009735, + 0.07357673346996307, + 0.037884071469306946, + 0.06657841801643372, + 0.044617801904678345, + 0.09479035437107086, + 0.1742672622203827, + 0.2624226212501526, + 0.16764315962791443 + ], + "sources": 9 + } + ], + "loss_nats": 1.3618223667144775, + "output_weights": { + "entropy_mean": 1.836374282836914, + "mean_weights": [ + 0.08612492680549622, + 0.0513404943048954, + 0.04786331206560135, + 0.0643724650144577, + 0.0475543811917305, + 0.059286996722221375, + 0.07609974592924118, + 0.2768789529800415, + 0.2904787063598633 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 7.052410364849493e-05, + 7.120969530660659e-05, + 9.769770258571953e-05, + 9.110000246437266e-05, + 9.818621765589342e-05, + 8.837205677991733e-05, + 7.198825915111229e-05, + 6.996823503868654e-05, + 0.00017213664250448346, + 0.00013030946138314903, + 0.0002552570658735931, + 0.00017121800919994712, + 0.00020642777963075787, + 0.00012888041965197772, + 9.31880422285758e-05, + 5.8820565755013376e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.263287600129843e-05, + "first_to_last_ratio": 0.6782682211585493, + "imbalance_abs_log_ratio": 0.3882124628447938, + "last_quartile_mean": 0.00012182920181658119, + "mean": 0.0001172052661786438, + "normalized": [ + 0.6017144617120052, + 0.6075639570500958, + 0.8335606903260565, + 0.7772688500661601, + 0.8377287203651615, + 0.7539939088165287, + 0.6142066947861206, + 0.5969717685896574, + 1.4686766910465736, + 1.1118055155007445, + 2.1778634543991506, + 1.4608388750977905, + 1.7612500390223207, + 1.0996128745233913, + 0.7950840885130448, + 0.5018594101851985 + ], + "population_cv": 0.46767509062432555 + }, + "activation_output_rms_by_block": [ + 0.17702145874500275, + 0.2651609778404236, + 0.2740711271762848, + 0.3786091208457947, + 0.05278070271015167, + 0.09138669073581696, + 0.05598878860473633, + 0.10357969254255295, + 0.06921031326055527, + 0.1325109452009201, + 0.06760682165622711, + 0.11279971897602081, + 0.060408592224121094, + 0.10416899621486664, + 0.073752261698246, + 0.19092032313346863 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.27371567115187645, + "first_to_last_ratio": 2.5506400527811586, + "imbalance_abs_log_ratio": 0.9363443287654816, + "last_quartile_mean": 0.10731254331767559, + "mean": 0.13812353322282434, + "normalized": [ + 1.2816169309789325, + 1.9197378727103604, + 1.984246426234597, + 2.7410906165788056, + 0.38212679243445447, + 0.661630126333285, + 0.40535300030599286, + 0.7499061899571856, + 0.5010754622740751, + 0.9593654470679528, + 0.4894663500039642, + 0.8166582214056853, + 0.43735191834883375, + 0.7541726871902314, + 0.5339586960845188, + 1.3822432620911254 + ], + "population_cv": 0.6672200194546598 + }, + "bits_per_byte": 1.8951255009474826, + "core_parameter_grad_rms_by_block": [ + 0.0004973569892103084, + 0.0002771506747073045, + 0.0003595853430528865, + 0.00025621221416812444, + 0.0002736875635381804, + 0.00034154014112545435, + 0.0002723491519532143, + 0.0002954443301970981, + 0.0007531989220686374, + 0.0008991377204894639, + 0.0016884293947290697, + 0.0013942943414501116, + 0.0012489939262520248, + 0.0008593069460315814, + 0.000558781441282585, + 0.00045362324645161216 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034757630528465595, + "first_to_last_ratio": 0.44550989973264, + "imbalance_abs_log_ratio": 0.8085358106960822, + "last_quartile_mean": 0.0007801763900044509, + "mean": 0.0006518182716692285, + "normalized": [ + 0.7630301432585446, + 0.42519623452339683, + 0.5516650248727019, + 0.39307307773376104, + 0.41988323346214174, + 0.523980618479459, + 0.417829882638547, + 0.45326181090398177, + 1.1555351465367567, + 1.3794300644363342, + 2.5903376264755575, + 2.139084468865058, + 1.916168939317244, + 1.3183228874989945, + 0.8572656913277571, + 0.6959351496697651 + ], + "population_cv": 0.6710654904340576 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2534913420677185, + "mean_weights": [ + 0.9232981204986572, + 0.07670190930366516 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5290617346763611, + "mean_weights": [ + 0.7383309602737427, + 0.26166898012161255 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4635298252105713, + "mean_weights": [ + 0.8211046457290649, + 0.17889533936977386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.680541455745697, + "mean_weights": [ + 0.4997203052043915, + 0.5002796649932861 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9984116554260254, + "mean_weights": [ + 0.45704665780067444, + 0.20008087158203125, + 0.3428725004196167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0224952697753906, + "mean_weights": [ + 0.22097927331924438, + 0.2672557532787323, + 0.5117650032043457 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9886391162872314, + "mean_weights": [ + 0.4091557264328003, + 0.17683157324790955, + 0.41401270031929016 + ], + "sources": 3 + }, + { + "entropy_mean": 1.083122730255127, + "mean_weights": [ + 0.2858731150627136, + 0.31187689304351807, + 0.4022499620914459 + ], + "sources": 3 + }, + { + "entropy_mean": 1.262911081314087, + "mean_weights": [ + 0.24318832159042358, + 0.1005454957485199, + 0.3312486708164215, + 0.325017511844635 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3697116374969482, + "mean_weights": [ + 0.2760387659072876, + 0.21072158217430115, + 0.21145658195018768, + 0.30178308486938477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1750009059906006, + "mean_weights": [ + 0.40358343720436096, + 0.10160854458808899, + 0.10028939694166183, + 0.3945186138153076 + ], + "sources": 4 + }, + { + "entropy_mean": 1.355452299118042, + "mean_weights": [ + 0.23967716097831726, + 0.20835967361927032, + 0.20370042324066162, + 0.348262757062912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5273280143737793, + "mean_weights": [ + 0.2132018804550171, + 0.09696971625089645, + 0.14153459668159485, + 0.2435910701751709, + 0.3047027587890625 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5883060693740845, + "mean_weights": [ + 0.2065391093492508, + 0.16525766253471375, + 0.16744214296340942, + 0.19437497854232788, + 0.26638609170913696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4673986434936523, + "mean_weights": [ + 0.19910070300102234, + 0.10532978177070618, + 0.08866854012012482, + 0.23074698448181152, + 0.37615397572517395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6001981496810913, + "mean_weights": [ + 0.22463412582874298, + 0.18846800923347473, + 0.19619622826576233, + 0.2022438496351242, + 0.18845780193805695 + ], + "sources": 5 + }, + { + "entropy_mean": 1.658735752105713, + "mean_weights": [ + 0.16469183564186096, + 0.09157001227140427, + 0.085207998752594, + 0.16954544186592102, + 0.1403937190771103, + 0.34859102964401245 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7217588424682617, + "mean_weights": [ + 0.1685413271188736, + 0.1354280561208725, + 0.15638956427574158, + 0.13712063431739807, + 0.10325069725513458, + 0.29926973581314087 + ], + "sources": 6 + }, + { + "entropy_mean": 1.383854627609253, + "mean_weights": [ + 0.14582018554210663, + 0.09375129640102386, + 0.03237149864435196, + 0.11780930310487747, + 0.08738291263580322, + 0.522864818572998 + ], + "sources": 6 + }, + { + "entropy_mean": 1.758126974105835, + "mean_weights": [ + 0.18006329238414764, + 0.15370121598243713, + 0.18850785493850708, + 0.14673401415348053, + 0.10883604735136032, + 0.2221575826406479 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4768933057785034, + "mean_weights": [ + 0.08639779686927795, + 0.05247560143470764, + 0.02685363218188286, + 0.07686029374599457, + 0.05649450421333313, + 0.2057446390390396, + 0.49517351388931274 + ], + "sources": 7 + }, + { + "entropy_mean": 1.754528522491455, + "mean_weights": [ + 0.1207476407289505, + 0.10044647008180618, + 0.12684977054595947, + 0.0954352393746376, + 0.07363080978393555, + 0.10486453771591187, + 0.37802553176879883 + ], + "sources": 7 + }, + { + "entropy_mean": 1.434983491897583, + "mean_weights": [ + 0.09186187386512756, + 0.054585013538599014, + 0.030214153230190277, + 0.07775386422872543, + 0.05580390244722366, + 0.14904344081878662, + 0.5407377481460571 + ], + "sources": 7 + }, + { + "entropy_mean": 1.814563274383545, + "mean_weights": [ + 0.13872049748897552, + 0.10858511924743652, + 0.09594539552927017, + 0.10938869416713715, + 0.08644531667232513, + 0.13206249475479126, + 0.32885247468948364 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5878477096557617, + "mean_weights": [ + 0.06617997586727142, + 0.04098459705710411, + 0.023611580953001976, + 0.05256733298301697, + 0.03869973123073578, + 0.09328385442495346, + 0.30499327182769775, + 0.37967967987060547 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8095037937164307, + "mean_weights": [ + 0.0948307067155838, + 0.0694318562746048, + 0.0552619993686676, + 0.07500732690095901, + 0.061008021235466, + 0.09275255352258682, + 0.18348734080791473, + 0.3682202100753784 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5925970077514648, + "mean_weights": [ + 0.07132496684789658, + 0.04804965853691101, + 0.030455447733402252, + 0.054331667721271515, + 0.038998860865831375, + 0.07944463938474655, + 0.22967059910297394, + 0.4477241635322571 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8567713499069214, + "mean_weights": [ + 0.09893427789211273, + 0.07625022530555725, + 0.05004327371716499, + 0.08632857352495193, + 0.05865295231342316, + 0.10661771893501282, + 0.2290721982717514, + 0.2941007912158966 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7706098556518555, + "mean_weights": [ + 0.0600857213139534, + 0.04508119821548462, + 0.026483699679374695, + 0.05021750181913376, + 0.03222868591547012, + 0.07324853539466858, + 0.17921699583530426, + 0.23399141430854797, + 0.2994462847709656 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0186803340911865, + "mean_weights": [ + 0.08094437420368195, + 0.07421140372753143, + 0.03662773221731186, + 0.07851459085941315, + 0.05622554570436478, + 0.11645162105560303, + 0.2003173828125, + 0.18591775000095367, + 0.17078962922096252 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8739352226257324, + "mean_weights": [ + 0.07170331478118896, + 0.05929408222436905, + 0.03108443133533001, + 0.05872419476509094, + 0.03831540048122406, + 0.07797900587320328, + 0.1519450843334198, + 0.2740318179130554, + 0.23692265152931213 + ], + "sources": 9 + } + ], + "loss_nats": 1.3136008977890015, + "output_weights": { + "entropy_mean": 1.685392141342163, + "mean_weights": [ + 0.08037909120321274, + 0.03995431959629059, + 0.0371873639523983, + 0.05237603187561035, + 0.03903363645076752, + 0.04661020264029503, + 0.0628754124045372, + 0.26517099142074585, + 0.37641292810440063 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.073347956844582, + "cross_entropy_nats": 5.596018373966217, + "step": 0 + }, + { + "bits_per_byte": 5.112499455104693, + "cross_entropy_nats": 3.5437145829200745, + "step": 100 + }, + { + "bits_per_byte": 3.1188867992707414, + "cross_entropy_nats": 2.1618475914001465, + "step": 500 + }, + { + "bits_per_byte": 1.962477669145846, + "cross_entropy_nats": 1.3602858632802963, + "step": 2000 + }, + { + "bits_per_byte": 1.8161008917328916, + "cross_entropy_nats": 1.2588252127170563, + "step": 4000 + }, + { + "bits_per_byte": 1.737390666804078, + "cross_entropy_nats": 1.20426744222641, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "0bff629038a5dd94843a123dbdfc2bdaca027f8a9025a2a7688adc4ec0647b31", + "final_model_state": "efa098925d29c4e5158cd73c7764d7014b6c4ee53df36a146dbd0cff04321b62", + "final_optimizer_state": "b86655a1b88951524b7f3bfca00df30a0958104385aa444bf7b109fbe1d9e51b", + "final_public_parameters": "87107ee0633f1863f2fcaf1d5f91af597c67acce2c9777fec3486aa4fd1463ef", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "seed": 2026073001, + "timing": { + "mean_ms": 54.63475258274364, + "measured_steps": 7980, + "median_ms": 54.539098986424506, + "p95_ms": 55.062895112496335, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00026696090935729444, + 0.00023596938990522176, + 0.00020828393462579697, + 0.00018593396816868335, + 0.00016940363275352865, + 0.00015857137623243034, + 0.00015041495498735458, + 0.00014485203428193927, + 0.00013822126493323594, + 0.00013424422650132328, + 0.00013012201816309243, + 0.00012678767961915582, + 0.00012444572348613292, + 0.00012238229101058096, + 0.00012059078289894387, + 0.00011903457198059186 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022428705051424913, + "first_to_last_ratio": 1.84426351740015, + "imbalance_abs_log_ratio": 0.6120800202390253, + "last_quartile_mean": 0.0001216133423440624, + "mean": 0.00015851367243158165, + "normalized": [ + 1.6841506808980231, + 1.488637454961949, + 1.3139808789408836, + 1.1729837894515815, + 1.0687004480742655, + 1.0003640304332333, + 0.9489083981211609, + 0.9138141338846396, + 0.8719832353445448, + 0.8468936744826624, + 0.8208882941580717, + 0.7998532724291052, + 0.785078798422557, + 0.772061419896786, + 0.7607595045215659, + 0.7509419859789702 + ], + "population_cv": 0.27150931413441437 + }, + "activation_output_rms_by_block": [ + 0.0048858714289963245, + 0.006843581795692444, + 0.004983869381248951, + 0.007334618363529444, + 0.005787155125290155, + 0.007929109036922455, + 0.005876626819372177, + 0.008892860263586044, + 0.006918553728610277, + 0.009880779311060905, + 0.007473294623196125, + 0.010543949902057648, + 0.008116732351481915, + 0.011967957951128483, + 0.008228716440498829, + 0.012017112225294113 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006011985242366791, + "first_to_last_ratio": 0.5962715478148781, + "imbalance_abs_log_ratio": 0.5170590985397028, + "last_quartile_mean": 0.010082629742100835, + "mean": 0.007980049296747893, + "normalized": [ + 0.612260807835794, + 0.8575864059488212, + 0.6245411771177937, + 0.9191194303171183, + 0.725202929216063, + 0.9936165482278163, + 0.736414851693595, + 1.114386632575083, + 0.866981326973731, + 1.2381852471873345, + 0.9364973003665171, + 1.321288817896729, + 1.017128096538166, + 1.499734839483483, + 1.031161103710509, + 1.5058944849114457 + ], + "population_cv": 0.2698221312597705 + }, + "bits_per_byte": 8.029409055280222, + "core_parameter_grad_rms_by_block": [ + 0.006887647168374071, + 0.005982369825085796, + 0.005174924477158116, + 0.004946754810206586, + 0.004967935165757302, + 0.004123664448940147, + 0.00409438714185196, + 0.004247982317789705, + 0.0044010147653324285, + 0.004095379628913903, + 0.00386485833459634, + 0.0038097011691932423, + 0.0037333099925226194, + 0.004058250822051008, + 0.003851730608686184, + 0.00381805442027398 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005747924070206143, + "first_to_last_ratio": 1.487043658003427, + "imbalance_abs_log_ratio": 0.39679002683413417, + "last_quartile_mean": 0.003865336460883448, + "mean": 0.004503622818545837, + "normalized": [ + 1.5293570189783356, + 1.3283461040410636, + 1.1490581440008412, + 1.0983945613375836, + 1.10309752080024, + 0.9156327283801325, + 0.909131893770355, + 0.9432366983079914, + 0.9772165526849029, + 0.9093522690331003, + 0.858166522889289, + 0.8459192349556811, + 0.8289570736583262, + 0.9011080602352411, + 0.8552515971863425, + 0.8477740197405745 + ], + "population_cv": 0.191198476656264 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.5655622482299805, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 8.836213237373158e-05, + 8.102934953058138e-05, + 6.889242649776861e-05, + 6.090942770242691e-05, + 5.060216426500119e-05, + 4.3459844164317474e-05, + 3.709086377057247e-05, + 3.2571624615229666e-05, + 2.8896074582007714e-05, + 2.6415440515847877e-05, + 2.491714622010477e-05, + 2.3614533347426914e-05, + 2.2646127035841346e-05, + 2.2026059014024213e-05, + 2.118862721545156e-05, + 2.081490310956724e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.479833402612712e-05, + "first_to_last_ratio": 3.4518703579034327, + "imbalance_abs_log_ratio": 1.2389162168680456, + "last_quartile_mean": 2.166892909372109e-05, + "mean": 4.083979649749381e-05, + "normalized": [ + 2.1636281262849595, + 1.984078190388482, + 1.6868944609456193, + 1.4914233890995197, + 1.2390405585910906, + 1.0641542782169457, + 0.9082039322318514, + 0.7975462026905171, + 0.7075469777079071, + 0.6468063697983631, + 0.6101192551641105, + 0.578223580249134, + 0.5545112605355675, + 0.5393283244047475, + 0.518823036171393, + 0.5096720575197914 + ], + "population_cv": 0.5343944992100478 + }, + "activation_output_rms_by_block": [ + 0.009077239781618118, + 0.016407418996095657, + 0.010880433954298496, + 0.01978805474936962, + 0.012077964842319489, + 0.02216547541320324, + 0.013358108699321747, + 0.02644582837820053, + 0.017588114365935326, + 0.03464006260037422, + 0.023338599130511284, + 0.0432722382247448, + 0.023765092715620995, + 0.04745635390281677, + 0.02874108962714672, + 0.05685042589902878 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014038286870345473, + "first_to_last_ratio": 0.3580899608898232, + "imbalance_abs_log_ratio": 1.0269710367470868, + "last_quartile_mean": 0.039203240536153316, + "mean": 0.025365781330037862, + "normalized": [ + 0.35785374254838964, + 0.6468327855789793, + 0.4289414078254319, + 0.7801082289523973, + 0.4761518947581916, + 0.8738337339112493, + 0.5266192484086122, + 1.042578899270242, + 0.6933795626909267, + 1.3656217464649458, + 0.9200820123318648, + 1.7059296404760187, + 0.9368957496877358, + 1.8708808250514843, + 1.133065418060348, + 2.2412251039831825 + ], + "population_cv": 0.5292067698975893 + }, + "bits_per_byte": 5.216988460540972, + "core_parameter_grad_rms_by_block": [ + 0.00035471167919676877, + 0.00030778414793465906, + 0.0002905596100869004, + 0.0002669019974562734, + 0.0002519467909729747, + 0.0002484952213280207, + 0.00022573236044592232, + 0.00021694664140763627, + 0.00020872399951088957, + 0.00018641204799172754, + 0.0001823569017298164, + 0.0001572320679108608, + 0.00014731276753985554, + 0.00014544881518036755, + 0.00014011786536140868, + 0.0001387938590477502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003049893586686504, + "first_to_last_ratio": 2.1340115402633257, + "imbalance_abs_log_ratio": 0.7580035606734332, + "last_quartile_mean": 0.0001429183267823455, + "mean": 0.00021684229831886448, + "normalized": [ + 1.6358048311919693, + 1.4193916515405378, + 1.339958173933553, + 1.230857630294052, + 1.1618895064582344, + 1.1459720878009276, + 1.0409978228232257, + 1.0004811934275772, + 0.9625612766931799, + 0.8596664462466195, + 0.840965545668873, + 0.7250986967480513, + 0.6793543911033149, + 0.6707585019528177, + 0.6461740465200508, + 0.6400681975970168 + ], + "population_cv": 0.294333589312772 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6924017667770386, + "mean_weights": [ + 0.4816230535507202, + 0.5183769464492798 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925060749053955, + "mean_weights": [ + 0.5170555114746094, + 0.4829444885253906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922789812088013, + "mean_weights": [ + 0.5104873776435852, + 0.4895126223564148 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927056312561035, + "mean_weights": [ + 0.5133966207504272, + 0.48660340905189514 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097900390625, + "mean_weights": [ + 0.3491937518119812, + 0.32761698961257935, + 0.32318928837776184 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982853174209595, + "mean_weights": [ + 0.3437661826610565, + 0.3295633792877197, + 0.32667046785354614 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097551703453064, + "mean_weights": [ + 0.3531244695186615, + 0.3277324438095093, + 0.3191430866718292 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098252296447754, + "mean_weights": [ + 0.3423447012901306, + 0.3336608111858368, + 0.323994517326355 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3856292963027954, + "mean_weights": [ + 0.2598890960216522, + 0.25528842210769653, + 0.2465861737728119, + 0.23823627829551697 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862417936325073, + "mean_weights": [ + 0.2525826096534729, + 0.2512264847755432, + 0.24808365106582642, + 0.24810725450515747 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854787349700928, + "mean_weights": [ + 0.2609134912490845, + 0.2559056282043457, + 0.24643474817276, + 0.23674613237380981 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386183738708496, + "mean_weights": [ + 0.25091099739074707, + 0.2541857361793518, + 0.24937650561332703, + 0.24552680552005768 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6086668968200684, + "mean_weights": [ + 0.20840778946876526, + 0.2072581648826599, + 0.20017966628074646, + 0.19401825964450836, + 0.1901361048221588 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093640327453613, + "mean_weights": [ + 0.20082800090312958, + 0.20345230400562286, + 0.19957157969474792, + 0.1978151500225067, + 0.19833296537399292 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085686683654785, + "mean_weights": [ + 0.2064795196056366, + 0.20873931050300598, + 0.20255593955516815, + 0.1941128820180893, + 0.18811233341693878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093556880950928, + "mean_weights": [ + 0.2007957398891449, + 0.20384852588176727, + 0.2003384232521057, + 0.19736894965171814, + 0.19764836132526398 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7904106378555298, + "mean_weights": [ + 0.17261356115341187, + 0.1785532385110855, + 0.16990220546722412, + 0.16298523545265198, + 0.1574374884366989, + 0.1585082709789276 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791741132736206, + "mean_weights": [ + 0.16599833965301514, + 0.16723501682281494, + 0.16713106632232666, + 0.1660757064819336, + 0.16713276505470276, + 0.1664271056652069 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893474102020264, + "mean_weights": [ + 0.17279052734375, + 0.18296070396900177, + 0.1714734584093094, + 0.16615013778209686, + 0.1573297083377838, + 0.14929547905921936 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916802167892456, + "mean_weights": [ + 0.1656593233346939, + 0.16598697006702423, + 0.1664597988128662, + 0.16510574519634247, + 0.16595518589019775, + 0.17083296179771423 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9424470663070679, + "mean_weights": [ + 0.14923666417598724, + 0.16057875752449036, + 0.15159368515014648, + 0.14453238248825073, + 0.13740569353103638, + 0.1286274939775467, + 0.12802532315254211 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9457793235778809, + "mean_weights": [ + 0.1414237767457962, + 0.14120322465896606, + 0.14217954874038696, + 0.14177344739437103, + 0.14084486663341522, + 0.145671084523201, + 0.14690405130386353 + ], + "sources": 7 + }, + { + "entropy_mean": 1.940324306488037, + "mean_weights": [ + 0.15125462412834167, + 0.16445259749889374, + 0.15269356966018677, + 0.1458570957183838, + 0.1393839716911316, + 0.12596309185028076, + 0.12039506435394287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458446502685547, + "mean_weights": [ + 0.1414341926574707, + 0.14160503447055817, + 0.14392146468162537, + 0.1416645050048828, + 0.14250624179840088, + 0.1432597041130066, + 0.1456088423728943 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073972225189209, + "mean_weights": [ + 0.13393926620483398, + 0.1460031270980835, + 0.13552871346473694, + 0.12954440712928772, + 0.12087934464216232, + 0.11472145467996597, + 0.10819212347269058, + 0.11119155585765839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0792579650878906, + "mean_weights": [ + 0.12344250082969666, + 0.1223982647061348, + 0.12281741201877594, + 0.12404577434062958, + 0.1245848536491394, + 0.1274431347846985, + 0.1261020004749298, + 0.12916606664657593 + ], + "sources": 8 + }, + { + "entropy_mean": 2.070988178253174, + "mean_weights": [ + 0.13450230658054352, + 0.1483522206544876, + 0.13837148249149323, + 0.13302284479141235, + 0.12319637835025787, + 0.11490452289581299, + 0.10592599213123322, + 0.10172426700592041 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0791521072387695, + "mean_weights": [ + 0.12388300150632858, + 0.12212474644184113, + 0.12333755195140839, + 0.12223256379365921, + 0.12343418598175049, + 0.12649230659008026, + 0.12949144840240479, + 0.12900421023368835 + ], + "sources": 8 + }, + { + "entropy_mean": 2.18794584274292, + "mean_weights": [ + 0.12259161472320557, + 0.13473162055015564, + 0.1250339150428772, + 0.11940491199493408, + 0.11259052157402039, + 0.10314525663852692, + 0.09605535864830017, + 0.09219000488519669, + 0.09425680339336395 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197138786315918, + "mean_weights": [ + 0.11026664823293686, + 0.11047966778278351, + 0.11075460910797119, + 0.10965213924646378, + 0.10944131761789322, + 0.111859992146492, + 0.11317223310470581, + 0.1125045120716095, + 0.11186887323856354 + ], + "sources": 9 + }, + { + "entropy_mean": 2.189896583557129, + "mean_weights": [ + 0.12157853692770004, + 0.13215823471546173, + 0.1242145225405693, + 0.11826580762863159, + 0.11163709312677383, + 0.1045699343085289, + 0.09787097573280334, + 0.09515326470136642, + 0.09455163776874542 + ], + "sources": 9 + } + ], + "loss_nats": 3.616140842437744, + "output_weights": { + "entropy_mean": 2.1926791667938232, + "mean_weights": [ + 0.09771968424320221, + 0.09518586099147797, + 0.10263510048389435, + 0.10697779804468155, + 0.11385782063007355, + 0.11843808740377426, + 0.1225852519273758, + 0.12233239412307739, + 0.12026800215244293 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00015875036478973925, + 0.00016262078133877367, + 0.00014673685654997826, + 0.00013248690811451524, + 0.00010898245091084391, + 9.717086504679173e-05, + 8.911665645428002e-05, + 7.969579746713862e-05, + 7.858985918574035e-05, + 6.784927973058075e-05, + 6.598868640139699e-05, + 5.512806092156097e-05, + 4.8474703362444416e-05, + 4.3001487938454375e-05, + 3.610895510064438e-05, + 3.5184959415346384e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001501487276982516, + "first_to_last_ratio": 3.689835475493601, + "imbalance_abs_log_ratio": 1.3055818704713245, + "last_quartile_mean": 4.069252645422239e-05, + "mean": 8.786791704551433e-05, + "normalized": [ + 1.806693160912291, + 1.8507412808538344, + 1.6699708094135275, + 1.5077961623454543, + 1.2402985591930278, + 1.1058742293442396, + 1.0142115512777987, + 0.9069954102344013, + 0.8944090383413996, + 0.7721735305895072, + 0.7509986422827776, + 0.6273969245567232, + 0.5516769372982315, + 0.489387815079083, + 0.4109458413807676, + 0.4004301068969357 + ], + "population_cv": 0.4737558240032668 + }, + "activation_output_rms_by_block": [ + 0.02468283288180828, + 0.04329179599881172, + 0.023283371701836586, + 0.03508218377828598, + 0.025414658710360527, + 0.03976430371403694, + 0.02767801284790039, + 0.03781749680638313, + 0.03550800308585167, + 0.04360951855778694, + 0.040389809757471085, + 0.05188606306910515, + 0.03616104647517204, + 0.06982707977294922, + 0.04418032616376877, + 0.09354322403669357 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03158504609018564, + "first_to_last_ratio": 0.5184002104527677, + "imbalance_abs_log_ratio": 0.6570077279781673, + "last_quartile_mean": 0.0609279191121459, + "mean": 0.042007482959888875, + "normalized": [ + 0.5875818102545408, + 1.0305734347413573, + 0.5542672414833535, + 0.8351412964157943, + 0.6050031308618963, + 0.9466004842995747, + 0.6588829155588524, + 0.9002561958423794, + 0.8452780453367492, + 1.038136910022145, + 0.9614908323842579, + 1.2351623904400295, + 0.8608239277202269, + 1.662253361850413, + 1.0517251463496313, + 2.2268228764387987 + ], + "population_cv": 0.4130647858768022 + }, + "bits_per_byte": 3.137961051112634, + "core_parameter_grad_rms_by_block": [ + 0.0008395536207241178, + 0.0006692968987216388, + 0.0005333404576068953, + 0.0004419910350210091, + 0.0004333787114417237, + 0.0004609433225170186, + 0.0004705681902150354, + 0.0005107414594492524, + 0.0005339389183678795, + 0.0004946080122888242, + 0.000448918054550765, + 0.00035668617352665593, + 0.00032572588316331664, + 0.000267052972482006, + 0.0002153120435976381, + 0.0002101978579413297 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006210455030184152, + "first_to_last_ratio": 2.439565392966498, + "imbalance_abs_log_ratio": 0.8918199058035992, + "last_quartile_mean": 0.00025457218929607263, + "mean": 0.0004507658507259441, + "normalized": [ + 1.8625049332642285, + 1.484799475478804, + 1.183187361571351, + 0.980533539329114, + 0.9614275587730993, + 1.022578178392804, + 1.04393043407614, + 1.1330526893878927, + 1.1845150148530281, + 1.0972614972768537, + 0.9959007627303548, + 0.7912892535053935, + 0.7226055004804499, + 0.5924427772244185, + 0.47765828589473863, + 0.46631273776132937 + ], + "population_cv": 0.34661834917809303 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6546831130981445, + "mean_weights": [ + 0.3866303563117981, + 0.6133696436882019 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6832369565963745, + "mean_weights": [ + 0.48925086855888367, + 0.510749101638794 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6867785453796387, + "mean_weights": [ + 0.5347544550895691, + 0.4652456045150757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6725893020629883, + "mean_weights": [ + 0.4844614565372467, + 0.5155385732650757 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0954453945159912, + "mean_weights": [ + 0.332688570022583, + 0.3308390974998474, + 0.3364723324775696 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0907697677612305, + "mean_weights": [ + 0.3209903836250305, + 0.33586689829826355, + 0.34314271807670593 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0933208465576172, + "mean_weights": [ + 0.33219069242477417, + 0.312181293964386, + 0.35562801361083984 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0861705541610718, + "mean_weights": [ + 0.319164514541626, + 0.32808834314346313, + 0.3527471721172333 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3827753067016602, + "mean_weights": [ + 0.24703660607337952, + 0.23925676941871643, + 0.2577625513076782, + 0.25594407320022583 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3791232109069824, + "mean_weights": [ + 0.24712958931922913, + 0.2454403042793274, + 0.24896711111068726, + 0.25846296548843384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3816200494766235, + "mean_weights": [ + 0.2438572347164154, + 0.2418915331363678, + 0.2618292272090912, + 0.252422034740448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765439987182617, + "mean_weights": [ + 0.24315571784973145, + 0.25071609020233154, + 0.23800383508205414, + 0.26812440156936646 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6069204807281494, + "mean_weights": [ + 0.1977769434452057, + 0.19419988989830017, + 0.19346478581428528, + 0.20617546141147614, + 0.20838290452957153 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6016991138458252, + "mean_weights": [ + 0.19928976893424988, + 0.19587931036949158, + 0.19414740800857544, + 0.18995173275470734, + 0.22073179483413696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6063685417175293, + "mean_weights": [ + 0.19720253348350525, + 0.19060829281806946, + 0.18910765647888184, + 0.20065464079380035, + 0.2224268764257431 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6036955118179321, + "mean_weights": [ + 0.19818633794784546, + 0.20014071464538574, + 0.19487890601158142, + 0.18811310827732086, + 0.2186809629201889 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7846078872680664, + "mean_weights": [ + 0.15616348385810852, + 0.1693512499332428, + 0.1684132218360901, + 0.14385344088077545, + 0.16338428854942322, + 0.19883427023887634 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7881443500518799, + "mean_weights": [ + 0.16932255029678345, + 0.1670200079679489, + 0.16782812774181366, + 0.15219610929489136, + 0.1653819978237152, + 0.1782512068748474 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7859947681427002, + "mean_weights": [ + 0.15620459616184235, + 0.18192027509212494, + 0.16651901602745056, + 0.14659695327281952, + 0.16306468844413757, + 0.18569447100162506 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7886135578155518, + "mean_weights": [ + 0.17137612402439117, + 0.16813044250011444, + 0.16593416035175323, + 0.15192663669586182, + 0.16282179951667786, + 0.17981085181236267 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937737226486206, + "mean_weights": [ + 0.12751945853233337, + 0.15645764768123627, + 0.14123380184173584, + 0.1275305449962616, + 0.13446006178855896, + 0.15007954835891724, + 0.16271892189979553 + ], + "sources": 7 + }, + { + "entropy_mean": 1.943496823310852, + "mean_weights": [ + 0.14288359880447388, + 0.1535882204771042, + 0.14250940084457397, + 0.1380811333656311, + 0.1316976547241211, + 0.13936954736709595, + 0.15187042951583862 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273982048034668, + "mean_weights": [ + 0.12311284244060516, + 0.16783711314201355, + 0.141703262925148, + 0.11705678701400757, + 0.12322613596916199, + 0.14240297675132751, + 0.184660866856575 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937564730644226, + "mean_weights": [ + 0.13963916897773743, + 0.16193091869354248, + 0.14834126830101013, + 0.12877905368804932, + 0.1304946392774582, + 0.13670659065246582, + 0.15410837531089783 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0582637786865234, + "mean_weights": [ + 0.10422320663928986, + 0.14683584868907928, + 0.12450192868709564, + 0.10290674865245819, + 0.1043052226305008, + 0.1136581152677536, + 0.13712583482265472, + 0.1664431095123291 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073244333267212, + "mean_weights": [ + 0.11922207474708557, + 0.13852310180664062, + 0.1310172826051712, + 0.1193084791302681, + 0.1166592389345169, + 0.11741948127746582, + 0.12622521817684174, + 0.13162514567375183 + ], + "sources": 8 + }, + { + "entropy_mean": 2.05503249168396, + "mean_weights": [ + 0.10400840640068054, + 0.1620255410671234, + 0.13802370429039001, + 0.10328710824251175, + 0.10247690975666046, + 0.11085900664329529, + 0.13948379456996918, + 0.13983553647994995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0654377937316895, + "mean_weights": [ + 0.12162524461746216, + 0.14755912125110626, + 0.1359378546476364, + 0.11907999217510223, + 0.10864339768886566, + 0.11126887798309326, + 0.12606029212474823, + 0.12982521951198578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.158114194869995, + "mean_weights": [ + 0.08706305176019669, + 0.1523345708847046, + 0.1240597814321518, + 0.09008736908435822, + 0.08798909187316895, + 0.09299354255199432, + 0.11455917358398438, + 0.09811963140964508, + 0.15279380977153778 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1800241470336914, + "mean_weights": [ + 0.10917439311742783, + 0.13041174411773682, + 0.1256810426712036, + 0.10147396475076675, + 0.10038554668426514, + 0.10393467545509338, + 0.11099927127361298, + 0.11076852679252625, + 0.10717083513736725 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1717891693115234, + "mean_weights": [ + 0.09730195999145508, + 0.15644855797290802, + 0.1323733627796173, + 0.10013202577829361, + 0.09525948762893677, + 0.09674295783042908, + 0.12173393368721008, + 0.10994285345077515, + 0.09006483852863312 + ], + "sources": 9 + } + ], + "loss_nats": 2.1750688552856445, + "output_weights": { + "entropy_mean": 2.173111915588379, + "mean_weights": [ + 0.1217261254787445, + 0.09291160106658936, + 0.09640607982873917, + 0.10597392916679382, + 0.09402626007795334, + 0.09312883019447327, + 0.11353092640638351, + 0.147413432598114, + 0.13488279283046722 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 7.532809104304761e-05, + 7.798708247719333e-05, + 7.679135887883604e-05, + 7.304617611225694e-05, + 7.868888496886939e-05, + 7.320931035792455e-05, + 8.902617992134765e-05, + 7.941434887470677e-05, + 8.246234938269481e-05, + 7.884583465056494e-05, + 0.0001634695363463834, + 0.00012072100798832253, + 0.000141287746373564, + 8.843849354889244e-05, + 5.323962977854535e-05, + 4.3788353650597855e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.578817712783348e-05, + "first_to_last_ratio": 0.9277698246768501, + "imbalance_abs_log_ratio": 0.07497161071478954, + "last_quartile_mean": 8.168855583789991e-05, + "mean": 8.723402402210922e-05, + "normalized": [ + 0.8635173246617158, + 0.8939984524550617, + 0.8802913741474712, + 0.8373587820933674, + 0.9020435071174282, + 0.8392288579897432, + 1.0205442305262016, + 0.9103598024387758, + 0.9453003034893238, + 0.9038426867776004, + 1.8739194732659872, + 1.383875263598136, + 1.6196403634636298, + 1.0138073365327953, + 0.6103080807673381, + 0.5019641606754243 + ], + "population_cv": 0.3374467083690701 + }, + "activation_output_rms_by_block": [ + 0.09266375005245209, + 0.20113667845726013, + 0.14358533918857574, + 0.17041823267936707, + 0.05199930816888809, + 0.07249397784471512, + 0.03721517324447632, + 0.07351206243038177, + 0.05156216770410538, + 0.08937478810548782, + 0.062482453882694244, + 0.09533937275409698, + 0.05472978577017784, + 0.08922836184501648, + 0.08201298862695694, + 0.20069743692874908 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15195100009441376, + "first_to_last_ratio": 1.4245342605399758, + "imbalance_abs_log_ratio": 0.353844925589721, + "last_quartile_mean": 0.10666714329272509, + "mean": 0.09802824235521257, + "normalized": [ + 0.9452760533712127, + 2.0518237767482, + 1.4647344044819621, + 1.7384605557023458, + 0.5304523157771692, + 0.739521347144304, + 0.37963725912400215, + 0.7499069723601224, + 0.5259929839123919, + 0.9117248861979151, + 0.6373923716420872, + 0.9725704599356962, + 0.5583063049509795, + 0.9102311711526044, + 0.8366261258645934, + 2.0473430116344136 + ], + "population_cv": 0.5182803671925184 + }, + "bits_per_byte": 2.0687232408845437, + "core_parameter_grad_rms_by_block": [ + 0.0003765937729119843, + 0.0003149609092301355, + 0.00027693435492303714, + 0.0001635333627570268, + 0.000180917699642147, + 0.00017837059454152616, + 0.000200147742296221, + 0.00024611271555124715, + 0.0002957900398296376, + 0.0002986151161329657, + 0.0009593337211615513, + 0.0007552262449127961, + 0.0007605792150238727, + 0.0005098371211048817, + 0.00029447700210242914, + 0.0002817503499702034 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00028300559995554593, + "first_to_last_ratio": 0.6130161476493403, + "imbalance_abs_log_ratio": 0.48936400138742103, + "last_quartile_mean": 0.00046166092205034674, + "mean": 0.00038082374763072894, + "normalized": [ + 0.9888925657996352, + 0.827051651031862, + 0.727198229222749, + 0.4294200762805349, + 0.47506937465878935, + 0.46838096534485474, + 0.5255652872002538, + 0.6462640974530133, + 0.7767111207477916, + 0.7841294509357175, + 2.5191016241239836, + 1.9831385243472572, + 1.997194817171378, + 1.3387744968027897, + 0.7732632324914067, + 0.7398444863879827 + ], + "population_cv": 0.6091669692594756 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.573702335357666, + "mean_weights": [ + 0.7102901935577393, + 0.28970980644226074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6441590189933777, + "mean_weights": [ + 0.564466118812561, + 0.43553388118743896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3999374806880951, + "mean_weights": [ + 0.8575316667556763, + 0.14246836304664612 + ], + "sources": 2 + }, + { + "entropy_mean": 0.676960825920105, + "mean_weights": [ + 0.46661365032196045, + 0.5333863496780396 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9920796155929565, + "mean_weights": [ + 0.5238763093948364, + 0.2048875242471695, + 0.2712361812591553 + ], + "sources": 3 + }, + { + "entropy_mean": 1.092700719833374, + "mean_weights": [ + 0.2899855673313141, + 0.34371256828308105, + 0.36630189418792725 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9979273080825806, + "mean_weights": [ + 0.49421679973602295, + 0.18891796469688416, + 0.3168652057647705 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091429591178894, + "mean_weights": [ + 0.3115330636501312, + 0.3673863708972931, + 0.3210805654525757 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2927134037017822, + "mean_weights": [ + 0.35746461153030396, + 0.11402665078639984, + 0.22846516966819763, + 0.3000435531139374 + ], + "sources": 4 + }, + { + "entropy_mean": 1.378767490386963, + "mean_weights": [ + 0.2549898624420166, + 0.23654252290725708, + 0.21668663620948792, + 0.2917810082435608 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2672696113586426, + "mean_weights": [ + 0.3839718699455261, + 0.12064628303050995, + 0.1715085655450821, + 0.32387328147888184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.375225305557251, + "mean_weights": [ + 0.23853997886180878, + 0.24968335032463074, + 0.2089165896177292, + 0.3028600811958313 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5798927545547485, + "mean_weights": [ + 0.22726407647132874, + 0.12221817672252655, + 0.18549029529094696, + 0.23680329322814941, + 0.22822415828704834 + ], + "sources": 5 + }, + { + "entropy_mean": 1.594498634338379, + "mean_weights": [ + 0.21032124757766724, + 0.19536492228507996, + 0.1600019335746765, + 0.1794220358133316, + 0.2548898458480835 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5424928665161133, + "mean_weights": [ + 0.23003876209259033, + 0.11985675245523453, + 0.1379462629556656, + 0.19240085780620575, + 0.319757342338562 + ], + "sources": 5 + }, + { + "entropy_mean": 1.595255732536316, + "mean_weights": [ + 0.2046268880367279, + 0.19130583107471466, + 0.17092418670654297, + 0.17953911423683167, + 0.253603994846344 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7457678318023682, + "mean_weights": [ + 0.17887650430202484, + 0.11389653384685516, + 0.11488981544971466, + 0.1474636197090149, + 0.20503896474838257, + 0.23983454704284668 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7712774276733398, + "mean_weights": [ + 0.16810789704322815, + 0.16247668862342834, + 0.1383969932794571, + 0.13383737206459045, + 0.1661752164363861, + 0.23100584745407104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.675686001777649, + "mean_weights": [ + 0.16882894933223724, + 0.11588791012763977, + 0.08043210208415985, + 0.1250714659690857, + 0.19613179564476013, + 0.3136477470397949 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7837879657745361, + "mean_weights": [ + 0.17464125156402588, + 0.18035069108009338, + 0.1619444340467453, + 0.15330034494400024, + 0.15676669776439667, + 0.17299658060073853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6526988744735718, + "mean_weights": [ + 0.09642067551612854, + 0.07173497974872589, + 0.05561239272356033, + 0.09171071648597717, + 0.11456114053726196, + 0.12951162457466125, + 0.44044846296310425 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8849880695343018, + "mean_weights": [ + 0.13606244325637817, + 0.13500544428825378, + 0.12342768162488937, + 0.11247340589761734, + 0.11233137547969818, + 0.11334770917892456, + 0.2673519253730774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.644601583480835, + "mean_weights": [ + 0.10291393101215363, + 0.08116044104099274, + 0.05550362914800644, + 0.08702410012483597, + 0.11089185625314713, + 0.11528079956769943, + 0.4472252428531647 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9121408462524414, + "mean_weights": [ + 0.14125728607177734, + 0.1494411677122116, + 0.12786200642585754, + 0.11922307312488556, + 0.1195257231593132, + 0.11599892377853394, + 0.2266918122768402 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7500674724578857, + "mean_weights": [ + 0.07664570212364197, + 0.0604618638753891, + 0.041497331112623215, + 0.06405743956565857, + 0.0716431736946106, + 0.07514288276433945, + 0.270142525434494, + 0.3404091000556946 + ], + "sources": 8 + }, + { + "entropy_mean": 1.952926754951477, + "mean_weights": [ + 0.10562807321548462, + 0.10351885855197906, + 0.07600137591362, + 0.08402621746063232, + 0.08850346505641937, + 0.09127135574817657, + 0.15880608558654785, + 0.2922445833683014 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7527103424072266, + "mean_weights": [ + 0.08054207265377045, + 0.06648321449756622, + 0.04810335487127304, + 0.06408451497554779, + 0.07014857977628708, + 0.07217346131801605, + 0.20459653437137604, + 0.39386826753616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9524379968643188, + "mean_weights": [ + 0.10777557641267776, + 0.09555133432149887, + 0.06806209683418274, + 0.08129536360502243, + 0.0918373167514801, + 0.10075562447309494, + 0.2000727653503418, + 0.25464993715286255 + ], + "sources": 8 + }, + { + "entropy_mean": 2.000408887863159, + "mean_weights": [ + 0.07838036119937897, + 0.07984089851379395, + 0.04941036179661751, + 0.06521518528461456, + 0.06765615195035934, + 0.06713232398033142, + 0.18892306089401245, + 0.23950761556625366, + 0.16393405199050903 + ], + "sources": 9 + }, + { + "entropy_mean": 2.096604347229004, + "mean_weights": [ + 0.09755367040634155, + 0.08867941796779633, + 0.060870569199323654, + 0.06980546563863754, + 0.08883253484964371, + 0.09541568160057068, + 0.17317476868629456, + 0.19020704925060272, + 0.13546085357666016 + ], + "sources": 9 + }, + { + "entropy_mean": 2.027402877807617, + "mean_weights": [ + 0.08629745244979858, + 0.08307671546936035, + 0.05121873319149017, + 0.0663638487458229, + 0.07480055093765259, + 0.08018200099468231, + 0.15043827891349792, + 0.25634968280792236, + 0.1512727439403534 + ], + "sources": 9 + } + ], + "loss_nats": 1.433929681777954, + "output_weights": { + "entropy_mean": 1.913235068321228, + "mean_weights": [ + 0.09498223662376404, + 0.06303347647190094, + 0.06038350611925125, + 0.06479769200086594, + 0.05589054524898529, + 0.06028280779719353, + 0.07675591856241226, + 0.2476816177368164, + 0.27619218826293945 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 5.492530908668414e-05, + 5.6656335800653324e-05, + 6.325786671368405e-05, + 5.9555983170866966e-05, + 6.814395601395518e-05, + 6.255660991882905e-05, + 7.046976679703221e-05, + 6.272071914281696e-05, + 7.296914554899558e-05, + 6.464503530878574e-05, + 0.00014864247350487858, + 0.00010719253623392433, + 0.0001445752423023805, + 9.124213829636574e-05, + 6.016199040459469e-05, + 4.548503056867048e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.859887369297212e-05, + "first_to_last_ratio": 0.6864419649392262, + "imbalance_abs_log_ratio": 0.37623359494524666, + "last_quartile_mean": 8.536610039300285e-05, + "mean": 7.707500867581984e-05, + "normalized": [ + 0.7126215102706234, + 0.7350804985173837, + 0.8207312305308824, + 0.772701608395031, + 0.8841251812318441, + 0.8116328625007921, + 0.9143011205283218, + 0.8137620769739057, + 0.9467289955931933, + 0.8387288748897194, + 1.9285430654970663, + 1.3907560709435642, + 1.8757732861305145, + 1.18380964029642, + 0.7805641729816484, + 0.5901398047190899 + ], + "population_cv": 0.3863401678553132 + }, + "activation_output_rms_by_block": [ + 0.14486487209796906, + 0.30995267629623413, + 0.2574450671672821, + 0.28713259100914, + 0.05978552997112274, + 0.08792455494403839, + 0.04697949066758156, + 0.09852420538663864, + 0.07244286686182022, + 0.13140366971492767, + 0.08212646842002869, + 0.12985831499099731, + 0.06840891391038895, + 0.11060241609811783, + 0.0924442708492279, + 0.23933660984039307 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24984880164265633, + "first_to_last_ratio": 1.9565592145673032, + "imbalance_abs_log_ratio": 0.6711874277699096, + "last_quartile_mean": 0.12769805267453194, + "mean": 0.13870203238911927, + "normalized": [ + 1.0444322235420485, + 2.234665714390418, + 1.8561016210998063, + 2.0701397525568246, + 0.43103571693454673, + 0.6339096365752733, + 0.3387080193301259, + 0.7103299330916477, + 0.5222913147990983, + 0.9473809968860692, + 0.5921071739571081, + 0.9362394534110972, + 0.4932077254533107, + 0.7974102051210767, + 0.6664954309384717, + 1.725545081913077 + ], + "population_cv": 0.5979328649415788 + }, + "bits_per_byte": 1.9415087059372296, + "core_parameter_grad_rms_by_block": [ + 0.0003274625120219605, + 0.0002950920627246982, + 0.0002675536202763891, + 0.00011636112532418193, + 0.00013949143839672265, + 0.0001705676155207084, + 0.00017763939173179324, + 0.00023071980568273914, + 0.00033685005198124334, + 0.0003711494248343089, + 0.0010020275600187679, + 0.0008489747430264115, + 0.0009447094037588545, + 0.0006663788253529681, + 0.0003910019531118643, + 0.00034755869426738895 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00025161733008680743, + "first_to_last_ratio": 0.4283488185904071, + "imbalance_abs_log_ratio": 0.8478174186232621, + "last_quartile_mean": 0.000587412219122769, + "mean": 0.00041459613925193755, + "normalized": [ + 0.789834928547107, + 0.7117578645501561, + 0.6453355324512677, + 0.2806613817826046, + 0.3364513684290676, + 0.4114066663246461, + 0.4284636901161473, + 0.5564928947457894, + 0.8124775416120065, + 0.895207141832, + 2.4168762444984226, + 2.047715023488232, + 2.278625665601732, + 1.607296263190791, + 0.9430911581023291, + 0.8383066347277007 + ], + "population_cv": 0.6738437321539833 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4194117784500122, + "mean_weights": [ + 0.8254866600036621, + 0.1745133101940155 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5809611082077026, + "mean_weights": [ + 0.6678193211555481, + 0.3321806788444519 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31704896688461304, + "mean_weights": [ + 0.8999109268188477, + 0.10008908808231354 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6732906103134155, + "mean_weights": [ + 0.4609490633010864, + 0.5390509366989136 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9927794337272644, + "mean_weights": [ + 0.4832823872566223, + 0.17606672644615173, + 0.34065088629722595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832945108413696, + "mean_weights": [ + 0.2762538492679596, + 0.3156396150588989, + 0.40810656547546387 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9905548095703125, + "mean_weights": [ + 0.4605642259120941, + 0.16346418857574463, + 0.37597161531448364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.09372079372406, + "mean_weights": [ + 0.32242417335510254, + 0.34579548239707947, + 0.3317803740501404 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2746227979660034, + "mean_weights": [ + 0.37224799394607544, + 0.09596331417560577, + 0.2427763193845749, + 0.2890123426914215 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3707606792449951, + "mean_weights": [ + 0.27432072162628174, + 0.2153838574886322, + 0.20746882259845734, + 0.30282658338546753 + ], + "sources": 4 + }, + { + "entropy_mean": 1.198214054107666, + "mean_weights": [ + 0.44253695011138916, + 0.10026887059211731, + 0.13153521716594696, + 0.325658917427063 + ], + "sources": 4 + }, + { + "entropy_mean": 1.372360110282898, + "mean_weights": [ + 0.24982473254203796, + 0.2343549281358719, + 0.2051662802696228, + 0.31065404415130615 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5764262676239014, + "mean_weights": [ + 0.2228686511516571, + 0.11472160369157791, + 0.20744341611862183, + 0.22938773036003113, + 0.22557863593101501 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5863620042800903, + "mean_weights": [ + 0.2179623544216156, + 0.17305074632167816, + 0.15634828805923462, + 0.18247520923614502, + 0.2701634168624878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4903430938720703, + "mean_weights": [ + 0.2547823488712311, + 0.09995707869529724, + 0.10266982018947601, + 0.1966245472431183, + 0.3459661602973938 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5923073291778564, + "mean_weights": [ + 0.22257764637470245, + 0.17832019925117493, + 0.1619993895292282, + 0.1890055239200592, + 0.2480972558259964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.691253662109375, + "mean_weights": [ + 0.18423792719841003, + 0.09657466411590576, + 0.08509969711303711, + 0.15257084369659424, + 0.18859976530075073, + 0.29291707277297974 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7595478296279907, + "mean_weights": [ + 0.1724337935447693, + 0.14752474427223206, + 0.13264453411102295, + 0.13973915576934814, + 0.15476800501346588, + 0.2528897523880005 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5512144565582275, + "mean_weights": [ + 0.16704371571540833, + 0.08849328011274338, + 0.04574323818087578, + 0.12566779553890228, + 0.1718127727508545, + 0.40123921632766724 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7799327373504639, + "mean_weights": [ + 0.18437626957893372, + 0.17012780904769897, + 0.17119532823562622, + 0.15378496050834656, + 0.1488233208656311, + 0.17169234156608582 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5764116048812866, + "mean_weights": [ + 0.09582588821649551, + 0.05901939421892166, + 0.040849827229976654, + 0.09081389009952545, + 0.10021352022886276, + 0.13485172390937805, + 0.4784257411956787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8431057929992676, + "mean_weights": [ + 0.13672879338264465, + 0.12035658955574036, + 0.11983893811702728, + 0.10482897609472275, + 0.10095320641994476, + 0.10664592683315277, + 0.3106476068496704 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5526533126831055, + "mean_weights": [ + 0.10306400060653687, + 0.06465981900691986, + 0.04082629829645157, + 0.08607284724712372, + 0.09878605604171753, + 0.11178258061408997, + 0.4948084056377411 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8949611186981201, + "mean_weights": [ + 0.1451701521873474, + 0.13929708302021027, + 0.1330569088459015, + 0.11840799450874329, + 0.10893036425113678, + 0.10698798298835754, + 0.24814949929714203 + ], + "sources": 7 + }, + { + "entropy_mean": 1.676889419555664, + "mean_weights": [ + 0.07736113667488098, + 0.04877929389476776, + 0.032506294548511505, + 0.06437233090400696, + 0.06250663101673126, + 0.06869478523731232, + 0.2701185345649719, + 0.3756609857082367 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8798964023590088, + "mean_weights": [ + 0.10042576491832733, + 0.08721707761287689, + 0.06748545169830322, + 0.0845440924167633, + 0.07678259164094925, + 0.08073122054338455, + 0.1554347574710846, + 0.34737905859947205 + ], + "sources": 8 + }, + { + "entropy_mean": 1.663856029510498, + "mean_weights": [ + 0.07920834422111511, + 0.05299394577741623, + 0.03748705983161926, + 0.06209969520568848, + 0.060945168137550354, + 0.06926221400499344, + 0.19488413631916046, + 0.44311946630477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9090958833694458, + "mean_weights": [ + 0.1065661758184433, + 0.0803120881319046, + 0.06028372794389725, + 0.08741366118192673, + 0.08287163078784943, + 0.09473372995853424, + 0.1887454390525818, + 0.29907354712486267 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8632590770721436, + "mean_weights": [ + 0.0691547840833664, + 0.05765335261821747, + 0.03405343368649483, + 0.05519368499517441, + 0.05223262310028076, + 0.057386673986911774, + 0.1615200936794281, + 0.25146448612213135, + 0.2613408863544464 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0701708793640137, + "mean_weights": [ + 0.09429951012134552, + 0.07856893539428711, + 0.04940951615571976, + 0.076503224670887, + 0.08159374445676804, + 0.0895766168832779, + 0.16711056232452393, + 0.2026878148317337, + 0.16025006771087646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9422186613082886, + "mean_weights": [ + 0.07884211093187332, + 0.06442099809646606, + 0.037228696048259735, + 0.06615173816680908, + 0.06501315534114838, + 0.0695233941078186, + 0.13319489359855652, + 0.2805612087249756, + 0.20506379008293152 + ], + "sources": 9 + } + ], + "loss_nats": 1.3457512855529785, + "output_weights": { + "entropy_mean": 1.7949464321136475, + "mean_weights": [ + 0.0834353119134903, + 0.05303390324115753, + 0.048570070415735245, + 0.05988207459449768, + 0.04784438759088516, + 0.05165069177746773, + 0.06509792059659958, + 0.2334408313035965, + 0.3570447862148285 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 6.899879372213036e-05, + 7.050983549561352e-05, + 8.400688238907605e-05, + 8.067955786827952e-05, + 9.6201452834066e-05, + 8.90634983079508e-05, + 0.00010480848868610337, + 9.241609222954139e-05, + 0.00012084992340533063, + 0.00010282631410518661, + 0.00023664520995225757, + 0.0001637248496990651, + 0.00023947370937094092, + 0.0001411983830621466, + 0.00010364947956986725, + 6.421437865355983e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.604876736877486e-05, + "first_to_last_ratio": 0.5545581271583456, + "imbalance_abs_log_ratio": 0.5895836496274593, + "last_quartile_mean": 0.00013713398766412865, + "mean": 0.00011620417808444472, + "normalized": [ + 0.5937720558721171, + 0.6067753901617422, + 0.7229248016196876, + 0.6942913688494942, + 0.8278656965686476, + 0.7664397251123708, + 0.9019339098972828, + 0.7952906147865294, + 1.0399791590755874, + 0.8848762221825061, + 2.0364604255476015, + 1.4089411619958057, + 2.060801197672231, + 1.2150886946555293, + 0.891960007621636, + 0.5525995683812308 + ], + "population_cv": 0.4517546138899352 + }, + "activation_output_rms_by_block": [ + 0.15090593695640564, + 0.3367030918598175, + 0.29793402552604675, + 0.3255155086517334, + 0.05378299206495285, + 0.08782409131526947, + 0.044081781059503555, + 0.09709836542606354, + 0.0669482871890068, + 0.12247904390096664, + 0.07360157370567322, + 0.12167256325483322, + 0.06349270045757294, + 0.10058031976222992, + 0.07551935315132141, + 0.19356463849544525 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2777646407485008, + "first_to_last_ratio": 2.565024996839381, + "imbalance_abs_log_ratio": 0.941968223931731, + "last_quartile_mean": 0.10828925296664238, + "mean": 0.13823151704855263, + "normalized": [ + 1.0916897982346618, + 2.4357910485895444, + 2.155326309711265, + 2.3548573842056504, + 0.38907908422984344, + 0.6353405734845695, + 0.3188981934128684, + 0.7024329002477675, + 0.4843199916954669, + 0.8860428252259355, + 0.5324514645948746, + 0.8802085505008006, + 0.45932144718683565, + 0.7276221943430017, + 0.5463251417894509, + 1.400293092547464 + ], + "population_cv": 0.6866316887805146 + }, + "bits_per_byte": 1.8676321823979032, + "core_parameter_grad_rms_by_block": [ + 0.00046247293295990515, + 0.0003824975682884986, + 0.00041690386457328876, + 0.00021732383526524047, + 0.00022655054548791778, + 0.0002917516444309179, + 0.000280629308661355, + 0.0003847878626551635, + 0.0005123546145807317, + 0.0005899763687075028, + 0.0015400377759773427, + 0.0012824907104132966, + 0.001537981149104749, + 0.001057571910144345, + 0.0006563117026664665, + 0.0005021677227557722 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036979955027173326, + "first_to_last_ratio": 0.394029142562531, + "imbalance_abs_log_ratio": 0.9313304065247964, + "last_quartile_mean": 0.0009385081211678332, + "mean": 0.0006463630947920308, + "normalized": [ + 0.7155002144865761, + 0.5917688855852271, + 0.6449994870258314, + 0.3362256246006202, + 0.3505004343739815, + 0.45137423033842894, + 0.434166664097133, + 0.595312241301416, + 0.792673063652533, + 0.9127630792369563, + 2.3826202151483513, + 1.9841645055956383, + 2.3794383706260316, + 1.6361885736756394, + 1.0153916706485797, + 0.7769127396070564 + ], + "population_cv": 0.675897465620864 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.43482720851898193, + "mean_weights": [ + 0.8150075674057007, + 0.18499243259429932 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611247420310974, + "mean_weights": [ + 0.6986296772956848, + 0.3013703525066376 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31192272901535034, + "mean_weights": [ + 0.9031083583831787, + 0.09689167141914368 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6764674186706543, + "mean_weights": [ + 0.4619978070259094, + 0.5380021929740906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9888706207275391, + "mean_weights": [ + 0.48646074533462524, + 0.1758481115102768, + 0.33769115805625916 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0781972408294678, + "mean_weights": [ + 0.276906281709671, + 0.2993447184562683, + 0.42374899983406067 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9839773178100586, + "mean_weights": [ + 0.46118396520614624, + 0.15739500522613525, + 0.3814210295677185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0957449674606323, + "mean_weights": [ + 0.3292894959449768, + 0.32970887422561646, + 0.3410016596317291 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2506439685821533, + "mean_weights": [ + 0.3648650050163269, + 0.0835169106721878, + 0.212466761469841, + 0.3391513228416443 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655824661254883, + "mean_weights": [ + 0.2861315608024597, + 0.20811524987220764, + 0.20031729340553284, + 0.3054359257221222 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1652674674987793, + "mean_weights": [ + 0.43905186653137207, + 0.08880466222763062, + 0.10791443288326263, + 0.3642290234565735 + ], + "sources": 4 + }, + { + "entropy_mean": 1.363007664680481, + "mean_weights": [ + 0.2524264454841614, + 0.21885012090206146, + 0.19597390294075012, + 0.33274954557418823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5649902820587158, + "mean_weights": [ + 0.22229184210300446, + 0.10384112596511841, + 0.1934116780757904, + 0.2539443373680115, + 0.22651101648807526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5797945261001587, + "mean_weights": [ + 0.21991753578186035, + 0.1605391800403595, + 0.15625253319740295, + 0.18265283107757568, + 0.2806379497051239 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4139833450317383, + "mean_weights": [ + 0.24990326166152954, + 0.08050692081451416, + 0.06835947930812836, + 0.20686665177345276, + 0.394363671541214 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5895863771438599, + "mean_weights": [ + 0.2244223803281784, + 0.16505929827690125, + 0.1643591672182083, + 0.19251328706741333, + 0.2536458969116211 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6177464723587036, + "mean_weights": [ + 0.1738494634628296, + 0.07661910355091095, + 0.06195354461669922, + 0.14467598497867584, + 0.19003543257713318, + 0.35286644101142883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7444759607315063, + "mean_weights": [ + 0.172578826546669, + 0.1303727626800537, + 0.13036483526229858, + 0.13626831769943237, + 0.1541597843170166, + 0.27625542879104614 + ], + "sources": 6 + }, + { + "entropy_mean": 1.470452904701233, + "mean_weights": [ + 0.15901926159858704, + 0.07292591035366058, + 0.03418249264359474, + 0.11740352213382721, + 0.16174395382404327, + 0.45472484827041626 + ], + "sources": 6 + }, + { + "entropy_mean": 1.778969645500183, + "mean_weights": [ + 0.18775978684425354, + 0.15501554310321808, + 0.17432880401611328, + 0.15116485953330994, + 0.14906319975852966, + 0.1826678216457367 + ], + "sources": 6 + }, + { + "entropy_mean": 1.531482219696045, + "mean_weights": [ + 0.09255115687847137, + 0.04949861764907837, + 0.03270748257637024, + 0.08605319261550903, + 0.09672325849533081, + 0.1471216380596161, + 0.4953446388244629 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8260389566421509, + "mean_weights": [ + 0.13673919439315796, + 0.10493747889995575, + 0.11646746844053268, + 0.09983028471469879, + 0.10169579833745956, + 0.1137751042842865, + 0.32655465602874756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4906136989593506, + "mean_weights": [ + 0.10088150203227997, + 0.05257141590118408, + 0.03239753842353821, + 0.08061188459396362, + 0.09571854770183563, + 0.11411523073911667, + 0.523703932762146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8863525390625, + "mean_weights": [ + 0.14810651540756226, + 0.12629631161689758, + 0.13384857773780823, + 0.11124427616596222, + 0.1082591712474823, + 0.11263860762119293, + 0.2596065402030945 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5897703170776367, + "mean_weights": [ + 0.07248121500015259, + 0.03783262521028519, + 0.02471475675702095, + 0.053565122187137604, + 0.058412034064531326, + 0.06658737361431122, + 0.26026394963264465, + 0.42614293098449707 + ], + "sources": 8 + }, + { + "entropy_mean": 1.814478874206543, + "mean_weights": [ + 0.09743618965148926, + 0.07339455187320709, + 0.06428516656160355, + 0.07672879844903946, + 0.07112542539834976, + 0.07841956615447998, + 0.14423531293869019, + 0.3943749666213989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5432119369506836, + "mean_weights": [ + 0.07432855665683746, + 0.04178062081336975, + 0.029931148514151573, + 0.0517733134329319, + 0.053720228374004364, + 0.06251072138547897, + 0.17899465560913086, + 0.5069607496261597 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8694422245025635, + "mean_weights": [ + 0.10448876023292542, + 0.07235300540924072, + 0.05752243846654892, + 0.08492527902126312, + 0.07930260896682739, + 0.08949960768222809, + 0.1757010817527771, + 0.33620721101760864 + ], + "sources": 8 + }, + { + "entropy_mean": 1.746185302734375, + "mean_weights": [ + 0.06162667274475098, + 0.04293470457196236, + 0.02593991532921791, + 0.04546971619129181, + 0.044149380177259445, + 0.052234895527362823, + 0.14164786040782928, + 0.2587880492210388, + 0.3272087574005127 + ], + "sources": 9 + }, + { + "entropy_mean": 2.061832904815674, + "mean_weights": [ + 0.09089995175600052, + 0.06807379424571991, + 0.04443599283695221, + 0.08077649772167206, + 0.08381937444210052, + 0.09205536544322968, + 0.15836367011070251, + 0.20683875679969788, + 0.17473658919334412 + ], + "sources": 9 + }, + { + "entropy_mean": 1.849208116531372, + "mean_weights": [ + 0.07359056174755096, + 0.050991274416446686, + 0.029770921915769577, + 0.0592363178730011, + 0.055861424654722214, + 0.06031768023967743, + 0.11290363222360611, + 0.2822529673576355, + 0.275075227022171 + ], + "sources": 9 + } + ], + "loss_nats": 1.294543981552124, + "output_weights": { + "entropy_mean": 1.636631727218628, + "mean_weights": [ + 0.07672396302223206, + 0.03940216451883316, + 0.03699969872832298, + 0.05207572132349014, + 0.04068697616457939, + 0.04208337888121605, + 0.051315970718860626, + 0.2188674658536911, + 0.4418446719646454 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.030247040746552, + "cross_entropy_nats": 5.566143095493317, + "step": 0 + }, + { + "bits_per_byte": 5.230785295754151, + "cross_entropy_nats": 3.6257040798664093, + "step": 100 + }, + { + "bits_per_byte": 3.16099769723908, + "cross_entropy_nats": 2.191036641597748, + "step": 500 + }, + { + "bits_per_byte": 1.983385019569791, + "cross_entropy_nats": 1.3747777342796326, + "step": 2000 + }, + { + "bits_per_byte": 1.818564586202164, + "cross_entropy_nats": 1.2605329155921936, + "step": 4000 + }, + { + "bits_per_byte": 1.7311772990915377, + "cross_entropy_nats": 1.1999606639146805, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "7517e8197003ce4ba5267d82e82d6a41d49e7ce8744b4095b5a5160f1d4d01c2", + "final_model_state": "9867cf3c0d2372bb731808e8fd98ce5f38db0a40bd3cc08683bc5f66c9472428", + "final_optimizer_state": "fa42da28c86335ac8bab241bdce833e1a77d2508025d738b471764f2dd16cf93", + "final_public_parameters": "8c73d93d872090c7e7d7cdebca989ac8c083cf6d1cb5fb16509c6e6993d3c9a0", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "seed": 2026073002, + "timing": { + "mean_ms": 54.937352295237, + "measured_steps": 7980, + "median_ms": 54.89947448950261, + "p95_ms": 55.28610254696105, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0003044046170543879, + 0.0002627764770295471, + 0.0002327302354387939, + 0.0002064981235889718, + 0.0001888095575850457, + 0.00017376731557305902, + 0.00016044666699599475, + 0.0001515327749075368, + 0.000144502759212628, + 0.00014044572890270501, + 0.00013478762411978096, + 0.00013080857752356678, + 0.00012769536988344043, + 0.00012482800229918212, + 0.00012209055421408266, + 0.0001202532512252219 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002516023632779252, + "first_to_last_ratio": 2.0336961080101905, + "imbalance_abs_log_ratio": 0.7098548803728145, + "last_quartile_mean": 0.00012371679440548178, + "mean": 0.00017039860222212155, + "normalized": [ + 1.7864267258341946, + 1.5421281254819637, + 1.3657989702017657, + 1.2118533890306977, + 1.1080463989893807, + 1.0197696067162934, + 0.941596145177564, + 0.8892841427772256, + 0.8480278437041564, + 0.8242187850791655, + 0.7910136724248463, + 0.7676622684560079, + 0.7493921206993488, + 0.7325647081098923, + 0.7164997401500549, + 0.7057173571674424 + ], + "population_cv": 0.31342321744292045 + }, + "activation_output_rms_by_block": [ + 0.005092422943562269, + 0.007266214583069086, + 0.00560376700013876, + 0.007757550571113825, + 0.005751485005021095, + 0.008497623726725578, + 0.006417509168386459, + 0.009013078175485134, + 0.006685016211122274, + 0.010579346679151058, + 0.007517023477703333, + 0.010370824486017227, + 0.008143824525177479, + 0.011447666212916374, + 0.009210918098688126, + 0.013363312929868698 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006429988774470985, + "first_to_last_ratio": 0.6099730781372781, + "imbalance_abs_log_ratio": 0.4943404569899061, + "last_quartile_mean": 0.01054143044166267, + "mean": 0.008294848987134174, + "normalized": [ + 0.6139259378273111, + 0.8759911837261217, + 0.675571913223562, + 0.93522505149235, + 0.6933803151741528, + 1.0244458627162376, + 0.7736740208701105, + 1.0865873736176486, + 0.8059237993954019, + 1.275411607319227, + 0.9062278878569946, + 1.250272850307826, + 0.9817929823447126, + 1.3800933845417096, + 1.1104383109294493, + 1.6110375186571844 + ], + "population_cv": 0.2672227042610828 + }, + "bits_per_byte": 8.019773211317682, + "core_parameter_grad_rms_by_block": [ + 0.006980641236819168, + 0.006790815718474037, + 0.006855853249106197, + 0.00601204909014858, + 0.005899153963056259, + 0.006300783956032697, + 0.005704077288608638, + 0.005731973180528048, + 0.005353243010845193, + 0.00629832195028187, + 0.005348066742481173, + 0.005021425359765903, + 0.004977505888369926, + 0.004996266682376707, + 0.0052721592596541, + 0.004713786845169383 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006659839823636995, + "first_to_last_ratio": 1.3346560503958143, + "imbalance_abs_log_ratio": 0.2886736185043388, + "last_quartile_mean": 0.004989929668892529, + "mean": 0.005766007713857367, + "normalized": [ + 1.2106541619850226, + 1.1777326801269727, + 1.189012153526888, + 1.0426710106023453, + 1.0230915836062624, + 1.092746362598529, + 0.9892593925776594, + 0.9940973833164453, + 0.9284141257702132, + 1.0923193763936871, + 0.9275164043967956, + 0.8708669167573224, + 0.8632499530667561, + 0.8665036417431853, + 0.914351752770568, + 0.8175131007613471 + ], + "population_cv": 0.12132893681311732 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.558883190155029, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.495960016967729e-05, + 6.749085150659084e-05, + 5.70677402720321e-05, + 5.0705322792055085e-05, + 4.370361421024427e-05, + 3.821287100436166e-05, + 3.268615546403453e-05, + 2.8849746740888804e-05, + 2.5781664589885622e-05, + 2.37614458455937e-05, + 2.279552791151218e-05, + 2.1788309823023155e-05, + 2.0940320609952323e-05, + 2.0449986550374888e-05, + 1.9821660316665657e-05, + 1.9541532310540788e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.255587868508883e-05, + "first_to_last_ratio": 3.098608919721194, + "imbalance_abs_log_ratio": 1.1309532752053548, + "last_quartile_mean": 2.0188374946883414e-05, + "mean": 3.5534771882339555e-05, + "normalized": [ + 2.1094718271409953, + 1.8992904113768394, + 1.605968949540217, + 1.4269212972563121, + 1.229883066506037, + 1.0753655920710468, + 0.9198358039911673, + 0.811873700397296, + 0.7255334204832441, + 0.6686815360535046, + 0.6414992049756577, + 0.6131546276747519, + 0.5892909817822544, + 0.575492270446749, + 0.5578102592665534, + 0.549927051037374 + ], + "population_cv": 0.4936925451995582 + }, + "activation_output_rms_by_block": [ + 0.009570118971168995, + 0.01747945137321949, + 0.011283415369689465, + 0.020727796480059624, + 0.012305145151913166, + 0.023130038753151894, + 0.015205477364361286, + 0.028237100690603256, + 0.017268717288970947, + 0.03324708342552185, + 0.021605173125863075, + 0.04313390702009201, + 0.02657386288046837, + 0.05039682611823082, + 0.03066091425716877, + 0.0630461722612381 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014765195548534393, + "first_to_last_ratio": 0.34603674681838265, + "imbalance_abs_log_ratio": 1.0612103048863237, + "last_quartile_mean": 0.042669443879276514, + "mean": 0.02649195003323257, + "normalized": [ + 0.3612463015808142, + 0.6598023683153774, + 0.42591864153205383, + 0.7824186763925585, + 0.46448619812724606, + 0.8730968737347248, + 0.5739659536307038, + 1.0658747527147492, + 0.6518477223197376, + 1.2549881523940432, + 0.8155372895827101, + 1.628189203361138, + 1.0030919901001265, + 1.9023449030747457, + 1.1573672084805566, + 2.3798237646587146 + ], + "population_cv": 0.544699846184418 + }, + "bits_per_byte": 5.090461856197109, + "core_parameter_grad_rms_by_block": [ + 0.00025280350140969286, + 0.00022135664689195577, + 0.00022214031151724322, + 0.0001810293778015987, + 0.00018338803861517842, + 0.00016050318916432303, + 0.0001626860438404638, + 0.00015310192441761306, + 0.00014075054270255738, + 0.00013076915091262195, + 0.00013639133469956174, + 0.0001377615870880476, + 0.00013681686670328103, + 0.0001370125878547912, + 0.0001387461931789955, + 0.0001464231652858264 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021933245940512265, + "first_to_last_ratio": 1.5694663694832551, + "imbalance_abs_log_ratio": 0.45073566953494, + "last_quartile_mean": 0.00013974970325572353, + "mean": 0.0001651050288802345, + "normalized": [ + 1.531167785283354, + 1.3407020270262369, + 1.3454484882976008, + 1.0964498115494445, + 1.1107356320939579, + 0.9721278040582907, + 0.9853488106560013, + 0.9273001885888748, + 0.8524909486836794, + 0.7920361469273026, + 0.8260883125401264, + 0.8343875896595401, + 0.8286656538034743, + 0.8298510880257848, + 0.8403511032946221, + 0.8868486095117084 + ], + "population_cv": 0.21861370307121916 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6929093599319458, + "mean_weights": [ + 0.4913179874420166, + 0.5086820125579834 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6919252872467041, + "mean_weights": [ + 0.5235024690628052, + 0.4764975309371948 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930201053619385, + "mean_weights": [ + 0.4991857707500458, + 0.5008142590522766 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6926943063735962, + "mean_weights": [ + 0.5129889249801636, + 0.48701101541519165 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097741961479187, + "mean_weights": [ + 0.3480756878852844, + 0.33060726523399353, + 0.32131701707839966 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982271432876587, + "mean_weights": [ + 0.3435906767845154, + 0.327891081571579, + 0.328518271446228 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097235918045044, + "mean_weights": [ + 0.35132721066474915, + 0.33742234110832214, + 0.3112504482269287 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098421573638916, + "mean_weights": [ + 0.340750515460968, + 0.3319299519062042, + 0.32731953263282776 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3854012489318848, + "mean_weights": [ + 0.26307183504104614, + 0.2547699213027954, + 0.2451326996088028, + 0.23702551424503326 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862006664276123, + "mean_weights": [ + 0.25451934337615967, + 0.25035756826400757, + 0.24778643250465393, + 0.24733665585517883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854570388793945, + "mean_weights": [ + 0.26253488659858704, + 0.2546469569206238, + 0.2452046275138855, + 0.2376135289669037 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861933946609497, + "mean_weights": [ + 0.2505362629890442, + 0.254558801651001, + 0.2493266463279724, + 0.24557828903198242 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6085797548294067, + "mean_weights": [ + 0.2100008726119995, + 0.20838744938373566, + 0.19692982733249664, + 0.19323000311851501, + 0.19145184755325317 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093010902404785, + "mean_weights": [ + 0.19938474893569946, + 0.20374906063079834, + 0.20229801535606384, + 0.19861868023872375, + 0.1959494650363922 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085379123687744, + "mean_weights": [ + 0.20589761435985565, + 0.21123698353767395, + 0.20005744695663452, + 0.1941205859184265, + 0.18868738412857056 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093740463256836, + "mean_weights": [ + 0.19956088066101074, + 0.2027459442615509, + 0.20118315517902374, + 0.1998024433851242, + 0.1967075765132904 + ], + "sources": 5 + }, + { + "entropy_mean": 1.789860725402832, + "mean_weights": [ + 0.17310243844985962, + 0.1810513734817505, + 0.17217648029327393, + 0.1649792194366455, + 0.1559385061264038, + 0.15275199711322784 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791734218597412, + "mean_weights": [ + 0.16535353660583496, + 0.16847901046276093, + 0.1662793755531311, + 0.16693073511123657, + 0.16613470017910004, + 0.1668226420879364 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890782356262207, + "mean_weights": [ + 0.17188119888305664, + 0.18350960314273834, + 0.17409591376781464, + 0.16590778529644012, + 0.15644267201423645, + 0.148162841796875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791684627532959, + "mean_weights": [ + 0.16539111733436584, + 0.16517111659049988, + 0.16458021104335785, + 0.16693061590194702, + 0.16811248660087585, + 0.16981446743011475 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9417986869812012, + "mean_weights": [ + 0.14831508696079254, + 0.16164575517177582, + 0.15476199984550476, + 0.14618372917175293, + 0.13434207439422607, + 0.1278611123561859, + 0.12689024209976196 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456918239593506, + "mean_weights": [ + 0.14120136201381683, + 0.1386352777481079, + 0.14049293100833893, + 0.1421712040901184, + 0.14506074786186218, + 0.1458473801612854, + 0.14659109711647034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9406665563583374, + "mean_weights": [ + 0.14782890677452087, + 0.1637636125087738, + 0.15569329261779785, + 0.14784488081932068, + 0.13445764780044556, + 0.12887068092823029, + 0.12154098600149155 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9455351829528809, + "mean_weights": [ + 0.14214086532592773, + 0.1382882297039032, + 0.1401882916688919, + 0.1396511197090149, + 0.14402973651885986, + 0.14592906832695007, + 0.14977267384529114 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0713937282562256, + "mean_weights": [ + 0.1321096122264862, + 0.1488187313079834, + 0.13977529108524323, + 0.13292929530143738, + 0.12093928456306458, + 0.11470195651054382, + 0.10486428439617157, + 0.10586154460906982 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0790562629699707, + "mean_weights": [ + 0.12411445379257202, + 0.1212267130613327, + 0.12224400043487549, + 0.12172792106866837, + 0.12399555742740631, + 0.12670958042144775, + 0.1301731914281845, + 0.12980858981609344 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068070888519287, + "mean_weights": [ + 0.13267414271831512, + 0.15216735005378723, + 0.1437777280807495, + 0.13539820909500122, + 0.1199566125869751, + 0.11211173236370087, + 0.10534092038869858, + 0.09857328981161118 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079178810119629, + "mean_weights": [ + 0.12415283918380737, + 0.12161671370267868, + 0.12301617860794067, + 0.12307952344417572, + 0.12411122024059296, + 0.12585631012916565, + 0.12885767221450806, + 0.1293095499277115 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1886847019195557, + "mean_weights": [ + 0.11892519891262054, + 0.13333868980407715, + 0.12725916504859924, + 0.12129361927509308, + 0.10894724726676941, + 0.10413059592247009, + 0.09661625325679779, + 0.09465864300727844, + 0.09483058750629425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969594955444336, + "mean_weights": [ + 0.1101110577583313, + 0.10734203457832336, + 0.10833269357681274, + 0.10946638882160187, + 0.11091698706150055, + 0.11324633657932281, + 0.11330728977918625, + 0.11495929211378098, + 0.11231791973114014 + ], + "sources": 9 + }, + { + "entropy_mean": 2.186551570892334, + "mean_weights": [ + 0.11992302536964417, + 0.13721907138824463, + 0.12899263203144073, + 0.12160105258226395, + 0.10788539052009583, + 0.10353681445121765, + 0.09616796672344208, + 0.09295656532049179, + 0.09171747416257858 + ], + "sources": 9 + } + ], + "loss_nats": 3.5284392833709717, + "output_weights": { + "entropy_mean": 2.193685531616211, + "mean_weights": [ + 0.10035791993141174, + 0.09579961001873016, + 0.10334210842847824, + 0.108812615275383, + 0.11718772351741791, + 0.11711248755455017, + 0.12047697603702545, + 0.11920724809169769, + 0.11770330369472504 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00015504666953347623, + 0.00015570323739666492, + 0.00013642148405779153, + 0.00012308824807405472, + 0.00010802906763274223, + 9.803942521102726e-05, + 8.508376777172089e-05, + 7.592589827254415e-05, + 7.549385918537155e-05, + 6.5492422436364e-05, + 6.254414620343596e-05, + 5.1708771934499964e-05, + 4.589578747982159e-05, + 4.085309410584159e-05, + 3.8330261304508895e-05, + 3.7662506656488404e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014256490976549685, + "first_to_last_ratio": 3.504079260905398, + "imbalance_abs_log_ratio": 1.2539277929397175, + "last_quartile_mean": 4.068541238666512e-05, + "mean": 8.470741545352212e-05, + "normalized": [ + 1.8303789426623265, + 1.8381299507608906, + 1.610502260367562, + 1.4530988510869118, + 1.2753200774023898, + 1.1573889332607517, + 1.0044429677872213, + 0.8963311873705286, + 0.8912308182368528, + 0.7731604380291695, + 0.7383550291149319, + 0.6104397313700582, + 0.5418154624845569, + 0.4822847431610892, + 0.45250183941145294, + 0.44461876749330576 + ], + "population_cv": 0.46442228143345987 + }, + "activation_output_rms_by_block": [ + 0.02604864165186882, + 0.04358876869082451, + 0.022780675441026688, + 0.03670342266559601, + 0.027025330811738968, + 0.05019363388419151, + 0.040244098752737045, + 0.04879867285490036, + 0.029798155650496483, + 0.04518449306488037, + 0.02838377095758915, + 0.051337067037820816, + 0.03289651870727539, + 0.0653979703783989, + 0.04346369579434395, + 0.08895192295312881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.032280377112329006, + "first_to_last_ratio": 0.5596699237065876, + "imbalance_abs_log_ratio": 0.5804080909828722, + "last_quartile_mean": 0.05767752695828676, + "mean": 0.04254980245605111, + "normalized": [ + 0.6121918351741814, + 1.0244176511946566, + 0.5353885124274412, + 0.8625991320055194, + 0.6351458585419503, + 1.1796443458470947, + 0.9458116472880084, + 1.1468601506506073, + 0.7003124322674456, + 1.0619201607704427, + 0.6670717446198772, + 1.2065171651700595, + 0.7731297634402319, + 1.5369747120670472, + 1.0214782040230805, + 2.0905366845123567 + ], + "population_cv": 0.38343105399742256 + }, + "bits_per_byte": 3.1146838872489564, + "core_parameter_grad_rms_by_block": [ + 0.0006567095640112879, + 0.00045868421249221203, + 0.00039439577454117915, + 0.0003702348543320087, + 0.0003691077163774564, + 0.00041151853265381964, + 0.0004306774133910072, + 0.00035317417015185023, + 0.00038561746855580054, + 0.0003833308661148803, + 0.00032325470855348767, + 0.0002967607909467712, + 0.00021968458301266372, + 0.00017220390788738176, + 0.00016255203232502606, + 0.0001597780110258732 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00047000610134417196, + "first_to_last_ratio": 2.6322817390176136, + "imbalance_abs_log_ratio": 0.967851051434004, + "last_quartile_mean": 0.0001785546335627362, + "mean": 0.00034673028789829414, + "normalized": [ + 1.8940069181493584, + 1.3228847565424027, + 1.137471366957324, + 1.0677891930819992, + 1.0645384301867686, + 1.1868548754371575, + 1.242110736854166, + 1.0185847111673332, + 1.1121539767789572, + 1.105559218487778, + 0.9322944081778846, + 0.8558836689623711, + 0.6335892498583898, + 0.49665089522809175, + 0.46881405518489694, + 0.46081353894511995 + ], + "population_cv": 0.3565995806724142 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6422993540763855, + "mean_weights": [ + 0.370745986700058, + 0.6292540431022644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6814000606536865, + "mean_weights": [ + 0.4577118158340454, + 0.5422881841659546 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6871446371078491, + "mean_weights": [ + 0.49052417278289795, + 0.5094758868217468 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6793373227119446, + "mean_weights": [ + 0.4676857590675354, + 0.5323142409324646 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0915337800979614, + "mean_weights": [ + 0.29751554131507874, + 0.351987361907959, + 0.3504970669746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0892891883850098, + "mean_weights": [ + 0.3054355978965759, + 0.34222692251205444, + 0.35233747959136963 + ], + "sources": 3 + }, + { + "entropy_mean": 1.093106746673584, + "mean_weights": [ + 0.30644628405570984, + 0.34674984216690063, + 0.3468038737773895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0885257720947266, + "mean_weights": [ + 0.3052448034286499, + 0.3325393795967102, + 0.3622158169746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.381313443183899, + "mean_weights": [ + 0.2415536344051361, + 0.24482353031635284, + 0.25347256660461426, + 0.2601502537727356 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3802430629730225, + "mean_weights": [ + 0.24572397768497467, + 0.2491092085838318, + 0.25083041191101074, + 0.254336416721344 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3813848495483398, + "mean_weights": [ + 0.2464662790298462, + 0.23372246325016022, + 0.2595708668231964, + 0.260240375995636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772881031036377, + "mean_weights": [ + 0.24203142523765564, + 0.2443695217370987, + 0.2359469085931778, + 0.27765214443206787 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6055738925933838, + "mean_weights": [ + 0.20364060997962952, + 0.18423235416412354, + 0.18416231870651245, + 0.20781230926513672, + 0.22015240788459778 + ], + "sources": 5 + }, + { + "entropy_mean": 1.602186679840088, + "mean_weights": [ + 0.19954648613929749, + 0.1888481080532074, + 0.18637022376060486, + 0.21049518883228302, + 0.21474000811576843 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6062731742858887, + "mean_weights": [ + 0.19444647431373596, + 0.1968603879213333, + 0.18490161001682281, + 0.20200669765472412, + 0.2217848002910614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6028468608856201, + "mean_weights": [ + 0.20196445286273956, + 0.19006873667240143, + 0.1860634982585907, + 0.1993759572505951, + 0.22252735495567322 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7881723642349243, + "mean_weights": [ + 0.15337401628494263, + 0.17425060272216797, + 0.1601826399564743, + 0.1535373479127884, + 0.17563557624816895, + 0.18301980197429657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7856398820877075, + "mean_weights": [ + 0.1621149629354477, + 0.15582670271396637, + 0.15492847561836243, + 0.1618887484073639, + 0.1784553974866867, + 0.1867857277393341 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7870616912841797, + "mean_weights": [ + 0.1564463973045349, + 0.18363508582115173, + 0.15811879932880402, + 0.15924012660980225, + 0.15980733931064606, + 0.18275225162506104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7877882719039917, + "mean_weights": [ + 0.17034202814102173, + 0.15881556272506714, + 0.1534808874130249, + 0.16709929704666138, + 0.16602207720279694, + 0.1842401623725891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9344044923782349, + "mean_weights": [ + 0.12371677905321121, + 0.1564711332321167, + 0.13837161660194397, + 0.1230623871088028, + 0.12980599701404572, + 0.152274027466774, + 0.17629806697368622 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9417431354522705, + "mean_weights": [ + 0.14328667521476746, + 0.1450044810771942, + 0.14678549766540527, + 0.13414078950881958, + 0.1328907310962677, + 0.1347782164812088, + 0.16311360895633698 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9288355112075806, + "mean_weights": [ + 0.12207633256912231, + 0.16755375266075134, + 0.13929706811904907, + 0.12109649926424026, + 0.12036697566509247, + 0.15116539597511292, + 0.1784439980983734 + ], + "sources": 7 + }, + { + "entropy_mean": 1.938415288925171, + "mean_weights": [ + 0.1377446949481964, + 0.1613207459449768, + 0.156096950173378, + 0.13170479238033295, + 0.12767548859119415, + 0.13129547238349915, + 0.15416184067726135 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0616660118103027, + "mean_weights": [ + 0.10379506647586823, + 0.1403447985649109, + 0.12122530490159988, + 0.10852085053920746, + 0.10464349389076233, + 0.1186104342341423, + 0.13291975855827332, + 0.1699402928352356 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0691847801208496, + "mean_weights": [ + 0.11521603912115097, + 0.14423911273479462, + 0.13052678108215332, + 0.12005230039358139, + 0.10937757790088654, + 0.12122473865747452, + 0.12272199243307114, + 0.1366414725780487 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054232597351074, + "mean_weights": [ + 0.10057584941387177, + 0.16718831658363342, + 0.1352493166923523, + 0.11277885735034943, + 0.09778623282909393, + 0.11311028897762299, + 0.13487182557582855, + 0.1384393274784088 + ], + "sources": 8 + }, + { + "entropy_mean": 2.060153007507324, + "mean_weights": [ + 0.11611463129520416, + 0.15179042518138885, + 0.14188171923160553, + 0.1159345731139183, + 0.10906632244586945, + 0.12251802533864975, + 0.12206383049488068, + 0.12063045799732208 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1772594451904297, + "mean_weights": [ + 0.09167024493217468, + 0.14425000548362732, + 0.1316833794116974, + 0.09630900621414185, + 0.09528810530900955, + 0.1022176742553711, + 0.12638677656650543, + 0.10320886224508286, + 0.10898594558238983 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191457986831665, + "mean_weights": [ + 0.10587693750858307, + 0.12391996383666992, + 0.11278913170099258, + 0.11095453053712845, + 0.10569566488265991, + 0.11266011744737625, + 0.11442379653453827, + 0.11124123632907867, + 0.10243862122297287 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1550254821777344, + "mean_weights": [ + 0.08980270475149155, + 0.17290523648262024, + 0.13377593457698822, + 0.10608047246932983, + 0.09299036115407944, + 0.1071130782365799, + 0.12428411841392517, + 0.09728670865297318, + 0.07576138526201248 + ], + "sources": 9 + } + ], + "loss_nats": 2.1589343547821045, + "output_weights": { + "entropy_mean": 2.1796395778656006, + "mean_weights": [ + 0.12411575019359589, + 0.09159804880619049, + 0.10466736555099487, + 0.10315616428852081, + 0.09333926439285278, + 0.09817550331354141, + 0.11351706087589264, + 0.1361408531665802, + 0.1352899968624115 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 7.91841812315397e-05, + 8.04664014140144e-05, + 6.887180643389001e-05, + 6.264931289479136e-05, + 6.203571683727205e-05, + 6.256609776755795e-05, + 7.726731564616784e-05, + 7.684857700951397e-05, + 0.0001297412090934813, + 0.00010362039029132575, + 0.0001698000414762646, + 0.00012187293032184243, + 0.00013834609126206487, + 8.384700049646199e-05, + 4.917434125673026e-05, + 4.311763041187078e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.279292549355887e-05, + "first_to_last_ratio": 0.9258681439467011, + "imbalance_abs_log_ratio": 0.07702344762093526, + "last_quartile_mean": 7.862126585678197e-05, + "mean": 8.808806524029933e-05, + "normalized": [ + 0.8989206541831708, + 0.91347677116652, + 0.7818517326497246, + 0.7112122706281212, + 0.7042465590320558, + 0.7102675895637067, + 0.8771598676323168, + 0.8724062311946045, + 1.4728579716168646, + 1.1763272358026606, + 1.9276168799151117, + 1.3835351019389504, + 1.570542966117651, + 0.9518542638861694, + 0.5582406779236824, + 0.4894832267486894 + ], + "population_cv": 0.3870844002085187 + }, + "activation_output_rms_by_block": [ + 0.10205306112766266, + 0.1637924462556839, + 0.1347782015800476, + 0.18643726408481598, + 0.08823998272418976, + 0.14544272422790527, + 0.051831819117069244, + 0.08038940280675888, + 0.045988935977220535, + 0.08743251860141754, + 0.060310039669275284, + 0.09679152816534042, + 0.05415637046098709, + 0.0906868502497673, + 0.09274918586015701, + 0.19966518878936768 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14676524326205254, + "first_to_last_ratio": 1.3425975426784762, + "imbalance_abs_log_ratio": 0.2946062022500781, + "last_quartile_mean": 0.10931439884006977, + "mean": 0.10504659498110414, + "normalized": [ + 0.9715027997434856, + 1.5592361302634037, + 1.2830325590685887, + 1.7748054005782146, + 0.840008024439654, + 1.3845543899263715, + 0.49341741278137374, + 0.7652737608603057, + 0.4377955895238019, + 0.8323213010107083, + 0.5741265547933648, + 0.9214151889716308, + 0.5155461771105363, + 0.8633011880687815, + 0.8829337674090323, + 1.9007297554507465 + ], + "population_cv": 0.43941183630186603 + }, + "bits_per_byte": 2.0898630043632984, + "core_parameter_grad_rms_by_block": [ + 0.00040335839120789034, + 0.00023897821552144955, + 0.00022638555923562337, + 0.00017534070905568056, + 0.00017358160175308344, + 0.00015895432307312814, + 0.00019058997613466408, + 0.00021161798453125184, + 0.00037807554491806804, + 0.0004740713531015684, + 0.0009894587672613622, + 0.0007545475239678793, + 0.0006624146206615605, + 0.0004575991184432586, + 0.0002840295434450018, + 0.00025831777406547436 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00026101571875516093, + "first_to_last_ratio": 0.6280602344874717, + "imbalance_abs_log_ratio": 0.46511920232449994, + "last_quartile_mean": 0.00041559026415382384, + "mean": 0.00037733256289855906, + "normalized": [ + 1.0689731840512475, + 0.633335786568984, + 0.5999629544203535, + 0.464684806709403, + 0.460022852042453, + 0.4212579000658921, + 0.5050981412009801, + 0.5608261924326488, + 1.001969037640965, + 1.2563754091613246, + 2.6222459033501577, + 1.9996883337384523, + 1.7555193635372572, + 1.2127209878949095, + 0.7527300089426935, + 0.6845891382422771 + ], + "population_cv": 0.6186471183066214 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4914427399635315, + "mean_weights": [ + 0.7962706685066223, + 0.2037293016910553 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6456852555274963, + "mean_weights": [ + 0.6051734089851379, + 0.39482662081718445 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5276303291320801, + "mean_weights": [ + 0.7698293924331665, + 0.2301705777645111 + ], + "sources": 2 + }, + { + "entropy_mean": 0.667390763759613, + "mean_weights": [ + 0.5093090534210205, + 0.4906909465789795 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9880077838897705, + "mean_weights": [ + 0.5180364847183228, + 0.2465059459209442, + 0.23545756936073303 + ], + "sources": 3 + }, + { + "entropy_mean": 1.08414626121521, + "mean_weights": [ + 0.2851155400276184, + 0.3203214704990387, + 0.3945629596710205 + ], + "sources": 3 + }, + { + "entropy_mean": 0.995106041431427, + "mean_weights": [ + 0.5032885074615479, + 0.23357318341732025, + 0.2631382942199707 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0889081954956055, + "mean_weights": [ + 0.2929039001464844, + 0.3457888960838318, + 0.36130720376968384 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3091163635253906, + "mean_weights": [ + 0.3480933904647827, + 0.15371239185333252, + 0.20259280502796173, + 0.29560136795043945 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786191940307617, + "mean_weights": [ + 0.22527548670768738, + 0.26118987798690796, + 0.22951865196228027, + 0.284015953540802 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308091163635254, + "mean_weights": [ + 0.28280675411224365, + 0.12650004029273987, + 0.24999231100082397, + 0.3407008647918701 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3780848979949951, + "mean_weights": [ + 0.23603014647960663, + 0.2631712555885315, + 0.21569286286830902, + 0.28510570526123047 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5637545585632324, + "mean_weights": [ + 0.21651385724544525, + 0.1117275133728981, + 0.18637539446353912, + 0.22181779146194458, + 0.2635654807090759 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585724115371704, + "mean_weights": [ + 0.18968582153320312, + 0.1962759643793106, + 0.1558789312839508, + 0.1798120141029358, + 0.2783472537994385 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5276048183441162, + "mean_weights": [ + 0.2346682846546173, + 0.15222865343093872, + 0.12689977884292603, + 0.14891111850738525, + 0.3372921645641327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.59938383102417, + "mean_weights": [ + 0.20912373065948486, + 0.21754522621631622, + 0.17016828060150146, + 0.17554700374603271, + 0.22761575877666473 + ], + "sources": 5 + }, + { + "entropy_mean": 1.762988805770874, + "mean_weights": [ + 0.15954776108264923, + 0.12860070168972015, + 0.13993129134178162, + 0.14435188472270966, + 0.1894117146730423, + 0.23815666139125824 + ], + "sources": 6 + }, + { + "entropy_mean": 1.730720043182373, + "mean_weights": [ + 0.17182672023773193, + 0.16310763359069824, + 0.12390270084142685, + 0.11688461899757385, + 0.1326894462108612, + 0.2915888726711273 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4424301385879517, + "mean_weights": [ + 0.1671508103609085, + 0.1365666687488556, + 0.058271970599889755, + 0.05150005966424942, + 0.10222888737916946, + 0.48428159952163696 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7727878093719482, + "mean_weights": [ + 0.1808200478553772, + 0.17958280444145203, + 0.1469326913356781, + 0.13797307014465332, + 0.14092227816581726, + 0.2137690931558609 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5771737098693848, + "mean_weights": [ + 0.09523316472768784, + 0.08390435576438904, + 0.04673727601766586, + 0.04574791342020035, + 0.07810044288635254, + 0.19550111889839172, + 0.45477572083473206 + ], + "sources": 7 + }, + { + "entropy_mean": 1.866133689880371, + "mean_weights": [ + 0.13646650314331055, + 0.12685352563858032, + 0.11049237847328186, + 0.10180146992206573, + 0.1026906818151474, + 0.1334708034992218, + 0.28822463750839233 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755802392959595, + "mean_weights": [ + 0.09840903431177139, + 0.08739377558231354, + 0.04908968508243561, + 0.048939045518636703, + 0.07940414547920227, + 0.16869762539863586, + 0.46806666254997253 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8938008546829224, + "mean_weights": [ + 0.14332836866378784, + 0.14485859870910645, + 0.1094474121928215, + 0.10118386149406433, + 0.10821370780467987, + 0.14364898204803467, + 0.24931909143924713 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7736961841583252, + "mean_weights": [ + 0.08116129040718079, + 0.07111318409442902, + 0.04280973970890045, + 0.041376810520887375, + 0.06107718497514725, + 0.1160595640540123, + 0.31332647800445557, + 0.2730756998062134 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9035927057266235, + "mean_weights": [ + 0.10087604820728302, + 0.09542917460203171, + 0.06220542639493942, + 0.057604290544986725, + 0.0788167417049408, + 0.11659102141857147, + 0.18421337008476257, + 0.3042638897895813 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7131891250610352, + "mean_weights": [ + 0.0740746557712555, + 0.07079474627971649, + 0.04342404007911682, + 0.04123558849096298, + 0.055328819900751114, + 0.09829520434141159, + 0.21916092932224274, + 0.39768603444099426 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9703106880187988, + "mean_weights": [ + 0.11310406029224396, + 0.11765976250171661, + 0.07340675592422485, + 0.0660770833492279, + 0.08407266438007355, + 0.12691649794578552, + 0.20392963290214539, + 0.2148335576057434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9610494375228882, + "mean_weights": [ + 0.07792562991380692, + 0.0730937123298645, + 0.040804408490657806, + 0.03727761656045914, + 0.05315954610705376, + 0.0937754213809967, + 0.19250980019569397, + 0.26475197076797485, + 0.16670189797878265 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9782097339630127, + "mean_weights": [ + 0.08397573232650757, + 0.08872128278017044, + 0.0407399907708168, + 0.036753468215465546, + 0.058452628552913666, + 0.11618129909038544, + 0.22768989205360413, + 0.21488600969314575, + 0.13259968161582947 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0315680503845215, + "mean_weights": [ + 0.08375388383865356, + 0.10381080955266953, + 0.0493716336786747, + 0.043391548097133636, + 0.06576120853424072, + 0.10571172833442688, + 0.18509769439697266, + 0.2181759774684906, + 0.14492549002170563 + ], + "sources": 9 + } + ], + "loss_nats": 1.448582649230957, + "output_weights": { + "entropy_mean": 1.893564224243164, + "mean_weights": [ + 0.09584413468837738, + 0.060246460139751434, + 0.05342919006943703, + 0.050279051065444946, + 0.05668472498655319, + 0.06780324131250381, + 0.0837026908993721, + 0.2589869201183319, + 0.2730236053466797 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 5.6704131566220894e-05, + 5.7463053963147104e-05, + 5.5973152484511957e-05, + 4.9604332161834463e-05, + 4.9622754886513576e-05, + 5.175438491278328e-05, + 6.153700815048069e-05, + 5.993711238261312e-05, + 0.00010211551852989942, + 8.014418563107029e-05, + 0.00014815112808719277, + 0.00010670819756342098, + 0.00014386174734681845, + 8.734197035664693e-05, + 5.547845285036601e-05, + 4.4348533265292645e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.4936167543928605e-05, + "first_to_last_ratio": 0.6638196023526066, + "imbalance_abs_log_ratio": 0.4097448496221764, + "last_quartile_mean": 8.275767595478101e-05, + "mean": 7.567160400867579e-05, + "normalized": [ + 0.7493449135784109, + 0.7593740705768433, + 0.7396850273993744, + 0.6555210876215509, + 0.6557645438680578, + 0.6839340277080636, + 0.8132113618660104, + 0.792068744515331, + 1.3494562440911366, + 1.0591051515424692, + 1.9578166741411636, + 1.4101484825296797, + 1.9011325216566657, + 1.1542238531990563, + 0.7331475733487212, + 0.5860657223574653 + ], + "population_cv": 0.42524285954419916 + }, + "activation_output_rms_by_block": [ + 0.16565655171871185, + 0.24343863129615784, + 0.21182885766029358, + 0.28418517112731934, + 0.12311498820781708, + 0.18273214995861053, + 0.06521740555763245, + 0.10707039386034012, + 0.06429450213909149, + 0.1264011114835739, + 0.0799982100725174, + 0.12911832332611084, + 0.06644809991121292, + 0.1084718257188797, + 0.10412788391113281, + 0.23786598443984985 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22627730295062065, + "first_to_last_ratio": 1.750986764798193, + "imbalance_abs_log_ratio": 0.5601794946219173, + "last_quartile_mean": 0.12922844849526882, + "mean": 0.14374813064932823, + "normalized": [ + 1.152408389385104, + 1.6935081534383452, + 1.4736112163924233, + 1.976966029705009, + 0.8564632294812553, + 1.2711967044940802, + 0.4536921994257406, + 0.7448472086328344, + 0.4472719182410596, + 0.8793235147657527, + 0.5565165245012613, + 0.8982261038308273, + 0.4622536627854468, + 0.7545964266032466, + 0.7243773080093228, + 1.654741410308291 + ], + "population_cv": 0.4691623243633533 + }, + "bits_per_byte": 1.9440733112268958, + "core_parameter_grad_rms_by_block": [ + 0.00039487297360913916, + 0.00021664788561845028, + 0.0001999236891396226, + 0.000155700993556381, + 0.00014909408964546676, + 0.0001300025003667294, + 0.0001780845905877945, + 0.00021758503566474802, + 0.0003627569164977402, + 0.0004938025952783663, + 0.0010348495424230683, + 0.0008427402552292494, + 0.0008322795274123101, + 0.0005665088497243569, + 0.00036612876328907985, + 0.0003194282437589591 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00024178638548089825, + "first_to_last_ratio": 0.464004453994026, + "imbalance_abs_log_ratio": 0.7678611276768971, + "last_quartile_mean": 0.0005210863460461766, + "mean": 0.00040377540323759136, + "normalized": [ + 0.9779520259107666, + 0.5365554312652605, + 0.49513587884892185, + 0.3856128736617538, + 0.36925005448570164, + 0.3219673593892314, + 0.44104863535485134, + 0.5388763998997622, + 0.8984126164918598, + 1.2229635369537375, + 2.5629335866556917, + 2.087151046031797, + 2.061243752687373, + 1.4030296179061932, + 0.9067634143965939, + 0.7911037700605048 + ], + "population_cv": 0.6734220009206653 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2750788629055023, + "mean_weights": [ + 0.9180313944816589, + 0.08196862041950226 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5785284042358398, + "mean_weights": [ + 0.7040213346481323, + 0.2959786057472229 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4888499975204468, + "mean_weights": [ + 0.7967724800109863, + 0.20322750508785248 + ], + "sources": 2 + }, + { + "entropy_mean": 0.651995062828064, + "mean_weights": [ + 0.5629274845123291, + 0.4370725154876709 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9925327301025391, + "mean_weights": [ + 0.4965394139289856, + 0.22532734274864197, + 0.27813321352005005 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0594394207000732, + "mean_weights": [ + 0.2700359523296356, + 0.27400293946266174, + 0.455961138010025 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9841134548187256, + "mean_weights": [ + 0.49647465348243713, + 0.21238623559474945, + 0.2911390960216522 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0744149684906006, + "mean_weights": [ + 0.26788532733917236, + 0.3177185654640198, + 0.41439610719680786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.307446002960205, + "mean_weights": [ + 0.33086591958999634, + 0.13986191153526306, + 0.23523268103599548, + 0.2940394878387451 + ], + "sources": 4 + }, + { + "entropy_mean": 1.376603603363037, + "mean_weights": [ + 0.21549805998802185, + 0.245200514793396, + 0.24268285930156708, + 0.29661858081817627 + ], + "sources": 4 + }, + { + "entropy_mean": 1.270529866218567, + "mean_weights": [ + 0.2375539094209671, + 0.10673755407333374, + 0.29245367646217346, + 0.3632548451423645 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3762142658233643, + "mean_weights": [ + 0.22679615020751953, + 0.24722415208816528, + 0.2286011129617691, + 0.2973785698413849 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5536365509033203, + "mean_weights": [ + 0.20543108880519867, + 0.10043682903051376, + 0.22657056152820587, + 0.26188796758651733, + 0.20567354559898376 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5861575603485107, + "mean_weights": [ + 0.1991366446018219, + 0.19079142808914185, + 0.16120794415473938, + 0.1751076579093933, + 0.27375632524490356 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4829623699188232, + "mean_weights": [ + 0.2499978393316269, + 0.13764306902885437, + 0.10969669371843338, + 0.13450437784194946, + 0.3681580424308777 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5994092226028442, + "mean_weights": [ + 0.2161637395620346, + 0.20309701561927795, + 0.17752361297607422, + 0.17640036344528198, + 0.22681528329849243 + ], + "sources": 5 + }, + { + "entropy_mean": 1.757448434829712, + "mean_weights": [ + 0.1632193624973297, + 0.11841119825839996, + 0.1406504064798355, + 0.1471654176712036, + 0.19204986095428467, + 0.23850372433662415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.719214677810669, + "mean_weights": [ + 0.17631781101226807, + 0.15250492095947266, + 0.11967972666025162, + 0.11400137841701508, + 0.13607722520828247, + 0.3014189600944519 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3921294212341309, + "mean_weights": [ + 0.18369951844215393, + 0.1281489133834839, + 0.04485193267464638, + 0.042511094361543655, + 0.10842494666576385, + 0.4923636019229889 + ], + "sources": 6 + }, + { + "entropy_mean": 1.769790768623352, + "mean_weights": [ + 0.1860106885433197, + 0.17010365426540375, + 0.1509144902229309, + 0.1396978795528412, + 0.13775232434272766, + 0.215520977973938 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5072013139724731, + "mean_weights": [ + 0.09493429213762283, + 0.07214362919330597, + 0.03675283491611481, + 0.03880048543214798, + 0.07758605480194092, + 0.19188526272773743, + 0.48789745569229126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8300648927688599, + "mean_weights": [ + 0.13595148921012878, + 0.11550325155258179, + 0.1085444986820221, + 0.09914489090442657, + 0.09707978367805481, + 0.12102536857128143, + 0.3227507174015045 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5086650848388672, + "mean_weights": [ + 0.10266698896884918, + 0.07744058966636658, + 0.03928206115961075, + 0.0407833606004715, + 0.07880876958370209, + 0.16221223771572113, + 0.4988059997558594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8771061897277832, + "mean_weights": [ + 0.1470566689968109, + 0.13075211644172668, + 0.10456536710262299, + 0.09934695065021515, + 0.1088101863861084, + 0.1398048996925354, + 0.26966381072998047 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6780993938446045, + "mean_weights": [ + 0.08037982881069183, + 0.05826176330447197, + 0.03183364123106003, + 0.03276165574789047, + 0.053941305726766586, + 0.10113498568534851, + 0.30644336342811584, + 0.33524346351623535 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294007778167725, + "mean_weights": [ + 0.09819990396499634, + 0.08131108433008194, + 0.05535554885864258, + 0.052511729300022125, + 0.07309524714946747, + 0.1031290739774704, + 0.1768367737531662, + 0.35956063866615295 + ], + "sources": 8 + }, + { + "entropy_mean": 1.620237112045288, + "mean_weights": [ + 0.07192602753639221, + 0.0598294772207737, + 0.03551005944609642, + 0.03596649691462517, + 0.052555397152900696, + 0.08522239327430725, + 0.2067522257566452, + 0.45223796367645264 + ], + "sources": 8 + }, + { + "entropy_mean": 1.922456979751587, + "mean_weights": [ + 0.11432191729545593, + 0.10134892165660858, + 0.06334333121776581, + 0.05966266989707947, + 0.07981826364994049, + 0.11505833268165588, + 0.20316767692565918, + 0.26327887177467346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8568212985992432, + "mean_weights": [ + 0.07325166463851929, + 0.05971681326627731, + 0.03186998516321182, + 0.03226577863097191, + 0.04611574858427048, + 0.07693300396203995, + 0.17066776752471924, + 0.3028550446033478, + 0.20632418990135193 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9784047603607178, + "mean_weights": [ + 0.08589711785316467, + 0.07969395816326141, + 0.03702075034379959, + 0.037476278841495514, + 0.0731067955493927, + 0.12099474668502808, + 0.22441011667251587, + 0.20659996569156647, + 0.1348002701997757 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9687256813049316, + "mean_weights": [ + 0.07957327365875244, + 0.0893080085515976, + 0.03895219415426254, + 0.03689703717827797, + 0.0598301999270916, + 0.09126800298690796, + 0.16929352283477783, + 0.23949456214904785, + 0.1953832060098648 + ], + "sources": 9 + } + ], + "loss_nats": 1.3475289344787598, + "output_weights": { + "entropy_mean": 1.7764334678649902, + "mean_weights": [ + 0.08629432320594788, + 0.05034743621945381, + 0.046479545533657074, + 0.04277106374502182, + 0.04524604231119156, + 0.05579439178109169, + 0.07300648093223572, + 0.2586928606033325, + 0.3413679003715515 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 7.570489106001332e-05, + 7.69118414609693e-05, + 7.822692714398727e-05, + 6.853767990833148e-05, + 6.88084473949857e-05, + 7.10485182935372e-05, + 8.91982635948807e-05, + 8.665756467962638e-05, + 0.000171030915225856, + 0.00012665522808674723, + 0.00023480529489461333, + 0.00016227969899773598, + 0.00023323859204538167, + 0.000136202055728063, + 9.528713417239487e-05, + 6.303648115135729e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.484533489332534e-05, + "first_to_last_ratio": 0.5672633797074002, + "imbalance_abs_log_ratio": 0.5669315686327029, + "last_quartile_mean": 0.0001319410657742992, + "mean": 0.00011485184586490504, + "normalized": [ + 0.6591525847051819, + 0.6696613439842941, + 0.6811116230208613, + 0.5967486146365397, + 0.5991061517280438, + 0.6186101560536372, + 0.7766376145125305, + 0.7545160813658816, + 1.4891438090340474, + 1.1027705051926284, + 2.0444189914962623, + 1.4129481139434028, + 2.0307779147035174, + 1.1858934847966764, + 0.8296526142424977, + 0.5488503965839976 + ], + "population_cv": 0.4839100882418841 + }, + "activation_output_rms_by_block": [ + 0.16869333386421204, + 0.24958865344524384, + 0.23942448198795319, + 0.31786683201789856, + 0.1331564038991928, + 0.19950589537620544, + 0.06165311112999916, + 0.10097606480121613, + 0.05752932280302048, + 0.11994150280952454, + 0.07362622767686844, + 0.12106145173311234, + 0.06137929856777191, + 0.09901073575019836, + 0.0852273628115654, + 0.19495683908462524 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2438933253288269, + "first_to_last_ratio": 2.2143221757549307, + "imbalance_abs_log_ratio": 0.7949463412042855, + "last_quartile_mean": 0.11014355905354023, + "mean": 0.142724844859913, + "normalized": [ + 1.1819479224502754, + 1.7487400577679353, + 1.6775249062134392, + 2.2271303383085868, + 0.9329588273848762, + 1.3978357837559683, + 0.43197182095740094, + 0.7074876480007805, + 0.4030785450107621, + 0.8403687734062649, + 0.5158613256797294, + 0.8482156827841455, + 0.430053356358641, + 0.6937175923882012, + 0.5971445468742151, + 1.3659628726587785 + ], + "population_cv": 0.529368044298578 + }, + "bits_per_byte": 1.883020158101202, + "core_parameter_grad_rms_by_block": [ + 0.0005343193187437567, + 0.0003255019836390766, + 0.00031277964721318437, + 0.00022314651478922115, + 0.0002396348782003974, + 0.00021519463019459167, + 0.00025456643186586784, + 0.0003008996254324453, + 0.00059437697364117, + 0.0008147654749400623, + 0.001560733102975404, + 0.001320101058102649, + 0.0015032932063515777, + 0.0009169853654825589, + 0.0006174439085696804, + 0.00045539974927406244 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003489368660963097, + "first_to_last_ratio": 0.3995701760811125, + "imbalance_abs_log_ratio": 0.9173658694246768, + "last_quartile_mean": 0.0008732805574194699, + "mean": 0.0006368213668384816, + "normalized": [ + 0.8390411292202721, + 0.5111354621391565, + 0.4911575890834005, + 0.3504067646113047, + 0.3762984263390404, + 0.33791992762889184, + 0.39974543117117817, + 0.4725024019314401, + 0.9333496088423855, + 1.2794254674352237, + 2.450817739869046, + 2.072953463631928, + 2.3606199236290095, + 1.4399412664731395, + 0.9695715953046595, + 0.7151138026899253 + ], + "population_cv": 0.7016607196207993 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2716493308544159, + "mean_weights": [ + 0.9192699193954468, + 0.0807301327586174 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5622239112854004, + "mean_weights": [ + 0.72159743309021, + 0.27840256690979004 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48637402057647705, + "mean_weights": [ + 0.7995253801345825, + 0.20047461986541748 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6475558280944824, + "mean_weights": [ + 0.5858595371246338, + 0.41414040327072144 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9933564066886902, + "mean_weights": [ + 0.4998217821121216, + 0.2229493409395218, + 0.2772288918495178 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489425659179688, + "mean_weights": [ + 0.2563193440437317, + 0.269240140914917, + 0.4744405150413513 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9815441966056824, + "mean_weights": [ + 0.5070900917053223, + 0.19802016019821167, + 0.29488980770111084 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071545958518982, + "mean_weights": [ + 0.2598530650138855, + 0.3191677927970886, + 0.4209791421890259 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3049530982971191, + "mean_weights": [ + 0.3487145006656647, + 0.13604819774627686, + 0.23624856770038605, + 0.27898871898651123 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3723504543304443, + "mean_weights": [ + 0.2135777771472931, + 0.23673927783966064, + 0.23828449845314026, + 0.311398446559906 + ], + "sources": 4 + }, + { + "entropy_mean": 1.265662670135498, + "mean_weights": [ + 0.24546673893928528, + 0.10275381803512573, + 0.2848597466945648, + 0.36691969633102417 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3751296997070312, + "mean_weights": [ + 0.23200613260269165, + 0.24026888608932495, + 0.22661332786083221, + 0.30111163854599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5548532009124756, + "mean_weights": [ + 0.2040066421031952, + 0.09975787997245789, + 0.22992464900016785, + 0.259371280670166, + 0.20693954825401306 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5716381072998047, + "mean_weights": [ + 0.1984836459159851, + 0.17718908190727234, + 0.15249167382717133, + 0.17045220732688904, + 0.301383376121521 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4117064476013184, + "mean_weights": [ + 0.25685301423072815, + 0.12795110046863556, + 0.08630626648664474, + 0.10788753628730774, + 0.4210020899772644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5975096225738525, + "mean_weights": [ + 0.2208021879196167, + 0.19316574931144714, + 0.17618897557258606, + 0.17661380767822266, + 0.23322927951812744 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7283259630203247, + "mean_weights": [ + 0.1650083363056183, + 0.10661210119724274, + 0.12329171597957611, + 0.13269363343715668, + 0.19600069522857666, + 0.27639350295066833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7028545141220093, + "mean_weights": [ + 0.17663681507110596, + 0.13835734128952026, + 0.11904168128967285, + 0.11434897780418396, + 0.12915222346782684, + 0.3224629759788513 + ], + "sources": 6 + }, + { + "entropy_mean": 1.27894127368927, + "mean_weights": [ + 0.17268770933151245, + 0.10967432707548141, + 0.03322022780776024, + 0.031795307993888855, + 0.09717383980751038, + 0.555448591709137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7646939754486084, + "mean_weights": [ + 0.1884503960609436, + 0.1533900499343872, + 0.15274858474731445, + 0.14330559968948364, + 0.1338358074426651, + 0.2282695174217224 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4344873428344727, + "mean_weights": [ + 0.09210026264190674, + 0.06219030171632767, + 0.029036182910203934, + 0.0315997451543808, + 0.07091090828180313, + 0.1949847936630249, + 0.5191777944564819 + ], + "sources": 7 + }, + { + "entropy_mean": 1.802895426750183, + "mean_weights": [ + 0.13523127138614655, + 0.10325458645820618, + 0.10260490328073502, + 0.09636874496936798, + 0.09404134750366211, + 0.12264730036258698, + 0.3458518385887146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4361302852630615, + "mean_weights": [ + 0.10137400776147842, + 0.06668317317962646, + 0.03099493309855461, + 0.033589236438274384, + 0.07286111265420914, + 0.16290563344955444, + 0.5315918922424316 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8628063201904297, + "mean_weights": [ + 0.14754432439804077, + 0.11970964819192886, + 0.10172373056411743, + 0.09891469776630402, + 0.10695713013410568, + 0.14022418856620789, + 0.28492626547813416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6038438081741333, + "mean_weights": [ + 0.07592754065990448, + 0.048247914761304855, + 0.024802275002002716, + 0.026826059445738792, + 0.049207836389541626, + 0.09689367562532425, + 0.29891306161880493, + 0.3791816532611847 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7748796939849854, + "mean_weights": [ + 0.09652430564165115, + 0.07308399677276611, + 0.04972345381975174, + 0.050231896340847015, + 0.0702696442604065, + 0.09669448435306549, + 0.16264159977436066, + 0.40083062648773193 + ], + "sources": 8 + }, + { + "entropy_mean": 1.526896357536316, + "mean_weights": [ + 0.06852421164512634, + 0.05069870874285698, + 0.02793668583035469, + 0.029434790834784508, + 0.046449482440948486, + 0.08027796447277069, + 0.19853916764259338, + 0.49813902378082275 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9032639265060425, + "mean_weights": [ + 0.11537295579910278, + 0.09104456007480621, + 0.05989263206720352, + 0.057612620294094086, + 0.07744243741035461, + 0.11209231615066528, + 0.20047283172607422, + 0.2860696315765381 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7657151222229004, + "mean_weights": [ + 0.06711487472057343, + 0.04881071671843529, + 0.02479729615151882, + 0.025201791897416115, + 0.040198031812906265, + 0.06934815645217896, + 0.15364184975624084, + 0.3216337561607361, + 0.24925348162651062 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9778118133544922, + "mean_weights": [ + 0.08567827194929123, + 0.08225203305482864, + 0.030744101852178574, + 0.03173041343688965, + 0.07107861340045929, + 0.128017395734787, + 0.2095082402229309, + 0.20655101537704468, + 0.15443992614746094 + ], + "sources": 9 + }, + { + "entropy_mean": 1.88002610206604, + "mean_weights": [ + 0.07342863082885742, + 0.07307932525873184, + 0.030178282409906387, + 0.029849819839000702, + 0.050495631992816925, + 0.08296258747577667, + 0.14574018120765686, + 0.2550249397754669, + 0.25924059748649597 + ], + "sources": 9 + } + ], + "loss_nats": 1.3052101135253906, + "output_weights": { + "entropy_mean": 1.609398603439331, + "mean_weights": [ + 0.0809592753648758, + 0.04013277217745781, + 0.03442784771323204, + 0.03263507038354874, + 0.03443540260195732, + 0.047240644693374634, + 0.05792279541492462, + 0.2393622249364853, + 0.43288397789001465 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.015965601418904, + "cross_entropy_nats": 5.55624395608902, + "step": 0 + }, + { + "bits_per_byte": 5.1077262486126465, + "cross_entropy_nats": 3.540406048297882, + "step": 100 + }, + { + "bits_per_byte": 3.1449978072640112, + "cross_entropy_nats": 2.1799463629722595, + "step": 500 + }, + { + "bits_per_byte": 1.978178932745523, + "cross_entropy_nats": 1.3711691498756409, + "step": 2000 + }, + { + "bits_per_byte": 1.8301036761704788, + "cross_entropy_nats": 1.2685312032699585, + "step": 4000 + }, + { + "bits_per_byte": 1.72661337398776, + "cross_entropy_nats": 1.1967971920967102, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "ad1807a65c19f598a4bde3ce03f75bb64bba14b79befbf1740b5aaddab905279", + "final_model_state": "6df5cb12d848445089dff93c7b22def426897ab70173e1fcca78c3db41e914dc", + "final_optimizer_state": "a8e9a566e68720356f0b3f878d687f8c4cac616f59f729e02056853409f8d145", + "final_public_parameters": "16b3a7903fb4172275d1673bd0f390d3b3c03a9d0d151ac66d732eb978cff06f", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "seed": 2026073003, + "timing": { + "mean_ms": 54.54310071476275, + "measured_steps": 7980, + "median_ms": 54.49852900346741, + "p95_ms": 54.8562506082817, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "baseline", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002768727717921138, + 0.0002554995589889586, + 0.00023783760843798518, + 0.00022115583124104887, + 0.00020655356638599187, + 0.00019376208365429193, + 0.00018488570640329272, + 0.00017590694187674671, + 0.0001684112212387845, + 0.0001647331955609843, + 0.00015843863366171718, + 0.0001538397918920964, + 0.00015132295084185898, + 0.00014730033581145108, + 0.00014253577683120966, + 0.0001394751889165491, + 0.0001364792260574177, + 0.0001337067224085331, + 0.00013106675760354847, + 0.00012915431580040604, + 0.00012709009752143174, + 0.00012574320135172457, + 0.0001242366706719622, + 0.0001228937617270276, + 0.00012132258416386321, + 0.00011988351616309956, + 0.00011879197700181976, + 0.00011764531518565491, + 0.00011639683361863717, + 0.00011549647024367005, + 0.00011481995898066089, + 0.00011415599146857858 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002190592585975537, + "first_to_last_ratio": 1.8672887091156776, + "imbalance_abs_log_ratio": 0.6244874905881238, + "last_quartile_mean": 0.00011731408085324802, + "mean": 0.0001546067051094724, + "normalized": [ + 1.7908199492128654, + 1.6525774791465027, + 1.538339545297078, + 1.4304413969915148, + 1.3359935860462033, + 1.2532579587482624, + 1.1958453307208163, + 1.1377704592578457, + 1.0892879524179597, + 1.065498391187767, + 1.0247850088360109, + 0.9950395863049215, + 0.978760596021445, + 0.9527422223192203, + 0.9219249367631522, + 0.9021289782858439, + 0.8827510162691898, + 0.8648183939620171, + 0.8477430361816715, + 0.835373315206192, + 0.8220219002238166, + 0.8133101424203405, + 0.80356586464988, + 0.7948798963150414, + 0.7847174809007043, + 0.775409553409819, + 0.7683494510649245, + 0.7609328140222236, + 0.7528576043077825, + 0.7470340316863389, + 0.7426583400723811, + 0.7383637817502684 + ], + "population_cv": 0.27888268733798954 + }, + "activation_output_rms_by_block": [ + 0.028270306065678596, + 0.0284538883715868, + 0.02867252007126808, + 0.02881420962512493, + 0.029061447829008102, + 0.02929086796939373, + 0.029533835127949715, + 0.029811719432473183, + 0.030115574598312378, + 0.030315518379211426, + 0.030812423676252365, + 0.03120259940624237, + 0.03179442882537842, + 0.03198441118001938, + 0.03240228444337845, + 0.0329107940196991, + 0.033366311341524124, + 0.033811621367931366, + 0.03420914337038994, + 0.03470747172832489, + 0.03500960022211075, + 0.03567802160978317, + 0.035978782922029495, + 0.036836784332990646, + 0.03769209608435631, + 0.038124579936265945, + 0.038542140275239944, + 0.03904740884900093, + 0.0400773324072361, + 0.04093807935714722, + 0.04163828864693642, + 0.042313966900110245 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.028988599311560392, + "first_to_last_ratio": 0.728416493900548, + "imbalance_abs_log_ratio": 0.31688228737002583, + "last_quartile_mean": 0.03979673655703664, + "mean": 0.03379432682413608, + "normalized": [ + 0.8365399971656723, + 0.8419723381282118, + 0.8484418174825138, + 0.8526345198433033, + 0.8599504875549783, + 0.8667392051281828, + 0.873928789339177, + 0.8821515986281313, + 0.8911429055838948, + 0.8970593951159853, + 0.9117632032322649, + 0.9233088017589188, + 0.9408214873116122, + 0.9464432106153376, + 0.9588084003566117, + 0.9738555879797565, + 0.9873346942272493, + 1.0005117587897308, + 1.0122747391422395, + 1.0270206567197142, + 1.0359608701276664, + 1.055739970659861, + 1.0646397281194921, + 1.0900286466627183, + 1.1153379761196018, + 1.1281355043603702, + 1.1404914344295491, + 1.1554427183059932, + 1.1859189447920195, + 1.2113891059344621, + 1.2321088348236646, + 1.2521026715611152 + ], + "population_cv": 0.12455242949918699 + }, + "bits_per_byte": 7.95915895791699, + "core_parameter_grad_rms_by_block": [ + 0.005888019831616423, + 0.005779358340731243, + 0.0060348395662513914, + 0.005586875351280001, + 0.005559071516712478, + 0.0054082748429522395, + 0.005481135467826832, + 0.0053805923812947415, + 0.004971746645687392, + 0.005439263395635174, + 0.005139424262259116, + 0.005696496994849179, + 0.005782818048228185, + 0.0054294510739720785, + 0.005067071208887471, + 0.005218073376917496, + 0.005181036474299639, + 0.005605812513830226, + 0.005170579144406511, + 0.004834742615880465, + 0.005150211089068095, + 0.00476651480071292, + 0.004776402486441839, + 0.00519066249960807, + 0.0053994045182471796, + 0.004795508114414784, + 0.004895792813412353, + 0.00496559115560922, + 0.00502742614163479, + 0.004657478744424888, + 0.005287606581362069, + 0.004870555415551657 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005639770912333169, + "first_to_last_ratio": 1.130799174678946, + "imbalance_abs_log_ratio": 0.12292461698112994, + "last_quartile_mean": 0.0049874204355821175, + "mean": 0.005263682419187692, + "normalized": [ + 1.1186122875029152, + 1.0979686615711766, + 1.146505257280075, + 1.061400537941684, + 1.0561183357962487, + 1.0274698228824493, + 1.0413119621059315, + 1.0222106792919112, + 0.9445377303850805, + 1.0333570611721248, + 0.976393302818647, + 1.0822265747043835, + 1.0986259404914112, + 1.0314929058371969, + 0.962647592570647, + 0.9913351455049914, + 0.9842988352437859, + 1.0649982402804865, + 0.9823121405573801, + 0.9185095586801336, + 0.9784425956805525, + 0.9055475655859389, + 0.9074260386664719, + 0.9861275977985596, + 1.0257846291342996, + 0.9110557462459602, + 0.9301079403205877, + 0.9433683038148654, + 0.9551157804103686, + 0.884832779319472, + 1.0045451378463044, + 0.9253133125579595 + ], + "population_cv": 0.06750436074287791 + }, + "loss_nats": 5.516868591308594, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.172436016844586e-05, + 6.834617670392618e-05, + 6.473411485785618e-05, + 6.156626477604732e-05, + 5.8372206694912165e-05, + 5.494635479408316e-05, + 5.167145354789682e-05, + 4.809222446056083e-05, + 4.4674157834379e-05, + 4.1754625272005796e-05, + 3.8718964788131416e-05, + 3.608527549658902e-05, + 3.353153806529008e-05, + 3.1093299185158685e-05, + 2.904756365751382e-05, + 2.7216165108256973e-05, + 2.5600058506825007e-05, + 2.4208331524278037e-05, + 2.304886766069103e-05, + 2.1986410501995124e-05, + 2.1054642274975777e-05, + 2.025439971475862e-05, + 1.956623418664094e-05, + 1.9002633052878082e-05, + 1.8555338101577945e-05, + 1.8124846974387765e-05, + 1.7765629308996722e-05, + 1.744573273754213e-05, + 1.7174377717310563e-05, + 1.6944877643254586e-05, + 1.6726755347917788e-05, + 1.656309177633375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.9931644500466064e-05, + "first_to_last_ratio": 3.4418587232383575, + "imbalance_abs_log_ratio": 1.2360116519842717, + "last_quartile_mean": 1.7412581200915156e-05, + "mean": 3.3612405388794286e-05, + "normalized": [ + 2.1338657361414946, + 2.033361668508004, + 1.9258995037420696, + 1.831653047852752, + 1.7366268798594315, + 1.634704632367703, + 1.537273305799265, + 1.4307879458276982, + 1.3290973174229443, + 1.242238536309156, + 1.1519248426368067, + 1.0735701619444094, + 0.9975941226886675, + 0.9250542716447356, + 0.8641917566303572, + 0.8097059640167944, + 0.761625305023828, + 0.7202201462305526, + 0.6857250290208349, + 0.654115950574753, + 0.626394988143127, + 0.602587035366146, + 0.582113477459246, + 0.5653458249439413, + 0.5520383884148895, + 0.5392308811207612, + 0.5285438249212427, + 0.5190266074607738, + 0.510953545830914, + 0.5041257073765888, + 0.49763636831222313, + 0.4927672264078892 + ], + "population_cv": 0.513984504821457 + }, + "activation_output_rms_by_block": [ + 0.029439672827720642, + 0.030752120539546013, + 0.03267423436045647, + 0.034982286393642426, + 0.03797784447669983, + 0.04108294099569321, + 0.044510453939437866, + 0.04866119846701622, + 0.05275992304086685, + 0.05702463164925575, + 0.061875153332948685, + 0.06669966131448746, + 0.071797676384449, + 0.07734881341457367, + 0.08311311900615692, + 0.08836977183818817, + 0.09436330199241638, + 0.10090246796607971, + 0.10806354880332947, + 0.11583591252565384, + 0.1238454207777977, + 0.13231202960014343, + 0.1417466551065445, + 0.15185882151126862, + 0.16391249001026154, + 0.1746179163455963, + 0.18763093650341034, + 0.20137503743171692, + 0.21650369465351105, + 0.23270079493522644, + 0.2497490495443344, + 0.2673082947731018 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037510094000026584, + "first_to_last_ratio": 0.17716440452291277, + "imbalance_abs_log_ratio": 1.7306771383999247, + "last_quartile_mean": 0.21172477677464485, + "mean": 0.11005612107692286, + "normalized": [ + 0.2674969146617844, + 0.27942217332966024, + 0.29688702491721536, + 0.3178586165979067, + 0.3450770761778485, + 0.37329083192909013, + 0.4044341514483109, + 0.4421489508339555, + 0.4793910827003499, + 0.5181413908763769, + 0.5622145567869099, + 0.6060513550888119, + 0.6523733135594211, + 0.702812462021184, + 0.7551885183020911, + 0.8029519028425762, + 0.857410756158323, + 0.9168274056792781, + 0.9818949436515149, + 1.0525167650120182, + 1.1252933463940358, + 1.2022232685055743, + 1.2879488548162785, + 1.3798307629352853, + 1.4893536897933755, + 1.586626119809806, + 1.7048659780791944, + 1.8297486360705681, + 1.9672117510137168, + 2.114383031658748, + 2.269288133185925, + 2.428836235162865 + ], + "population_cv": 0.6268498746310547 + }, + "bits_per_byte": 5.1517691996807775, + "core_parameter_grad_rms_by_block": [ + 0.0004018231655400003, + 0.0003405186139403606, + 0.0003221632759320079, + 0.0003173401597627506, + 0.0002956526561473367, + 0.0002913580859417252, + 0.00030034368266542127, + 0.00026684124508166604, + 0.0002497190139849492, + 0.00026578754174316435, + 0.0002404550275593826, + 0.00024493210822064133, + 0.0002532169075836586, + 0.00023865218150919594, + 0.0002121496513808785, + 0.0002091708005241373, + 0.0001980500112256327, + 0.00020258502349527826, + 0.00019483821933596074, + 0.0001851504169444552, + 0.00018439346409065922, + 0.00017525536186022104, + 0.000171573793780652, + 0.0001670148315601783, + 0.00018406810665917416, + 0.00014791927954418835, + 0.00017020274695916728, + 0.00015649953794173187, + 0.00016066786897129027, + 0.00016871534344057424, + 0.00016946426752177343, + 0.0001617385113992835 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031700511062640856, + "first_to_last_ratio": 1.9222979375866573, + "imbalance_abs_log_ratio": 0.653521312887717, + "last_quartile_mean": 0.0001649094578046479, + "mean": 0.00022650815319523427, + "normalized": [ + 1.773989853661721, + 1.5033393241560593, + 1.4223032212634112, + 1.401009876625613, + 1.305262755343313, + 1.286302862972861, + 1.3259729437047942, + 1.1780646361619815, + 1.1024725179305521, + 1.1734126917456873, + 1.061573387833537, + 1.0813390368757583, + 1.117915200895233, + 1.0536140891294732, + 0.9366093378458668, + 0.9234581518301755, + 0.8743615116358631, + 0.8943829201621014, + 0.8601819254074433, + 0.8174117104953322, + 0.814069875585118, + 0.7737265055936556, + 0.7574729269580301, + 0.7373457829406393, + 0.8126334706394442, + 0.6530417446682029, + 0.7514199579935843, + 0.6909223166321972, + 0.7093248817088087, + 0.7448532914184036, + 0.7481596804849096, + 0.7140516097002307 + ], + "population_cv": 0.27857383691000986 + }, + "loss_nats": 3.570934295654297, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00014451092283707112, + 0.00014379274216480553, + 0.00014628663484472781, + 0.0001445338421035558, + 0.00013843078340869397, + 0.0001292457600357011, + 0.00012031046207994223, + 0.00011329296830808744, + 0.0001077206979971379, + 0.0001003394863801077, + 9.358100214740261e-05, + 8.73711978783831e-05, + 8.221827738452703e-05, + 7.71504346630536e-05, + 7.201205153251067e-05, + 6.728569132974371e-05, + 6.256035703700036e-05, + 5.796365803689696e-05, + 5.370515646063723e-05, + 4.9705678975442424e-05, + 4.572505713440478e-05, + 4.197436646791175e-05, + 3.853226371575147e-05, + 3.558541357051581e-05, + 3.2570587791269645e-05, + 2.9896507840021513e-05, + 2.8078287868993357e-05, + 2.6684616386773996e-05, + 2.568897070887033e-05, + 2.5203555196640082e-05, + 2.4927356207626872e-05, + 2.5036461011040956e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013505051447282312, + "first_to_last_ratio": 4.954020049421058, + "imbalance_abs_log_ratio": 1.6001993781766728, + "last_quartile_mean": 2.7260792876404594e-05, + "mean": 7.412253904703903e-05, + "normalized": [ + 1.9496218652920512, + 1.9399327655728709, + 1.9735783032458267, + 1.9499310730820074, + 1.8675936521932173, + 1.743676912547211, + 1.6231292617160862, + 1.5284550389752622, + 1.4532785760182485, + 1.3536973728926245, + 1.2625174926619152, + 1.1787399487615544, + 1.1092210067487078, + 1.040849863684419, + 0.9715270477554875, + 0.90776290443914, + 0.8440126018524382, + 0.7819977402570161, + 0.7245455586263081, + 0.670587915828121, + 0.616884657787925, + 0.5662834410094119, + 0.5198454371793504, + 0.4800889719648283, + 0.4394154357097774, + 0.4033389603808477, + 0.3788090401379066, + 0.36000677701879086, + 0.34657434889767885, + 0.34002552422881266, + 0.3362992758762309, + 0.3377712276579264 + ], + "population_cv": 0.5690574657123064 + }, + "activation_output_rms_by_block": [ + 0.049516160041093826, + 0.057937320321798325, + 0.06586314737796783, + 0.0719481110572815, + 0.076947420835495, + 0.08064281940460205, + 0.08439230173826218, + 0.08712083846330643, + 0.0897950753569603, + 0.09305135160684586, + 0.09555979818105698, + 0.09745962917804718, + 0.10008277744054794, + 0.10209457576274872, + 0.1030229851603508, + 0.10483334213495255, + 0.10759729146957397, + 0.11058001965284348, + 0.11383581161499023, + 0.1180567741394043, + 0.1237289309501648, + 0.13110940158367157, + 0.14003770053386688, + 0.15068796277046204, + 0.16247524321079254, + 0.1767880618572235, + 0.1922861486673355, + 0.20968382060527802, + 0.22965215146541595, + 0.2505997121334076, + 0.27601101994514465, + 0.307341605424881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07179601490497589, + "first_to_last_ratio": 0.31823808816289667, + "imbalance_abs_log_ratio": 1.1449554714268964, + "last_quartile_mean": 0.22560472041368484, + "mean": 0.13002310344018042, + "normalized": [ + 0.3808258588741859, + 0.44559250463094413, + 0.506549571847971, + 0.5533486676879896, + 0.5917980635718029, + 0.6202191554495797, + 0.6490562023624398, + 0.6700412169702441, + 0.6906086147856962, + 0.7156524428726306, + 0.7349447571448028, + 0.7495562450013764, + 0.7697307231756155, + 0.785203345109504, + 0.7923436868875267, + 0.8062670353287107, + 0.8275244062227459, + 0.8504643923048483, + 0.8755044957634206, + 0.9079676689436854, + 0.9515918915678598, + 1.008354654786339, + 1.0770216740619014, + 1.1589322111496045, + 1.2495874880077935, + 1.359666529868349, + 1.4788613990882042, + 1.6126658651994643, + 1.7662411170724883, + 1.9273475674936549, + 2.1227844332459633, + 2.363746113522658 + ], + "population_cv": 0.4886061625493153 + }, + "bits_per_byte": 3.0864900833624125, + "core_parameter_grad_rms_by_block": [ + 0.0006755996472372422, + 0.0003902215048764158, + 0.00033650809592979206, + 0.00029675556364760884, + 0.00026186200490798707, + 0.0002455230107742763, + 0.00021966924842671773, + 0.00022418062520577193, + 0.00022548545903395049, + 0.00022091596273961857, + 0.00024014950731391624, + 0.00021478395214610848, + 0.0001956916368008953, + 0.00020741213919383669, + 0.0001699546796737774, + 0.00016808908215382032, + 0.00015850476160987511, + 0.00015061268581609982, + 0.00013300713992511992, + 0.0001284985461118983, + 0.0001312583398227583, + 0.00012527717278626568, + 0.0001234119340024874, + 0.00012326054887191204, + 0.00010914072106764207, + 0.00010077507233480893, + 9.59134879196041e-05, + 8.362219426315277e-05, + 8.733675859901561e-05, + 7.698995894839292e-05, + 8.92787325176121e-05, + 9.631544502550806e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003312899626257265, + "first_to_last_ratio": 3.584553340265607, + "imbalance_abs_log_ratio": 1.276633875010675, + "last_quartile_mean": 9.242154633446708e-05, + "mean": 0.00019081267561512152, + "normalized": [ + 3.540643435030279, + 2.045050223306504, + 1.763552105985586, + 1.5552193411206041, + 1.3723512028947988, + 1.2867227503769625, + 1.1512298526215383, + 1.17487281430902, + 1.1817111117332983, + 1.1577635606620644, + 1.2585616051960276, + 1.1256272753039647, + 1.0255693767201026, + 1.0869935056735807, + 0.8906886249872848, + 0.8809115097409158, + 0.8306825586869656, + 0.7893222257408777, + 0.697056102254977, + 0.6734277253733716, + 0.6878910921385158, + 0.6565453389425546, + 0.6467701037399387, + 0.6459767333305183, + 0.5719783589628198, + 0.5281361524329609, + 0.5026578428839092, + 0.43824234419217944, + 0.45770941745598764, + 0.4034845101364514, + 0.4678868017012227, + 0.5047643963642175 + ], + "population_cv": 0.6065148734399883 + }, + "loss_nats": 2.1393918991088867, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.00010485891834832728, + 0.00010495351307326928, + 0.00011622957390500233, + 0.00011138651461806148, + 0.00010787966311909258, + 0.000104747356090229, + 0.00010223284334642813, + 9.973083797376603e-05, + 9.735774074215442e-05, + 9.589978435542434e-05, + 9.408760524820536e-05, + 9.229916759068146e-05, + 9.143509669229388e-05, + 8.827183046378195e-05, + 8.612661622464657e-05, + 8.33196536405012e-05, + 8.087792957667261e-05, + 7.82960414653644e-05, + 8.221158350352198e-05, + 7.83153300289996e-05, + 7.789687515469268e-05, + 7.347232894971967e-05, + 6.715180643368512e-05, + 6.132270209491253e-05, + 5.5257889471249655e-05, + 5.0392085540806875e-05, + 4.541657472145744e-05, + 4.046381582156755e-05, + 3.4924181818496436e-05, + 2.9514063498936594e-05, + 2.4251618015114218e-05, + 2.1234091036603786e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010650240255927201, + "first_to_last_ratio": 2.8263626166920495, + "imbalance_abs_log_ratio": 1.038990590500678, + "last_quartile_mean": 3.768178999052907e-05, + "mean": 7.755673851761458e-05, + "normalized": [ + 1.3520284678359944, + 1.353248151989171, + 1.4986392688316106, + 1.4361938981325721, + 1.3909773048874547, + 1.3505900079390127, + 1.3181684183793927, + 1.2859080961883835, + 1.255309887999515, + 1.236511310150616, + 1.213145460298544, + 1.1900857276214447, + 1.1789445822496427, + 1.1381581040019337, + 1.1104981703825207, + 1.0743057951254327, + 1.0428227272386361, + 1.0095324140994133, + 1.060018575753417, + 1.009781116713833, + 1.0043856490561534, + 0.947336496531926, + 0.8658410309303259, + 0.7906818062106236, + 0.7124834092746121, + 0.6497447740064765, + 0.585591601574923, + 0.5217317875271078, + 0.4503049314092096, + 0.380548022816011, + 0.3126951761851902, + 0.2737878286589002 + ], + "population_cv": 0.34367475536074504 + }, + "activation_output_rms_by_block": [ + 0.09794864058494568, + 0.13700027763843536, + 0.20758503675460815, + 0.22207145392894745, + 0.22574232518672943, + 0.2265917956829071, + 0.22746813297271729, + 0.22716933488845825, + 0.22722382843494415, + 0.22613705694675446, + 0.22211141884326935, + 0.21741008758544922, + 0.21311214566230774, + 0.20640577375888824, + 0.19932369887828827, + 0.19335229694843292, + 0.19345322251319885, + 0.19645750522613525, + 0.20112650096416473, + 0.2038118690252304, + 0.20720544457435608, + 0.21031972765922546, + 0.2129739224910736, + 0.21647654473781586, + 0.22099705040454865, + 0.23361735045909882, + 0.2512694001197815, + 0.2773953676223755, + 0.3133004307746887, + 0.35808902978897095, + 0.4303363263607025, + 0.54518723487854 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1964471247047186, + "first_to_last_ratio": 0.5975141297159507, + "imbalance_abs_log_ratio": 0.5149773473959137, + "last_quartile_mean": 0.32877402380108833, + "mean": 0.2358959447592497, + "normalized": [ + 0.4152196880065486, + 0.5807657176059338, + 0.879985609614718, + 0.9413958097312304, + 0.9569572101678865, + 0.9605582491643161, + 0.964273180723248, + 0.9630065286637394, + 0.9632375353753703, + 0.958630540162719, + 0.9415652272867661, + 0.9216355448896473, + 0.9034158933075572, + 0.8749865283591098, + 0.8449644994182233, + 0.8196507877478102, + 0.8200786270854848, + 0.832814253872128, + 0.8526068609167068, + 0.8639905583507864, + 0.8783764586789548, + 0.8915783943376951, + 0.9028299435517223, + 0.9176781099766134, + 0.9368412442617167, + 0.9903406804959011, + 1.065170494457722, + 1.1759225785143497, + 1.32812978660755, + 1.5179956999872501, + 1.824263349672647, + 2.311134409007948 + ], + "population_cv": 0.335075401661643 + }, + "bits_per_byte": 2.08044695422856, + "core_parameter_grad_rms_by_block": [ + 0.00040070945427075937, + 0.00021125547826331077, + 0.00020379424936248455, + 0.00011927406183950472, + 9.446000613801463e-05, + 0.00010295610627477164, + 9.818733926828671e-05, + 0.00012673421109679544, + 0.00011573807851122185, + 0.00011146965874368272, + 0.0001232427596852727, + 0.00012517422740451753, + 0.000137325623762558, + 0.0001763842480315643, + 0.00017420922346804142, + 0.00019637547238758496, + 0.00020814888113781, + 0.0002138469295217413, + 0.0002110578549773679, + 0.0002602958562879386, + 0.0002719585176535882, + 0.00027346390406242154, + 0.00028937535779988523, + 0.00028561131716422746, + 0.00028061956919165, + 0.00023874030969169658, + 0.00021716224392712217, + 0.0001981302869037355, + 0.00018664421118786767, + 0.0001673158872138231, + 0.0001540361578610053, + 0.00014575183058398037 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016967136331424098, + "first_to_last_ratio": 0.8545520537501935, + "imbalance_abs_log_ratio": 0.1571778610785998, + "last_quartile_mean": 0.0001985500620701101, + "mean": 0.00019123279105231976, + "normalized": [ + 2.0954013799920355, + 1.104703210682696, + 1.0656867383519393, + 0.6237113477409442, + 0.4939529754192163, + 0.5383810261212141, + 0.5134440528111281, + 0.6627221743687356, + 0.605220882225982, + 0.582900339059453, + 0.6444645764311125, + 0.6545646628682571, + 0.71810709348998, + 0.9223535726323577, + 0.9109798717542076, + 1.0268922568507521, + 1.0884581038241612, + 1.1182545019867147, + 1.1036697933233854, + 1.3611465630741315, + 1.4221332866452938, + 1.4300052964640568, + 1.5132099270606496, + 1.493526897727945, + 1.4674239059496585, + 1.2484276800958218, + 1.1355910392360917, + 1.0360685832877303, + 0.976005266465014, + 0.8749330399515364, + 0.8054902980465429, + 0.7621696560612549 + ], + "population_cv": 0.3661316865844204 + }, + "loss_nats": 1.4420559406280518, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.00010598741209832951, + 0.00010775360715342686, + 0.00011871542665176094, + 0.00011485091090435162, + 0.00011244894267292693, + 0.00011016445205314085, + 0.00010846564691746607, + 0.00010656306403689086, + 0.0001046601973939687, + 0.00010342278255848214, + 0.00010179441596847028, + 0.00010021102934842929, + 9.965883509721607e-05, + 9.537018195260316e-05, + 9.32025650399737e-05, + 9.038933058036491e-05, + 8.758632611716166e-05, + 8.443772094324231e-05, + 8.744140359340236e-05, + 8.20954010123387e-05, + 8.16298124846071e-05, + 7.812595140421763e-05, + 7.176945655373856e-05, + 6.528423546114936e-05, + 5.917211092310026e-05, + 5.3287756600184366e-05, + 4.791216633748263e-05, + 4.251671634847298e-05, + 3.64475381502416e-05, + 3.0126682759146206e-05, + 2.3703445549472235e-05, + 1.9818620785372332e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001106186828110367, + "first_to_last_ratio": 2.827449739094469, + "imbalance_abs_log_ratio": 1.0393751531099071, + "last_quartile_mean": 3.912312968168408e-05, + "mean": 8.203169204534788e-05, + "normalized": [ + 1.292030061256553, + 1.3135607040003474, + 1.4471897835062812, + 1.4000797501636437, + 1.3707987714158585, + 1.342949892978447, + 1.3222407610159408, + 1.2990474946924393, + 1.275850769189444, + 1.2607661743867122, + 1.2409157172108236, + 1.2216135843331344, + 1.2148821097353903, + 1.1626016674126585, + 1.1361775274420824, + 1.1018830445481589, + 1.067713269509827, + 1.0293304799389529, + 1.06594660445458, + 1.000776638460116, + 0.9951009231832177, + 0.9523874182801066, + 0.8748991367149143, + 0.7958416294163435, + 0.7213323222735595, + 0.6495996275528049, + 0.5840689755734447, + 0.5182962253779837, + 0.444310451747028, + 0.36725662982170226, + 0.28895473150022005, + 0.24159712290728336 + ], + "population_cv": 0.3450366052372569 + }, + "activation_output_rms_by_block": [ + 0.11079900711774826, + 0.18173734843730927, + 0.25566670298576355, + 0.2647266387939453, + 0.2669948637485504, + 0.2675777077674866, + 0.2675747275352478, + 0.26725223660469055, + 0.2656837999820709, + 0.265005499124527, + 0.25976043939590454, + 0.2535160183906555, + 0.24569463729858398, + 0.23344992101192474, + 0.22194796800613403, + 0.21299737691879272, + 0.21236176788806915, + 0.21684354543685913, + 0.22108736634254456, + 0.22472712397575378, + 0.22933396697044373, + 0.23535853624343872, + 0.2425893247127533, + 0.2497948557138443, + 0.26040923595428467, + 0.27719539403915405, + 0.2986113131046295, + 0.3278995454311371, + 0.36809566617012024, + 0.4197922646999359, + 0.5092319846153259, + 0.6714946031570435 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23529115412384272, + "first_to_last_ratio": 0.6008590681870453, + "imbalance_abs_log_ratio": 0.5093948674743478, + "last_quartile_mean": 0.39159125089645386, + "mean": 0.2751628558617085, + "normalized": [ + 0.40266701975732394, + 0.6604719516672225, + 0.9291468580855864, + 0.962072580490267, + 0.9703157895800326, + 0.9724339679842758, + 0.972423137190165, + 0.9712511369594387, + 0.9655511066348229, + 0.9630860178952118, + 0.9440243617999629, + 0.9213308155155495, + 0.8929062628353637, + 0.8484063747661208, + 0.806605845513105, + 0.7740775049443485, + 0.7717675673303741, + 0.7880552945919432, + 0.803478237097738, + 0.8167058859452209, + 0.8334481274816485, + 0.8553426860845104, + 0.881620900295765, + 0.9078073235269309, + 0.9463822256778773, + 1.007386673506787, + 1.0852166516788362, + 1.191656280802424, + 1.3377374828349577, + 1.5256138528774221, + 1.850656706627787, + 2.4403533720209807 + ], + "population_cv": 0.3576152457433685 + }, + "bits_per_byte": 1.9501283044158226, + "core_parameter_grad_rms_by_block": [ + 0.00032837002715831827, + 0.0002079177996867109, + 0.00018809406473136247, + 0.00010899326506499365, + 9.474430680287842e-05, + 8.318366756728252e-05, + 9.647881676005136e-05, + 0.00013404889675397192, + 0.00012865721885494213, + 0.0001321695767247016, + 0.00014056789256563647, + 0.00015176892928157556, + 0.00016698296163968515, + 0.00022933649843535653, + 0.000217708018553423, + 0.0002507549770035681, + 0.00025689149388127776, + 0.0002828493196393162, + 0.00027498157120012506, + 0.0003306027661240732, + 0.00032889440594708014, + 0.00033897343531885953, + 0.000372930739966371, + 0.0003793341532875068, + 0.0003649148263870391, + 0.0003190518951646816, + 0.00028800578855572547, + 0.0002568521570555004, + 0.00024004963937954607, + 0.00021095610212654392, + 0.00019368945551758278, + 0.00017448279084866967 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015522885556569618, + "first_to_last_ratio": 0.6063619309634984, + "imbalance_abs_log_ratio": 0.5002782253787689, + "last_quartile_mean": 0.00025600033187941115, + "mean": 0.00022728867056201116, + "normalized": [ + 1.4447267712304608, + 0.9147741467825811, + 0.8275558313851142, + 0.4795367265578555, + 0.41684570800914306, + 0.36598246345317736, + 0.42447701648080627, + 0.5897737728084399, + 0.56605205414249, + 0.581505344713791, + 0.6184553423540983, + 0.6677364468114501, + 0.7346734935216545, + 1.0090098106002459, + 0.9578480881387607, + 1.1032445056919573, + 1.1302432859766764, + 1.244449707677561, + 1.2098340428503753, + 1.4545501335662696, + 1.4470338760565185, + 1.491378494496396, + 1.6407801543483633, + 1.668953196609125, + 1.605512608634311, + 1.4037298664107178, + 1.2671365794149818, + 1.1300702160841907, + 1.05614432424626, + 0.928141739774877, + 0.8521738238806694, + 0.7676704272906798 + ], + "population_cv": 0.3864254083322711 + }, + "loss_nats": 1.3517259359359741, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.00018891454965341836, + 0.0001931450387928635, + 0.0002085021696984768, + 0.00020282011246308684, + 0.00019955210154876113, + 0.00019673256610985845, + 0.00019503913063090295, + 0.0001924382959259674, + 0.00019001582404598594, + 0.00018850783817470074, + 0.00018595026631373912, + 0.0001838041644077748, + 0.00018427071336191148, + 0.00017771514831110835, + 0.0001728432544041425, + 0.0001671368081588298, + 0.0001615467481315136, + 0.00015400326810777187, + 0.000160209892783314, + 0.00015067463391460478, + 0.00014840754738543183, + 0.00014097621897235513, + 0.00012854386295657605, + 0.00011642893514363095, + 0.00010425484651932493, + 9.334496280644089e-05, + 8.331386925419793e-05, + 7.313644891837612e-05, + 6.15216777077876e-05, + 4.904071465716697e-05, + 3.525121428538114e-05, + 2.4962779207271524e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019714299560291693, + "first_to_last_ratio": 3.0050767723956184, + "imbalance_abs_log_ratio": 1.1003031158789967, + "last_quartile_mean": 6.560331416949339e-05, + "mean": 0.00014728142508602104, + "normalized": [ + 1.282677360998382, + 1.311401208129643, + 1.4156718647765612, + 1.3770922731405397, + 1.3549033860326305, + 1.3357595229334251, + 1.324261566025645, + 1.3066026202121013, + 1.2901547084773621, + 1.2799159029361715, + 1.262550699826086, + 1.247979263511487, + 1.2511470014245618, + 1.206636534154339, + 1.1735577266664268, + 1.1348125404219307, + 1.0968575842959203, + 1.045639448544342, + 1.0877807075110928, + 1.023038946198422, + 1.0076460578701834, + 0.9571894004285788, + 0.8727771535446431, + 0.7905201560592559, + 0.707861473084158, + 0.6337863905914947, + 0.5656780493910737, + 0.4965761899415362, + 0.41771511697320496, + 0.33297284181304126, + 0.23934596141225786, + 0.1694903426735027 + ], + "population_cv": 0.361085575415688 + }, + "activation_output_rms_by_block": [ + 0.1016591340303421, + 0.18086770176887512, + 0.2325568050146103, + 0.2376081645488739, + 0.23870401084423065, + 0.23941391706466675, + 0.23885947465896606, + 0.2381371706724167, + 0.23675362765789032, + 0.23536528646945953, + 0.22870509326457977, + 0.22131286561489105, + 0.21254637837409973, + 0.19929367303848267, + 0.18746964633464813, + 0.17704539000988007, + 0.17596758902072906, + 0.1784123033285141, + 0.18117663264274597, + 0.18366514146327972, + 0.1872515082359314, + 0.19141052663326263, + 0.1969655156135559, + 0.2033752053976059, + 0.21314463019371033, + 0.22718842327594757, + 0.2457481175661087, + 0.269707053899765, + 0.30305343866348267, + 0.347743958234787, + 0.4299577474594116, + 0.6035616993904114 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2134757973253727, + "first_to_last_ratio": 0.6468706108937196, + "imbalance_abs_log_ratio": 0.4356089876432565, + "last_quartile_mean": 0.33001313358545303, + "mean": 0.23576961969956756, + "normalized": [ + 0.4311799550759871, + 0.7671374369579427, + 0.986373076017804, + 1.0077980566438083, + 1.0124460104249322, + 1.015457026947505, + 1.0131053990897376, + 1.010041798327817, + 1.0041735994636487, + 0.9982850494876174, + 0.9700363157730422, + 0.9386827102529232, + 0.9015002808459363, + 0.8452898778580343, + 0.7951391132306771, + 0.7509253746750002, + 0.7463539587711004, + 0.7567230398719659, + 0.7684477451913126, + 0.7790025775895867, + 0.7942138960674365, + 0.8118540755045961, + 0.8354151644496934, + 0.8626014057992686, + 0.9040377232033228, + 0.9636034683579899, + 1.0423230858973958, + 1.143943202875089, + 1.2853795117863462, + 1.4749311581276, + 1.8236350722679655, + 2.5599638331669174 + ], + "population_cv": 0.3675345504070441 + }, + "bits_per_byte": 1.8720084529317582, + "core_parameter_grad_rms_by_block": [ + 0.0004570302696620844, + 0.00031493934518203267, + 0.0002143615651481701, + 0.00013786409436507824, + 0.0001291171736265367, + 0.00013682107280354788, + 0.00015837516984797755, + 0.00019563631023119442, + 0.00019674727625245423, + 0.000191343414336242, + 0.00021955065404037036, + 0.0002332172506517699, + 0.00027919484030541734, + 0.0003858941801364767, + 0.0003704235701666674, + 0.00042305543536477747, + 0.0004472346343569547, + 0.0004940425030518999, + 0.0004673224693273912, + 0.0005594278228367747, + 0.0005764247107952261, + 0.0005972083894683241, + 0.0006512089060709392, + 0.0006542639335938355, + 0.0006197665059356166, + 0.0005424206044372668, + 0.0004994517842756415, + 0.0004394000632885711, + 0.0004055610719669202, + 0.0003512581615295656, + 0.00030757820471853463, + 0.00025263519960351045 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021801812510832775, + "first_to_last_ratio": 0.5102716406035511, + "imbalance_abs_log_ratio": 0.6728120664256351, + "last_quartile_mean": 0.0004272589494694534, + "mean": 0.0003721492683555553, + "normalized": [ + 1.2280832142478726, + 0.8462715688618158, + 0.5760096374645209, + 0.3704537562958782, + 0.34694995961452973, + 0.36765105950128485, + 0.42556894051626687, + 0.5256931206546978, + 0.5286783905872947, + 0.5141577066152674, + 0.5899532061705127, + 0.6266766334979106, + 0.7502227306239702, + 1.0369338675356186, + 0.9953628870573538, + 1.1367896468913272, + 1.2017614231332092, + 1.3275385579419885, + 1.2557393203871798, + 1.5032350468100115, + 1.5489072794427852, + 1.6047549740115168, + 1.7498594285795221, + 1.7580685741634856, + 1.6653707494152192, + 1.4575350553128925, + 1.3420738041017997, + 1.1807091956143891, + 1.0897806510785424, + 0.9438636359052839, + 0.8264914937967075, + 0.6788544841693472 + ], + "population_cv": 0.43653500885793567 + }, + "loss_nats": 1.2975773811340332, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 7.95552135286859, + "cross_entropy_nats": 5.514347195625305, + "step": 0 + }, + { + "bits_per_byte": 5.166060485015187, + "cross_entropy_nats": 3.5808402597904205, + "step": 100 + }, + { + "bits_per_byte": 3.1277970484292927, + "cross_entropy_nats": 2.168023705482483, + "step": 500 + }, + { + "bits_per_byte": 1.9613801618593651, + "cross_entropy_nats": 1.359525129199028, + "step": 2000 + }, + { + "bits_per_byte": 1.7971937855298066, + "cross_entropy_nats": 1.2457198053598404, + "step": 4000 + }, + { + "bits_per_byte": 1.7178972072507075, + "cross_entropy_nats": 1.1907556056976318, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "final_optimizer_state": "06d1b38e173978179caca1f5bf426eaad5609e9e9a1b567ef88c8b18eeab7e4d", + "final_public_parameters": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "seed": 2026073001, + "timing": { + "mean_ms": 42.00224438344842, + "measured_steps": 7980, + "median_ms": 41.91726399585605, + "p95_ms": 42.462232409161516, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "baseline", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00042041894630528986, + 0.000374794501112774, + 0.0003388339828234166, + 0.000304876797599718, + 0.0002806047268677503, + 0.0002645689237397164, + 0.00024906310136429965, + 0.00023202023294288665, + 0.00021741738601122051, + 0.00020566635066643357, + 0.0001967942516785115, + 0.0001872276043286547, + 0.00017897134239319712, + 0.00017287296941503882, + 0.00016580727242399007, + 0.00016062518989201635, + 0.0001559370430186391, + 0.00015203763905446976, + 0.00014800428471062332, + 0.00014486612053588033, + 0.00014247088984120637, + 0.00013994764594826847, + 0.00013763502647634596, + 0.00013544136891141534, + 0.00013363662583287805, + 0.0001319239236181602, + 0.00013088916603010148, + 0.00012932090612594038, + 0.00012785944272764027, + 0.00012667682312894613, + 0.0001251104404218495, + 0.00012353518104646355 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00030814765159448143, + "first_to_last_ratio": 2.395816319369912, + "imbalance_abs_log_ratio": 0.8737240159497589, + "last_quartile_mean": 0.00012861906361649744, + "mean": 0.00019174550334355445, + "normalized": [ + 2.1925882952885543, + 1.9546455826984732, + 1.7671026277801187, + 1.5900075479395406, + 1.4634227242606306, + 1.3797920635754504, + 1.2989253829752037, + 1.2100426288859099, + 1.1338851875012121, + 1.0726006455438846, + 1.0263304653664347, + 0.9764380445115058, + 0.9333796061570758, + 0.9015750899007978, + 0.8647257408008662, + 0.8376999047754503, + 0.8132500647967916, + 0.7929137132465669, + 0.7718787774931073, + 0.7555124788314888, + 0.7430207611488979, + 0.7298614231256383, + 0.7178005433052705, + 0.7063600791135226, + 0.6969478997034861, + 0.6880157360550423, + 0.6826192205174408, + 0.6744403590744623, + 0.6668184676985713, + 0.660650815377801, + 0.6524817439796045, + 0.644266378571199 + ], + "population_cv": 0.4034880502878489 + }, + "activation_output_rms_by_block": [ + 0.02817646972835064, + 0.028377581387758255, + 0.028678538277745247, + 0.028959807008504868, + 0.029224006459116936, + 0.029430421069264412, + 0.02957010455429554, + 0.029960129410028458, + 0.030406322330236435, + 0.03066711500287056, + 0.030792171135544777, + 0.031063809990882874, + 0.03129913657903671, + 0.03160789608955383, + 0.03180097043514252, + 0.032092027366161346, + 0.03248445317149162, + 0.032903194427490234, + 0.03316524624824524, + 0.033593542873859406, + 0.034045662730932236, + 0.03455357626080513, + 0.03491735830903053, + 0.03521261364221573, + 0.03560211881995201, + 0.0359802283346653, + 0.03664350509643555, + 0.03706779330968857, + 0.0375242717564106, + 0.03805854171514511, + 0.038408540189266205, + 0.039021097123622894 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029047132236883044, + "first_to_last_ratio": 0.7789886319526242, + "imbalance_abs_log_ratio": 0.24975882634672159, + "last_quartile_mean": 0.03728826204314828, + "mean": 0.03285275783855468, + "normalized": [ + 0.8576591915605899, + 0.863780798166523, + 0.8729415782588962, + 0.8815030735264118, + 0.8895449996231614, + 0.8958280219241153, + 0.9000798258584322, + 0.9119517319446604, + 0.9255333290331198, + 0.93347155674343, + 0.9372781209682287, + 0.945546493951495, + 0.9527095634663979, + 0.96210784631481, + 0.9679848063720901, + 0.9768442431490312, + 0.9887892313676379, + 1.0015352315073034, + 1.0095117862319563, + 1.0225486407851998, + 1.0363106469855512, + 1.051770948138011, + 1.0628440534770849, + 1.0718312847663467, + 1.0836873724546434, + 1.095196589323784, + 1.1153859677933096, + 1.128300810904444, + 1.1421954875390588, + 1.1584580479414466, + 1.1691115972063533, + 1.187757122716477 + ], + "population_cv": 0.09694557255182189 + }, + "bits_per_byte": 8.04236897991864, + "core_parameter_grad_rms_by_block": [ + 0.010411069784541356, + 0.009271735997530202, + 0.009345779773643629, + 0.008099773113238128, + 0.008225801589217078, + 0.008647498843153682, + 0.007992995665779355, + 0.0076659621960069, + 0.007406648616870928, + 0.006601475371863238, + 0.006568536350729326, + 0.006644454140273569, + 0.005847021988988907, + 0.006236102346072616, + 0.006448411665711761, + 0.005769055287848533, + 0.005701484582581849, + 0.005836584711958408, + 0.005498613632501928, + 0.004978853123018873, + 0.005187902378445178, + 0.005078997897134883, + 0.005158829428828408, + 0.005043411661395455, + 0.00572819320620205, + 0.004439465860302503, + 0.004811331873512697, + 0.004686191415610403, + 0.004548827932369003, + 0.00413026790671143, + 0.004686908144290253, + 0.004087931241888514 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00870757712038879, + "first_to_last_ratio": 1.8766776125890325, + "imbalance_abs_log_ratio": 0.6295029861071718, + "last_quartile_mean": 0.004639889697610857, + "mean": 0.006274566179006908, + "normalized": [ + 1.6592493389222875, + 1.4776696480708191, + 1.489470268862924, + 1.2908897415629936, + 1.3109753494573861, + 1.3781827454599171, + 1.2738722387727572, + 1.2217517478188766, + 1.1804240174646141, + 1.0521006844983298, + 1.0468510751716935, + 1.0589503641708666, + 0.9318607569319367, + 0.9938698817038568, + 1.027706375507919, + 0.9194349255810409, + 0.9086659411861104, + 0.930197330850717, + 0.8763336740154054, + 0.7934975870804967, + 0.8268145128188418, + 0.8094580170542961, + 0.8221810531042816, + 0.8037865116905484, + 0.9129225898305317, + 0.707533514453289, + 0.766799127820212, + 0.7468550465352011, + 0.7249629380893642, + 0.6582555333514927, + 0.7469692741422425, + 0.6515081880187487 + ], + "population_cv": 0.26225105130829346 + }, + "loss_nats": 5.574545383453369, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.223192369565368e-05, + 6.862939335405827e-05, + 6.46814878564328e-05, + 6.090753959142603e-05, + 5.703999340767041e-05, + 5.302598947309889e-05, + 4.9269601731793955e-05, + 4.545194315141998e-05, + 4.1738676372915506e-05, + 3.8814047002233565e-05, + 3.584829028113745e-05, + 3.317802838864736e-05, + 3.07806913042441e-05, + 2.884516288759187e-05, + 2.6981158953276463e-05, + 2.530650635890197e-05, + 2.3913053155411035e-05, + 2.265583498228807e-05, + 2.164491161238402e-05, + 2.069839865725953e-05, + 1.9972700101789087e-05, + 1.928487472468987e-05, + 1.8700015061767772e-05, + 1.8183285646955483e-05, + 1.7775813830667175e-05, + 1.7374133676639758e-05, + 1.7024083717842586e-05, + 1.673021324677393e-05, + 1.646951204747893e-05, + 1.624442847969476e-05, + 1.604618228157051e-05, + 1.58860220835777e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.890473403269425e-05, + "first_to_last_ratio": 3.528539860533838, + "imbalance_abs_log_ratio": 1.2608841481311845, + "last_quartile_mean": 1.669379867053067e-05, + "mean": 3.222918428491539e-05, + "normalized": [ + 2.2411961487173366, + 2.1294176342582674, + 2.0069228958644927, + 1.8898256640002307, + 1.7698242966194933, + 1.64527867054697, + 1.528726302727252, + 1.4102728368677142, + 1.2950584167422121, + 1.2043136636380887, + 1.1122928202038285, + 1.029440524939878, + 0.9550564802426835, + 0.8950013327235407, + 0.8371654310191393, + 0.7852046808006393, + 0.7419689230733446, + 0.7029602357293278, + 0.6715935290523237, + 0.6422253344757239, + 0.6197085202413003, + 0.5983668266067783, + 0.5802199303728628, + 0.5641869644049918, + 0.5515440190333021, + 0.5390807760769664, + 0.5282195033961989, + 0.5191013554322059, + 0.5110123762948401, + 0.5040285331483811, + 0.49787739397055714, + 0.49290797877912856 + ], + "population_cv": 0.5370990514498413 + }, + "activation_output_rms_by_block": [ + 0.029305610805749893, + 0.03049159236252308, + 0.0324031226336956, + 0.03488599509000778, + 0.037802305072546005, + 0.04105908423662186, + 0.04460485652089119, + 0.04878050833940506, + 0.053197599947452545, + 0.057943083345890045, + 0.06266378611326218, + 0.06765592098236084, + 0.07297883927822113, + 0.07823853194713593, + 0.08444638550281525, + 0.09052036702632904, + 0.09696295112371445, + 0.10418187081813812, + 0.11156722903251648, + 0.1197449341416359, + 0.12838369607925415, + 0.13761533796787262, + 0.14720149338245392, + 0.15857642889022827, + 0.16927987337112427, + 0.1816006898880005, + 0.19378916919231415, + 0.20805081725120544, + 0.22281049191951752, + 0.2388531118631363, + 0.25510889291763306, + 0.2763713300228119 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03741663438268006, + "first_to_last_ratio": 0.17145265067740043, + "imbalance_abs_log_ratio": 1.763448139840426, + "last_quartile_mean": 0.2182330470532179, + "mean": 0.11303362209582701, + "normalized": [ + 0.2592645467992289, + 0.2697568369230271, + 0.286668002253569, + 0.30863378916082446, + 0.3344341654423688, + 0.36324664710658405, + 0.39461582928906175, + 0.4315575085973109, + 0.47063518766436574, + 0.5126181243379725, + 0.554381828622084, + 0.5985468724076087, + 0.6456383324277757, + 0.6921704400555022, + 0.747090856127956, + 0.8008269163451932, + 0.8578239759627604, + 0.9216892185390237, + 0.9870269302520681, + 1.0593744756769734, + 1.135800956377508, + 1.2174726016583441, + 1.3022806015864932, + 1.4029138052020595, + 1.497606377928977, + 1.6066077200821192, + 1.714438284814271, + 1.8406100184494245, + 1.9711877562467606, + 2.1131156149330748, + 2.256929294023315, + 2.445036484706394 + ], + "population_cv": 0.6334590447760624 + }, + "bits_per_byte": 5.11848367743279, + "core_parameter_grad_rms_by_block": [ + 0.00028524855645157025, + 0.00022469127596424853, + 0.00022670239393809648, + 0.0001993403227560908, + 0.0002100203582478305, + 0.00019183315773228503, + 0.00020704106188439305, + 0.00020121077118127333, + 0.00020796371696885172, + 0.00017825031863398375, + 0.00018469615204762195, + 0.0001848551977445252, + 0.00018214824531137669, + 0.0001649843875278976, + 0.0001966104199015976, + 0.00016919176207710993, + 0.0001681445657707544, + 0.00017333427923610245, + 0.0001624042419733422, + 0.00015992743704390023, + 0.00016485453322609178, + 0.00015449118786750607, + 0.00015016403044324942, + 0.00015560032515501767, + 0.00015156764536984935, + 0.00014581309890539414, + 0.000137484036700603, + 0.0001500871171184727, + 0.00014447415143049674, + 0.00014218802742214665, + 0.00014221421491139387, + 0.00016697872359292516 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002182609872694735, + "first_to_last_ratio": 1.4787241905811908, + "imbalance_abs_log_ratio": 0.3911796826204884, + "last_quartile_mean": 0.0001476008769314102, + "mean": 0.00017764111607924995, + "normalized": [ + 1.6057575112527105, + 1.2648607536557492, + 1.2761819951466593, + 1.122151938446279, + 1.1822733547459552, + 1.0798916487706802, + 1.1655019201299495, + 1.1326813049942133, + 1.170695847666675, + 1.0034294006262718, + 1.0397151072008834, + 1.0406104274984227, + 1.0253721060281789, + 0.9287511313219521, + 1.1067844215405909, + 0.9524358201038854, + 0.9465408092557868, + 0.9757554054027329, + 0.9142266472863736, + 0.9002839014620511, + 0.9280201389443322, + 0.869681475084765, + 0.8453224892837178, + 0.8759251719945322, + 0.8532238972323919, + 0.8208296712138615, + 0.7739426532271284, + 0.8448895189980412, + 0.8132922975221759, + 0.8004229570293466, + 0.8005703749797919, + 0.9399779019539145 + ], + "population_cv": 0.17622384112301237 + }, + "loss_nats": 3.5478625297546387, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00014093925710767508, + 0.00014282441406976432, + 0.00014458753867074847, + 0.0001413008722011, + 0.00013349838263820857, + 0.00012304198753554374, + 0.00011454652121756226, + 0.00010784466576296836, + 0.0001000517513602972, + 9.189492993755266e-05, + 8.563220035284758e-05, + 8.04013543529436e-05, + 7.555109914392233e-05, + 7.068048580549657e-05, + 6.581390334758908e-05, + 6.151734851300716e-05, + 5.7112174545181915e-05, + 5.2947088988730684e-05, + 4.8879119276534766e-05, + 4.527245255303569e-05, + 4.1345181671204045e-05, + 3.832591755781323e-05, + 3.484322223812342e-05, + 3.232451854273677e-05, + 3.010137334058527e-05, + 2.8228545488673262e-05, + 2.6716850697994232e-05, + 2.5478970201220363e-05, + 2.4533494070055895e-05, + 2.4085495169856586e-05, + 2.3973674615263008e-05, + 2.415898779872805e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013107295490044635, + "first_to_last_ratio": 5.058842318548806, + "imbalance_abs_log_ratio": 1.621137666322135, + "last_quartile_mean": 2.5909673922797083e-05, + "mean": 6.995168058665513e-05, + "normalized": [ + 2.01480873548251, + 2.041758151797876, + 2.06696304446375, + 2.019978233776078, + 1.9084371028489115, + 1.7589568471213655, + 1.6375092100276802, + 1.5417022844700914, + 1.430298036032952, + 1.3136915248764403, + 1.2241621592889058, + 1.14938417031087, + 1.0800469482692516, + 1.010418694915251, + 0.9408480653450457, + 0.8794263125215463, + 0.8164517859500485, + 0.756909462999114, + 0.6987554675828556, + 0.6471960669615769, + 0.591053443240527, + 0.5478913049177658, + 0.49810414769034944, + 0.46209781197027316, + 0.4303166569857621, + 0.4035434924784137, + 0.38193293533380923, + 0.3642367129358298, + 0.3507205811826659, + 0.34431617607842635, + 0.34271763614835193, + 0.34536679599570513 + ], + "population_cv": 0.5909500348424317 + }, + "activation_output_rms_by_block": [ + 0.04959565028548241, + 0.05870900675654411, + 0.06652694195508957, + 0.07286322116851807, + 0.07801168411970139, + 0.08261121064424515, + 0.08867742121219635, + 0.09177010506391525, + 0.09494514763355255, + 0.0981190949678421, + 0.10163456201553345, + 0.10214314609766006, + 0.10418473184108734, + 0.10623370856046677, + 0.10849270969629288, + 0.11100933700799942, + 0.11405495554208755, + 0.11736668646335602, + 0.12177617102861404, + 0.12738163769245148, + 0.13297238945960999, + 0.14011575281620026, + 0.15017743408679962, + 0.16121913492679596, + 0.17427310347557068, + 0.189283087849617, + 0.2044389545917511, + 0.22237628698349, + 0.24115440249443054, + 0.26191622018814087, + 0.2883519232273102, + 0.3199674189090729 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07359565515071154, + "first_to_last_ratio": 0.309589437408787, + "imbalance_abs_log_ratio": 1.1725082547452423, + "last_quartile_mean": 0.2377201747149229, + "mean": 0.13694853871129453, + "normalized": [ + 0.3621480795062482, + 0.4286939262660582, + 0.48578059014807806, + 0.5320481828734462, + 0.5696423258970309, + 0.6032281280372068, + 0.6475236760221296, + 0.6701064934864231, + 0.6932906953736058, + 0.7164668998381211, + 0.7421368856647125, + 0.745850573206854, + 0.7607582586968847, + 0.775719913188861, + 0.7922151686847111, + 0.8105916138471667, + 0.8328307597537082, + 0.8570130617514684, + 0.8892111750482725, + 0.9301423650893323, + 0.9709661067646236, + 1.0231270383365143, + 1.0965975650415178, + 1.177224207310945, + 1.2725444544024032, + 1.3821475543353592, + 1.4928158892059098, + 1.6237945222058072, + 1.7609125644108963, + 1.9125156255978355, + 2.105549470923482, + 2.336406229084387 + ], + "population_cv": 0.48939845088217687 + }, + "bits_per_byte": 3.103317553853433, + "core_parameter_grad_rms_by_block": [ + 0.0005239687790313407, + 0.0004396407496139251, + 0.0004479555677531567, + 0.000484081397596248, + 0.0004298002646761996, + 0.0003778969963663625, + 0.0003311985651517251, + 0.0003381159607123189, + 0.00027120323619484496, + 0.00025249768611013433, + 0.0002492173720055997, + 0.0002512649978189958, + 0.00021872457711399188, + 0.0001974354513181695, + 0.0001883229322305711, + 0.00018548264077727127, + 0.00019625685908145198, + 0.00017137558371316787, + 0.00015305597823747372, + 0.00014260825992090635, + 0.00013921721401447933, + 0.00013266258711253505, + 0.00012717821667718618, + 0.00011256530678442586, + 0.0001087245205863228, + 0.00010436458537579927, + 0.00010507936295752309, + 9.703474953173527e-05, + 9.385369360633854e-05, + 9.122967598255955e-05, + 0.00010232602359400015, + 0.0001188300325189452 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004215822851126596, + "first_to_last_ratio": 4.105774523524949, + "imbalance_abs_log_ratio": 1.4123944031885163, + "last_quartile_mean": 0.00010268033051915298, + "mean": 0.00022447405700517828, + "normalized": [ + 2.3342063934775927, + 1.9585370152764836, + 1.995578347580821, + 2.156513781835738, + 1.9146990543601428, + 1.6834773755509973, + 1.475442505786247, + 1.506258519239571, + 1.2081718476206305, + 1.1248412822347198, + 1.1102279494144423, + 1.11934983120655, + 0.9743868847567816, + 0.8795468570054622, + 0.8389518804225415, + 0.8262987859349487, + 0.874296396206376, + 0.7634538529733728, + 0.6818426159328645, + 0.6352995166725061, + 0.6201928894227087, + 0.5909929587519093, + 0.5665608684314569, + 0.5014624330589309, + 0.48435227677029374, + 0.4649293854629791, + 0.4681136179362547, + 0.4322760092026885, + 0.41810485745430026, + 0.4064152321194739, + 0.45584788264257925, + 0.5293708952576376 + ], + "population_cv": 0.5698198347315865 + }, + "loss_nats": 2.1510558128356934, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.00010639085667207837, + 0.00010674773511709645, + 0.000114928770926781, + 0.00011333057773299515, + 0.00011114918743260205, + 0.00010892991122091189, + 0.00010597817890811712, + 0.00010435423610033467, + 0.00010266973549732938, + 0.0001015385496430099, + 0.00010008511890191585, + 9.485893679084256e-05, + 9.186805982608348e-05, + 8.913136116461828e-05, + 8.655320561956614e-05, + 8.490345499012619e-05, + 8.653065015096217e-05, + 8.836571214487776e-05, + 8.356331090908498e-05, + 8.248459926107898e-05, + 7.508730777772143e-05, + 7.080488285282627e-05, + 6.500465678982437e-05, + 6.0319798649288714e-05, + 5.538603363675065e-05, + 5.071960185887292e-05, + 4.5680528273805976e-05, + 4.0155850001610816e-05, + 3.449051291681826e-05, + 2.892977681767661e-05, + 2.4221473722718656e-05, + 2.1527655917452648e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010897618176386459, + "first_to_last_ratio": 2.8953050536910427, + "imbalance_abs_log_ratio": 1.0630904780785062, + "last_quartile_mean": 3.763892914321332e-05, + "mean": 7.927156963205562e-05, + "normalized": [ + 1.3421060938480058, + 1.3466080665814153, + 1.4498107134781197, + 1.429649725103549, + 1.4021317850587363, + 1.3741359194288378, + 1.3369002203440912, + 1.3164144041136308, + 1.29516465958575, + 1.2808949048733107, + 1.262560075023968, + 1.1966325030668217, + 1.1589029995557718, + 1.1243799205481557, + 1.0918568412522767, + 1.0710454628842516, + 1.0915723071032915, + 1.1147213629682524, + 1.054139728744489, + 1.0405319289618822, + 0.9472161094607345, + 0.8931939052231708, + 0.8200248473891447, + 0.760926003222431, + 0.6986872329364575, + 0.6398208348124227, + 0.5762536112989216, + 0.5065605511281904, + 0.4350930992903045, + 0.3649451745683369, + 0.305550575510795, + 0.2715684326344833 + ], + "population_cv": 0.35248901334980626 + }, + "activation_output_rms_by_block": [ + 0.09771591424942017, + 0.13586144149303436, + 0.1991395801305771, + 0.2241763025522232, + 0.22696681320667267, + 0.22763192653656006, + 0.22912801802158356, + 0.2290782928466797, + 0.22818906605243683, + 0.22590669989585876, + 0.22214065492153168, + 0.21565376222133636, + 0.20698806643486023, + 0.20169880986213684, + 0.1979832947254181, + 0.1976672261953354, + 0.2011667788028717, + 0.20515663921833038, + 0.205980584025383, + 0.20982402563095093, + 0.2095191776752472, + 0.2096652388572693, + 0.2105090171098709, + 0.2148621529340744, + 0.2245095670223236, + 0.2386746108531952, + 0.25724881887435913, + 0.2833212912082672, + 0.3200184106826782, + 0.3660464584827423, + 0.43435168266296387, + 0.5434449911117554 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19621228612959385, + "first_to_last_ratio": 0.5884274155428802, + "imbalance_abs_log_ratio": 0.5303016979635413, + "last_quartile_mean": 0.3334519788622856, + "mean": 0.23750704107806087, + "normalized": [ + 0.4114232311003534, + 0.5720312159014314, + 0.8384575852011324, + 0.9438722386278381, + 0.9556214088494162, + 0.9584218030055994, + 0.9647209488255828, + 0.9645115858750019, + 0.960767584054228, + 0.951157906184392, + 0.9353013448073788, + 0.9079889221071891, + 0.8715028636428086, + 0.8492329698800171, + 0.8335891594066359, + 0.8322583839961553, + 0.8469929055145556, + 0.8637918197587332, + 0.8672609582032722, + 0.8834433904719, + 0.8821598581845203, + 0.8827748344031583, + 0.8863274796164187, + 0.9046559291834054, + 0.9452754158498172, + 1.0049159375226715, + 1.0831208106786603, + 1.1928963870807885, + 1.3474059936500937, + 1.5412025547589332, + 1.8287949725254948, + 2.288121601132417 + ], + "population_cv": 0.33570692978684213 + }, + "bits_per_byte": 2.0753701983577404, + "core_parameter_grad_rms_by_block": [ + 0.00037331716215226093, + 0.0002052857320094383, + 0.00022573696371192266, + 0.00015002318346068796, + 0.00011232796492243865, + 0.00011469567077281986, + 0.0001328622706326584, + 0.00011987742098506258, + 0.00012268315232766492, + 0.00013855849890850403, + 0.00016581239426236876, + 0.00020970093409220108, + 0.00019415593885306704, + 0.0001873084275669645, + 0.00019997003245353254, + 0.00019081336552079618, + 0.00019544152058884818, + 0.0002331889851483562, + 0.00028258169221688257, + 0.00026717547757069314, + 0.00031847213451059053, + 0.00028331189819324093, + 0.0003090653888381074, + 0.000275741819401602, + 0.000255221776165695, + 0.00023469865916502512, + 0.00021558263955965886, + 0.0002009260839676739, + 0.00018496297858040776, + 0.0001671377375581853, + 0.00015254274522576858, + 0.00014770686125523138 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017926579608091117, + "first_to_last_ratio": 0.9200315924660546, + "imbalance_abs_log_ratio": 0.08334726989162704, + "last_quartile_mean": 0.00019484743518470574, + "mean": 0.0002052152347055736, + "normalized": [ + 1.8191493564689118, + 1.0003435286077362, + 1.1000010015620527, + 0.7310528561679606, + 0.5473665982138111, + 0.5589042691561181, + 0.6474288852057137, + 0.584154588508271, + 0.5978267281358565, + 0.6751862214678978, + 0.8079926156567426, + 1.021858510617222, + 0.946108797096017, + 0.9127413363617942, + 0.9744404831368078, + 0.9298206626547972, + 0.9523733502010805, + 1.1363141994936055, + 1.377001530233894, + 1.301928085183418, + 1.5518932352537513, + 1.3805597747151384, + 1.5060547979370522, + 1.3436712912528852, + 1.2436785043365166, + 1.1436707391717382, + 1.0505196647264496, + 0.9790992576937408, + 0.9013121216160088, + 0.8144509241625318, + 0.7433305107421713, + 0.7197655742623075 + ], + "population_cv": 0.31225216605899336 + }, + "loss_nats": 1.4385370016098022, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.0001095466359402053, + 0.00011157916014781222, + 0.0001209566617035307, + 0.00011857408389914781, + 0.00011687575170071796, + 0.00011450610327301547, + 0.00011203528265468776, + 0.00011082263517891988, + 0.00010936566832242534, + 0.00010819115414051339, + 0.00010705165914259851, + 0.00010176753858104348, + 9.876617696136236e-05, + 9.541869076201692e-05, + 9.238059283234179e-05, + 9.015990508487448e-05, + 9.102027252083644e-05, + 9.192643483402207e-05, + 8.731557318242267e-05, + 8.749608969083056e-05, + 7.95004380051978e-05, + 7.466241368092597e-05, + 6.735928764101118e-05, + 6.187916733324528e-05, + 5.64747242606245e-05, + 5.17304906679783e-05, + 4.670304406317882e-05, + 4.1035571484826505e-05, + 3.507353176246397e-05, + 2.8758000553352758e-05, + 2.3154751033871435e-05, + 1.9675369912874885e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011436203931225464, + "first_to_last_ratio": 3.02339634825199, + "imbalance_abs_log_ratio": 1.1063808181034647, + "last_quartile_mean": 3.78256854673964e-05, + "mean": 8.318008940477739e-05, + "normalized": [ + 1.3169814642434559, + 1.341416689333394, + 1.45415402374626, + 1.4255104165869963, + 1.4050928838507029, + 1.376604715051423, + 1.3469002432720767, + 1.3323216646188754, + 1.3148058520377592, + 1.300685715953312, + 1.286986584273256, + 1.223460317358093, + 1.1873776244786027, + 1.1471337845969738, + 1.1106094438392846, + 1.0839120963928204, + 1.0942555264386233, + 1.1051495074341955, + 1.0497172316985723, + 1.0518874206187772, + 0.955762834280288, + 0.8975995844101339, + 0.8098006160250947, + 0.743918019036071, + 0.6789452219244779, + 0.6219095343387212, + 0.5614690293960716, + 0.49333406321717244, + 0.4216577790844446, + 0.34573178219860223, + 0.2783689125554313, + 0.23653941771003728 + ], + "population_cv": 0.36398106608216985 + }, + "activation_output_rms_by_block": [ + 0.11107808351516724, + 0.1828140765428543, + 0.2578570544719696, + 0.27254435420036316, + 0.2737041115760803, + 0.2742544412612915, + 0.27536308765411377, + 0.2744285464286804, + 0.2742423117160797, + 0.27115195989608765, + 0.26583269238471985, + 0.255132257938385, + 0.23856717348098755, + 0.22743643820285797, + 0.22118839621543884, + 0.22011704742908478, + 0.22253327071666718, + 0.2256278693675995, + 0.22584785521030426, + 0.22925184667110443, + 0.23080435395240784, + 0.23398378491401672, + 0.24050307273864746, + 0.24922524392604828, + 0.2630935311317444, + 0.27895617485046387, + 0.2995731830596924, + 0.3278428912162781, + 0.36820048093795776, + 0.42407870292663574, + 0.5123529434204102, + 0.6704596281051636 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24025546945631504, + "first_to_last_ratio": 0.6112286812568156, + "imbalance_abs_log_ratio": 0.4922841160907453, + "last_quartile_mean": 0.39306969195604324, + "mean": 0.2780639645643532, + "normalized": [ + 0.39946953820209985, + 0.6574533195240588, + 0.9273299935716515, + 0.9801498537480837, + 0.9843206832100536, + 0.9862998310153919, + 0.9902868503134846, + 0.9869259645298935, + 0.9862562095945767, + 0.975142393301143, + 0.9560127390156568, + 0.9175308218672072, + 0.8579578941656613, + 0.817928488357655, + 0.7954586872196038, + 0.7916058011110692, + 0.8002952524442108, + 0.8114243415938267, + 0.8122154755440653, + 0.824457232458282, + 0.8300405063777765, + 0.8414746775282531, + 0.8649199586701105, + 0.896287457875072, + 0.9461619075450383, + 1.0032086512450777, + 1.077353491413524, + 1.179019696888499, + 1.3241574884211362, + 1.5251120496359385, + 1.842572244926166, + 2.411170498685733 + ], + "population_cv": 0.35267348500337414 + }, + "bits_per_byte": 1.9134731260894795, + "core_parameter_grad_rms_by_block": [ + 0.00037295757895409754, + 0.00022454763424927058, + 0.00021936409959100382, + 0.00010566073392567048, + 9.515815728211845e-05, + 0.00010551160558417538, + 0.00011265453132986426, + 0.00010566938037237213, + 0.00012588745811984293, + 0.0001281493774985002, + 0.0001672472035341268, + 0.0002455532725992925, + 0.00022493633083964273, + 0.0002220131528127045, + 0.0002449305149794571, + 0.0002379224969510024, + 0.00024418235837095776, + 0.00027770634611442576, + 0.000335912973129177, + 0.00033241348484404164, + 0.00039774019686300875, + 0.00036753844348542204, + 0.0003869475308616486, + 0.00034605021914329437, + 0.00032032390763334626, + 0.00028824702486555996, + 0.00027123288540817104, + 0.0002526057589974623, + 0.00022790421762734515, + 0.00020876185354851858, + 0.00018495058171630387, + 0.00016871013519521616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016769046516107157, + "first_to_last_ratio": 0.6977158937201501, + "imbalance_abs_log_ratio": 0.359943288132801, + "last_quartile_mean": 0.00024034204562399042, + "mean": 0.00023591848270084503, + "normalized": [ + 1.5808747779504164, + 0.9518017905108639, + 0.9298300713012261, + 0.4478696739485771, + 0.40335185354164543, + 0.44723755585512437, + 0.47751464845047825, + 0.44790632409401143, + 0.5336057466912151, + 0.5431934625529073, + 0.7089194607367985, + 1.040839487386263, + 0.9534493790589178, + 0.9410587516122122, + 1.0381997721223044, + 1.0084945199172823, + 1.035028521612661, + 1.1771284001795213, + 1.4238518715599293, + 1.4090184077080408, + 1.6859221554394308, + 1.5579044052749225, + 1.640174718113608, + 1.466821145938367, + 1.357773685072106, + 1.2218077259807991, + 1.1496890040281678, + 1.0707332299935883, + 0.9660295158660277, + 0.8848897770050418, + 0.7839596948867686, + 0.7151204656107762 + ], + "population_cv": 0.3768493091252366 + }, + "loss_nats": 1.3263185024261475, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.00018927091150544584, + 0.0001941375230671838, + 0.00020724961359519511, + 0.00020481550018303096, + 0.00020258830045349896, + 0.00020023793331347406, + 0.00019735096429940313, + 0.0001957487838808447, + 0.00019380047160666436, + 0.00019302338478155434, + 0.0001922047813422978, + 0.00018312338215764612, + 0.00017868881695903838, + 0.00017335887241642922, + 0.00016814567788969725, + 0.0001643431605771184, + 0.00016659151879139245, + 0.0001678116386756301, + 0.0001584039127919823, + 0.0001587578735779971, + 0.00014523210120387375, + 0.00013619904348161072, + 0.00012270425213500857, + 0.00011189868382643908, + 0.00010116446355823427, + 9.155859152087942e-05, + 8.147361222654581e-05, + 7.049509440548718e-05, + 5.894774221815169e-05, + 4.627152156899683e-05, + 3.385082527529448e-05, + 2.4755347112659365e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019892494128725957, + "first_to_last_ratio": 3.129490087873211, + "imbalance_abs_log_ratio": 1.140870080057971, + "last_quartile_mean": 6.356464973578113e-05, + "mean": 0.00014731888438745955, + "normalized": [ + 1.2847701928535473, + 1.3178047327360134, + 1.4068095509745602, + 1.3902867988353147, + 1.3751685759490058, + 1.3592142931712237, + 1.3396175590113453, + 1.3287419647165597, + 1.3155168287655152, + 1.310241966494184, + 1.304685289611514, + 1.2430407881451104, + 1.212938977253409, + 1.1767593349436627, + 1.141372191276318, + 1.1155607189155992, + 1.1308225655119981, + 1.1391047344227307, + 1.0752451286242999, + 1.077647813028859, + 0.9858349240678649, + 0.9245185642554635, + 0.8329159743857911, + 0.759567819778877, + 0.6867039753855606, + 0.6214993542855888, + 0.553042555034996, + 0.4785204198266929, + 0.40013703920751104, + 0.314090903969238, + 0.2297792670372409, + 0.16803919752440544 + ], + "population_cv": 0.3716245052237534 + }, + "activation_output_rms_by_block": [ + 0.10094190388917923, + 0.17939583957195282, + 0.23316815495491028, + 0.2387540638446808, + 0.23957562446594238, + 0.2399372160434723, + 0.24013783037662506, + 0.23953619599342346, + 0.2386971116065979, + 0.235844686627388, + 0.23058047890663147, + 0.2186841070652008, + 0.2011982500553131, + 0.1896955370903015, + 0.18346232175827026, + 0.1819538027048111, + 0.18335868418216705, + 0.18572981655597687, + 0.1859624832868576, + 0.18881060183048248, + 0.18956796824932098, + 0.19203881919384003, + 0.19704653322696686, + 0.20452122390270233, + 0.21571911871433258, + 0.22943545877933502, + 0.2464856505393982, + 0.26912161707878113, + 0.30166757106781006, + 0.3485357463359833, + 0.4315659701824188, + 0.6013737916946411 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2139308536425233, + "first_to_last_ratio": 0.6473178416327895, + "imbalance_abs_log_ratio": 0.4349178506515776, + "last_quartile_mean": 0.3304881155490875, + "mean": 0.2363282556179911, + "normalized": [ + 0.42712583658360814, + 0.7590960255803447, + 0.9866283417748029, + 1.0102645712860119, + 1.0137409250513, + 1.015270964599827, + 1.0161198446147375, + 1.0135740873093813, + 1.0100235834365736, + 0.9979538249062154, + 0.9756788425644265, + 0.9253405035862031, + 0.8513508024217668, + 0.8026781926446058, + 0.7763029489576774, + 0.7699197974825631, + 0.7758644166466246, + 0.7858976323855102, + 0.7868821390001439, + 0.7989336752676848, + 0.8021383975166516, + 0.8125935626768981, + 0.8337832169568393, + 0.8654116426657729, + 0.9127944441100949, + 0.9708338013978413, + 1.0429800274827308, + 1.138761915603517, + 1.2764769505828182, + 1.4747950702067896, + 1.8261293769290816, + 2.5446546377709565 + ], + "population_cv": 0.36483989965797836 + }, + "bits_per_byte": 1.8402207275901343, + "core_parameter_grad_rms_by_block": [ + 0.0004888726102119165, + 0.00034371112095624357, + 0.00027329239576199496, + 0.00013109360072049493, + 0.0001309003833444521, + 0.00015153657998531148, + 0.00016751171475037753, + 0.00015920301567702626, + 0.00019282855273444342, + 0.00018189099839941241, + 0.00022228035276414997, + 0.00039117902483509027, + 0.00035836801505216564, + 0.00037533451853890784, + 0.0004202645380768536, + 0.0004297743688669142, + 0.0004147505595850994, + 0.0004900318658719625, + 0.0005824864404877667, + 0.0005699482810090311, + 0.0006776225188364896, + 0.0006405760434101032, + 0.0006565350519436726, + 0.0006182198906110644, + 0.0005596621809681081, + 0.0005013865499612654, + 0.000471541721969693, + 0.0004314515507572245, + 0.0003811454676239625, + 0.0003426357383233983, + 0.00029149764497300414, + 0.0002412795793653285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023076517767597717, + "first_to_last_ratio": 0.5732227450358324, + "imbalance_abs_log_ratio": 0.5564809030131627, + "last_quartile_mean": 0.00040257505424274806, + "mean": 0.000384025402386654, + "normalized": [ + 1.2730215427772602, + 0.8950218366288691, + 0.7116518700677824, + 0.3413670030830513, + 0.34086386611648084, + 0.3946004067531388, + 0.4361995683340739, + 0.4145637624167724, + 0.5021244728500928, + 0.4736431425342962, + 0.5788167954065396, + 1.0186279928462485, + 0.9331883068801387, + 0.9773689870676945, + 1.0943665066554955, + 1.1191300528453012, + 1.0800081375020862, + 1.2760402380324216, + 1.5167914332429844, + 1.4841421360850018, + 1.7645252491766907, + 1.6680564343635333, + 1.7096136033278435, + 1.6098411375105153, + 1.4573571891075452, + 1.305607771895378, + 1.227892006724398, + 1.1234974251073624, + 0.9925006659851322, + 0.8922215462674452, + 0.7590582371931511, + 0.6282906752152749 + ], + "population_cv": 0.4280332727708177 + }, + "loss_nats": 1.2755438089370728, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.037726566232575, + "cross_entropy_nats": 5.57132750749588, + "step": 0 + }, + { + "bits_per_byte": 5.132850850111892, + "cross_entropy_nats": 3.5578210949897766, + "step": 100 + }, + { + "bits_per_byte": 3.1258965111502706, + "cross_entropy_nats": 2.1667063534259796, + "step": 500 + }, + { + "bits_per_byte": 1.9712387018569424, + "cross_entropy_nats": 1.3663585484027863, + "step": 2000 + }, + { + "bits_per_byte": 1.7961544728728112, + "cross_entropy_nats": 1.2449994087219238, + "step": 4000 + }, + { + "bits_per_byte": 1.7269767303333703, + "cross_entropy_nats": 1.1970490515232086, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "final_optimizer_state": "4b960a87623e39ce5348146d7169b13bd4e5894a99361177367c7518bf6f603e", + "final_public_parameters": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "seed": 2026073002, + "timing": { + "mean_ms": 41.92772524377444, + "measured_steps": 7980, + "median_ms": 41.88774450449273, + "p95_ms": 42.070325661916286, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "baseline", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002897916710935533, + 0.0002662435290403664, + 0.00024211003619711846, + 0.00022891115804668516, + 0.00021424151782412082, + 0.00020374931045807898, + 0.00019545880786608905, + 0.00018862746946979314, + 0.00018024678865913302, + 0.00017406532424502075, + 0.0001698832056717947, + 0.00016633344057481736, + 0.00016165875422302634, + 0.0001582534605404362, + 0.00015452370280399919, + 0.0001515774056315422, + 0.00014964381989557296, + 0.00014728632231708616, + 0.00014459768135566264, + 0.00014173447561915964, + 0.00013920740457251668, + 0.00013723202573601156, + 0.00013503432273864746, + 0.0001331205276073888, + 0.00013141198724042624, + 0.00013009003305342048, + 0.0001286487968172878, + 0.00012732617324218154, + 0.00012632299331016839, + 0.00012548104859888554, + 0.00012477955897338688, + 0.00012385686568450183 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022864168749947567, + "first_to_last_ratio": 1.796936959436678, + "imbalance_abs_log_ratio": 0.5860835261825511, + "last_quartile_mean": 0.00012723968211503234, + "mean": 0.00016535780059712124, + "normalized": [ + 1.7525128542293782, + 1.6101056501655084, + 1.4641585417974736, + 1.3843384298779209, + 1.2956238958819981, + 1.2321723542664615, + 1.1820356049746095, + 1.1407231396924917, + 1.0900410383317047, + 1.052658680851196, + 1.0273673516358577, + 1.0059001751001344, + 0.9776300461137163, + 0.9570365593214796, + 0.9344808787127115, + 0.9166631697094613, + 0.9049698251621409, + 0.8907128770776013, + 0.874453341986335, + 0.8571381277892198, + 0.8418556854882369, + 0.8299095974937676, + 0.816619005883163, + 0.8050453448623477, + 0.7947129604160569, + 0.786718452855893, + 0.7780025880407573, + 0.7700040323613145, + 0.7639373096038118, + 0.7588456555769529, + 0.7546034025779078, + 0.7490234221623898 + ], + "population_cv": 0.25804396357350073 + }, + "activation_output_rms_by_block": [ + 0.028274519369006157, + 0.02845057100057602, + 0.02867918275296688, + 0.02882254309952259, + 0.0290658138692379, + 0.029274335131049156, + 0.029572302475571632, + 0.02980491891503334, + 0.03008633479475975, + 0.03044024296104908, + 0.030671589076519012, + 0.030981140211224556, + 0.0313686840236187, + 0.031716227531433105, + 0.03204340860247612, + 0.03237249702215195, + 0.032637808471918106, + 0.03308072313666344, + 0.03343126177787781, + 0.033881161361932755, + 0.03435666114091873, + 0.03482793644070625, + 0.03533180430531502, + 0.03546451777219772, + 0.036012887954711914, + 0.03628316894173622, + 0.03689942881464958, + 0.03742557764053345, + 0.03777860850095749, + 0.03856590390205383, + 0.038892220705747604, + 0.039198413491249084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02899302332662046, + "first_to_last_ratio": 0.7704348189669382, + "imbalance_abs_log_ratio": 0.2608002235603661, + "last_quartile_mean": 0.0376320262439549, + "mean": 0.032990387349855155, + "normalized": [ + 0.8570532703712039, + 0.8623897227657417, + 0.8693193701798957, + 0.8736648889225345, + 0.8810388784163674, + 0.887359545694382, + 0.8963914901017818, + 0.9034425270294439, + 0.9119727657545023, + 0.9227003805150147, + 0.9297129115597873, + 0.9390959821925394, + 0.9508431559460433, + 0.961377846070436, + 0.9712953128638184, + 0.9812705949417742, + 0.9893126784418129, + 1.002738245715344, + 1.0133637239037931, + 1.0270010170729782, + 1.0414142997647942, + 1.0556995306349188, + 1.070972702764284, + 1.0749954947817073, + 1.0916176149374623, + 1.099810334354729, + 1.1184903172957619, + 1.1344388668021435, + 1.1451398887901616, + 1.1690042767025335, + 1.1788955459450936, + 1.1881768187672155 + ], + "population_cv": 0.10167083190861884 + }, + "bits_per_byte": 8.0691508062416, + "core_parameter_grad_rms_by_block": [ + 0.006421208939522194, + 0.005746349410576503, + 0.005404164999391892, + 0.005101115831054325, + 0.00485388847333966, + 0.0045855144167447825, + 0.004662942770173713, + 0.005059104716899598, + 0.00461737020280362, + 0.004446534400320367, + 0.004114212128857249, + 0.004678028852169936, + 0.004404060169276539, + 0.004656150674369456, + 0.004513829588134647, + 0.004839518675960965, + 0.004348161380019098, + 0.0047201153472655015, + 0.004169246752947831, + 0.004575240937826365, + 0.0047925402340120975, + 0.004508705150890575, + 0.004155367099556682, + 0.004478773084339979, + 0.004234175381969994, + 0.004428191743895327, + 0.004153362780335295, + 0.00410615213012631, + 0.004175626130102292, + 0.004125604583408443, + 0.0038633837013440408, + 0.003947405979968418 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005229286194712833, + "first_to_last_ratio": 1.266404707857162, + "imbalance_abs_log_ratio": 0.236181947089728, + "last_quartile_mean": 0.004129237803893765, + "mean": 0.004590188958362615, + "normalized": [ + 1.3988986069568536, + 1.2518764396632391, + 1.1773295279154767, + 1.1113084618795224, + 1.057448509716931, + 0.9989816232707989, + 1.0158498511653973, + 1.1021560904769923, + 1.0059215959707901, + 0.9687039990411438, + 0.8963056131625672, + 1.0191364439686714, + 0.9594507348663767, + 1.0143701526462583, + 0.983364656461723, + 1.0543179637831923, + 0.9472728507390573, + 1.0283052375580706, + 0.9082952337620236, + 0.9967434847079623, + 1.044083430439357, + 0.9822482672911342, + 0.9052714686148694, + 0.9757273883421174, + 0.9224403222564466, + 0.9647079421050512, + 0.9048348157364001, + 0.894549694440255, + 0.9096850190654888, + 0.8987875272307099, + 0.8416611464993293, + 0.859965900265795 + ], + "population_cv": 0.11345956482154346 + }, + "loss_nats": 5.593109130859375, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.023746729828417e-05, + 6.667003617621958e-05, + 6.32333685643971e-05, + 6.012299854774028e-05, + 5.672227416653186e-05, + 5.335300011211075e-05, + 5.0173275667475536e-05, + 4.719031130662188e-05, + 4.408728636917658e-05, + 4.096491466043517e-05, + 3.824968007393181e-05, + 3.570610351744108e-05, + 3.324189310660586e-05, + 3.1057134037837386e-05, + 2.914190736191813e-05, + 2.7255719032837078e-05, + 2.56736693700077e-05, + 2.4158533051377162e-05, + 2.2910870029591024e-05, + 2.187793143093586e-05, + 2.0998755644541234e-05, + 2.0234831026755273e-05, + 1.963501563295722e-05, + 1.908456215460319e-05, + 1.8524928236729465e-05, + 1.8092870959662832e-05, + 1.7702994227875024e-05, + 1.74019587575458e-05, + 1.7127178580267355e-05, + 1.6885946024558507e-05, + 1.666023126745131e-05, + 1.649606565479189e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8462841479922645e-05, + "first_to_last_ratio": 3.3673800283354014, + "imbalance_abs_log_ratio": 1.2141350023382471, + "last_quartile_mean": 1.7361521713610273e-05, + "mean": 3.315230350153797e-05, + "normalized": [ + 2.1186300763392136, + 2.011022738529366, + 1.9073597272498315, + 1.8135390967614395, + 1.710960270495245, + 1.6093301061157226, + 1.5134174813870156, + 1.4234398917237432, + 1.3298408168570044, + 1.235658169530656, + 1.1537563316575383, + 1.0770323551057208, + 1.0027023644092703, + 0.9368016927208878, + 0.8790311466763148, + 0.8221365079977823, + 0.7744158522443748, + 0.7287135583280486, + 0.691079279861448, + 0.6599219094963015, + 0.6334026123876152, + 0.610359730382432, + 0.5922670088986827, + 0.5756632311754095, + 0.5587825363589011, + 0.5457500399277982, + 0.5339898697251538, + 0.5249094910324558, + 0.516621072181992, + 0.5093445776331997, + 0.5025361591142052, + 0.49758429769523016 + ], + "population_cv": 0.5033867184228629 + }, + "activation_output_rms_by_block": [ + 0.029364673420786858, + 0.03058232180774212, + 0.03253435716032982, + 0.035004038363695145, + 0.03789948299527168, + 0.041081421077251434, + 0.04464675113558769, + 0.04816310852766037, + 0.05221380665898323, + 0.056122854351997375, + 0.06031070277094841, + 0.06517986208200455, + 0.07041798532009125, + 0.07559357583522797, + 0.08127844333648682, + 0.08752159774303436, + 0.09348469227552414, + 0.10084687173366547, + 0.10794128477573395, + 0.11579020321369171, + 0.12420836836099625, + 0.13254153728485107, + 0.14211823046207428, + 0.151936337351799, + 0.16282601654529572, + 0.17539408802986145, + 0.19021794199943542, + 0.2045798897743225, + 0.2205442190170288, + 0.23628318309783936, + 0.2528626322746277, + 0.27021101117134094 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03740951931104064, + "first_to_last_ratio": 0.17471705179812935, + "imbalance_abs_log_ratio": 1.74458746043398, + "last_quartile_mean": 0.214114872738719, + "mean": 0.11030317156109959, + "normalized": [ + 0.2662178521722837, + 0.2772569580268309, + 0.29495395916434053, + 0.31734389744455865, + 0.34359377394944846, + 0.37244097785978386, + 0.40476398369793726, + 0.43664300714129206, + 0.47336632229165543, + 0.5088054455524842, + 0.5467721545752731, + 0.590915575314168, + 0.6384039944045038, + 0.685325496677626, + 0.736864064615447, + 0.793464018344695, + 0.8475249709727586, + 0.9142699190458359, + 0.9785873175545334, + 1.0497450034748341, + 1.126063435920283, + 1.2016112991949022, + 1.2884328569224464, + 1.3774430526472923, + 1.476168039783901, + 1.5901092012817277, + 1.7245011118657556, + 1.8547054167069055, + 1.9994367876798904, + 2.142125015570893, + 2.292433016167273, + 2.4497120739784397 + ], + "population_cv": 0.6367597776091559 + }, + "bits_per_byte": 5.184434896455056, + "core_parameter_grad_rms_by_block": [ + 0.00026099601887828554, + 0.00022299818257146475, + 0.00021593606328411676, + 0.0001980902028289896, + 0.00019039206376599677, + 0.0001764249768268048, + 0.00019262760932023515, + 0.00017741019012756782, + 0.0001866267032514166, + 0.00018194420611625288, + 0.00016335812718032435, + 0.00018628060025801724, + 0.00018197064718745634, + 0.00018231380553041538, + 0.00016812738233283717, + 0.0001758928024359223, + 0.00015172497688937343, + 0.0001676233054310719, + 0.0001547068922126687, + 0.0001733627635453279, + 0.0001619137110647745, + 0.00014190991701657472, + 0.0001580141865100647, + 0.00014951069781870242, + 0.00014861801255322757, + 0.00015877260520542335, + 0.00016243531213177502, + 0.00014972906733594748, + 0.0001506252547372875, + 0.00014728724932094392, + 0.00014196626632039671, + 0.00013988030321371655 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00020435941345043265, + "first_to_last_ratio": 1.3631752910956885, + "imbalance_abs_log_ratio": 0.30981675126526836, + "last_quartile_mean": 0.00014991425885233975, + "mean": 0.00017248344072510563, + "normalized": [ + 1.513165656836857, + 1.2928671971871588, + 1.251923444803397, + 1.1484592491675452, + 1.1038280716433115, + 1.0228516783125923, + 1.1167890002103689, + 1.028563608087539, + 1.0819977991327974, + 1.0548502821567973, + 0.9470945529239257, + 1.0799912123442663, + 1.0550035784447904, + 1.0569930931571387, + 0.9747450632132802, + 1.0197663131978583, + 0.8796495260735443, + 0.9718225977311089, + 0.8969376513031868, + 1.0050980129833083, + 0.9387203222761739, + 0.8227451656808188, + 0.9161122125450802, + 0.8668118933050746, + 0.8616364094341472, + 0.9205092647616311, + 0.941744386875116, + 0.8680779250837024, + 0.8732737131406461, + 0.853921099334295, + 0.823071859672921, + 0.8109781589796199 + ], + "population_cv": 0.15033028761203962 + }, + "loss_nats": 3.593576431274414, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.0001506932603660971, + 0.00015340163372457027, + 0.00015237691695801914, + 0.000148982901009731, + 0.0001392244448652491, + 0.00012851000064983964, + 0.00011962141434196383, + 0.0001119004882639274, + 0.00010405662760604173, + 9.720238449517637e-05, + 9.135912114288658e-05, + 8.468296437058598e-05, + 7.889774860814214e-05, + 7.427640230162069e-05, + 6.881110311951488e-05, + 6.43495368422009e-05, + 5.969387711957097e-05, + 5.503925422090106e-05, + 5.034419518779032e-05, + 4.6255379857029766e-05, + 4.248250479577109e-05, + 3.87999571103137e-05, + 3.563905556802638e-05, + 3.2581570849288255e-05, + 3.028466016985476e-05, + 2.807797682180535e-05, + 2.6328352760174312e-05, + 2.5215304049197584e-05, + 2.4513139578630216e-05, + 2.4017937903408892e-05, + 2.3832226361264475e-05, + 2.3948901798576117e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013808888252242468, + "first_to_last_ratio": 5.356993010635397, + "imbalance_abs_log_ratio": 1.678402812181787, + "last_quartile_mean": 2.5777312430363963e-05, + "mean": 7.298128883803656e-05, + "normalized": [ + 2.0648204870774824, + 2.1019310040551114, + 2.087890189171378, + 2.041384899907163, + 1.907673145841839, + 1.7608623072556988, + 1.6390696334156716, + 1.533276406124618, + 1.425798711735127, + 1.3318808977310936, + 1.2518156721736573, + 1.1603380225103768, + 1.0810681732852876, + 1.0177458288857888, + 0.9428595221471582, + 0.881726506433839, + 0.8179339947263247, + 0.7541556897281809, + 0.6898233230645799, + 0.6337977938383883, + 0.5821013231220159, + 0.5316425309563984, + 0.488331408439747, + 0.44643731794864266, + 0.4149647211227507, + 0.3847284320247795, + 0.3607548342781861, + 0.3455036825282238, + 0.3358825250816124, + 0.32909720129375364, + 0.3265525553289974, + 0.3281512587661287 + ], + "population_cv": 0.6061250608697893 + }, + "activation_output_rms_by_block": [ + 0.048818644136190414, + 0.05689374357461929, + 0.06500593572854996, + 0.07087047398090363, + 0.07611831277608871, + 0.0806790366768837, + 0.08478696644306183, + 0.08861920237541199, + 0.0918315127491951, + 0.09433750063180923, + 0.09700070321559906, + 0.09972338378429413, + 0.10201811045408249, + 0.10424874722957611, + 0.10649008303880692, + 0.10806933790445328, + 0.11109450459480286, + 0.11401215940713882, + 0.11832662671804428, + 0.12293888628482819, + 0.12948566675186157, + 0.1387748122215271, + 0.14864373207092285, + 0.16069285571575165, + 0.1741981953382492, + 0.18964411318302155, + 0.20650440454483032, + 0.22468066215515137, + 0.24238750338554382, + 0.2638821303844452, + 0.2911032736301422, + 0.32429978251457214 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07147403946146369, + "first_to_last_ratio": 0.2983212272448851, + "imbalance_abs_log_ratio": 1.2095844292661684, + "last_quartile_mean": 0.23958750814199448, + "mean": 0.13550565636251122, + "normalized": [ + 0.3602701573253034, + 0.41986249948426085, + 0.4797285771941724, + 0.523007495652488, + 0.5617353165867361, + 0.5953923905659501, + 0.6257079499045831, + 0.6539889533344176, + 0.67769505136762, + 0.6961886548811884, + 0.7158424660598531, + 0.7359352108336301, + 0.7528697560872202, + 0.7693313329348029, + 0.7858718661449788, + 0.7975263970925391, + 0.8198514185828341, + 0.8413830275994386, + 0.8732227856340641, + 0.907260180755379, + 0.9555738869339544, + 1.024125604397429, + 1.0969559209636535, + 1.185875630799193, + 1.285541873412471, + 1.3995291287005507, + 1.523954129209041, + 1.6580906523494112, + 1.7887629925727762, + 1.9473883044298541, + 2.148273964677672, + 2.393256423532534 + ], + "population_cv": 0.5081939166956976 + }, + "bits_per_byte": 3.1154419867739596, + "core_parameter_grad_rms_by_block": [ + 0.0005931392614171548, + 0.0004012275990867258, + 0.00038098222807989436, + 0.0003649111519840816, + 0.00034656151495199636, + 0.00031005860528648927, + 0.00027821812913176474, + 0.0002561842642921938, + 0.0002305368063696604, + 0.00020943363497294808, + 0.0001942700943631853, + 0.0001900904574854747, + 0.00018678033884870243, + 0.00017357948080148642, + 0.00015815775362626997, + 0.00014490835987546657, + 0.0001375100278294081, + 0.0001295570650502436, + 0.00012744106699623042, + 0.00011906627060222178, + 0.00011368520556157287, + 0.0001091753996736031, + 0.00010941555658600093, + 9.636520203422524e-05, + 9.391483800802352e-05, + 9.101856383525677e-05, + 7.998410113295477e-05, + 7.102545922046693e-05, + 6.763159533414917e-05, + 6.887797791066819e-05, + 7.001610738319118e-05, + 7.664349539138475e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036641034427878756, + "first_to_last_ratio": 4.73465560322638, + "imbalance_abs_log_ratio": 1.5549089897216843, + "last_quartile_mean": 7.738901727701191e-05, + "mean": 0.00018688648791009675, + "normalized": [ + 3.173794253667441, + 2.146905341170195, + 2.038575567128047, + 1.9525817840807467, + 1.8543957823141959, + 1.6590744935805388, + 1.4887011481836172, + 1.3708014268823612, + 1.2335659412710565, + 1.1206462131906392, + 1.0395085088047697, + 1.017143933792149, + 0.9994320131830752, + 0.9287963123636216, + 0.8462770925544549, + 0.7753816848715992, + 0.7357943817509076, + 0.6932393374130293, + 0.681916966932018, + 0.6371047577259816, + 0.6083115308810451, + 0.5841802737826761, + 0.5854653153877886, + 0.5156349349375247, + 0.5025234250921448, + 0.4870259206703161, + 0.4279822582541749, + 0.38004598414110224, + 0.3618859559642637, + 0.3685551517443171, + 0.3746451023086966, + 0.4101072059755049 + ], + "population_cv": 0.6522632271008562 + }, + "loss_nats": 2.1594598293304443, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 0.00010946426482405514, + 0.00010930908320005983, + 0.00012598594184964895, + 0.00012082153261872008, + 0.00011562183499336243, + 0.00011443938274169341, + 0.00011117216490674764, + 0.00010893856233451515, + 0.00010224300058325753, + 9.949541708920151e-05, + 9.518271690467373e-05, + 9.17384895728901e-05, + 8.801270450931042e-05, + 8.554522355552763e-05, + 8.286484808195382e-05, + 8.027214062167332e-05, + 7.707033364567906e-05, + 7.422274211421609e-05, + 7.329088111873716e-05, + 7.053956505842507e-05, + 6.641655636485666e-05, + 6.314766505965963e-05, + 5.928793325438164e-05, + 5.485894507728517e-05, + 5.013645932194777e-05, + 4.561002060654573e-05, + 4.094795440323651e-05, + 3.615552486735396e-05, + 3.135877341264859e-05, + 2.6529152819421142e-05, + 2.2670756152365357e-05, + 2.0441382730496116e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011446909593360033, + "first_to_last_ratio": 3.3439937416939496, + "imbalance_abs_log_ratio": 1.2071658237176983, + "last_quartile_mean": 3.4231253039251897e-05, + "mean": 7.668099857482957e-05, + "normalized": [ + 1.427527899460436, + 1.4255041696332367, + 1.642987756956567, + 1.5756384875558933, + 1.5078290207779732, + 1.4924086131979242, + 1.4498006934307157, + 1.4206721920582042, + 1.3333551007878848, + 1.297523753451233, + 1.2412816561300926, + 1.196365348364192, + 1.1477772348442066, + 1.115598716050207, + 1.0806438312235815, + 1.046832228499728, + 1.0050773343864263, + 0.9679417781940518, + 0.955789341308797, + 0.91990931742482, + 0.866140994500009, + 0.8235112509396527, + 0.7731763325503017, + 0.7154177188205337, + 0.6538315913168741, + 0.5948021211804766, + 0.5340039274955088, + 0.47150566032432917, + 0.4089510308351939, + 0.3459677535828191, + 0.29565024678495777, + 0.26657689793317285 + ], + "population_cv": 0.3996524250074872 + }, + "activation_output_rms_by_block": [ + 0.09914769232273102, + 0.13720469176769257, + 0.1984461098909378, + 0.2206047922372818, + 0.22331808507442474, + 0.2258162647485733, + 0.22567619383335114, + 0.2251615822315216, + 0.22496475279331207, + 0.22249442338943481, + 0.21478813886642456, + 0.205490842461586, + 0.19144563376903534, + 0.188800647854805, + 0.18874111771583557, + 0.190367192029953, + 0.19439123570919037, + 0.198538675904274, + 0.20280276238918304, + 0.2084609866142273, + 0.21558508276939392, + 0.21985429525375366, + 0.22511647641658783, + 0.2311476618051529, + 0.24068132042884827, + 0.2547627091407776, + 0.2762547731399536, + 0.30550503730773926, + 0.34050679206848145, + 0.39206719398498535, + 0.46342816948890686, + 0.575099527835846 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19442192651331425, + "first_to_last_ratio": 0.5460704265504183, + "imbalance_abs_log_ratio": 0.6050073251991793, + "last_quartile_mean": 0.3560381904244423, + "mean": 0.2414584644138813, + "normalized": [ + 0.41062007316000754, + 0.5682331000519887, + 0.8218643747803495, + 0.9136345365766327, + 0.9248716362729685, + 0.9352178450099977, + 0.9346377414482436, + 0.932506477990246, + 0.9316913090596918, + 0.9214604421904198, + 0.8895448721907656, + 0.851040128000468, + 0.7928719096004871, + 0.7819177029602238, + 0.7816711589464782, + 0.7884055441669947, + 0.8050711172253067, + 0.8222477368362661, + 0.8399074469452479, + 0.8633409771748842, + 0.8928454146045669, + 0.9105263540353832, + 0.9323196723007322, + 0.9572978208332561, + 0.9967814589274413, + 1.0550995168431603, + 1.144108879390653, + 1.2652488205345183, + 1.410208554481745, + 1.6237459098263265, + 1.9192873217919157, + 2.381774145842633 + ], + "population_cv": 0.36935148393532197 + }, + "bits_per_byte": 2.073311222061611, + "core_parameter_grad_rms_by_block": [ + 0.00035888588058483766, + 0.00018018950310225553, + 0.0002078431087578172, + 0.0001449691382379752, + 0.00012177099939920197, + 0.00012306582379429542, + 0.00012204348242118167, + 0.00015411618529826362, + 0.00023712554584861813, + 0.0001667297606409996, + 0.00020124714337423614, + 0.00019657670547036952, + 0.00021589641757185544, + 0.00019691649893044806, + 0.0002141921139315525, + 0.00021492094770207205, + 0.00022724548428173296, + 0.00022961612017464663, + 0.00022356249125846063, + 0.00024496232741861765, + 0.0002428330999302831, + 0.0002472582804061272, + 0.00023198586071939664, + 0.00023918771709558322, + 0.0002286007324212131, + 0.00020791382420134994, + 0.00018978221308207498, + 0.00017514362262291224, + 0.0001585770655414503, + 0.0001491152323589239, + 0.00013960034115182495, + 0.00014044566543596396 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017661051519947854, + "first_to_last_ratio": 1.0170643451662862, + "imbalance_abs_log_ratio": 0.016920384648356147, + "last_quartile_mean": 0.00017364733710196415, + "mean": 0.0001978849791614544, + "normalized": [ + 1.813608501795493, + 0.9105769617573595, + 1.0503228170150194, + 0.7325929378383735, + 0.6153625197587583, + 0.6219058380064613, + 0.6167394965416064, + 0.7788169973857398, + 1.1982998752782916, + 0.8425589519099616, + 1.0169904973436037, + 0.9933887165330652, + 1.0910197352357174, + 0.9951058426207471, + 1.0824071379202214, + 1.0860902561316592, + 1.1483715704177997, + 1.160351438232726, + 1.1297597837178734, + 1.2379025859196358, + 1.227142661152444, + 1.2495050481036718, + 1.1723267814588358, + 1.2087209353087376, + 1.155220236472308, + 1.0506801733127658, + 0.959053152423619, + 0.8850779041698386, + 0.8013597909928638, + 0.7535449784556955, + 0.7054620403397319, + 0.7097338364493765 + ], + "population_cv": 0.24613765810690832 + }, + "loss_nats": 1.4371098279953003, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 0.00010021305934060365, + 0.00010125436529051512, + 0.000114263420982752, + 0.00011024689592886716, + 0.00010690757335396484, + 0.00010680041305022314, + 0.00010475025192135945, + 0.00010345032933400944, + 9.713075269246474e-05, + 9.490915545029566e-05, + 9.159108594758436e-05, + 8.836394408717752e-05, + 8.501022239215672e-05, + 8.242073090514168e-05, + 7.99787012510933e-05, + 7.76422893977724e-05, + 7.462384382961318e-05, + 7.162441761465743e-05, + 7.150838064262643e-05, + 6.863063026685268e-05, + 6.474268593592569e-05, + 6.149821274448186e-05, + 5.7751010899664834e-05, + 5.37736268597655e-05, + 4.920278661302291e-05, + 4.476245885598473e-05, + 4.037534017697908e-05, + 3.581849159672856e-05, + 3.1086256058188155e-05, + 2.5875511710182764e-05, + 2.1458168703247793e-05, + 1.863625948317349e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010598578865028685, + "first_to_last_ratio": 3.1730458332581706, + "imbalance_abs_log_ratio": 1.1546919572676668, + "last_quartile_mean": 3.3401909149688436e-05, + "mean": 7.300941479115863e-05, + "normalized": [ + 1.3726046103404645, + 1.3868672359606, + 1.565050498070684, + 1.5100367020366523, + 1.4642984560247596, + 1.4628306959550723, + 1.434749918500163, + 1.4169450560578547, + 1.330386676434875, + 1.2999577619103044, + 1.2545106163304838, + 1.2103088942698703, + 1.1643734254729483, + 1.12890551363689, + 1.0954573664214418, + 1.0634558518222064, + 1.0221126144220245, + 0.9810298825094963, + 0.9794405399245307, + 0.9400243853915095, + 0.8867717419886232, + 0.8423326350498187, + 0.7910077222897892, + 0.7365300353876748, + 0.6739238597346121, + 0.613105295858438, + 0.5530155294693376, + 0.4906009957645373, + 0.425784210804997, + 0.35441335592401246, + 0.29390961103616947, + 0.255258305199158 + ], + "population_cv": 0.383084243994172 + }, + "activation_output_rms_by_block": [ + 0.11517385393381119, + 0.17733007669448853, + 0.24607165157794952, + 0.2600470185279846, + 0.2621006965637207, + 0.26366734504699707, + 0.2635972201824188, + 0.26321178674697876, + 0.2646143436431885, + 0.2614729404449463, + 0.2522071599960327, + 0.23732660710811615, + 0.2133668214082718, + 0.2109174132347107, + 0.21198545396327972, + 0.21621736884117126, + 0.2226637899875641, + 0.22916270792484283, + 0.23635683953762054, + 0.24468488991260529, + 0.25434690713882446, + 0.26204079389572144, + 0.2696947753429413, + 0.27905911207199097, + 0.29096299409866333, + 0.30846187472343445, + 0.3323265314102173, + 0.36396095156669617, + 0.403833270072937, + 0.4673614203929901, + 0.5606572031974792, + 0.7190070152282715 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23139995615929365, + "first_to_last_ratio": 0.5371134119256165, + "imbalance_abs_log_ratio": 0.6215460113796702, + "last_quartile_mean": 0.43082140758633614, + "mean": 0.28637152607552707, + "normalized": [ + 0.4021833298588333, + 0.6192308262090269, + 0.859274156722729, + 0.9080756808880515, + 0.9152470573998156, + 0.9207177426482616, + 0.92047286891539, + 0.9191269479688416, + 0.9240246307637429, + 0.9130549535709983, + 0.8806991513867063, + 0.8287367475407597, + 0.7450699597557016, + 0.7365167065495323, + 0.740246269831209, + 0.7550239781316335, + 0.7775346698709116, + 0.8002286786864554, + 0.8253503509119278, + 0.8544316303572463, + 0.8881710784044343, + 0.9150378792429639, + 0.9417653320456605, + 0.9744652895357776, + 1.0160332561203216, + 1.0771387747610122, + 1.1604733751447416, + 1.2709397353656795, + 1.410172567109311, + 1.6320107896122644, + 1.9577966108600218, + 2.510748973830038 + ], + "population_cv": 0.3918463566300126 + }, + "bits_per_byte": 1.9262875534052675, + "core_parameter_grad_rms_by_block": [ + 0.0003320807822012488, + 0.00017565239824668803, + 0.00017577232236880354, + 0.00012480421087066412, + 0.00013414970937401124, + 0.0001166490159724099, + 0.00011491447201106907, + 0.00015823331904477844, + 0.000249176250949333, + 0.0001712841712530518, + 0.00020010962025834788, + 0.00021729074089037623, + 0.0002489951018894741, + 0.00022387049947650086, + 0.00024825867897268754, + 0.000256508706958601, + 0.0002733879022225402, + 0.00028045470195987973, + 0.0002710271146389, + 0.0002966387758490912, + 0.00030658370753967526, + 0.0003196905817287839, + 0.0002949547776417074, + 0.00030211663360406435, + 0.0002930543360663917, + 0.00026176751657805116, + 0.0002415189744957938, + 0.00021641616631898222, + 0.00019981422740886822, + 0.0001881612684369175, + 0.00017039642606795488, + 0.0001626151127852574 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016653202876120915, + "first_to_last_ratio": 0.7684272928714567, + "imbalance_abs_log_ratio": 0.2634093296250088, + "last_quartile_mean": 0.0002167180035197771, + "mean": 0.00022582338200252827, + "normalized": [ + 1.4705332072191308, + 0.7778308724679426, + 0.7783619251917662, + 0.5526629251760424, + 0.5940470299594988, + 0.5165497697271398, + 0.5088687937982548, + 0.7006950194511165, + 1.103412094618897, + 0.758487317540635, + 0.8861333068517568, + 0.9622154223513644, + 1.1026099232128512, + 0.991352168634134, + 1.0993488662299287, + 1.1358819652950252, + 1.2106270829806252, + 1.2419205640837485, + 1.2001729503628886, + 1.313587517902686, + 1.3576260563498372, + 1.4156664331827458, + 1.306130370673508, + 1.3378447834984684, + 1.2977147603923083, + 1.159169233304284, + 1.069503841250105, + 0.9583425967668807, + 0.8848252365941058, + 0.8332231444253673, + 0.7545561693254915, + 0.7200986511814654 + ], + "population_cv": 0.2750960622596733 + }, + "loss_nats": 1.3352007865905762, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 0.00018638069741427898, + 0.00018956589337904006, + 0.00020923581905663013, + 0.00020349136320874095, + 0.00019885810615960509, + 0.0001991111203096807, + 0.00019671664631459862, + 0.00019502552459016442, + 0.00018589819956105202, + 0.00018270809960085899, + 0.00017816871695686132, + 0.00017320679035037756, + 0.0001670292258495465, + 0.00016241244156844914, + 0.00015707056445535272, + 0.0001517482742201537, + 0.0001448393304599449, + 0.00013789460354018956, + 0.00013790397497359663, + 0.00013200659304857254, + 0.00012367918679956347, + 0.00011618347343755886, + 0.00010830583050847054, + 9.980145841836929e-05, + 8.979889389593154e-05, + 8.034214988583699e-05, + 7.120905502233654e-05, + 6.192603905219585e-05, + 5.214559132582508e-05, + 4.129187436774373e-05, + 3.1231873435899615e-05, + 2.3505688659497537e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019729814630409237, + "first_to_last_ratio": 3.49624785701179, + "imbalance_abs_log_ratio": 1.2516903525947785, + "last_quartile_mean": 5.643139570565836e-05, + "mean": 0.00013714665936959136, + "normalized": [ + 1.3589882412812453, + 1.3822129846282736, + 1.525635549698432, + 1.4837500537316024, + 1.449966824373825, + 1.4518116680706283, + 1.434352445905914, + 1.4220216918634343, + 1.3554701253063843, + 1.3322096246506874, + 1.2991108771867432, + 1.2629311654147486, + 1.217887673074309, + 1.1842245543201309, + 1.1452744469121132, + 1.1064671565292232, + 1.0560908382727927, + 1.0054536083783316, + 1.0055219398524642, + 0.9625213888209478, + 0.9018024016630627, + 0.8471476736772742, + 0.7897081198062669, + 0.7276987924978771, + 0.6547654482340389, + 0.5858119348669365, + 0.519218297767195, + 0.451531516238494, + 0.38021772871113024, + 0.3010782366668357, + 0.22772609686200243, + 0.17139089473665517 + ], + "population_cv": 0.4031953558435787 + }, + "activation_output_rms_by_block": [ + 0.1026567816734314, + 0.17673343420028687, + 0.23246681690216064, + 0.24062344431877136, + 0.2416554093360901, + 0.24258241057395935, + 0.24218299984931946, + 0.24160927534103394, + 0.24130181968212128, + 0.23599836230278015, + 0.22310413420200348, + 0.20329499244689941, + 0.176778644323349, + 0.17310521006584167, + 0.1729687601327896, + 0.17606647312641144, + 0.1812635064125061, + 0.18683874607086182, + 0.19253391027450562, + 0.1985689103603363, + 0.20644955337047577, + 0.2135031372308731, + 0.22015690803527832, + 0.228074312210083, + 0.2383035272359848, + 0.2532499134540558, + 0.27322494983673096, + 0.2992607355117798, + 0.3339390158653259, + 0.3918353319168091, + 0.48234984278678894, + 0.651733934879303 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.21506382152438164, + "first_to_last_ratio": 0.5884305856918151, + "imbalance_abs_log_ratio": 0.5302963104844547, + "last_quartile_mean": 0.3654871564358473, + "mean": 0.2460754751227796, + "normalized": [ + 0.41717599700746566, + 0.7182082412402355, + 0.9446972185512233, + 0.9778440708027163, + 0.9820377638832796, + 0.9858049058034841, + 0.9841817829610285, + 0.9818502848383519, + 0.9806008484255633, + 0.9590486909962422, + 0.906649206267652, + 0.8261489380258848, + 0.7183919658598449, + 0.7034638863522286, + 0.7029093819548116, + 0.7154978489366439, + 0.7366175207913933, + 0.7592741453720182, + 0.7824181185810598, + 0.8069431147547724, + 0.838968423275288, + 0.8676327339176954, + 0.8946722867259763, + 0.9268469850410127, + 0.9684164060521798, + 1.0291554382967116, + 1.1103298681041054, + 1.2161339335521484, + 1.357059315637638, + 1.5923380081712835, + 1.9601703198830358, + 2.648512349937025 + ], + "population_cv": 0.40510862299171807 + }, + "bits_per_byte": 1.8611962476373152, + "core_parameter_grad_rms_by_block": [ + 0.0005542142737822822, + 0.0003797328422683109, + 0.000340619215402376, + 0.0001793765938241596, + 0.00020684039436717747, + 0.00016374338231517116, + 0.00018705111879548875, + 0.0002256278252430331, + 0.000426778694633347, + 0.00027155262856832466, + 0.0003392540980461625, + 0.0003897844238340517, + 0.00045204461946776503, + 0.0004208641344247883, + 0.00047738343422760874, + 0.00048450285457785075, + 0.0005178591522769467, + 0.0005253914687520978, + 0.0005086607978429822, + 0.0005610796431247281, + 0.0005682688177114847, + 0.000574059459111366, + 0.0005453911942509562, + 0.0005508569019095279, + 0.0005214428582378581, + 0.000461966711188474, + 0.00041497592078977314, + 0.00037256322877604213, + 0.00033686502618798456, + 0.00030674152201740854, + 0.00025927963978097604, + 0.00021950976285567438 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002796507057497499, + "first_to_last_ratio": 0.7732247282264533, + "imbalance_abs_log_ratio": 0.25718555049038655, + "last_quartile_mean": 0.00036166808372927383, + "mean": 0.00039825883245600557, + "normalized": [ + 1.3915931766397287, + 0.9534825377922003, + 0.8552709636138532, + 0.4504020481302817, + 0.5193617253674505, + 0.41114815032572916, + 0.46967224214958675, + 0.5665356468094339, + 1.0716113739435822, + 0.681849607436697, + 0.8518432496626144, + 0.9787213542266133, + 1.1350523394046785, + 1.056760328024313, + 1.1986763263570401, + 1.2165526916000595, + 1.3003080159789122, + 1.319221134436827, + 1.2772115930389878, + 1.4088316376177517, + 1.4268831508570738, + 1.441423045337684, + 1.3694390426638032, + 1.383163051306287, + 1.3093064503358134, + 1.159966015918821, + 1.0419754365036318, + 0.9354801410893958, + 0.8458444577627716, + 0.7702064512311685, + 0.6510329932472191, + 0.5511736211899907 + ], + "population_cv": 0.3252608102570666 + }, + "loss_nats": 1.2900829315185547, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.06581270897968, + "cross_entropy_nats": 5.590795338153839, + "step": 0 + }, + { + "bits_per_byte": 5.193364149696528, + "cross_entropy_nats": 3.599765717983246, + "step": 100 + }, + { + "bits_per_byte": 3.118855971380574, + "cross_entropy_nats": 2.1618262231349945, + "step": 500 + }, + { + "bits_per_byte": 1.9576097217110056, + "cross_entropy_nats": 1.3569116592407227, + "step": 2000 + }, + { + "bits_per_byte": 1.792987261872353, + "cross_entropy_nats": 1.2428040653467178, + "step": 4000 + }, + { + "bits_per_byte": 1.709514213904039, + "cross_entropy_nats": 1.1849449574947357, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "final_optimizer_state": "e774625d2eec7b16bb949a8bd26f25b05b7288a59bf24a716a88b489cd22bcd6", + "final_public_parameters": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "seed": 2026073003, + "timing": { + "mean_ms": 42.39461278025134, + "measured_steps": 7980, + "median_ms": 42.272048493032344, + "p95_ms": 43.13050716154976, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.516995429992676, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "population_cv": 0.5072011380661324 + }, + "activation_output_rms_by_block": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013632480811793357, + "first_to_last_ratio": 0.32005493218522185, + "imbalance_abs_log_ratio": 1.1392626348419668, + "last_quartile_mean": 0.04259419068694115, + "mean": 0.025403792154975235, + "normalized": [ + 0.23092935972382672, + 0.3992639012357397, + 0.5833875213879504, + 0.7700561688360058, + 0.2642123711098415, + 0.4687595594389148, + 0.6729013896370168, + 0.9035435305603466, + 0.2950160146386087, + 0.5418894364067922, + 0.8025599005701267, + 1.0558320747576517, + 0.36189473544610173, + 0.6606003816028687, + 0.9564482474371727, + 1.2361787191078082, + 0.389077999650742, + 0.7376930678145923, + 1.1018562250488806, + 1.4948807199009364, + 0.4969232792693754, + 0.9255949835565679, + 1.3737978229029388, + 1.8632120600111322, + 0.6868422339014575, + 1.1618129342705898, + 1.7580201402629136, + 2.3690874226600394, + 0.7468506348084945, + 1.4805931380763324, + 2.2249315569778956, + 2.985352468990339 + ], + "population_cv": 0.6518934572295025 + }, + "bits_per_byte": 5.150682269327325, + "core_parameter_grad_rms_by_block": [ + 0.00039664652254938686, + 0.00033413090452743513, + 0.0003161152806455274, + 0.0003097753114302562, + 0.00028318594507881776, + 0.00028174654548060826, + 0.00029366070958977545, + 0.0002617199768643518, + 0.00023850476284541366, + 0.0002578321956415913, + 0.00023580491915923377, + 0.00024129375767696618, + 0.00024677521358199467, + 0.00023282043614108414, + 0.00020802360655182162, + 0.00020816264493602737, + 0.00019546204845585216, + 0.00020244473639840523, + 0.00019508742079131332, + 0.00018509966437476374, + 0.00018490270602469487, + 0.0001750081453781441, + 0.00017332318168420213, + 0.00016951527760705314, + 0.0001849276373753642, + 0.00014904354377519617, + 0.00017214209100244258, + 0.0001582098123018746, + 0.0001606899226932654, + 0.0001683991686835421, + 0.00016911992547040899, + 0.00016138660409180947 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030962264952076986, + "first_to_last_ratio": 1.8709465967014842, + "imbalance_abs_log_ratio": 0.6264445042344079, + "last_quartile_mean": 0.00016548983817423794, + "mean": 0.0002234675193377695, + "normalized": [ + 1.774962749507496, + 1.4952101563466982, + 1.4145916220053505, + 1.3862207463001954, + 1.2672353723620313, + 1.2607941696204648, + 1.314109139708619, + 1.1711768119140573, + 1.0672905107292818, + 1.1537792892929548, + 1.0552089174213117, + 1.0797710485712801, + 1.104300143095945, + 1.0418535849461763, + 0.9308896754583627, + 0.9315118615577912, + 0.8746776669606858, + 0.9059246596477921, + 0.8730012369110346, + 0.828306793413619, + 0.8274254199117672, + 0.7831480203331949, + 0.775607937107967, + 0.7585678585836542, + 0.827536985792765, + 0.6669584206996899, + 0.7703226469447271, + 0.7079767689314242, + 0.719075072607683, + 0.7535733568018358, + 0.7567986881117407, + 0.7221926684024036 + ], + "population_cv": 0.268339881652204 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6930912733078003, + "mean_weights": [ + 0.5023526549339294, + 0.49764734506607056 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692903995513916, + "mean_weights": [ + 0.5103387832641602, + 0.48966121673583984 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928629875183105, + "mean_weights": [ + 0.5053091049194336, + 0.4946908950805664 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927823424339294, + "mean_weights": [ + 0.5126834511756897, + 0.4873165488243103 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928155422210693, + "mean_weights": [ + 0.502953290939331, + 0.49704670906066895 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692751407623291, + "mean_weights": [ + 0.5129156708717346, + 0.48708435893058777 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925709247589111, + "mean_weights": [ + 0.5112693905830383, + 0.4887305796146393 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929247379302979, + "mean_weights": [ + 0.5095131397247314, + 0.49048691987991333 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0983518362045288, + "mean_weights": [ + 0.3398353159427643, + 0.32833361625671387, + 0.33183106780052185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098447322845459, + "mean_weights": [ + 0.3408135175704956, + 0.3303682208061218, + 0.3288182020187378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980432033538818, + "mean_weights": [ + 0.3472674489021301, + 0.3313220739364624, + 0.32141047716140747 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985240936279297, + "mean_weights": [ + 0.3373557925224304, + 0.33420175313949585, + 0.32844245433807373 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098268985748291, + "mean_weights": [ + 0.3445510268211365, + 0.33061009645462036, + 0.32483887672424316 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985302925109863, + "mean_weights": [ + 0.3371829390525818, + 0.3332410156726837, + 0.3295760750770569 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983161926269531, + "mean_weights": [ + 0.3423102796077728, + 0.3327076733112335, + 0.32498204708099365 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098489761352539, + "mean_weights": [ + 0.33793729543685913, + 0.3344462811946869, + 0.327616423368454 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3861088752746582, + "mean_weights": [ + 0.2561652958393097, + 0.2513129711151123, + 0.2458309531211853, + 0.2466908097267151 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861844539642334, + "mean_weights": [ + 0.2536832094192505, + 0.25214672088623047, + 0.24721932411193848, + 0.24695077538490295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3860987424850464, + "mean_weights": [ + 0.25705933570861816, + 0.2498370260000229, + 0.2462591528892517, + 0.24684447050094604 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862221240997314, + "mean_weights": [ + 0.2514326870441437, + 0.2528519928455353, + 0.24950753152370453, + 0.24620777368545532 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386016845703125, + "mean_weights": [ + 0.25482749938964844, + 0.2536851465702057, + 0.24858617782592773, + 0.24290117621421814 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862277269363403, + "mean_weights": [ + 0.25171467661857605, + 0.2530350983142853, + 0.24868467450141907, + 0.24656552076339722 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859888315200806, + "mean_weights": [ + 0.25458407402038574, + 0.2546530067920685, + 0.248786062002182, + 0.24197682738304138 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862541913986206, + "mean_weights": [ + 0.2501963675022125, + 0.2521768808364868, + 0.24995486438274384, + 0.24767187237739563 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6090729236602783, + "mean_weights": [ + 0.20360878109931946, + 0.20631682872772217, + 0.19986985623836517, + 0.19557830691337585, + 0.19462622702121735 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093083620071411, + "mean_weights": [ + 0.2017374038696289, + 0.20362424850463867, + 0.20011676847934723, + 0.1968517303466797, + 0.1976698487997055 + ], + "sources": 5 + }, + { + "entropy_mean": 1.608644723892212, + "mean_weights": [ + 0.20431089401245117, + 0.20831988751888275, + 0.20422273874282837, + 0.19545811414718628, + 0.18768835067749023 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609345555305481, + "mean_weights": [ + 0.2002599537372589, + 0.20386311411857605, + 0.2001732438802719, + 0.19764204323291779, + 0.19806164503097534 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6086196899414062, + "mean_weights": [ + 0.20515945553779602, + 0.21025606989860535, + 0.20179016888141632, + 0.1938556730747223, + 0.18893863260746002 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609370470046997, + "mean_weights": [ + 0.2002836912870407, + 0.20340535044670105, + 0.20010054111480713, + 0.1980183869600296, + 0.1981920301914215 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6083989143371582, + "mean_weights": [ + 0.20363344252109528, + 0.21095217764377594, + 0.20392383635044098, + 0.19413790106773376, + 0.18735264241695404 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093392372131348, + "mean_weights": [ + 0.20084109902381897, + 0.20331618189811707, + 0.2011999487876892, + 0.19900763034820557, + 0.19563516974449158 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7908520698547363, + "mean_weights": [ + 0.17160072922706604, + 0.17730243504047394, + 0.16903036832809448, + 0.16099891066551208, + 0.15854957699775696, + 0.1625179797410965 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916775941848755, + "mean_weights": [ + 0.16706335544586182, + 0.16981256008148193, + 0.16858771443367004, + 0.1652718484401703, + 0.16462719440460205, + 0.16463729739189148 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898297309875488, + "mean_weights": [ + 0.17284834384918213, + 0.18101516366004944, + 0.1735592484474182, + 0.16231873631477356, + 0.15689018368721008, + 0.153368279337883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917213439941406, + "mean_weights": [ + 0.16615921258926392, + 0.16885727643966675, + 0.16764026880264282, + 0.16661542654037476, + 0.16609220206737518, + 0.16463561356067657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890949249267578, + "mean_weights": [ + 0.17241932451725006, + 0.18413974344730377, + 0.17484498023986816, + 0.1621202528476715, + 0.15553176403045654, + 0.15094394981861115 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917208671569824, + "mean_weights": [ + 0.16663745045661926, + 0.168935626745224, + 0.16597260534763336, + 0.16514870524406433, + 0.16661295294761658, + 0.16669265925884247 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789101004600525, + "mean_weights": [ + 0.17229264974594116, + 0.18365857005119324, + 0.1747075319290161, + 0.1640234887599945, + 0.155386283993721, + 0.14993149042129517 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917447090148926, + "mean_weights": [ + 0.1658121943473816, + 0.167683407664299, + 0.16629494726657867, + 0.1669958382844925, + 0.16702724993228912, + 0.1661863625049591 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9436898231506348, + "mean_weights": [ + 0.14778709411621094, + 0.15856821835041046, + 0.1500256359577179, + 0.1408001184463501, + 0.13533659279346466, + 0.130964457988739, + 0.13651786744594574 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458942413330078, + "mean_weights": [ + 0.142832413315773, + 0.1429906189441681, + 0.14157448709011078, + 0.14285019040107727, + 0.14392998814582825, + 0.14335544407367706, + 0.14246684312820435 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9425597190856934, + "mean_weights": [ + 0.1483290046453476, + 0.1615946888923645, + 0.15280696749687195, + 0.14239417016506195, + 0.13462956249713898, + 0.1307675987482071, + 0.12947802245616913 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458987712860107, + "mean_weights": [ + 0.1426449418067932, + 0.14291533827781677, + 0.1416938602924347, + 0.14274153113365173, + 0.14325542747974396, + 0.1432831734418869, + 0.14346569776535034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941068410873413, + "mean_weights": [ + 0.14912807941436768, + 0.16456490755081177, + 0.15460123121738434, + 0.14255177974700928, + 0.13539987802505493, + 0.1283380389213562, + 0.1254161298274994 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458861351013184, + "mean_weights": [ + 0.14264336228370667, + 0.1430070698261261, + 0.14314037561416626, + 0.14221979677677155, + 0.14132913947105408, + 0.1431397795677185, + 0.14452049136161804 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415628910064697, + "mean_weights": [ + 0.14895881712436676, + 0.16342055797576904, + 0.15406624972820282, + 0.14293311536312103, + 0.1363803744316101, + 0.12938986718654633, + 0.1248510330915451 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458723068237305, + "mean_weights": [ + 0.14191991090774536, + 0.1418565809726715, + 0.14164230227470398, + 0.14289972186088562, + 0.14397987723350525, + 0.14332248270511627, + 0.14437910914421082 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073223114013672, + "mean_weights": [ + 0.13280923664569855, + 0.14827430248260498, + 0.13796664774417877, + 0.12756125628948212, + 0.12019773572683334, + 0.11310221254825592, + 0.10922321677207947, + 0.11086538434028625 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079413890838623, + "mean_weights": [ + 0.12475188821554184, + 0.12460128962993622, + 0.12424701452255249, + 0.12420951575040817, + 0.12484367191791534, + 0.12469655275344849, + 0.12630701065063477, + 0.12634305655956268 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072925567626953, + "mean_weights": [ + 0.13228744268417358, + 0.14843900501728058, + 0.13848665356636047, + 0.12843599915504456, + 0.12169650197029114, + 0.11442162096500397, + 0.11023421585559845, + 0.10599856078624725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793263912200928, + "mean_weights": [ + 0.12420415878295898, + 0.12273682653903961, + 0.12360996007919312, + 0.1238352432847023, + 0.1241140365600586, + 0.12607696652412415, + 0.1275939792394638, + 0.12782883644104004 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073427200317383, + "mean_weights": [ + 0.13135744631290436, + 0.1465209424495697, + 0.13772746920585632, + 0.12966200709342957, + 0.12215200066566467, + 0.11654195189476013, + 0.11090746521949768, + 0.10513069480657578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079422950744629, + "mean_weights": [ + 0.12477636337280273, + 0.12493626773357391, + 0.1242394968867302, + 0.12470392882823944, + 0.12560302019119263, + 0.12461308389902115, + 0.12591606378555298, + 0.12521176040172577 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0725789070129395, + "mean_weights": [ + 0.13186156749725342, + 0.14827941358089447, + 0.13866659998893738, + 0.12988534569740295, + 0.12183371186256409, + 0.11524046957492828, + 0.1099783331155777, + 0.10425455868244171 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793380737304688, + "mean_weights": [ + 0.12417487055063248, + 0.1225123256444931, + 0.12277349829673767, + 0.1243998259305954, + 0.12567630410194397, + 0.1261572688817978, + 0.12747105956077576, + 0.12683483958244324 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1906280517578125, + "mean_weights": [ + 0.11854428052902222, + 0.13291707634925842, + 0.12378416955471039, + 0.11745279282331467, + 0.11140251904726028, + 0.10409101843833923, + 0.09838292002677917, + 0.09486069530248642, + 0.09856453537940979 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197185516357422, + "mean_weights": [ + 0.11084984987974167, + 0.11021925508975983, + 0.11018812656402588, + 0.11046157777309418, + 0.11131840199232101, + 0.11167057603597641, + 0.11249847710132599, + 0.1118280440568924, + 0.11096569895744324 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1899101734161377, + "mean_weights": [ + 0.1190595030784607, + 0.13352498412132263, + 0.1239522397518158, + 0.11779220402240753, + 0.11234381794929504, + 0.10513705015182495, + 0.09944089502096176, + 0.0946362093091011, + 0.09411308169364929 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197190284729004, + "mean_weights": [ + 0.11067462712526321, + 0.11021950840950012, + 0.11038416624069214, + 0.11103623360395432, + 0.11016053706407547, + 0.11144962906837463, + 0.11242866516113281, + 0.11139006912708282, + 0.11225657165050507 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1896841526031494, + "mean_weights": [ + 0.11968795955181122, + 0.13428261876106262, + 0.12373492866754532, + 0.11790731549263, + 0.11133000254631042, + 0.10504785925149918, + 0.09994535893201828, + 0.09480959177017212, + 0.09325435757637024 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971938610076904, + "mean_weights": [ + 0.11116788536310196, + 0.1117495447397232, + 0.11143069714307785, + 0.11117202788591385, + 0.11217275261878967, + 0.11135637760162354, + 0.11160033196210861, + 0.10969291627407074, + 0.10965746641159058 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188610792160034, + "mean_weights": [ + 0.11987313628196716, + 0.1360916942358017, + 0.12462760508060455, + 0.11755925416946411, + 0.11237476021051407, + 0.10438287258148193, + 0.09920360147953033, + 0.09431086480617523, + 0.09157621115446091 + ], + "sources": 9 + } + ], + "loss_nats": 3.570180892944336, + "output_weights": { + "entropy_mean": 2.1930694580078125, + "mean_weights": [ + 0.10071488469839096, + 0.09411877393722534, + 0.10181896388530731, + 0.10989432036876678, + 0.1145806610584259, + 0.11861918121576309, + 0.11984598636627197, + 0.12094368040561676, + 0.11946355551481247 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "population_cv": 0.5868596491376813 + }, + "activation_output_rms_by_block": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.036073930095881224, + "first_to_last_ratio": 0.5177916953272103, + "imbalance_abs_log_ratio": 0.6581822501725932, + "last_quartile_mean": 0.0696688077878207, + "mean": 0.04647241474594921, + "normalized": [ + 0.46329906589537145, + 0.8300990481085756, + 1.06603234209322, + 1.2414755802942754, + 0.4606131805713009, + 0.5586764120892566, + 0.7186330436105184, + 0.8711228081179162, + 0.4630334112674251, + 0.7541501214633191, + 0.8015850238486076, + 0.9304351337283648, + 0.4571868851775181, + 0.6517799422454194, + 0.8562272308804426, + 1.1237390363826774, + 0.8427218914862018, + 0.8798161433899512, + 1.0235622313138022, + 1.1926338466716728, + 0.4420161942758587, + 0.7919574083446129, + 1.1031381372276345, + 1.482919724824311, + 0.6253151219931478, + 1.1530933116414785, + 1.6957169376190813, + 2.3138065984257823, + 0.6272181919070765, + 1.163956934601208, + 1.8566997944110037, + 2.5573392660929675 + ], + "population_cv": 0.5056023373965206 + }, + "bits_per_byte": 3.0609520355704922, + "core_parameter_grad_rms_by_block": [ + 0.0006151943096300597, + 0.0003715258289130363, + 0.0003398511463891951, + 0.0003030496529751275, + 0.00028984271802693696, + 0.0002579116693919001, + 0.00022400787473702318, + 0.00028477639847268295, + 0.000278695861793727, + 0.00022792247112618847, + 0.00023417680672174036, + 0.00019296669269337194, + 0.00020274366236211453, + 0.00017465616055496573, + 0.00015881266908702134, + 0.00017808900330691153, + 0.00016226510475368574, + 0.00014508607560593053, + 0.00013101353173913878, + 0.00011930204103094746, + 0.00012786456583361604, + 0.00012508100764064805, + 0.00012354146515800217, + 0.00012514897623388166, + 0.00011397604830293932, + 9.556324385288696e-05, + 9.289013346881659e-05, + 9.744025747256133e-05, + 8.302691520063822e-05, + 7.404706874242776e-05, + 8.914255818565412e-05, + 8.643783624226864e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003357699498169952, + "first_to_last_ratio": 3.666991625028822, + "imbalance_abs_log_ratio": 1.2993716052111575, + "last_quartile_mean": 9.156550768352412e-05, + "mean": 0.00019143905486393895, + "normalized": [ + 3.2135256312631473, + 1.940700288021637, + 1.7752445896201103, + 1.5830085098911153, + 1.514020836724963, + 1.3472259857070703, + 1.1701263093689624, + 1.487556437609251, + 1.4557941794677367, + 1.1905745736583337, + 1.223244686870066, + 1.0079797606111183, + 1.0590506859021536, + 0.9123329650739171, + 0.8295729896906037, + 0.9302647436986373, + 0.8476071137574935, + 0.7578708310540252, + 0.6843615678747147, + 0.6231854890619823, + 0.6679126467924371, + 0.6533724674390322, + 0.645330521746498, + 0.6537275078109245, + 0.5953646625760102, + 0.49918363795102527, + 0.4852203931681348, + 0.5089883960292994, + 0.4336989401647838, + 0.386791863316788, + 0.46564457941463516, + 0.45151620866339115 + ], + "population_cv": 0.578773657781326 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.658259391784668, + "mean_weights": [ + 0.3887256383895874, + 0.6112743616104126 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6694332361221313, + "mean_weights": [ + 0.42782124876976013, + 0.5721787214279175 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6890689730644226, + "mean_weights": [ + 0.504529595375061, + 0.49547040462493896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740086078643799, + "mean_weights": [ + 0.45225459337234497, + 0.547745406627655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.679557204246521, + "mean_weights": [ + 0.4461178183555603, + 0.5538821816444397 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740646362304688, + "mean_weights": [ + 0.44189155101776123, + 0.5581084489822388 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6792672276496887, + "mean_weights": [ + 0.4455937147140503, + 0.5544062852859497 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674970269203186, + "mean_weights": [ + 0.44674086570739746, + 0.5532591342926025 + ], + "sources": 2 + }, + { + "entropy_mean": 1.093062162399292, + "mean_weights": [ + 0.3082374036312103, + 0.33985745906829834, + 0.35190510749816895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0853323936462402, + "mean_weights": [ + 0.30180174112319946, + 0.3268374502658844, + 0.3713608384132385 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0921990871429443, + "mean_weights": [ + 0.3034760355949402, + 0.3359980285167694, + 0.3605259656906128 + ], + "sources": 3 + }, + { + "entropy_mean": 1.076724648475647, + "mean_weights": [ + 0.29931002855300903, + 0.30481812357902527, + 0.3958718776702881 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856893062591553, + "mean_weights": [ + 0.2801036834716797, + 0.3310527801513672, + 0.3888435363769531 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0752873420715332, + "mean_weights": [ + 0.29463040828704834, + 0.31052330136299133, + 0.3948462903499603 + ], + "sources": 3 + }, + { + "entropy_mean": 1.088913917541504, + "mean_weights": [ + 0.29400017857551575, + 0.32166045904159546, + 0.3843393623828888 + ], + "sources": 3 + }, + { + "entropy_mean": 1.054490327835083, + "mean_weights": [ + 0.2850497364997864, + 0.2890807092189789, + 0.42586958408355713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3700100183486938, + "mean_weights": [ + 0.21610887348651886, + 0.2282472848892212, + 0.2728016972541809, + 0.28284209966659546 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3620216846466064, + "mean_weights": [ + 0.22791308164596558, + 0.2214205265045166, + 0.2562018632888794, + 0.2944645285606384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3784711360931396, + "mean_weights": [ + 0.23347607254981995, + 0.23307976126670837, + 0.2570483684539795, + 0.2763957977294922 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3619427680969238, + "mean_weights": [ + 0.24715131521224976, + 0.22003202140331268, + 0.23014183342456818, + 0.3026748299598694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804662227630615, + "mean_weights": [ + 0.24302782118320465, + 0.23108476400375366, + 0.24228686094284058, + 0.2836005389690399 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3502743244171143, + "mean_weights": [ + 0.2455797642469406, + 0.21441739797592163, + 0.2208172082901001, + 0.31918564438819885 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3820215463638306, + "mean_weights": [ + 0.24522465467453003, + 0.22846508026123047, + 0.25027886033058167, + 0.27603137493133545 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3491160869598389, + "mean_weights": [ + 0.24498313665390015, + 0.21178750693798065, + 0.2205154001712799, + 0.3227139711380005 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6032840013504028, + "mean_weights": [ + 0.20074918866157532, + 0.17959122359752655, + 0.18619459867477417, + 0.2124267816543579, + 0.22103822231292725 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5699318647384644, + "mean_weights": [ + 0.1938496232032776, + 0.17019830644130707, + 0.16257275640964508, + 0.20397180318832397, + 0.2694075107574463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6068798303604126, + "mean_weights": [ + 0.19717518985271454, + 0.1942940056324005, + 0.19104912877082825, + 0.20343774557113647, + 0.21404394507408142 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5761134624481201, + "mean_weights": [ + 0.1940101534128189, + 0.18250694870948792, + 0.16849923133850098, + 0.18937936425209045, + 0.26560431718826294 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6027262210845947, + "mean_weights": [ + 0.20452985167503357, + 0.18702910840511322, + 0.17977532744407654, + 0.20095866918563843, + 0.22770702838897705 + ], + "sources": 5 + }, + { + "entropy_mean": 1.577937126159668, + "mean_weights": [ + 0.1962222307920456, + 0.1818992793560028, + 0.17167484760284424, + 0.18149182200431824, + 0.2687118649482727 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5977532863616943, + "mean_weights": [ + 0.2153339385986328, + 0.18402469158172607, + 0.17019470036029816, + 0.1919252574443817, + 0.23852142691612244 + ], + "sources": 5 + }, + { + "entropy_mean": 1.578121542930603, + "mean_weights": [ + 0.20422281324863434, + 0.1835813820362091, + 0.16642534732818604, + 0.17722904682159424, + 0.26854145526885986 + ], + "sources": 5 + }, + { + "entropy_mean": 1.782798409461975, + "mean_weights": [ + 0.15961389243602753, + 0.16475999355316162, + 0.14714640378952026, + 0.15768738090991974, + 0.1760568618774414, + 0.19473549723625183 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7707297801971436, + "mean_weights": [ + 0.16306114196777344, + 0.1578725278377533, + 0.139655202627182, + 0.14299748837947845, + 0.18966570496559143, + 0.20674791932106018 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781312108039856, + "mean_weights": [ + 0.15919369459152222, + 0.16844786703586578, + 0.14742718636989594, + 0.1539241373538971, + 0.1669306755065918, + 0.20407646894454956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7736209630966187, + "mean_weights": [ + 0.15695160627365112, + 0.16567477583885193, + 0.1478763222694397, + 0.14320015907287598, + 0.17375096678733826, + 0.2125461995601654 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781569480895996, + "mean_weights": [ + 0.15368077158927917, + 0.1725173443555832, + 0.14978237450122833, + 0.15674246847629547, + 0.16688688099384308, + 0.20039016008377075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7793617248535156, + "mean_weights": [ + 0.1678246557712555, + 0.1683025360107422, + 0.14878368377685547, + 0.14351718127727509, + 0.16739681363105774, + 0.20417512953281403 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7772380113601685, + "mean_weights": [ + 0.1536770462989807, + 0.18459689617156982, + 0.14832255244255066, + 0.15016385912895203, + 0.1652815043926239, + 0.19795814156532288 + ], + "sources": 6 + }, + { + "entropy_mean": 1.780207633972168, + "mean_weights": [ + 0.16951784491539001, + 0.17200809717178345, + 0.15038040280342102, + 0.14669056236743927, + 0.16062122583389282, + 0.20078186690807343 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937079668045044, + "mean_weights": [ + 0.13008031249046326, + 0.15163566172122955, + 0.1323973685503006, + 0.13641127943992615, + 0.13806134462356567, + 0.1444631665945053, + 0.16695083677768707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939245343208313, + "mean_weights": [ + 0.14692582190036774, + 0.14671875536441803, + 0.13336658477783203, + 0.1307726800441742, + 0.131200909614563, + 0.147438645362854, + 0.16357660293579102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9312366247177124, + "mean_weights": [ + 0.12334205210208893, + 0.16950437426567078, + 0.13870283961296082, + 0.1300538182258606, + 0.12885476648807526, + 0.14288470149040222, + 0.1666574478149414 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9345850944519043, + "mean_weights": [ + 0.1465124636888504, + 0.153631791472435, + 0.13159149885177612, + 0.13120220601558685, + 0.1253916621208191, + 0.14393125474452972, + 0.1677391082048416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9284589290618896, + "mean_weights": [ + 0.12412485480308533, + 0.1698783040046692, + 0.13807369768619537, + 0.13034150004386902, + 0.13028821349143982, + 0.1419651210308075, + 0.16532832384109497 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9329564571380615, + "mean_weights": [ + 0.1454196274280548, + 0.1566689908504486, + 0.1286500096321106, + 0.12666445970535278, + 0.12782412767410278, + 0.14222374558448792, + 0.17254900932312012 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9259672164916992, + "mean_weights": [ + 0.12082186341285706, + 0.17791953682899475, + 0.1424643099308014, + 0.13298648595809937, + 0.12929362058639526, + 0.1344166398048401, + 0.16209754347801208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9201786518096924, + "mean_weights": [ + 0.13756656646728516, + 0.17355048656463623, + 0.14306491613388062, + 0.13006147742271423, + 0.11658851057291031, + 0.139857679605484, + 0.15931037068367004 + ], + "sources": 7 + }, + { + "entropy_mean": 2.043663501739502, + "mean_weights": [ + 0.10298450291156769, + 0.1751018464565277, + 0.1414714753627777, + 0.12628036737442017, + 0.11767096072435379, + 0.12069195508956909, + 0.12702946364879608, + 0.08876943588256836 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0607244968414307, + "mean_weights": [ + 0.1220984011888504, + 0.1498527079820633, + 0.12913095951080322, + 0.1179434061050415, + 0.10775719583034515, + 0.12020602822303772, + 0.13844498991966248, + 0.11456632614135742 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0388383865356445, + "mean_weights": [ + 0.1025313287973404, + 0.18260905146598816, + 0.14208021759986877, + 0.12606360018253326, + 0.11357533931732178, + 0.11596501618623734, + 0.12797299027442932, + 0.08920245617628098 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0634398460388184, + "mean_weights": [ + 0.12216806411743164, + 0.15090107917785645, + 0.12766355276107788, + 0.1134132519364357, + 0.10988184064626694, + 0.11335282772779465, + 0.13999530673027039, + 0.12262405455112457 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0386834144592285, + "mean_weights": [ + 0.10029368102550507, + 0.17931804060935974, + 0.14185777306556702, + 0.13114282488822937, + 0.11872631311416626, + 0.11815309524536133, + 0.12660562992095947, + 0.08390261977910995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0626261234283447, + "mean_weights": [ + 0.11644256860017776, + 0.15398627519607544, + 0.1359611600637436, + 0.12036904692649841, + 0.10964122414588928, + 0.11886263638734818, + 0.13317911326885223, + 0.1115579903125763 + ], + "sources": 8 + }, + { + "entropy_mean": 2.020051956176758, + "mean_weights": [ + 0.09590239822864532, + 0.1913527548313141, + 0.14906750619411469, + 0.1341754049062729, + 0.11847006529569626, + 0.11603336781263351, + 0.12013453245162964, + 0.07486395537853241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0452136993408203, + "mean_weights": [ + 0.11420649290084839, + 0.15673109889030457, + 0.13583534955978394, + 0.12758757174015045, + 0.1112617775797844, + 0.13417255878448486, + 0.12519711256027222, + 0.09500804543495178 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1403071880340576, + "mean_weights": [ + 0.08488726615905762, + 0.1673685610294342, + 0.13726134598255157, + 0.12220048159360886, + 0.10616928339004517, + 0.1037440225481987, + 0.10401669889688492, + 0.06658264249563217, + 0.10776971280574799 + ], + "sources": 9 + }, + { + "entropy_mean": 2.18704891204834, + "mean_weights": [ + 0.10885682702064514, + 0.12684625387191772, + 0.11236055195331573, + 0.10673345625400543, + 0.10181289166212082, + 0.11441421508789062, + 0.11840946972370148, + 0.10466625541448593, + 0.1059001013636589 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1283018589019775, + "mean_weights": [ + 0.08667369186878204, + 0.1812124401330948, + 0.14178472757339478, + 0.12440058588981628, + 0.11160320788621902, + 0.10330478847026825, + 0.11197598278522491, + 0.06831350177526474, + 0.07073107361793518 + ], + "sources": 9 + }, + { + "entropy_mean": 2.173035144805908, + "mean_weights": [ + 0.10752685368061066, + 0.13639645278453827, + 0.11839058995246887, + 0.11235916614532471, + 0.10138258337974548, + 0.11953261494636536, + 0.11256720125675201, + 0.09452059864997864, + 0.097323939204216 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1028335094451904, + "mean_weights": [ + 0.08200174570083618, + 0.19083696603775024, + 0.15512439608573914, + 0.1322997808456421, + 0.11209174990653992, + 0.10435720533132553, + 0.1024213582277298, + 0.06072331964969635, + 0.06014348566532135 + ], + "sources": 9 + }, + { + "entropy_mean": 2.147671937942505, + "mean_weights": [ + 0.09926989674568176, + 0.14984291791915894, + 0.12505242228507996, + 0.13004425168037415, + 0.1122550517320633, + 0.12226949632167816, + 0.10689133405685425, + 0.07532045990228653, + 0.07905416935682297 + ], + "sources": 9 + }, + { + "entropy_mean": 2.109609603881836, + "mean_weights": [ + 0.08386656641960144, + 0.18956288695335388, + 0.1487923562526703, + 0.12990039587020874, + 0.11159682273864746, + 0.10738210380077362, + 0.10535432398319244, + 0.0630122721195221, + 0.06053228676319122 + ], + "sources": 9 + } + ], + "loss_nats": 2.121690273284912, + "output_weights": { + "entropy_mean": 2.1622862815856934, + "mean_weights": [ + 0.12356589734554291, + 0.09235234558582306, + 0.08736804127693176, + 0.08735492080450058, + 0.08889827132225037, + 0.10504580289125443, + 0.14004790782928467, + 0.14956718683242798, + 0.12579965591430664 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "population_cv": 0.5384894543542103 + }, + "activation_output_rms_by_block": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14976411033421755, + "first_to_last_ratio": 1.036636701844262, + "imbalance_abs_log_ratio": 0.035981532131895505, + "last_quartile_mean": 0.14447116339579225, + "mean": 0.12951817945577204, + "normalized": [ + 0.7297249643070838, + 1.5686735699681784, + 1.7647660354854744, + 1.8765703797812938, + 0.636469098825333, + 0.749653246535227, + 0.8711234054002422, + 1.0535576234146717, + 0.42020993720306404, + 0.6840666215997374, + 0.9431112804194469, + 1.192130706187048, + 0.4699981697046636, + 0.7312780916470571, + 0.9733494312076132, + 1.2651958587237788, + 0.6835788065012903, + 0.9366911620158862, + 1.2091399212742004, + 1.6196823469097898, + 0.34397948019882096, + 0.5722161994862464, + 0.7727653276358534, + 1.008461533628578, + 0.3829837776042769, + 0.608007158744095, + 0.8769750006177236, + 1.2205488119398642, + 0.5370088925855727, + 1.0271058497013745, + 1.7339584411053826, + 2.5370188696411313 + ], + "population_cv": 0.4929365839545065 + }, + "bits_per_byte": 2.073383110809672, + "core_parameter_grad_rms_by_block": [ + 0.0002879944983597963, + 0.00022853159560210997, + 0.00014071406888316784, + 0.00014318671869519129, + 0.00013886536811003923, + 0.00010286844600842235, + 9.677924556041764e-05, + 0.0001337288949114584, + 0.00010758657043361882, + 9.13787469069405e-05, + 9.373485859236601e-05, + 9.381017142422257e-05, + 0.00010148426099734976, + 9.552468088000852e-05, + 0.00010200246393672037, + 0.00010753255944705701, + 0.00013288008273706816, + 0.00012966970691820486, + 0.0001395588859415414, + 0.00019785772756123382, + 0.0004949688435464942, + 0.0005919211037074493, + 0.000516172452953512, + 0.0004666614668327092, + 0.0003441119880144902, + 0.00027788329882105144, + 0.00022798541204965527, + 0.00021216050695355764, + 0.00012431994828037904, + 0.00011340006302052245, + 0.00011536177905162644, + 0.00011797314476595472 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015908360451632537, + "first_to_last_ratio": 0.8300756844691923, + "imbalance_abs_log_ratio": 0.18623839624123317, + "last_quartile_mean": 0.00019164951761965464, + "mean": 0.00019589404874701053, + "normalized": [ + 1.4701544033720488, + 1.1666081591751138, + 0.7183172219024098, + 0.730939605419618, + 0.7088799736299207, + 0.5251228746681984, + 0.4940387223575344, + 0.682659303673712, + 0.5492079576779928, + 0.4664702551783621, + 0.4784977348312449, + 0.4788821917983569, + 0.5180568866000248, + 0.4876344201930039, + 0.5207022091235275, + 0.5489322423772612, + 0.6783262870260932, + 0.661937959563377, + 0.7124202436684344, + 1.01002418821186, + 2.526717104028653, + 3.0216390313719645, + 2.6349572958192744, + 2.3822135987162336, + 1.7566229817369148, + 1.4185387488719823, + 1.1638200012093756, + 1.0830370208313709, + 0.6346285100316312, + 0.5788846764149236, + 0.5888988450109153, + 0.6022293455086652 + ], + "population_cv": 0.7015245062301059 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4738388657569885, + "mean_weights": [ + 0.8067666292190552, + 0.19323334097862244 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6344536542892456, + "mean_weights": [ + 0.4895974397659302, + 0.5104025602340698 + ], + "sources": 2 + }, + { + "entropy_mean": 0.41773349046707153, + "mean_weights": [ + 0.8461704254150391, + 0.15382957458496094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868985891342163, + "mean_weights": [ + 0.5008196830749512, + 0.49918028712272644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5841774344444275, + "mean_weights": [ + 0.684928297996521, + 0.315071702003479 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6896515488624573, + "mean_weights": [ + 0.5092520117759705, + 0.49074798822402954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611414909362793, + "mean_weights": [ + 0.7280503511428833, + 0.2719496488571167 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6817618608474731, + "mean_weights": [ + 0.5313050746917725, + 0.46869492530822754 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0218309164047241, + "mean_weights": [ + 0.4974328279495239, + 0.24236315488815308, + 0.2602040767669678 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0932960510253906, + "mean_weights": [ + 0.3533782958984375, + 0.29999595880508423, + 0.34662577509880066 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0456881523132324, + "mean_weights": [ + 0.4338536858558655, + 0.23105540871620178, + 0.3350909352302551 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0891339778900146, + "mean_weights": [ + 0.33998236060142517, + 0.2821658253669739, + 0.37785181403160095 + ], + "sources": 3 + }, + { + "entropy_mean": 1.038641333580017, + "mean_weights": [ + 0.34165239334106445, + 0.23904743790626526, + 0.4193001389503479 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0754427909851074, + "mean_weights": [ + 0.3781457543373108, + 0.24021697044372559, + 0.3816372752189636 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0433809757232666, + "mean_weights": [ + 0.37802231311798096, + 0.21331989765167236, + 0.4086577892303467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0922105312347412, + "mean_weights": [ + 0.3460429906845093, + 0.3074926435947418, + 0.3464643955230713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3404325246810913, + "mean_weights": [ + 0.2880150079727173, + 0.16260021924972534, + 0.2385387420654297, + 0.3108460605144501 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765960931777954, + "mean_weights": [ + 0.28703436255455017, + 0.2552259564399719, + 0.2101576328277588, + 0.24758204817771912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3274900913238525, + "mean_weights": [ + 0.33539247512817383, + 0.1987970769405365, + 0.20543015003204346, + 0.2603802978992462 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3747133016586304, + "mean_weights": [ + 0.27988845109939575, + 0.23608103394508362, + 0.20066317915916443, + 0.2833673357963562 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3239198923110962, + "mean_weights": [ + 0.3361120820045471, + 0.20296622812747955, + 0.1885247528553009, + 0.2723969519138336 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3692121505737305, + "mean_weights": [ + 0.28167375922203064, + 0.21946553885936737, + 0.20140981674194336, + 0.2974509000778198 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3342361450195312, + "mean_weights": [ + 0.36468517780303955, + 0.20593920350074768, + 0.18943950533866882, + 0.23993611335754395 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3645737171173096, + "mean_weights": [ + 0.2597482204437256, + 0.22607669234275818, + 0.209133118391037, + 0.30504196882247925 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5910131931304932, + "mean_weights": [ + 0.2617476284503937, + 0.18299485743045807, + 0.17935843765735626, + 0.17898105084896088, + 0.19691801071166992 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5884685516357422, + "mean_weights": [ + 0.2046908289194107, + 0.17294223606586456, + 0.15044087171554565, + 0.20923088490962982, + 0.26269516348838806 + ], + "sources": 5 + }, + { + "entropy_mean": 1.599259853363037, + "mean_weights": [ + 0.21657392382621765, + 0.20580387115478516, + 0.19366897642612457, + 0.1731838881969452, + 0.21076934039592743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5786192417144775, + "mean_weights": [ + 0.20599734783172607, + 0.20593789219856262, + 0.13473063707351685, + 0.17552703619003296, + 0.2778070569038391 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580681324005127, + "mean_weights": [ + 0.21277281641960144, + 0.18522506952285767, + 0.18489867448806763, + 0.17205308377742767, + 0.2450503408908844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5705549716949463, + "mean_weights": [ + 0.2125053107738495, + 0.20224159955978394, + 0.12915469706058502, + 0.1714920997619629, + 0.28460633754730225 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5599372386932373, + "mean_weights": [ + 0.2665032148361206, + 0.181569442152977, + 0.15844659507274628, + 0.14596089720726013, + 0.2475198209285736 + ], + "sources": 5 + }, + { + "entropy_mean": 1.562828540802002, + "mean_weights": [ + 0.21806953847408295, + 0.22324788570404053, + 0.11591021716594696, + 0.17848318815231323, + 0.26428914070129395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.761409044265747, + "mean_weights": [ + 0.1612042486667633, + 0.1651904582977295, + 0.1752963364124298, + 0.14996090531349182, + 0.2072838991880417, + 0.14106415212154388 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7366840839385986, + "mean_weights": [ + 0.1851401925086975, + 0.1663871705532074, + 0.09063514322042465, + 0.11971023678779602, + 0.18503433465957642, + 0.2530929446220398 + ], + "sources": 6 + }, + { + "entropy_mean": 1.777672290802002, + "mean_weights": [ + 0.15483248233795166, + 0.16576308012008667, + 0.16957050561904907, + 0.14225167036056519, + 0.16980668902397156, + 0.19777557253837585 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7414222955703735, + "mean_weights": [ + 0.18020270764827728, + 0.1900298297405243, + 0.09944288432598114, + 0.11960913985967636, + 0.15577411651611328, + 0.25494131445884705 + ], + "sources": 6 + }, + { + "entropy_mean": 1.733978509902954, + "mean_weights": [ + 0.15204599499702454, + 0.14166373014450073, + 0.1616208702325821, + 0.1256796270608902, + 0.1571422517299652, + 0.26184749603271484 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7344707250595093, + "mean_weights": [ + 0.1829337775707245, + 0.18416841328144073, + 0.09014187753200531, + 0.12335793673992157, + 0.15779981017112732, + 0.2615981996059418 + ], + "sources": 6 + }, + { + "entropy_mean": 1.735896348953247, + "mean_weights": [ + 0.17276549339294434, + 0.15101772546768188, + 0.13220663368701935, + 0.11932596564292908, + 0.15458261966705322, + 0.27010154724121094 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7644665241241455, + "mean_weights": [ + 0.19160008430480957, + 0.1915181279182434, + 0.11520625650882721, + 0.1341419219970703, + 0.1580837219953537, + 0.2094498574733734 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6835086345672607, + "mean_weights": [ + 0.10065847635269165, + 0.09339895844459534, + 0.09682720899581909, + 0.08155890554189682, + 0.09126688539981842, + 0.10349369794130325, + 0.4327958822250366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7937740087509155, + "mean_weights": [ + 0.13723835349082947, + 0.12937892973423004, + 0.07450413703918457, + 0.08537043631076813, + 0.09880411624908447, + 0.12944120168685913, + 0.3452628552913666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5790722370147705, + "mean_weights": [ + 0.09500400722026825, + 0.08892714232206345, + 0.09081116318702698, + 0.07538282871246338, + 0.07685283571481705, + 0.08106325566768646, + 0.49195876717567444 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8034920692443848, + "mean_weights": [ + 0.142653226852417, + 0.13419847190380096, + 0.07437048852443695, + 0.09070360660552979, + 0.1013825386762619, + 0.12073953449726105, + 0.33595213294029236 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5278494358062744, + "mean_weights": [ + 0.10045769065618515, + 0.08844951540231705, + 0.08561251312494278, + 0.06598107516765594, + 0.06767018139362335, + 0.07361933588981628, + 0.51820969581604 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8435695171356201, + "mean_weights": [ + 0.14839640259742737, + 0.1482572704553604, + 0.08113027364015579, + 0.09534022957086563, + 0.10612697899341583, + 0.12550383806228638, + 0.2952450215816498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6421003341674805, + "mean_weights": [ + 0.11422763019800186, + 0.10543526709079742, + 0.09285669028759003, + 0.07746599614620209, + 0.07571054995059967, + 0.08126918226480484, + 0.4530346691608429 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8733552694320679, + "mean_weights": [ + 0.14798465371131897, + 0.16052685678005219, + 0.09668852388858795, + 0.10477185249328613, + 0.10857058316469193, + 0.11694246530532837, + 0.26451510190963745 + ], + "sources": 7 + }, + { + "entropy_mean": 1.784506916999817, + "mean_weights": [ + 0.08589775860309601, + 0.08786045759916306, + 0.07714233547449112, + 0.06314891576766968, + 0.05684572085738182, + 0.056072384119033813, + 0.29875272512435913, + 0.2742796838283539 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9675533771514893, + "mean_weights": [ + 0.11549945175647736, + 0.12592051923274994, + 0.07530814409255981, + 0.08277833461761475, + 0.08296641707420349, + 0.09016790241003036, + 0.17876628041267395, + 0.24859298765659332 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7119876146316528, + "mean_weights": [ + 0.08804735541343689, + 0.07527872174978256, + 0.05666808784008026, + 0.05041901767253876, + 0.05142981559038162, + 0.05313319340348244, + 0.2591783106327057, + 0.3658455014228821 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9094492197036743, + "mean_weights": [ + 0.10337945818901062, + 0.10583682358264923, + 0.08642245084047318, + 0.08043928444385529, + 0.0717030018568039, + 0.0645938515663147, + 0.21096080541610718, + 0.2766643166542053 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7860023975372314, + "mean_weights": [ + 0.09247160702943802, + 0.0784597396850586, + 0.07236532866954803, + 0.05986309051513672, + 0.05389959365129471, + 0.06059400737285614, + 0.2579019069671631, + 0.3244447112083435 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9232951402664185, + "mean_weights": [ + 0.1160542219877243, + 0.11346358060836792, + 0.08816123008728027, + 0.08138670027256012, + 0.07161372900009155, + 0.06013647839426994, + 0.21822679042816162, + 0.2509572505950928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294003009796143, + "mean_weights": [ + 0.08831655979156494, + 0.10072161257266998, + 0.0818098857998848, + 0.07514279335737228, + 0.06651484966278076, + 0.06491033732891083, + 0.31431835889816284, + 0.20826560258865356 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9393219947814941, + "mean_weights": [ + 0.11041727662086487, + 0.12080886960029602, + 0.09995828568935394, + 0.09934850037097931, + 0.08031868189573288, + 0.075287364423275, + 0.2604207992553711, + 0.15344023704528809 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9925593137741089, + "mean_weights": [ + 0.08698172867298126, + 0.08857265114784241, + 0.07299254834651947, + 0.060478877276182175, + 0.054115522652864456, + 0.056821923702955246, + 0.2503892779350281, + 0.14502599835395813, + 0.18462148308753967 + ], + "sources": 9 + }, + { + "entropy_mean": 2.068819999694824, + "mean_weights": [ + 0.09429103136062622, + 0.09028280526399612, + 0.09188265353441238, + 0.0735517293214798, + 0.06746295094490051, + 0.060166649520397186, + 0.21415996551513672, + 0.15325874090194702, + 0.15494348108768463 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0139355659484863, + "mean_weights": [ + 0.0956040769815445, + 0.09831400215625763, + 0.0714857280254364, + 0.0677608773112297, + 0.05654233694076538, + 0.05388312786817551, + 0.24272149801254272, + 0.17641445994377136, + 0.137273907661438 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0349321365356445, + "mean_weights": [ + 0.10256819427013397, + 0.0938093364238739, + 0.0912778377532959, + 0.08394120633602142, + 0.06161238253116608, + 0.053358763456344604, + 0.2261163890361786, + 0.15197426080703735, + 0.13534162938594818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.046657085418701, + "mean_weights": [ + 0.08380690217018127, + 0.1217331811785698, + 0.10242430865764618, + 0.09225839376449585, + 0.06788390129804611, + 0.06336672604084015, + 0.2572976052761078, + 0.11757771670818329, + 0.09365126490592957 + ], + "sources": 9 + }, + { + "entropy_mean": 2.094015121459961, + "mean_weights": [ + 0.10284869372844696, + 0.13057856261730194, + 0.10152992606163025, + 0.10079468786716461, + 0.0796234980225563, + 0.07438632845878601, + 0.2018478661775589, + 0.1111527606844902, + 0.09723767638206482 + ], + "sources": 9 + }, + { + "entropy_mean": 2.057201385498047, + "mean_weights": [ + 0.09265108406543732, + 0.11481083929538727, + 0.09007447212934494, + 0.08496075868606567, + 0.06770835071802139, + 0.05925413966178894, + 0.24551761150360107, + 0.15022093057632446, + 0.09480181336402893 + ], + "sources": 9 + } + ], + "loss_nats": 1.4371596574783325, + "output_weights": { + "entropy_mean": 1.8435449600219727, + "mean_weights": [ + 0.1112452894449234, + 0.06898629665374756, + 0.05835210159420967, + 0.042265649884939194, + 0.041034020483493805, + 0.04826357960700989, + 0.10493715852499008, + 0.31774088740348816, + 0.20717504620552063 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "population_cv": 0.6027314599912318 + }, + "activation_output_rms_by_block": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22688884753733873, + "first_to_last_ratio": 1.3228372499356764, + "imbalance_abs_log_ratio": 0.2797788616441042, + "last_quartile_mean": 0.17151682684198022, + "mean": 0.17372163233812898, + "normalized": [ + 0.9268367409782202, + 1.919387740182172, + 2.055289136269615, + 2.170611306021289, + 0.6406640960204442, + 0.7450635822219737, + 0.8704616111421293, + 1.1200724150644035, + 0.4345583639666029, + 0.677281348405711, + 0.9287495476341928, + 1.2372530952113672, + 0.4794804996179952, + 0.7087278040538177, + 0.9675973641757485, + 1.32977546738468, + 0.6641820538556276, + 0.930123337383164, + 1.1558946518222766, + 1.5614450026347633, + 0.3131140120506511, + 0.5373702635317168, + 0.742418848247821, + 0.9851745133746388, + 0.3553571676266622, + 0.5579738073870804, + 0.7927337262517246, + 1.0584297759573136, + 0.4595608055861626, + 0.8937492173058054, + 1.5039930398498347, + 2.276669658784396 + ], + "population_cv": 0.5173518438224235 + }, + "bits_per_byte": 1.9433676664104056, + "core_parameter_grad_rms_by_block": [ + 0.0002458031044227025, + 0.00017276993155813553, + 0.00010939491852011586, + 0.000125258738441463, + 0.00011215544691093282, + 7.31087164016768e-05, + 9.049562402296399e-05, + 0.00013630654451635742, + 9.25970647966133e-05, + 7.580293890923953e-05, + 7.957502622146584e-05, + 8.776523713070475e-05, + 0.00010318467043462154, + 0.00010632361913360551, + 0.00011315585331317908, + 0.00012408116098017353, + 0.00012849590289080713, + 0.00014012462333747852, + 0.00013373629199672077, + 0.00019202792118308513, + 0.0005640243510171648, + 0.0006705621295490249, + 0.000579819507581129, + 0.0005486985299520294, + 0.0004559472160958734, + 0.00038131537611027277, + 0.00030455316817913544, + 0.000254991251417659, + 0.00017923215935365346, + 0.0001463414879582755, + 0.00013146832622599984, + 0.0001387805380572583 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0001331616280992935, + "first_to_last_ratio": 0.5346167023449758, + "imbalance_abs_log_ratio": 0.6262052330699333, + "last_quartile_mean": 0.000249078690424766, + "mean": 0.00021243429301935995, + "normalized": [ + 1.1570782707870122, + 0.8132864477883067, + 0.5149588466403882, + 0.5896352074852985, + 0.5279535865742339, + 0.34414743195447317, + 0.4259934812630124, + 0.6416409637964399, + 0.4358856730733892, + 0.3568300476907056, + 0.37458653727914765, + 0.4131406275478619, + 0.4857251104237578, + 0.5005012026185237, + 0.5326628375526298, + 0.584091952464876, + 0.6048736344046712, + 0.6596139509580424, + 0.6295419166835406, + 0.9039403123373536, + 2.655053207279312, + 3.156562530551101, + 2.7294063464992813, + 2.5829093888434693, + 2.146297613325176, + 1.7949803239890372, + 1.433634672869791, + 1.200329983419488, + 0.8437063376453977, + 0.6888788334421074, + 0.6188658354422025, + 0.6532868873699723 + ], + "population_cv": 0.7868568919954686 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.24809812009334564, + "mean_weights": [ + 0.9288002252578735, + 0.07119979709386826 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6067080497741699, + "mean_weights": [ + 0.5604246854782104, + 0.43957534432411194 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3847596049308777, + "mean_weights": [ + 0.8648147583007812, + 0.13518527150154114 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6843481659889221, + "mean_weights": [ + 0.49438929557800293, + 0.5056107044219971 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5728601813316345, + "mean_weights": [ + 0.6954652070999146, + 0.3045347332954407 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686518132686615, + "mean_weights": [ + 0.5263072848320007, + 0.4736926555633545 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044870376586914, + "mean_weights": [ + 0.777843713760376, + 0.2221563160419464 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6724032163619995, + "mean_weights": [ + 0.5694063901901245, + 0.4305936098098755 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9718273282051086, + "mean_weights": [ + 0.5544795989990234, + 0.20597517490386963, + 0.23954525589942932 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856361389160156, + "mean_weights": [ + 0.3703838884830475, + 0.26754841208457947, + 0.36206769943237305 + ], + "sources": 3 + }, + { + "entropy_mean": 1.022249460220337, + "mean_weights": [ + 0.46441033482551575, + 0.21309645473957062, + 0.32249322533607483 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0742855072021484, + "mean_weights": [ + 0.38770636916160583, + 0.24731171131134033, + 0.36498188972473145 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0207322835922241, + "mean_weights": [ + 0.3404029607772827, + 0.22699853777885437, + 0.4325985312461853 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0628657341003418, + "mean_weights": [ + 0.4176652133464813, + 0.22035521268844604, + 0.361979603767395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.029329538345337, + "mean_weights": [ + 0.42153143882751465, + 0.19259268045425415, + 0.3858758807182312 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0849454402923584, + "mean_weights": [ + 0.37970077991485596, + 0.2838303744792938, + 0.3364688754081726 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3144235610961914, + "mean_weights": [ + 0.3205832540988922, + 0.13048580288887024, + 0.2396528720855713, + 0.30927807092666626 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647149801254272, + "mean_weights": [ + 0.3188267648220062, + 0.2379944771528244, + 0.19448311626911163, + 0.24869562685489655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2778496742248535, + "mean_weights": [ + 0.37896889448165894, + 0.16353651881217957, + 0.1984466314315796, + 0.2590479254722595 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3567583560943604, + "mean_weights": [ + 0.29770219326019287, + 0.21492166817188263, + 0.17857596278190613, + 0.3088001608848572 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2773678302764893, + "mean_weights": [ + 0.3856483995914459, + 0.18032220005989075, + 0.1928499937057495, + 0.24117937684059143 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655540943145752, + "mean_weights": [ + 0.29819539189338684, + 0.20063453912734985, + 0.20929867029190063, + 0.29187142848968506 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2754871845245361, + "mean_weights": [ + 0.42651012539863586, + 0.1713070571422577, + 0.1726072132587433, + 0.22957557439804077 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3596348762512207, + "mean_weights": [ + 0.2750997543334961, + 0.22746169567108154, + 0.18702520430088043, + 0.31041333079338074 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5752925872802734, + "mean_weights": [ + 0.28211507201194763, + 0.1675817370414734, + 0.1759152114391327, + 0.16564446687698364, + 0.20874354243278503 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5704567432403564, + "mean_weights": [ + 0.21000899374485016, + 0.18079039454460144, + 0.13544103503227234, + 0.1749160885810852, + 0.29884350299835205 + ], + "sources": 5 + }, + { + "entropy_mean": 1.579707384109497, + "mean_weights": [ + 0.21335941553115845, + 0.18788449466228485, + 0.20250459015369415, + 0.1511598825454712, + 0.24509160220623016 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5663042068481445, + "mean_weights": [ + 0.21676909923553467, + 0.19467836618423462, + 0.13713446259498596, + 0.15313217043876648, + 0.29828590154647827 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5384478569030762, + "mean_weights": [ + 0.21904908120632172, + 0.15864285826683044, + 0.18178997933864594, + 0.14981383085250854, + 0.29070425033569336 + ], + "sources": 5 + }, + { + "entropy_mean": 1.552668571472168, + "mean_weights": [ + 0.22710487246513367, + 0.1922147125005722, + 0.13047492504119873, + 0.14686013758182526, + 0.30334532260894775 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5210195779800415, + "mean_weights": [ + 0.2812899947166443, + 0.14511609077453613, + 0.15909329056739807, + 0.13197475671768188, + 0.282525897026062 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5376527309417725, + "mean_weights": [ + 0.23905543982982635, + 0.21673071384429932, + 0.0950920507311821, + 0.16199833154678345, + 0.2871234714984894 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7209875583648682, + "mean_weights": [ + 0.1703755110502243, + 0.14151470363140106, + 0.17087554931640625, + 0.12742051482200623, + 0.25328749418258667, + 0.1365262269973755 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7200093269348145, + "mean_weights": [ + 0.18992580473423004, + 0.16036251187324524, + 0.08439569175243378, + 0.11250905692577362, + 0.1775888204574585, + 0.27521812915802 + ], + "sources": 6 + }, + { + "entropy_mean": 1.753357172012329, + "mean_weights": [ + 0.15539541840553284, + 0.1388990879058838, + 0.15745995938777924, + 0.12427005916833878, + 0.17903099954128265, + 0.24494445323944092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7199265956878662, + "mean_weights": [ + 0.19207912683486938, + 0.17904852330684662, + 0.08410340547561646, + 0.11219910532236099, + 0.15951165556907654, + 0.2730581760406494 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6756346225738525, + "mean_weights": [ + 0.15863081812858582, + 0.11242729425430298, + 0.15970338881015778, + 0.09774985909461975, + 0.15905189514160156, + 0.3124367296695709 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7271867990493774, + "mean_weights": [ + 0.19560003280639648, + 0.18333327770233154, + 0.08475087583065033, + 0.11598999053239822, + 0.15889793634414673, + 0.2614278793334961 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6916961669921875, + "mean_weights": [ + 0.19065657258033752, + 0.12983909249305725, + 0.11857748031616211, + 0.09883742034435272, + 0.1529499739408493, + 0.3091394901275635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7504222393035889, + "mean_weights": [ + 0.20809361338615417, + 0.19103100895881653, + 0.10743936896324158, + 0.1194620355963707, + 0.15873855352401733, + 0.2152354121208191 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5379457473754883, + "mean_weights": [ + 0.09571269154548645, + 0.07078731805086136, + 0.0824701339006424, + 0.060012347996234894, + 0.08145605027675629, + 0.09452696144580841, + 0.5150344967842102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7319592237472534, + "mean_weights": [ + 0.13678354024887085, + 0.11740625649690628, + 0.06668869405984879, + 0.07360264658927917, + 0.09287553280591965, + 0.12232495844364166, + 0.390318363904953 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4663677215576172, + "mean_weights": [ + 0.09485072642564774, + 0.07167200744152069, + 0.0841417908668518, + 0.05784624442458153, + 0.07052354514598846, + 0.0724712610244751, + 0.5484944581985474 + ], + "sources": 7 + }, + { + "entropy_mean": 1.749481201171875, + "mean_weights": [ + 0.14530614018440247, + 0.12230368703603745, + 0.0660901889204979, + 0.07630561292171478, + 0.09453228116035461, + 0.12125740200281143, + 0.3742046654224396 + ], + "sources": 7 + }, + { + "entropy_mean": 1.472255825996399, + "mean_weights": [ + 0.10688216984272003, + 0.07465942203998566, + 0.08327433466911316, + 0.054589346051216125, + 0.0637175589799881, + 0.07299347221851349, + 0.5438836812973022 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7924723625183105, + "mean_weights": [ + 0.15522946417331696, + 0.13724805414676666, + 0.06889298558235168, + 0.08198928833007812, + 0.09816981852054596, + 0.123870849609375, + 0.3345995247364044 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755163431167603, + "mean_weights": [ + 0.12274101376533508, + 0.08900541067123413, + 0.08551642298698425, + 0.06285787373781204, + 0.07429315149784088, + 0.07904880493879318, + 0.4865373373031616 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8390922546386719, + "mean_weights": [ + 0.15884171426296234, + 0.14726482331752777, + 0.08053631335496902, + 0.09097073972225189, + 0.10738551616668701, + 0.12150256335735321, + 0.29349833726882935 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6550344228744507, + "mean_weights": [ + 0.0844201147556305, + 0.06695197522640228, + 0.061310961842536926, + 0.0447731651365757, + 0.04748314991593361, + 0.045562684535980225, + 0.2905467748641968, + 0.3589511513710022 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8679088354110718, + "mean_weights": [ + 0.11674155294895172, + 0.1088850349187851, + 0.05779466778039932, + 0.06273136287927628, + 0.07066237181425095, + 0.08129343390464783, + 0.17753683030605316, + 0.32435476779937744 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5923268795013428, + "mean_weights": [ + 0.08656083047389984, + 0.05770102143287659, + 0.05105586349964142, + 0.039375390857458115, + 0.04410157352685928, + 0.04500962793827057, + 0.2357216477394104, + 0.4404740333557129 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8469831943511963, + "mean_weights": [ + 0.10824646055698395, + 0.09116974472999573, + 0.0701109915971756, + 0.0629766583442688, + 0.07049155235290527, + 0.07297641038894653, + 0.18795298039913177, + 0.33607521653175354 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6590619087219238, + "mean_weights": [ + 0.0944472923874855, + 0.0654030293226242, + 0.060137998312711716, + 0.0449567511677742, + 0.044405773282051086, + 0.05244143307209015, + 0.2202538251876831, + 0.4179539084434509 + ], + "sources": 8 + }, + { + "entropy_mean": 1.884438157081604, + "mean_weights": [ + 0.11613816022872925, + 0.09775452315807343, + 0.08394932001829147, + 0.07057420164346695, + 0.06920696794986725, + 0.06340071558952332, + 0.21605131030082703, + 0.2829247713088989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7424441576004028, + "mean_weights": [ + 0.08763163536787033, + 0.07771545648574829, + 0.07502467930316925, + 0.06006643921136856, + 0.06314834952354431, + 0.054570816457271576, + 0.3030073344707489, + 0.278835266828537 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8953055143356323, + "mean_weights": [ + 0.1135263741016388, + 0.10274903476238251, + 0.0858360230922699, + 0.08475605398416519, + 0.08078759908676147, + 0.07204630225896835, + 0.2779633104801178, + 0.18233531713485718 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801485300064087, + "mean_weights": [ + 0.07824120670557022, + 0.06023171544075012, + 0.054100219160318375, + 0.039871927350759506, + 0.04207881540060043, + 0.045095670968294144, + 0.2047339826822281, + 0.15412676334381104, + 0.321519672870636 + ], + "sources": 9 + }, + { + "entropy_mean": 2.008213520050049, + "mean_weights": [ + 0.09438058733940125, + 0.07153058052062988, + 0.07423336803913116, + 0.06403130292892456, + 0.0645662397146225, + 0.05577655881643295, + 0.21496367454528809, + 0.1609123945236206, + 0.1996053010225296 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8966764211654663, + "mean_weights": [ + 0.09610478579998016, + 0.07267358899116516, + 0.060228414833545685, + 0.046177469193935394, + 0.04206448048353195, + 0.04096413776278496, + 0.21562573313713074, + 0.21000833809375763, + 0.21615305542945862 + ], + "sources": 9 + }, + { + "entropy_mean": 1.981720209121704, + "mean_weights": [ + 0.10501760244369507, + 0.06946457922458649, + 0.07816961407661438, + 0.07385893911123276, + 0.06364607810974121, + 0.05328303575515747, + 0.23084522783756256, + 0.15472577512264252, + 0.17098914086818695 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0028607845306396, + "mean_weights": [ + 0.0842779204249382, + 0.10298711061477661, + 0.09207960218191147, + 0.07771934568881989, + 0.06265503168106079, + 0.0552704893052578, + 0.2620389461517334, + 0.1382768750190735, + 0.12469469010829926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0655057430267334, + "mean_weights": [ + 0.10472239553928375, + 0.11792458593845367, + 0.0928182601928711, + 0.10183010995388031, + 0.08287280797958374, + 0.06570011377334595, + 0.21871674060821533, + 0.11616810411214828, + 0.09924688935279846 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0258567333221436, + "mean_weights": [ + 0.09674200415611267, + 0.09482249617576599, + 0.08996410667896271, + 0.06836152076721191, + 0.06293784081935883, + 0.05016916245222092, + 0.22868779301643372, + 0.1848844438791275, + 0.12343062460422516 + ], + "sources": 9 + } + ], + "loss_nats": 1.347039818763733, + "output_weights": { + "entropy_mean": 1.7394249439239502, + "mean_weights": [ + 0.10602211207151413, + 0.05902107432484627, + 0.049884356558322906, + 0.034865088760852814, + 0.03282830864191055, + 0.043302543461322784, + 0.08693542331457138, + 0.3314756751060486, + 0.2556654214859009 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "population_cv": 0.6420341358971389 + }, + "activation_output_rms_by_block": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23796202801167965, + "first_to_last_ratio": 1.7210004980371292, + "imbalance_abs_log_ratio": 0.542905806617582, + "last_quartile_mean": 0.13826958695426583, + "mean": 0.15974999987520278, + "normalized": [ + 1.0588703589137207, + 2.3439518181213, + 2.5018026404306735, + 2.625406629974646, + 0.6192476728139908, + 0.7301872144203714, + 0.8733231516990753, + 1.1639318005158055, + 0.41362918444307983, + 0.6526334563206251, + 0.9108721190988135, + 1.2435622068278698, + 0.40590548584340497, + 0.6529420199592338, + 0.9214830517691831, + 1.2794736788600816, + 0.6292358348878756, + 0.8817240486108118, + 1.0928942572485663, + 1.5151234871428112, + 0.30958389010160414, + 0.5326764501933169, + 0.737604494775539, + 0.9796364773984299, + 0.3464072479049888, + 0.5325333617345424, + 0.7475662125351719, + 0.9760454607844148, + 0.41495429177381643, + 0.7752670281456541, + 1.247487625087717, + 1.8840373416628649 + ], + "population_cv": 0.591409330949146 + }, + "bits_per_byte": 1.8740129107275643, + "core_parameter_grad_rms_by_block": [ + 0.0003065916678101079, + 0.00023586292031870838, + 0.0001322871105193671, + 0.00016391009158296018, + 0.0001640597280926294, + 0.00011452252186811768, + 0.00012670462239247783, + 0.00021513822087305528, + 0.00015124888511926465, + 0.00012797922905715274, + 0.0001317390201682035, + 0.00014536342393434786, + 0.00017054401801644127, + 0.00018163687748006386, + 0.00019727429589294548, + 0.0002118946661868627, + 0.0002209032617907253, + 0.0002444386191807451, + 0.00023590577825104205, + 0.00032383840230939837, + 0.0010316902394974224, + 0.0010821684520459678, + 0.0009765210521856827, + 0.0008761640307174725, + 0.0008326776627851861, + 0.0006767392019523396, + 0.00052552098450176, + 0.0004066875433884613, + 0.0003364842674135882, + 0.00025378360531775436, + 0.000205309661176443, + 0.00020659565641729775 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018238461043217796, + "first_to_last_ratio": 0.42368241008054586, + "imbalance_abs_log_ratio": 0.8587711372443304, + "last_quartile_mean": 0.0004304748228691038, + "mean": 0.00035038080369512474, + "normalized": [ + 0.875024158220954, + 0.6731616510701846, + 0.37755239192405493, + 0.46780556998089007, + 0.46823263821091626, + 0.3268515873596963, + 0.3616197607181892, + 0.6140125788975943, + 0.43167001024082974, + 0.3652575361078021, + 0.37598812143496585, + 0.4148726825252455, + 0.486739045683667, + 0.5183984840622454, + 0.5630282647122382, + 0.6047553517550511, + 0.630466222638552, + 0.69763701836085, + 0.6732839692219831, + 0.9242469875466649, + 2.944482770223686, + 3.0885494885375917, + 2.7870278333951726, + 2.500605117282182, + 2.3764933866346176, + 1.9314391508194257, + 1.499856667259172, + 1.160701554136312, + 0.9603387624693381, + 0.7243079604856947, + 0.5859614996348036, + 0.5896317784494322 + ], + "population_cv": 0.8248093894360681 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22886337339878082, + "mean_weights": [ + 0.9364937543869019, + 0.06350618600845337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.603729784488678, + "mean_weights": [ + 0.5866844654083252, + 0.4133155643939972 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3967982828617096, + "mean_weights": [ + 0.858893632888794, + 0.14110638201236725 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6852496862411499, + "mean_weights": [ + 0.48102498054504395, + 0.518975019454956 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5890734195709229, + "mean_weights": [ + 0.6876576542854309, + 0.3123423457145691 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873323321342468, + "mean_weights": [ + 0.5168514847755432, + 0.4831485152244568 + ], + "sources": 2 + }, + { + "entropy_mean": 0.487301766872406, + "mean_weights": [ + 0.7939615249633789, + 0.2060384750366211 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674697756767273, + "mean_weights": [ + 0.5704737305641174, + 0.42952626943588257 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9691112041473389, + "mean_weights": [ + 0.5506306886672974, + 0.18125343322753906, + 0.2681158185005188 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0745112895965576, + "mean_weights": [ + 0.342629998922348, + 0.24457570910453796, + 0.412794291973114 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0074045658111572, + "mean_weights": [ + 0.4399106502532959, + 0.18251104652881622, + 0.3775783181190491 + ], + "sources": 3 + }, + { + "entropy_mean": 1.060497522354126, + "mean_weights": [ + 0.3786153495311737, + 0.2188822329044342, + 0.4025024175643921 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9941019415855408, + "mean_weights": [ + 0.32296958565711975, + 0.19686520099639893, + 0.4801652133464813 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0503522157669067, + "mean_weights": [ + 0.40130966901779175, + 0.19853851199150085, + 0.4001518487930298 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0129402875900269, + "mean_weights": [ + 0.4093267321586609, + 0.16906049847602844, + 0.42161279916763306 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0774019956588745, + "mean_weights": [ + 0.38510820269584656, + 0.25884830951690674, + 0.3560434877872467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2740377187728882, + "mean_weights": [ + 0.29550763964653015, + 0.09541245549917221, + 0.24569378793239594, + 0.3633860945701599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.360302448272705, + "mean_weights": [ + 0.3109527826309204, + 0.20984822511672974, + 0.1935039907693863, + 0.28569501638412476 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2722704410552979, + "mean_weights": [ + 0.350364625453949, + 0.12631991505622864, + 0.22418734431266785, + 0.29912811517715454 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3558036088943481, + "mean_weights": [ + 0.2989768385887146, + 0.1941717267036438, + 0.1926535815000534, + 0.314197838306427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2696614265441895, + "mean_weights": [ + 0.3667730689048767, + 0.15310388803482056, + 0.19384080171585083, + 0.2862822115421295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3606936931610107, + "mean_weights": [ + 0.2982274293899536, + 0.18611131608486176, + 0.21284276247024536, + 0.3028184771537781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.269316554069519, + "mean_weights": [ + 0.4081852436065674, + 0.14420126378536224, + 0.18228203058242798, + 0.2653314471244812 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3636136054992676, + "mean_weights": [ + 0.27427661418914795, + 0.21341991424560547, + 0.19867971539497375, + 0.3136237859725952 + ], + "sources": 4 + }, + { + "entropy_mean": 1.570773959159851, + "mean_weights": [ + 0.26587405800819397, + 0.13419361412525177, + 0.1907704770565033, + 0.17973750829696655, + 0.2294243425130844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5560038089752197, + "mean_weights": [ + 0.20477107167243958, + 0.15721666812896729, + 0.13811248540878296, + 0.1752161681652069, + 0.32468360662460327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5659973621368408, + "mean_weights": [ + 0.20531430840492249, + 0.15986156463623047, + 0.21557101607322693, + 0.15222224593162537, + 0.26703086495399475 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5552496910095215, + "mean_weights": [ + 0.2162160575389862, + 0.17982497811317444, + 0.1307673156261444, + 0.1554446816444397, + 0.31774696707725525 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5068919658660889, + "mean_weights": [ + 0.20646695792675018, + 0.1197461187839508, + 0.19215428829193115, + 0.15179987251758575, + 0.3298327624797821 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5435223579406738, + "mean_weights": [ + 0.22410975396633148, + 0.16986176371574402, + 0.13080401718616486, + 0.15279728174209595, + 0.3224271535873413 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4949920177459717, + "mean_weights": [ + 0.27174079418182373, + 0.12178683280944824, + 0.1529826819896698, + 0.1350194811820984, + 0.31847020983695984 + ], + "sources": 5 + }, + { + "entropy_mean": 1.544264793395996, + "mean_weights": [ + 0.24016115069389343, + 0.20730282366275787, + 0.09692574292421341, + 0.1695065051317215, + 0.2861037850379944 + ], + "sources": 5 + }, + { + "entropy_mean": 1.686682939529419, + "mean_weights": [ + 0.16755425930023193, + 0.10962115228176117, + 0.1783454418182373, + 0.13310004770755768, + 0.2885603904724121, + 0.12281867861747742 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7070834636688232, + "mean_weights": [ + 0.181177020072937, + 0.14557164907455444, + 0.08318156003952026, + 0.11424209922552109, + 0.17698991298675537, + 0.2988377511501312 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7419235706329346, + "mean_weights": [ + 0.1532188206911087, + 0.11420352011919022, + 0.1659742295742035, + 0.1250365972518921, + 0.18140509724617004, + 0.26016172766685486 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7127439975738525, + "mean_weights": [ + 0.18731769919395447, + 0.16715484857559204, + 0.08496320992708206, + 0.11362075805664062, + 0.1561521738767624, + 0.2907913327217102 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6076834201812744, + "mean_weights": [ + 0.15032559633255005, + 0.08624927699565887, + 0.15736976265907288, + 0.08584291487932205, + 0.14632999897003174, + 0.3738824725151062 + ], + "sources": 6 + }, + { + "entropy_mean": 1.709290862083435, + "mean_weights": [ + 0.18991519510746002, + 0.17091524600982666, + 0.07822999358177185, + 0.116765096783638, + 0.1540035903453827, + 0.2901708781719208 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6690887212753296, + "mean_weights": [ + 0.19029225409030914, + 0.11002412438392639, + 0.11471512168645859, + 0.0999990925192833, + 0.1507222205400467, + 0.3342471718788147 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7458693981170654, + "mean_weights": [ + 0.20528709888458252, + 0.1722964346408844, + 0.10435444861650467, + 0.12075909972190857, + 0.16116347908973694, + 0.2361394464969635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4612125158309937, + "mean_weights": [ + 0.08934466540813446, + 0.054379623383283615, + 0.07567165046930313, + 0.05758609622716904, + 0.07884950935840607, + 0.08982288837432861, + 0.5543456077575684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7241814136505127, + "mean_weights": [ + 0.13360008597373962, + 0.10609637945890427, + 0.06363911926746368, + 0.07426095008850098, + 0.09383077174425125, + 0.13373199105262756, + 0.39484071731567383 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3979300260543823, + "mean_weights": [ + 0.08959861099720001, + 0.05414731800556183, + 0.07668881118297577, + 0.05603286623954773, + 0.06979228556156158, + 0.07073447853326797, + 0.5830056071281433 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7459560632705688, + "mean_weights": [ + 0.14318783581256866, + 0.1094653457403183, + 0.06349228322505951, + 0.07859592139720917, + 0.09567482024431229, + 0.13266830146312714, + 0.37691545486450195 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4055393934249878, + "mean_weights": [ + 0.10154954344034195, + 0.05619275942444801, + 0.07791627943515778, + 0.053915880620479584, + 0.06398355215787888, + 0.06915920972824097, + 0.5772827863693237 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7853620052337646, + "mean_weights": [ + 0.15284112095832825, + 0.12206703424453735, + 0.06508444249629974, + 0.08347312361001968, + 0.09811970591545105, + 0.13767032325267792, + 0.3407442569732666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.509842872619629, + "mean_weights": [ + 0.11788781732320786, + 0.06822571903467178, + 0.08556114137172699, + 0.06067090481519699, + 0.07228855788707733, + 0.07258446514606476, + 0.5227814316749573 + ], + "sources": 7 + }, + { + "entropy_mean": 1.839367389678955, + "mean_weights": [ + 0.15906387567520142, + 0.13139380514621735, + 0.07717745006084442, + 0.09607303887605667, + 0.11078508198261261, + 0.12967455387115479, + 0.29583218693733215 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5841834545135498, + "mean_weights": [ + 0.07896879315376282, + 0.048752427101135254, + 0.055707160383462906, + 0.04139826446771622, + 0.04449019208550453, + 0.04285799711942673, + 0.2801477909088135, + 0.40767738223075867 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8125314712524414, + "mean_weights": [ + 0.10889512300491333, + 0.08656036853790283, + 0.048123255372047424, + 0.06561170518398285, + 0.06917259097099304, + 0.08286689221858978, + 0.16869932413101196, + 0.3700707256793976 + ], + "sources": 8 + }, + { + "entropy_mean": 1.50089693069458, + "mean_weights": [ + 0.07974998652935028, + 0.04166398197412491, + 0.04607272148132324, + 0.03586764261126518, + 0.04223340004682541, + 0.04095568507909775, + 0.22249138355255127, + 0.49096518754959106 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8059929609298706, + "mean_weights": [ + 0.10711923241615295, + 0.07651708275079727, + 0.058284737169742584, + 0.06609588116407394, + 0.07139427959918976, + 0.07493151724338531, + 0.17425751686096191, + 0.37139976024627686 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5622732639312744, + "mean_weights": [ + 0.08775827288627625, + 0.047765426337718964, + 0.053928621113300323, + 0.04293380677700043, + 0.04174989461898804, + 0.04734005779027939, + 0.2016349583864212, + 0.4768889844417572 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8757455348968506, + "mean_weights": [ + 0.11563669890165329, + 0.08108697831630707, + 0.07280858606100082, + 0.07553227990865707, + 0.07606013864278793, + 0.06935927271842957, + 0.19856664538383484, + 0.3109493851661682 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6924852132797241, + "mean_weights": [ + 0.08638577163219452, + 0.059912849217653275, + 0.06609142571687698, + 0.056745607405900955, + 0.06277773529291153, + 0.05352145433425903, + 0.2830069661140442, + 0.3315581977367401 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8980522155761719, + "mean_weights": [ + 0.11603894829750061, + 0.08804173022508621, + 0.07975024729967117, + 0.09026999771595001, + 0.08430293202400208, + 0.07433462888002396, + 0.26602888107299805, + 0.2012326419353485 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7512003183364868, + "mean_weights": [ + 0.07744461297988892, + 0.046086352318525314, + 0.05025433003902435, + 0.037484876811504364, + 0.04005061089992523, + 0.041623204946517944, + 0.198423832654953, + 0.17380405962467194, + 0.33482813835144043 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0160694122314453, + "mean_weights": [ + 0.0938527062535286, + 0.06208505481481552, + 0.06560768187046051, + 0.07029052078723907, + 0.06935097277164459, + 0.056832704693078995, + 0.1942581683397293, + 0.1706070452928543, + 0.21711516380310059 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7837426662445068, + "mean_weights": [ + 0.08836328238248825, + 0.04953558370471001, + 0.04813014343380928, + 0.0373263955116272, + 0.037015400826931, + 0.03390160948038101, + 0.1840541511774063, + 0.23140482604503632, + 0.29026859998703003 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9973840713500977, + "mean_weights": [ + 0.10261696577072144, + 0.06039813905954361, + 0.06671924889087677, + 0.08377155661582947, + 0.07002593576908112, + 0.05768897384405136, + 0.20831166207790375, + 0.1594475358724594, + 0.19101998209953308 + ], + "sources": 9 + }, + { + "entropy_mean": 1.969987154006958, + "mean_weights": [ + 0.08120635896921158, + 0.07894632965326309, + 0.08615730702877045, + 0.07609778642654419, + 0.058728985488414764, + 0.05076971277594566, + 0.24503237009048462, + 0.1544169783592224, + 0.16864418983459473 + ], + "sources": 9 + }, + { + "entropy_mean": 2.071648359298706, + "mean_weights": [ + 0.10569609701633453, + 0.09922641515731812, + 0.08180510997772217, + 0.10646798461675644, + 0.0826798751950264, + 0.06615597754716873, + 0.20955613255500793, + 0.13320739567279816, + 0.11520501971244812 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781701564788818, + "mean_weights": [ + 0.09446848928928375, + 0.07056106626987457, + 0.08033449202775955, + 0.0658978745341301, + 0.058234505355358124, + 0.04316519945859909, + 0.2036968618631363, + 0.22251945734024048, + 0.16112206876277924 + ], + "sources": 9 + } + ], + "loss_nats": 1.2989667654037476, + "output_weights": { + "entropy_mean": 1.6036605834960938, + "mean_weights": [ + 0.09759747982025146, + 0.04215940088033676, + 0.041145484894514084, + 0.029554029926657677, + 0.02899969555437565, + 0.03647579252719879, + 0.06642461568117142, + 0.341248482465744, + 0.31639501452445984 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "seed": 2026073001, + "timing": { + "mean_ms": 109.14386622172245, + "measured_steps": 7980, + "median_ms": 109.04977201425936, + "p95_ms": 109.64786536205793, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "population_cv": 0.39438035124793563 + }, + "activation_output_rms_by_block": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005765057983808219, + "first_to_last_ratio": 0.622587017371887, + "imbalance_abs_log_ratio": 0.4738718734697682, + "last_quartile_mean": 0.009259842918254435, + "mean": 0.007398182220640592, + "normalized": [ + 0.49149180917985336, + 0.6965082091758006, + 0.8632718983507628, + 0.9846233376628902, + 0.5069511565390686, + 0.730127144119427, + 0.8901300989627224, + 1.0709228264321597, + 0.6068057253534959, + 0.8292953231982888, + 0.9889841312125875, + 1.119997332175246, + 0.6022405569783624, + 0.8738552664669654, + 1.0668989020126547, + 1.2430080659586231, + 0.6705962885532487, + 0.9336251153230055, + 1.1108121094231989, + 1.3080235563150786, + 0.7274038039370601, + 1.01412651484229, + 1.232101109614035, + 1.4250990487866828, + 0.8294838994154143, + 1.1448566626512724, + 1.427936504016329, + 1.6627087330393202, + 0.7804514393709395, + 1.1025360304896303, + 1.4135242707919937, + 1.6516031296515918 + ], + "population_cv": 0.30590027610070303 + }, + "bits_per_byte": 8.038491115106986, + "core_parameter_grad_rms_by_block": [ + 0.010012756197717748, + 0.008904953530422711, + 0.008980942528705073, + 0.007787912846174446, + 0.007907551845183929, + 0.008289745083748344, + 0.007663833460373458, + 0.007350080301903749, + 0.00709564946584838, + 0.006310668475383808, + 0.006285078227083486, + 0.006362421438950662, + 0.005604026016423943, + 0.005959901601525332, + 0.006167718108848569, + 0.005523196426059091, + 0.0054641843609823, + 0.00557989785603791, + 0.005260407000687673, + 0.004763357766419076, + 0.004970214176820314, + 0.004845025235106639, + 0.004921304265254701, + 0.004820106373346748, + 0.0054804096177630225, + 0.0042302744374162005, + 0.004588429583035324, + 0.004469681656036737, + 0.004349111730454971, + 0.003928502929307311, + 0.004468266803351117, + 0.003904642007649288 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.008362221974278682, + "first_to_last_ratio": 1.8887369414995314, + "imbalance_abs_log_ratio": 0.6359083207487641, + "last_quartile_mean": 0.004427414845626747, + "mean": 0.00600782035481319, + "normalized": [ + 1.6666204390908572, + 1.4822269982304765, + 1.4948753455169401, + 1.2962958920592769, + 1.3162097696294732, + 1.3798257261648947, + 1.2756429133626719, + 1.2234187888150154, + 1.1810688480662828, + 1.0504089840715678, + 1.0461494944748422, + 1.0590232502297414, + 0.9327885465041001, + 0.9920239370590588, + 1.0266149359654666, + 0.9193344840336578, + 0.9095119424808776, + 0.9287724210274617, + 0.875593258455752, + 0.7928595538984271, + 0.8272907449435313, + 0.8064530809788657, + 0.8191497039874, + 0.8023053434820334, + 0.9122126318860999, + 0.7041279844573081, + 0.7637428072161453, + 0.7439772483303089, + 0.7239084182952745, + 0.6538982022256998, + 0.7437417465006834, + 0.6499265585598059 + ], + "population_cv": 0.2648157248292805 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.571857452392578, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 6.95402777637355e-05, + 6.603818474104628e-05, + 6.243807729333639e-05, + 5.902352859266102e-05, + 5.3786254284204915e-05, + 5.022022014600225e-05, + 4.6831908548483625e-05, + 4.3414886022219434e-05, + 3.877186099998653e-05, + 3.6122120945947245e-05, + 3.3541862649144605e-05, + 3.1072035199031234e-05, + 2.835266786860302e-05, + 2.6703228286351077e-05, + 2.5144798200926743e-05, + 2.3762264390825294e-05, + 2.2584137695957907e-05, + 2.1571571778622456e-05, + 2.0759547624038532e-05, + 1.9997522031189874e-05, + 1.9395642084418796e-05, + 1.8848633771995082e-05, + 1.8380347682978027e-05, + 1.797073673515115e-05, + 1.755586163199041e-05, + 1.7233234757441096e-05, + 1.6949628843576647e-05, + 1.6709716874174774e-05, + 1.630509359529242e-05, + 1.611644620425068e-05, + 1.594743662280962e-05, + 1.580910975462757e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.641166717396118e-05, + "first_to_last_ratio": 3.40273807382454, + "imbalance_abs_log_ratio": 1.2245804233600555, + "last_quartile_mean": 1.6578316035520402e-05, + "mean": 3.084058886315688e-05, + "normalized": [ + 2.254829765809383, + 2.1412750915381364, + 2.0245423189025695, + 1.913826251974467, + 1.7440086673720954, + 1.6283807150646488, + 1.518515381022133, + 1.407719100788124, + 1.2571699318719751, + 1.1712526341901288, + 1.0875882687575642, + 1.0075046017085245, + 0.9193296533476374, + 0.8658469008110146, + 0.8153151132261776, + 0.770486727612793, + 0.7322862023214354, + 0.6994539525278819, + 0.6731242297658762, + 0.648415700488765, + 0.6288998621420432, + 0.6111632256967778, + 0.5959791417905044, + 0.5826975877435189, + 0.5692453444999938, + 0.5587842318415797, + 0.5495883661230995, + 0.5418092679201958, + 0.5286894380532074, + 0.5225725836741036, + 0.517092481391006, + 0.5126072600226392 + ], + "population_cv": 0.5330470009230729 + }, + "activation_output_rms_by_block": [ + 0.005864492617547512, + 0.009892112575471401, + 0.014509912580251694, + 0.019168438389897346, + 0.006793031003326178, + 0.01223041582852602, + 0.017751524224877357, + 0.023574935272336006, + 0.008267601951956749, + 0.014755276031792164, + 0.02151178941130638, + 0.02830074541270733, + 0.009282662533223629, + 0.01676790602505207, + 0.02516039088368416, + 0.0331411249935627, + 0.010437680408358574, + 0.01991143450140953, + 0.029664477333426476, + 0.04036801680922508, + 0.01357664167881012, + 0.025029508396983147, + 0.03699437528848648, + 0.05095698684453964, + 0.015438608825206757, + 0.029965175315737724, + 0.04443887993693352, + 0.060807984322309494, + 0.019359532743692398, + 0.037317242473363876, + 0.05564062297344208, + 0.07935244590044022 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.01372310781152919, + "first_to_last_ratio": 0.3207078305283738, + "imbalance_abs_log_ratio": 1.137224755592113, + "last_quartile_mean": 0.04279006156139076, + "mean": 0.02613224917149637, + "normalized": [ + 0.224415915333617, + 0.37854041994445614, + 0.5552492816453899, + 0.7335165933901026, + 0.2599481949963819, + 0.4680200218611986, + 0.6792956897196492, + 0.9021395409795135, + 0.31637544467372514, + 0.5646385787522038, + 0.8231893577216564, + 1.0829816150527234, + 0.35521865998999624, + 0.6416556766702496, + 0.9628100022529916, + 1.268207905720998, + 0.3994176061868959, + 0.7619487465750876, + 1.1351673994362057, + 1.5447586062839287, + 0.519535903309034, + 0.9578015360531602, + 1.4156598249812316, + 1.9499655965365867, + 0.5907876020884704, + 1.1466741771474498, + 1.700537892674199, + 2.3269326694097012, + 0.7408291807047638, + 1.4280149492094805, + 2.1291938021979306, + 3.0365716085010215 + ], + "population_cv": 0.6534988385002395 + }, + "bits_per_byte": 5.118322357706281, + "core_parameter_grad_rms_by_block": [ + 0.0002932670153511829, + 0.0002289733734981176, + 0.00023382191596673618, + 0.00020236837455671756, + 0.0002080571281121399, + 0.00019123637931049836, + 0.00021124835961838567, + 0.00020444780259018514, + 0.00020817260281626697, + 0.00017921191649714578, + 0.0001893830879389061, + 0.0001891854298096901, + 0.00018418310100442068, + 0.00017033524110907422, + 0.00020556753369166975, + 0.00017823357128584385, + 0.0001749817385153215, + 0.00018220334134876856, + 0.00017289789528954973, + 0.00017141475273833558, + 0.00017603112434287998, + 0.00016603952795395127, + 0.0001609386975915838, + 0.00016666055126527177, + 0.00016172141650657888, + 0.00015460675258205218, + 0.00014513294616478987, + 0.00015851211030580876, + 0.00015100080713109572, + 0.00014721653732759288, + 0.00014725738129728224, + 0.00017222312685716471 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002216775436254954, + "first_to_last_ratio": 1.4328688617518024, + "imbalance_abs_log_ratio": 0.35967863157527663, + "last_quartile_mean": 0.00015470888477154567, + "mean": 0.000183954110636719, + "normalized": [ + 1.5942400761586677, + 1.2447309424376207, + 1.2710882902122171, + 1.1001024867357483, + 1.1310273382420937, + 1.0395874201917712, + 1.1483753142954691, + 1.1114065452657267, + 1.1316550747124958, + 0.974220776458003, + 1.02951267184732, + 1.0284381749060347, + 1.0012448233252798, + 0.9259659407419281, + 1.1174935584757544, + 0.9689023565112261, + 0.9512249435995668, + 0.9904825758886794, + 0.9398968834732723, + 0.9318343153355962, + 0.956929550166532, + 0.9026138496130359, + 0.8748850299377811, + 0.9059898182672343, + 0.879139998268121, + 0.8404637006855296, + 0.7889627780669988, + 0.8616937656742324, + 0.8208612822428254, + 0.8002894679441159, + 0.8005115014151157, + 0.936228748904007 + ], + "population_cv": 0.1635293006829388 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.692298173904419, + "mean_weights": [ + 0.48010045289993286, + 0.5198994874954224 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927915811538696, + "mean_weights": [ + 0.5130036473274231, + 0.4869963526725769 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928644776344299, + "mean_weights": [ + 0.5081723928451538, + 0.4918276071548462 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692646861076355, + "mean_weights": [ + 0.5149973034858704, + 0.48500269651412964 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925361156463623, + "mean_weights": [ + 0.5161343812942505, + 0.4838656485080719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928029656410217, + "mean_weights": [ + 0.5107344388961792, + 0.4892655611038208 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924350261688232, + "mean_weights": [ + 0.5171386003494263, + 0.48286139965057373 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929639577865601, + "mean_weights": [ + 0.5072796940803528, + 0.49272027611732483 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0978727340698242, + "mean_weights": [ + 0.34929555654525757, + 0.32294148206710815, + 0.3277629613876343 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098320722579956, + "mean_weights": [ + 0.3432672619819641, + 0.3268624544143677, + 0.329870343208313 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098150610923767, + "mean_weights": [ + 0.3442404866218567, + 0.33018797636032104, + 0.32557153701782227 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984500646591187, + "mean_weights": [ + 0.34007447957992554, + 0.32975536584854126, + 0.3301700949668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982636213302612, + "mean_weights": [ + 0.3434537351131439, + 0.33091628551483154, + 0.3256300091743469 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984963178634644, + "mean_weights": [ + 0.33822712302207947, + 0.33379626274108887, + 0.32797661423683167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980933904647827, + "mean_weights": [ + 0.34460341930389404, + 0.33179372549057007, + 0.3236028254032135 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985530614852905, + "mean_weights": [ + 0.3358767628669739, + 0.3345070779323578, + 0.32961612939834595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860312700271606, + "mean_weights": [ + 0.25749218463897705, + 0.24992284178733826, + 0.24571210145950317, + 0.24687285721302032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862019777297974, + "mean_weights": [ + 0.25314557552337646, + 0.25236955285072327, + 0.24678170680999756, + 0.2477031946182251 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385892391204834, + "mean_weights": [ + 0.25815317034721375, + 0.25234687328338623, + 0.2448991984128952, + 0.24460072815418243 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861613273620605, + "mean_weights": [ + 0.25378289818763733, + 0.25339171290397644, + 0.24767157435417175, + 0.24515381455421448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3855845928192139, + "mean_weights": [ + 0.2592235505580902, + 0.2543467879295349, + 0.2462996244430542, + 0.24013003706932068 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861030340194702, + "mean_weights": [ + 0.25138330459594727, + 0.25586679577827454, + 0.24899962544441223, + 0.24375030398368835 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3856689929962158, + "mean_weights": [ + 0.2574368119239807, + 0.25204822421073914, + 0.2460038810968399, + 0.24451106786727905 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386252999305725, + "mean_weights": [ + 0.25002047419548035, + 0.25257009267807007, + 0.2502796947956085, + 0.24712975323200226 + ], + "sources": 4 + }, + { + "entropy_mean": 1.608945608139038, + "mean_weights": [ + 0.20495271682739258, + 0.2066430151462555, + 0.2000001072883606, + 0.19580814242362976, + 0.19259600341320038 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093038320541382, + "mean_weights": [ + 0.20154938101768494, + 0.20420850813388824, + 0.2010108083486557, + 0.19671520590782166, + 0.19651609659194946 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085759401321411, + "mean_weights": [ + 0.20549985766410828, + 0.209174245595932, + 0.20157773792743683, + 0.19417354464530945, + 0.18957462906837463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093413829803467, + "mean_weights": [ + 0.20104944705963135, + 0.20404505729675293, + 0.2002137303352356, + 0.19850106537342072, + 0.1961907148361206 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6080443859100342, + "mean_weights": [ + 0.20538949966430664, + 0.21283748745918274, + 0.20341980457305908, + 0.19302619993686676, + 0.18532699346542358 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093816757202148, + "mean_weights": [ + 0.2010989934206009, + 0.20253092050552368, + 0.19950580596923828, + 0.19837552309036255, + 0.1984887421131134 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6076874732971191, + "mean_weights": [ + 0.20581099390983582, + 0.21552260220050812, + 0.20295079052448273, + 0.19259145855903625, + 0.18312416970729828 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609355092048645, + "mean_weights": [ + 0.20059309899806976, + 0.203348308801651, + 0.2011984884738922, + 0.1984248012304306, + 0.19643528759479523 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7901467084884644, + "mean_weights": [ + 0.17125868797302246, + 0.18124164640903473, + 0.17160004377365112, + 0.1620369851589203, + 0.15427924692630768, + 0.15958338975906372 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916979789733887, + "mean_weights": [ + 0.16689208149909973, + 0.16927623748779297, + 0.1683787703514099, + 0.16569086909294128, + 0.1651553511619568, + 0.1646067053079605 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789665699005127, + "mean_weights": [ + 0.1720278263092041, + 0.18269017338752747, + 0.17356479167938232, + 0.16241812705993652, + 0.15660038590431213, + 0.15269872546195984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917176485061646, + "mean_weights": [ + 0.16644492745399475, + 0.16854418814182281, + 0.16665583848953247, + 0.16677430272102356, + 0.16461026668548584, + 0.16697049140930176 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893871068954468, + "mean_weights": [ + 0.17303556203842163, + 0.18359911441802979, + 0.17276406288146973, + 0.16385790705680847, + 0.1554735004901886, + 0.15126986801624298 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917413711547852, + "mean_weights": [ + 0.16618916392326355, + 0.16740113496780396, + 0.16675519943237305, + 0.1668717861175537, + 0.1653555929660797, + 0.16742712259292603 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7882882356643677, + "mean_weights": [ + 0.17288538813591003, + 0.18687990307807922, + 0.17607632279396057, + 0.162863090634346, + 0.15389539301395416, + 0.14739990234375 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917335033416748, + "mean_weights": [ + 0.16660982370376587, + 0.16799160838127136, + 0.16742658615112305, + 0.1664734035730362, + 0.16542598605155945, + 0.1660725623369217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941649079322815, + "mean_weights": [ + 0.15037035942077637, + 0.16432975232601166, + 0.15334898233413696, + 0.1418340504169464, + 0.13212506473064423, + 0.12697139382362366, + 0.13102038204669952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458887577056885, + "mean_weights": [ + 0.1431310474872589, + 0.14324922859668732, + 0.14309079945087433, + 0.142366424202919, + 0.14222830533981323, + 0.14392223954200745, + 0.14201194047927856 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9409761428833008, + "mean_weights": [ + 0.1505419909954071, + 0.16567251086235046, + 0.1550290882587433, + 0.14243587851524353, + 0.13249853253364563, + 0.12776729464530945, + 0.12605470418930054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458999633789062, + "mean_weights": [ + 0.14249610900878906, + 0.14256571233272552, + 0.1435922086238861, + 0.14210085570812225, + 0.14355877041816711, + 0.1427185833454132, + 0.14296773076057434 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941575527191162, + "mean_weights": [ + 0.15046924352645874, + 0.1640411913394928, + 0.15395450592041016, + 0.14198558032512665, + 0.13400861620903015, + 0.1288735270500183, + 0.126667320728302 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458911418914795, + "mean_weights": [ + 0.14260169863700867, + 0.14355596899986267, + 0.1428615152835846, + 0.14244115352630615, + 0.14214617013931274, + 0.14205437898635864, + 0.14433911442756653 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9400205612182617, + "mean_weights": [ + 0.1507154405117035, + 0.16689133644104004, + 0.15648439526557922, + 0.1427520215511322, + 0.13353285193443298, + 0.12712189555168152, + 0.12250205874443054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458441734313965, + "mean_weights": [ + 0.1422245353460312, + 0.14142528176307678, + 0.14198407530784607, + 0.14165860414505005, + 0.14288201928138733, + 0.14370128512382507, + 0.14612418413162231 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0741422176361084, + "mean_weights": [ + 0.13300861418247223, + 0.14654949307441711, + 0.1379268318414688, + 0.12711459398269653, + 0.1185426190495491, + 0.1134917140007019, + 0.10956315696239471, + 0.11380299925804138 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794055461883545, + "mean_weights": [ + 0.12428364902734756, + 0.1234029084444046, + 0.12436957657337189, + 0.12428068369626999, + 0.12555459141731262, + 0.12575605511665344, + 0.1263071894645691, + 0.1260453760623932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.071772813796997, + "mean_weights": [ + 0.134201318025589, + 0.1495765745639801, + 0.14145472645759583, + 0.12840476632118225, + 0.11872729659080505, + 0.11272042989730835, + 0.10793697834014893, + 0.1069779098033905 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079404592514038, + "mean_weights": [ + 0.12455718219280243, + 0.12374450266361237, + 0.12391386926174164, + 0.12458803504705429, + 0.1251734495162964, + 0.1251622885465622, + 0.1266230344772339, + 0.1262376308441162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0732176303863525, + "mean_weights": [ + 0.13281872868537903, + 0.1473538726568222, + 0.13968363404273987, + 0.12848341464996338, + 0.12016644328832626, + 0.11505335569381714, + 0.10907343029975891, + 0.1073671206831932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794081687927246, + "mean_weights": [ + 0.12450858950614929, + 0.12424902617931366, + 0.12409761548042297, + 0.12468484044075012, + 0.12504076957702637, + 0.1255234032869339, + 0.12684062123298645, + 0.12505511939525604 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072478771209717, + "mean_weights": [ + 0.13302236795425415, + 0.14877822995185852, + 0.14031866192817688, + 0.12882456183433533, + 0.11896270513534546, + 0.11445862054824829, + 0.11000129580497742, + 0.10563355684280396 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079291820526123, + "mean_weights": [ + 0.12426379323005676, + 0.12249694019556046, + 0.12262053787708282, + 0.12315947562456131, + 0.1255876123905182, + 0.126260906457901, + 0.1273583620786667, + 0.12825237214565277 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1905503273010254, + "mean_weights": [ + 0.11892962455749512, + 0.13242119550704956, + 0.12570281326770782, + 0.11684690415859222, + 0.10851435363292694, + 0.10355254262685776, + 0.09824737906455994, + 0.0950399786233902, + 0.10074522346258163 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971964836120605, + "mean_weights": [ + 0.11044953763484955, + 0.11010265350341797, + 0.1108412891626358, + 0.11044170707464218, + 0.11131919175386429, + 0.11133328080177307, + 0.11097346246242523, + 0.11225177347660065, + 0.11228710412979126 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1902995109558105, + "mean_weights": [ + 0.11926670372486115, + 0.13237422704696655, + 0.12644407153129578, + 0.11611802130937576, + 0.11051423847675323, + 0.10466914623975754, + 0.09992008656263351, + 0.09535142779350281, + 0.09534208476543427 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197113037109375, + "mean_weights": [ + 0.11037027835845947, + 0.10917964577674866, + 0.10961420089006424, + 0.10947252064943314, + 0.11109665036201477, + 0.11174711585044861, + 0.1118166521191597, + 0.11308293044567108, + 0.11361998319625854 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1894307136535645, + "mean_weights": [ + 0.1198490560054779, + 0.13425537943840027, + 0.12682178616523743, + 0.11624304950237274, + 0.10967148095369339, + 0.10377678275108337, + 0.09994889795780182, + 0.0946216955780983, + 0.09481187164783478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.196977138519287, + "mean_weights": [ + 0.10978136211633682, + 0.1075383648276329, + 0.10839901119470596, + 0.11003260314464569, + 0.11053797602653503, + 0.11152469366788864, + 0.11391061544418335, + 0.11424341052770615, + 0.11403194814920425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1905651092529297, + "mean_weights": [ + 0.11943811178207397, + 0.13202990591526031, + 0.12506362795829773, + 0.11625386774539948, + 0.11043960601091385, + 0.1047937273979187, + 0.1010652631521225, + 0.09624719619750977, + 0.0946686640381813 + ], + "sources": 9 + } + ], + "loss_nats": 3.54775071144104, + "output_weights": { + "entropy_mean": 2.1935477256774902, + "mean_weights": [ + 0.10072024166584015, + 0.09480113536119461, + 0.10205776989459991, + 0.11059839278459549, + 0.1157325804233551, + 0.11904929578304291, + 0.11986880004405975, + 0.11937490850687027, + 0.11779685318470001 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.00014133188233245164, + 0.0001488620473537594, + 0.00014702342741657048, + 0.0001400294277118519, + 0.00013315585965756327, + 0.00011939816613448784, + 0.0001078194472938776, + 9.923135075950995e-05, + 8.953637006925419e-05, + 8.09328121249564e-05, + 7.468360854545608e-05, + 6.987994856899604e-05, + 7.115858170436695e-05, + 6.467441562563181e-05, + 5.974166560918093e-05, + 5.531677743420005e-05, + 5.705624425900169e-05, + 5.295562368701212e-05, + 4.8012458137236536e-05, + 4.413425631355494e-05, + 5.2346804295666516e-05, + 4.681985228671692e-05, + 4.172014814685099e-05, + 3.8159589166752994e-05, + 3.1379553547594696e-05, + 3.02761654893402e-05, + 2.9319184250198305e-05, + 2.8667578590102494e-05, + 2.2355405235430226e-05, + 2.2064739823690616e-05, + 2.201247843913734e-05, + 2.2052488930057734e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000129606451082509, + "first_to_last_ratio": 4.9818074922726145, + "imbalance_abs_log_ratio": 1.6057927754415957, + "last_quartile_mean": 2.601594928819395e-05, + "mean": 6.850338621688934e-05, + "normalized": [ + 2.0631371693799077, + 2.173061151786652, + 2.146221311616304, + 2.04412417319785, + 1.9437850741564366, + 1.7429527608527258, + 1.5739287245233284, + 1.448561249882341, + 1.3070356812110293, + 1.181442503713818, + 1.0902177639657034, + 1.0200948073975256, + 1.038760061861336, + 0.9441053821903842, + 0.8720979926456799, + 0.8075042780959905, + 0.832896699126015, + 0.7730365841967101, + 0.7008771505867426, + 0.6442638641806842, + 0.7641491492103867, + 0.6834677068149604, + 0.6090231512754765, + 0.5570467574542299, + 0.45807302792658366, + 0.4419659692950432, + 0.4279961308389998, + 0.4184841096663127, + 0.3263401485679923, + 0.32209706763737483, + 0.3213341654300619, + 0.32191823131541414 + ], + "population_cv": 0.5858213540607353 + }, + "activation_output_rms_by_block": [ + 0.021054552868008614, + 0.038058556616306305, + 0.04883526638150215, + 0.05791458487510681, + 0.016990283504128456, + 0.027475843206048012, + 0.03445403650403023, + 0.040619369596242905, + 0.019055280834436417, + 0.029903952032327652, + 0.037492990493774414, + 0.04475866258144379, + 0.0285621490329504, + 0.03507128730416298, + 0.04228900745511055, + 0.05219251662492752, + 0.027010982856154442, + 0.04389137029647827, + 0.054182786494493484, + 0.06662839651107788, + 0.03837457299232483, + 0.041143495589494705, + 0.05716610327363014, + 0.07329611480236053, + 0.028997479006648064, + 0.053451184183359146, + 0.08026835322380066, + 0.10854664444923401, + 0.027061885222792625, + 0.05247335880994797, + 0.08323372155427933, + 0.11460508406162262 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.035675311693921685, + "first_to_last_ratio": 0.5202021080271609, + "imbalance_abs_log_ratio": 0.6535378736360024, + "last_quartile_mean": 0.06857971381396055, + "mean": 0.047658121038693935, + "normalized": [ + 0.4417831087153916, + 0.798574425236075, + 1.0246997849926245, + 1.2152091524566315, + 0.3565034276180115, + 0.5765196488493577, + 0.7229415628043072, + 0.8523074076559539, + 0.3998328179779751, + 0.6274681288431082, + 0.7867072741565622, + 0.9391612931005807, + 0.5993133680146686, + 0.7358932022453922, + 0.8873410561187638, + 1.095144237485823, + 0.5667655851187262, + 0.92096308750491, + 1.136905637771622, + 1.398049168933493, + 0.8052053281993276, + 0.8633050295056751, + 1.1995039256209161, + 1.5379564532737442, + 0.6084478022770731, + 1.121554585418124, + 1.6842534173479116, + 2.277610658655726, + 0.567833658419326, + 1.1010370880409766, + 1.746475096798326, + 2.404733580842896 + ], + "population_cv": 0.48875915937608194 + }, + "bits_per_byte": 3.059021014366605, + "core_parameter_grad_rms_by_block": [ + 0.00048079443816738296, + 0.0004352800282264352, + 0.00039416580024316875, + 0.0003828745093102654, + 0.0003643213588911513, + 0.0002989310042525185, + 0.0002673721517191021, + 0.00023887606182770654, + 0.0002716851528017292, + 0.00026407625379019375, + 0.00023115090959535868, + 0.00019271062698547964, + 0.00023181255047936123, + 0.00019176941572124543, + 0.00020689480536159778, + 0.0001935784151883457, + 0.00017612030874433946, + 0.00017372524780256995, + 0.0001906158264031438, + 0.00020699034417808569, + 0.0002469606938485226, + 0.00020806313134476826, + 0.00017307229934302834, + 0.00014129096626273445, + 0.00011634997010006574, + 0.00010725656087507846, + 0.00011122939555071074, + 0.00011004876520390145, + 7.494181919924042e-05, + 7.493373834181984e-05, + 7.712439386045853e-05, + 7.424519704760952e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00035782691907971636, + "first_to_last_ratio": 3.8366182378544447, + "imbalance_abs_log_ratio": 1.3445913113590247, + "last_quartile_mean": 9.326623002236059e-05, + "mean": 0.0002159144418958475, + "normalized": [ + 2.226782210331757, + 2.015983851772199, + 1.8255647782620577, + 1.7732695689478515, + 1.6873413176636518, + 1.384488233523149, + 1.2383245389767708, + 1.1063459198478962, + 1.258300048927641, + 1.2230597058328248, + 1.070567154127008, + 0.892532362788586, + 1.0736315201703026, + 0.8881731765481019, + 0.9582258766247762, + 0.896551492751577, + 0.8156948983954307, + 0.8046022594744787, + 0.8828303689620391, + 0.9586683612295531, + 1.1437896033265396, + 0.9636369365469933, + 0.801578152083568, + 0.6543840468572741, + 0.5388707169305075, + 0.49675491798189547, + 0.5151549594091785, + 0.5096869122677149, + 0.34709034996089216, + 0.3470529237593392, + 0.3571988662882573, + 0.34386396943018666 + ], + "population_cv": 0.48652454891086055 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6572662591934204, + "mean_weights": [ + 0.3911391496658325, + 0.6088608503341675 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6751130819320679, + "mean_weights": [ + 0.44147175550460815, + 0.5585282444953918 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6885842084884644, + "mean_weights": [ + 0.4896422326564789, + 0.5103577375411987 + ], + "sources": 2 + }, + { + "entropy_mean": 0.668085515499115, + "mean_weights": [ + 0.4332469701766968, + 0.5667530298233032 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6886032223701477, + "mean_weights": [ + 0.51185142993927, + 0.4881485104560852 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6789880990982056, + "mean_weights": [ + 0.4699174761772156, + 0.5300825834274292 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772009134292603, + "mean_weights": [ + 0.4395230710506439, + 0.5604768991470337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6781555414199829, + "mean_weights": [ + 0.4541339576244354, + 0.5458660125732422 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0917184352874756, + "mean_weights": [ + 0.30861830711364746, + 0.355538010597229, + 0.33584368228912354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0860013961791992, + "mean_weights": [ + 0.30104559659957886, + 0.3248867988586426, + 0.37406760454177856 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0827364921569824, + "mean_weights": [ + 0.2735002934932709, + 0.34776103496551514, + 0.3787387013435364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0724949836730957, + "mean_weights": [ + 0.2886848449707031, + 0.3024986684322357, + 0.40881651639938354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0897927284240723, + "mean_weights": [ + 0.2973601222038269, + 0.3298739790916443, + 0.3727658987045288 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0697213411331177, + "mean_weights": [ + 0.30285951495170593, + 0.28946101665496826, + 0.4076794683933258 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0826778411865234, + "mean_weights": [ + 0.2862996757030487, + 0.3235614597797394, + 0.3901388645172119 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0579445362091064, + "mean_weights": [ + 0.28999701142311096, + 0.2830814719200134, + 0.4269214868545532 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3809051513671875, + "mean_weights": [ + 0.23795948922634125, + 0.2348281741142273, + 0.26517200469970703, + 0.26204031705856323 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3605632781982422, + "mean_weights": [ + 0.24186556041240692, + 0.22099661827087402, + 0.2607640027999878, + 0.27637380361557007 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3829561471939087, + "mean_weights": [ + 0.23731295764446259, + 0.2382979840040207, + 0.2565648853778839, + 0.2678241431713104 + ], + "sources": 4 + }, + { + "entropy_mean": 1.35977041721344, + "mean_weights": [ + 0.23858636617660522, + 0.21978360414505005, + 0.2459542453289032, + 0.29567578434944153 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3790044784545898, + "mean_weights": [ + 0.24452988803386688, + 0.2358427345752716, + 0.24750955402851105, + 0.27211782336235046 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3556556701660156, + "mean_weights": [ + 0.2410905957221985, + 0.22088873386383057, + 0.23317404091358185, + 0.3048466145992279 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804312944412231, + "mean_weights": [ + 0.26568183302879333, + 0.22685569524765015, + 0.2384241223335266, + 0.2690383493900299 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533532619476318, + "mean_weights": [ + 0.24537262320518494, + 0.2171444594860077, + 0.23335666954517365, + 0.30412623286247253 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6053518056869507, + "mean_weights": [ + 0.20177139341831207, + 0.18913303315639496, + 0.18395328521728516, + 0.1997503787279129, + 0.22539189457893372 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585533618927002, + "mean_weights": [ + 0.19244730472564697, + 0.1780712604522705, + 0.17078758776187897, + 0.20516318082809448, + 0.25353068113327026 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6022908687591553, + "mean_weights": [ + 0.1985350400209427, + 0.1897844821214676, + 0.17792725563049316, + 0.1998080015182495, + 0.23394520580768585 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5833771228790283, + "mean_weights": [ + 0.1926397681236267, + 0.1801614761352539, + 0.17439623177051544, + 0.19375719130039215, + 0.2590453326702118 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601109504699707, + "mean_weights": [ + 0.19747966527938843, + 0.19028550386428833, + 0.18151578307151794, + 0.19478440284729004, + 0.23593464493751526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5755603313446045, + "mean_weights": [ + 0.1969345360994339, + 0.17712506651878357, + 0.16844677925109863, + 0.18347150087356567, + 0.27402210235595703 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5980476140975952, + "mean_weights": [ + 0.19043894112110138, + 0.1923028826713562, + 0.1810823678970337, + 0.19125500321388245, + 0.24492081999778748 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5842485427856445, + "mean_weights": [ + 0.1964084804058075, + 0.17743000388145447, + 0.17621047794818878, + 0.18783687055110931, + 0.26211416721343994 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7856926918029785, + "mean_weights": [ + 0.15997162461280823, + 0.16431675851345062, + 0.15234553813934326, + 0.15752777457237244, + 0.18046650290489197, + 0.18537181615829468 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7751184701919556, + "mean_weights": [ + 0.16064736247062683, + 0.15155571699142456, + 0.14907033741474152, + 0.14832919836044312, + 0.18695974349975586, + 0.2034376710653305 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7829411029815674, + "mean_weights": [ + 0.14932186901569366, + 0.16724151372909546, + 0.15592646598815918, + 0.1555134654045105, + 0.17433607578277588, + 0.19766061007976532 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7661566734313965, + "mean_weights": [ + 0.165928453207016, + 0.143815815448761, + 0.1375015527009964, + 0.1444939523935318, + 0.17902101576328278, + 0.22923919558525085 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7807259559631348, + "mean_weights": [ + 0.14958028495311737, + 0.16952179372310638, + 0.15695258975028992, + 0.1499582827091217, + 0.1711740493774414, + 0.20281299948692322 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781292200088501, + "mean_weights": [ + 0.1682473123073578, + 0.15929323434829712, + 0.14874719083309174, + 0.15058940649032593, + 0.1693388819694519, + 0.20378397405147552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7818934917449951, + "mean_weights": [ + 0.14812925457954407, + 0.18386004865169525, + 0.16644306480884552, + 0.15062814950942993, + 0.15809310972690582, + 0.19284634292125702 + ], + "sources": 6 + }, + { + "entropy_mean": 1.77958083152771, + "mean_weights": [ + 0.19125157594680786, + 0.15093471109867096, + 0.14643779397010803, + 0.1490234136581421, + 0.16375312209129333, + 0.19859938323497772 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9289655685424805, + "mean_weights": [ + 0.12320417165756226, + 0.15015369653701782, + 0.13329030573368073, + 0.12274879962205887, + 0.13076716661453247, + 0.1441647708415985, + 0.19567111134529114 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9340521097183228, + "mean_weights": [ + 0.14550700783729553, + 0.14042077958583832, + 0.13418063521385193, + 0.1260203719139099, + 0.12906627357006073, + 0.1355920135974884, + 0.18921294808387756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9230798482894897, + "mean_weights": [ + 0.12067099660634995, + 0.15888366103172302, + 0.14002522826194763, + 0.12021138519048691, + 0.12209220230579376, + 0.13571104407310486, + 0.20240548253059387 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9308109283447266, + "mean_weights": [ + 0.1450628787279129, + 0.14527732133865356, + 0.13644321262836456, + 0.12497064471244812, + 0.12634633481502533, + 0.1283716857433319, + 0.19352790713310242 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273775815963745, + "mean_weights": [ + 0.1260491907596588, + 0.16657930612564087, + 0.14038512110710144, + 0.12195964902639389, + 0.12248437106609344, + 0.13317090272903442, + 0.18937145173549652 + ], + "sources": 7 + }, + { + "entropy_mean": 1.918284296989441, + "mean_weights": [ + 0.13994047045707703, + 0.15779416263103485, + 0.13759920001029968, + 0.11949826776981354, + 0.11786683648824692, + 0.12190048396587372, + 0.20540057122707367 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9060933589935303, + "mean_weights": [ + 0.11515206843614578, + 0.18610316514968872, + 0.15217435359954834, + 0.11993157863616943, + 0.11246243864297867, + 0.12337306141853333, + 0.19080331921577454 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9199097156524658, + "mean_weights": [ + 0.13630829751491547, + 0.1662534922361374, + 0.14684012532234192, + 0.12322510778903961, + 0.12947364151477814, + 0.1381947100162506, + 0.15970461070537567 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0490760803222656, + "mean_weights": [ + 0.10742150247097015, + 0.1722080111503601, + 0.14153704047203064, + 0.1159135103225708, + 0.10868977755308151, + 0.11180940270423889, + 0.1477845162153244, + 0.09463626146316528 + ], + "sources": 8 + }, + { + "entropy_mean": 2.051286220550537, + "mean_weights": [ + 0.1131708174943924, + 0.16089829802513123, + 0.14448881149291992, + 0.11956015229225159, + 0.10795272141695023, + 0.11352235078811646, + 0.13960374891757965, + 0.10080308467149734 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0247201919555664, + "mean_weights": [ + 0.1011081114411354, + 0.1894242763519287, + 0.1531166434288025, + 0.11718504130840302, + 0.10284577310085297, + 0.10429200530052185, + 0.14699342846870422, + 0.08503472805023193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.047537326812744, + "mean_weights": [ + 0.1111423671245575, + 0.16533835232257843, + 0.14553973078727722, + 0.12043843418359756, + 0.10876712203025818, + 0.11067016422748566, + 0.13975468277931213, + 0.09834913164377213 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0282247066497803, + "mean_weights": [ + 0.100978784263134, + 0.18681633472442627, + 0.15267831087112427, + 0.11643697321414948, + 0.10204557329416275, + 0.10493149608373642, + 0.14740025997161865, + 0.08871228992938995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.053830623626709, + "mean_weights": [ + 0.11781682819128036, + 0.1550440490245819, + 0.1398259550333023, + 0.11913557350635529, + 0.10805961489677429, + 0.11164884269237518, + 0.13677509129047394, + 0.11169403791427612 + ], + "sources": 8 + }, + { + "entropy_mean": 2.017721176147461, + "mean_weights": [ + 0.09720280021429062, + 0.19308976829051971, + 0.16317345201969147, + 0.11856384575366974, + 0.10023905336856842, + 0.10130266845226288, + 0.14337101578712463, + 0.08305738866329193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0608584880828857, + "mean_weights": [ + 0.12434182316064835, + 0.14232873916625977, + 0.13082580268383026, + 0.11684279143810272, + 0.10329097509384155, + 0.10883669555187225, + 0.1425117552280426, + 0.13102145493030548 + ], + "sources": 8 + }, + { + "entropy_mean": 2.142296314239502, + "mean_weights": [ + 0.08636107295751572, + 0.16583400964736938, + 0.14278221130371094, + 0.1055997684597969, + 0.08843953907489777, + 0.08854378759860992, + 0.12657995522022247, + 0.07546606659889221, + 0.12039357423782349 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191580057144165, + "mean_weights": [ + 0.1060912162065506, + 0.12397187203168869, + 0.11601399630308151, + 0.1091354638338089, + 0.10782144963741302, + 0.10360677540302277, + 0.12169313430786133, + 0.10359254479408264, + 0.10807354003190994 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1432788372039795, + "mean_weights": [ + 0.08927536010742188, + 0.16873395442962646, + 0.14930182695388794, + 0.112123504281044, + 0.09766747802495956, + 0.0965762585401535, + 0.1252625584602356, + 0.07491002976894379, + 0.08614903688430786 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1724743843078613, + "mean_weights": [ + 0.1105542778968811, + 0.12893787026405334, + 0.11523330211639404, + 0.10665617883205414, + 0.10366716235876083, + 0.10828493535518646, + 0.11819782853126526, + 0.10312190651893616, + 0.10534654557704926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1452555656433105, + "mean_weights": [ + 0.09202822297811508, + 0.1701684445142746, + 0.14628148078918457, + 0.11057832837104797, + 0.09616327285766602, + 0.09830240160226822, + 0.12624149024486542, + 0.08024715632200241, + 0.07998920977115631 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1762189865112305, + "mean_weights": [ + 0.09908370673656464, + 0.13803134858608246, + 0.13144394755363464, + 0.11672185361385345, + 0.1074555367231369, + 0.10124591737985611, + 0.12084682285785675, + 0.08855879306793213, + 0.09661206603050232 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1389729976654053, + "mean_weights": [ + 0.09089216589927673, + 0.17082780599594116, + 0.14892268180847168, + 0.11308471858501434, + 0.10078474879264832, + 0.09951519966125488, + 0.12574949860572815, + 0.07681785523891449, + 0.07340531051158905 + ], + "sources": 9 + } + ], + "loss_nats": 2.120351791381836, + "output_weights": { + "entropy_mean": 2.1635239124298096, + "mean_weights": [ + 0.1242082342505455, + 0.0880277156829834, + 0.0938715785741806, + 0.09339353442192078, + 0.09749531745910645, + 0.10569863021373749, + 0.13438579440116882, + 0.14857196807861328, + 0.11434720456600189 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 5.134936509421095e-05, + 5.603071258519776e-05, + 4.972576425643638e-05, + 4.7571254981448874e-05, + 4.3214979086769745e-05, + 3.9845475839683786e-05, + 3.5890228900825605e-05, + 3.406162795727141e-05, + 3.5466175177134573e-05, + 3.383069270057604e-05, + 3.1626921554561704e-05, + 3.0198520107660443e-05, + 2.883204069803469e-05, + 2.77392227872042e-05, + 2.7800235329777934e-05, + 3.112062404397875e-05, + 6.127587403170764e-05, + 6.331028998829424e-05, + 5.8612553402781487e-05, + 5.708334720111452e-05, + 0.00015888184134382755, + 0.00012872362276539207, + 0.00010434311116114259, + 8.657111175125465e-05, + 5.805831096949987e-05, + 5.078948743175715e-05, + 4.287513002054766e-05, + 3.959577952628024e-05, + 2.2001073375577107e-05, + 1.997212712012697e-05, + 1.8872458895202726e-05, + 1.86532452062238e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.4711176087730564e-05, + "first_to_last_ratio": 1.3207760209544162, + "imbalance_abs_log_ratio": 0.27821945851834845, + "last_quartile_mean": 3.385220156815194e-05, + "mean": 4.981010016535947e-05, + "normalized": [ + 1.0309026668033479, + 1.1248865671658381, + 0.9983068512481783, + 0.9550523854303026, + 0.8675947035501784, + 0.799947715571839, + 0.7205411910772573, + 0.68382974224492, + 0.7120277827066004, + 0.6791934284063869, + 0.6349499689734955, + 0.6062730250974693, + 0.5788392434931497, + 0.5568995584252106, + 0.5581244614543389, + 0.6247854138149606, + 1.2301897371875206, + 1.271033179578376, + 1.1767202476646232, + 1.146019522378179, + 3.189751492495938, + 2.584287570955602, + 2.0948183363362958, + 1.7380232415485222, + 1.165593138274326, + 1.019662423146035, + 0.8607718088943986, + 0.7949347500773989, + 0.4416990390008851, + 0.4009654076948961, + 0.3788881940118565, + 0.3744872052916737 + ], + "population_cv": 0.6175097196118516 + }, + "activation_output_rms_by_block": [ + 0.0955304205417633, + 0.19925470650196075, + 0.2359117865562439, + 0.25103330612182617, + 0.09304249286651611, + 0.11085781455039978, + 0.12910984456539154, + 0.14378803968429565, + 0.04010729491710663, + 0.06472750008106232, + 0.09987020492553711, + 0.14899125695228577, + 0.05702618882060051, + 0.10170809924602509, + 0.15923865139484406, + 0.38537684082984924, + 0.05919196084141731, + 0.08540526777505875, + 0.1145314872264862, + 0.15640178322792053, + 0.06262174993753433, + 0.08427268266677856, + 0.11030438542366028, + 0.13170360028743744, + 0.05311359465122223, + 0.08614693582057953, + 0.12623678147792816, + 0.17503507435321808, + 0.07505694776773453, + 0.13522638380527496, + 0.2187158316373825, + 0.3292153775691986 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15731605142354965, + "first_to_last_ratio": 1.0498699791884785, + "imbalance_abs_log_ratio": 0.04866632715763954, + "last_quartile_mean": 0.14984336588531733, + "mean": 0.13496107165701687, + "normalized": [ + 0.7078368552417796, + 1.4763865169086419, + 1.7479987648273716, + 1.86004232953773, + 0.6894024456398029, + 0.8214058556983611, + 0.9566450753555414, + 1.06540380806779, + 0.2971767667867461, + 0.479601260469815, + 0.7399926786247083, + 1.1039572754054923, + 0.4225380557552472, + 0.7536106374770113, + 1.1798857955094264, + 2.8554666623367226, + 0.4385854388578413, + 0.632814090214868, + 0.8486260950679959, + 1.158865895977708, + 0.4639986121085199, + 0.624422151010662, + 0.8173051982276947, + 0.9758636225277033, + 0.39354751706627217, + 0.6383095122385285, + 0.935356987967166, + 1.2969300866107762, + 0.5561377576971261, + 1.0019658420387498, + 1.6205845800722127, + 2.439335828673987 + ], + "population_cv": 0.5772709429538598 + }, + "bits_per_byte": 2.064953725142933, + "core_parameter_grad_rms_by_block": [ + 0.00020065673277085515, + 0.0001817740596446822, + 0.0001465045892684325, + 9.755666249346124e-05, + 0.00012624719322665892, + 7.565538032235173e-05, + 8.200655559893429e-05, + 7.97391790009874e-05, + 8.09162329894288e-05, + 7.508564486922528e-05, + 9.13364471873345e-05, + 0.00011039800739152168, + 8.766456722124108e-05, + 8.422239419368561e-05, + 9.398125423258295e-05, + 0.00017161898601766447, + 0.0001497631172902175, + 0.00014237628599353527, + 0.00022365597941099996, + 0.00025566927173363617, + 0.0008281082302532628, + 0.0006115617552883831, + 0.0005788343529396405, + 0.00045039612289279527, + 0.0002758540055807746, + 0.00022394859978287228, + 0.00021964130134669458, + 0.00019854079025805258, + 0.00011545923673689007, + 0.00010570174900205617, + 9.906174415914337e-05, + 0.0001053958635417218 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00012376754404079543, + "first_to_last_ratio": 0.7369290916409895, + "imbalance_abs_log_ratio": 0.30526360357753907, + "last_quartile_mean": 0.00016795041130102567, + "mean": 0.00019904163414499135, + "normalized": [ + 1.0081143758329842, + 0.9132464191500225, + 0.7360499721465892, + 0.49013194105106844, + 0.6342753051087516, + 0.3800982676179234, + 0.4120070454164219, + 0.40061557645221907, + 0.4065291833892681, + 0.3772358742519635, + 0.4588811159016144, + 0.5546478145929135, + 0.4404333173700838, + 0.42313958361261256, + 0.4721688235543854, + 0.8622265726207266, + 0.7524210597121753, + 0.7153090689047581, + 1.123664304564935, + 1.2845014704179658, + 4.16047744890413, + 3.072531824386513, + 2.908106916555911, + 2.2628236792142964, + 1.385911077176092, + 1.1251344511155768, + 1.1034942628470255, + 0.9974837229954918, + 0.5800758079225982, + 0.5310534625386868, + 0.49769358347903286, + 0.5295166711952659 + ], + "population_cv": 0.8756839263617585 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4580664336681366, + "mean_weights": [ + 0.81428062915802, + 0.18571940064430237 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6350213885307312, + "mean_weights": [ + 0.5189062356948853, + 0.48109376430511475 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4277062714099884, + "mean_weights": [ + 0.8402144908905029, + 0.15978553891181946 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868079900741577, + "mean_weights": [ + 0.493515282869339, + 0.5064846873283386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.38686758279800415, + "mean_weights": [ + 0.8638634085655212, + 0.13613657653331757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772255897521973, + "mean_weights": [ + 0.5679300427436829, + 0.43206995725631714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5690979361534119, + "mean_weights": [ + 0.7181376218795776, + 0.28186243772506714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6592018008232117, + "mean_weights": [ + 0.5172787308692932, + 0.4827212691307068 + ], + "sources": 2 + }, + { + "entropy_mean": 1.038456916809082, + "mean_weights": [ + 0.4540913999080658, + 0.23118522763252258, + 0.3147233724594116 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0903832912445068, + "mean_weights": [ + 0.3152569830417633, + 0.30281925201416016, + 0.38192376494407654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0411981344223022, + "mean_weights": [ + 0.4020236134529114, + 0.2803056836128235, + 0.31767070293426514 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0905810594558716, + "mean_weights": [ + 0.3278552293777466, + 0.2872077226638794, + 0.384937047958374 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0150185823440552, + "mean_weights": [ + 0.48287349939346313, + 0.22136211395263672, + 0.29576438665390015 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0816482305526733, + "mean_weights": [ + 0.3306047320365906, + 0.2639211118221283, + 0.40547415614128113 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0114538669586182, + "mean_weights": [ + 0.4296638071537018, + 0.22373110055923462, + 0.346605122089386 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0918333530426025, + "mean_weights": [ + 0.36806726455688477, + 0.3041593134403229, + 0.32777342200279236 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3290088176727295, + "mean_weights": [ + 0.35130923986434937, + 0.17179939150810242, + 0.21658103168010712, + 0.2603103518486023 + ], + "sources": 4 + }, + { + "entropy_mean": 1.379033088684082, + "mean_weights": [ + 0.29040855169296265, + 0.23445811867713928, + 0.2306380271911621, + 0.24449528753757477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3394957780838013, + "mean_weights": [ + 0.3262045383453369, + 0.18669262528419495, + 0.22525924444198608, + 0.26184359192848206 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3677098751068115, + "mean_weights": [ + 0.29889944195747375, + 0.22409284114837646, + 0.1924189329147339, + 0.2845887541770935 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2910308837890625, + "mean_weights": [ + 0.3990757167339325, + 0.18164023756980896, + 0.16051796078681946, + 0.2587660551071167 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3623619079589844, + "mean_weights": [ + 0.3024129569530487, + 0.21113446354866028, + 0.18768057227134705, + 0.29877203702926636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2281851768493652, + "mean_weights": [ + 0.46696996688842773, + 0.16045212745666504, + 0.13728894293308258, + 0.23528897762298584 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772211074829102, + "mean_weights": [ + 0.28748467564582825, + 0.24638661742210388, + 0.22340166568756104, + 0.24272704124450684 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5519046783447266, + "mean_weights": [ + 0.30400484800338745, + 0.1470816731452942, + 0.13316048681735992, + 0.1807912439107895, + 0.23496176302433014 + ], + "sources": 5 + }, + { + "entropy_mean": 1.572178602218628, + "mean_weights": [ + 0.21885336935520172, + 0.2029750645160675, + 0.14561443030834198, + 0.14960341155529022, + 0.28295373916625977 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5391805171966553, + "mean_weights": [ + 0.2898171544075012, + 0.15984714031219482, + 0.1228363886475563, + 0.16408921778202057, + 0.26341012120246887 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5521924495697021, + "mean_weights": [ + 0.210292249917984, + 0.185291588306427, + 0.14173434674739838, + 0.14211969077587128, + 0.32056212425231934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5468778610229492, + "mean_weights": [ + 0.2756267488002777, + 0.16966477036476135, + 0.1333748996257782, + 0.1590600460767746, + 0.26227355003356934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5309922695159912, + "mean_weights": [ + 0.21606209874153137, + 0.27386677265167236, + 0.12764990329742432, + 0.10704417526721954, + 0.2753770351409912 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5832164287567139, + "mean_weights": [ + 0.22763225436210632, + 0.24636119604110718, + 0.17577087879180908, + 0.1817018985748291, + 0.1685338020324707 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5793015956878662, + "mean_weights": [ + 0.21581211686134338, + 0.21387425065040588, + 0.1464191973209381, + 0.17280855774879456, + 0.25108590722084045 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7719972133636475, + "mean_weights": [ + 0.16962555050849915, + 0.20328447222709656, + 0.1659279465675354, + 0.14008209109306335, + 0.12920910120010376, + 0.1918708235025406 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689546823501587, + "mean_weights": [ + 0.16601097583770752, + 0.1523149460554123, + 0.0875597670674324, + 0.101267509162426, + 0.1655988097190857, + 0.3272479772567749 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5251271724700928, + "mean_weights": [ + 0.13745036721229553, + 0.11702707409858704, + 0.07592719793319702, + 0.08887732028961182, + 0.11522199213504791, + 0.4654960632324219 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7189464569091797, + "mean_weights": [ + 0.1744517982006073, + 0.15982553362846375, + 0.09849169105291367, + 0.10780535638332367, + 0.16236206889152527, + 0.29706352949142456 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5877950191497803, + "mean_weights": [ + 0.14238224923610687, + 0.1503014713525772, + 0.10275131464004517, + 0.10159259289503098, + 0.08505517989397049, + 0.4179171919822693 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7369492053985596, + "mean_weights": [ + 0.17980654537677765, + 0.1706826537847519, + 0.10907262563705444, + 0.1106589287519455, + 0.15735921263694763, + 0.2724200189113617 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5987625122070312, + "mean_weights": [ + 0.15493065118789673, + 0.15333226323127747, + 0.10736657679080963, + 0.1186048835515976, + 0.07128532975912094, + 0.39448028802871704 + ], + "sources": 6 + }, + { + "entropy_mean": 1.75795578956604, + "mean_weights": [ + 0.1946164071559906, + 0.17554126679897308, + 0.11572675406932831, + 0.1155121773481369, + 0.1708213835954666, + 0.2277819961309433 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5344648361206055, + "mean_weights": [ + 0.0905577763915062, + 0.0786280632019043, + 0.06272733211517334, + 0.07359343767166138, + 0.050107162445783615, + 0.13821640610694885, + 0.5061697959899902 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7138500213623047, + "mean_weights": [ + 0.13302072882652283, + 0.10984362661838531, + 0.07243296504020691, + 0.07083481550216675, + 0.08885611593723297, + 0.11790697276592255, + 0.4071047604084015 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4565383195877075, + "mean_weights": [ + 0.09418222308158875, + 0.07457256317138672, + 0.057202234864234924, + 0.06210128590464592, + 0.04410415142774582, + 0.11846546828746796, + 0.5493720769882202 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7785513401031494, + "mean_weights": [ + 0.1471671462059021, + 0.12308044731616974, + 0.07605892419815063, + 0.07563325762748718, + 0.10904586315155029, + 0.11226382851600647, + 0.3567505478858948 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5135828256607056, + "mean_weights": [ + 0.10121430456638336, + 0.07933001220226288, + 0.061728835105895996, + 0.06817515194416046, + 0.048567138612270355, + 0.12093988060951233, + 0.5200446844100952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7546210289001465, + "mean_weights": [ + 0.14427465200424194, + 0.12224892526865005, + 0.08193168044090271, + 0.07876674830913544, + 0.09007281810045242, + 0.10526382178068161, + 0.3774413466453552 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5284571647644043, + "mean_weights": [ + 0.10831618309020996, + 0.08754566311836243, + 0.06345342099666595, + 0.06688661873340607, + 0.05242554843425751, + 0.11085730791091919, + 0.5105152726173401 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8309729099273682, + "mean_weights": [ + 0.1468513160943985, + 0.13658970594406128, + 0.09501184523105621, + 0.0928751677274704, + 0.10057065635919571, + 0.11721575260162354, + 0.31088554859161377 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6937754154205322, + "mean_weights": [ + 0.09603364765644073, + 0.06767555326223373, + 0.045112669467926025, + 0.047895483672618866, + 0.03816477954387665, + 0.07518067210912704, + 0.3669250011444092, + 0.26301223039627075 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9070472717285156, + "mean_weights": [ + 0.10919073224067688, + 0.11743452399969101, + 0.07601121068000793, + 0.0681401938199997, + 0.0658918172121048, + 0.07979628443717957, + 0.24526360630989075, + 0.2382715940475464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.776512861251831, + "mean_weights": [ + 0.08731270581483841, + 0.08430957794189453, + 0.056895699352025986, + 0.06295455247163773, + 0.04594648256897926, + 0.09297705441713333, + 0.33455103635787964, + 0.23505286872386932 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9573358297348022, + "mean_weights": [ + 0.11900999397039413, + 0.11752337962388992, + 0.07979564368724823, + 0.07266241312026978, + 0.07412992417812347, + 0.08811671286821365, + 0.22605092823505402, + 0.2227109968662262 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7231189012527466, + "mean_weights": [ + 0.09091638028621674, + 0.07795695960521698, + 0.05235600098967552, + 0.053361356258392334, + 0.04162060469388962, + 0.07854442298412323, + 0.3417898118495941, + 0.2634544372558594 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8967883586883545, + "mean_weights": [ + 0.1089041531085968, + 0.1000017300248146, + 0.07706406712532043, + 0.07702124863862991, + 0.06017591059207916, + 0.07969298958778381, + 0.261149525642395, + 0.23599036037921906 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801592230796814, + "mean_weights": [ + 0.09432460367679596, + 0.09541776776313782, + 0.06706254929304123, + 0.07247316837310791, + 0.048089414834976196, + 0.08817503601312637, + 0.3473600149154663, + 0.1870974600315094 + ], + "sources": 8 + }, + { + "entropy_mean": 1.919801950454712, + "mean_weights": [ + 0.10887958109378815, + 0.10445678234100342, + 0.0838727131485939, + 0.0852852314710617, + 0.05865198373794556, + 0.08557502925395966, + 0.27338963747024536, + 0.19988907873630524 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9332969188690186, + "mean_weights": [ + 0.07704077661037445, + 0.08837630599737167, + 0.05746292322874069, + 0.06001049652695656, + 0.04015304520726204, + 0.06584024429321289, + 0.2511133551597595, + 0.10944834351539612, + 0.25055450201034546 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0115504264831543, + "mean_weights": [ + 0.08485299348831177, + 0.08635596185922623, + 0.06204982101917267, + 0.06870189309120178, + 0.0476498119533062, + 0.08640170097351074, + 0.2653427720069885, + 0.13704031705856323, + 0.16160473227500916 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781036376953125, + "mean_weights": [ + 0.08389372378587723, + 0.08841055631637573, + 0.060679104179143906, + 0.0658775195479393, + 0.045975662767887115, + 0.08423998951911926, + 0.29481440782546997, + 0.12572166323661804, + 0.15038736164569855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0586161613464355, + "mean_weights": [ + 0.1038442850112915, + 0.09918913245201111, + 0.08148118108510971, + 0.0698266476392746, + 0.059573929756879807, + 0.08160553872585297, + 0.22791330516338348, + 0.13585451245307922, + 0.1407114714384079 + ], + "sources": 9 + }, + { + "entropy_mean": 2.022735118865967, + "mean_weights": [ + 0.09707076847553253, + 0.09551473706960678, + 0.06701318919658661, + 0.0673459842801094, + 0.04480960965156555, + 0.08329527080059052, + 0.24251294136047363, + 0.15929152071475983, + 0.14314597845077515 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9317231178283691, + "mean_weights": [ + 0.07746735960245132, + 0.08971680700778961, + 0.06909166276454926, + 0.08098623156547546, + 0.04421798139810562, + 0.0916394367814064, + 0.3483121693134308, + 0.10497690737247467, + 0.09359145164489746 + ], + "sources": 9 + }, + { + "entropy_mean": 2.053220272064209, + "mean_weights": [ + 0.09178322553634644, + 0.10647953301668167, + 0.07523074746131897, + 0.08394691348075867, + 0.04770101606845856, + 0.09626083076000214, + 0.26201331615448, + 0.13223087787628174, + 0.10435353964567184 + ], + "sources": 9 + } + ], + "loss_nats": 1.43131685256958, + "output_weights": { + "entropy_mean": 1.843801498413086, + "mean_weights": [ + 0.11151862144470215, + 0.061874739825725555, + 0.05308101326227188, + 0.040848225355148315, + 0.042743273079395294, + 0.053944505751132965, + 0.1199670359492302, + 0.3250349462032318, + 0.19098761677742004 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 3.57800890924409e-05, + 3.942943658330478e-05, + 3.388081677258015e-05, + 3.245583502575755e-05, + 3.3783475373638794e-05, + 2.915231198130641e-05, + 2.5950168492272496e-05, + 2.450849387969356e-05, + 2.7269536076346412e-05, + 2.568827403592877e-05, + 2.400098310317844e-05, + 2.2796592020313255e-05, + 2.010442767641507e-05, + 1.8936527339974418e-05, + 1.9039980543311685e-05, + 2.3072669137036428e-05, + 4.836148946196772e-05, + 4.9383146688342094e-05, + 4.5993543608346954e-05, + 4.280430221115239e-05, + 0.00014426946290768683, + 0.00011744296352844685, + 9.347966988570988e-05, + 7.698025729041547e-05, + 6.582303467439488e-05, + 5.6237244280055165e-05, + 4.4205091398907825e-05, + 3.852568261208944e-05, + 2.7434407456894405e-05, + 2.285257687617559e-05, + 2.028334711212665e-05, + 1.9826158677460626e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.186757840012433e-05, + "first_to_last_ratio": 0.8636564556008468, + "imbalance_abs_log_ratio": 0.146580210082559, + "last_quartile_mean": 3.689844288601307e-05, + "mean": 4.2179749868864747e-05, + "normalized": [ + 0.8482764645044092, + 0.9347954102594115, + 0.8032484042203742, + 0.7694648528419803, + 0.8009406285876599, + 0.6911447334785021, + 0.6152281265998635, + 0.5810488197746441, + 0.646507771172811, + 0.6090191173677569, + 0.569016724323795, + 0.5404629494292167, + 0.4766369582304063, + 0.4489483081063157, + 0.4514009827584627, + 0.5470082020108817, + 1.146557047216301, + 1.170778557053383, + 1.090417647125437, + 1.014806923801809, + 3.420348943656972, + 2.7843447126541196, + 2.216221532283493, + 1.825052484420704, + 1.5605363919661974, + 1.3332759073938238, + 1.048016916561615, + 0.913369157756135, + 0.6504165515961319, + 0.5417902416971032, + 0.48087879077488155, + 0.470039740375403 + ], + "population_cv": 0.6827037752697015 + }, + "activation_output_rms_by_block": [ + 0.15143299102783203, + 0.30262914299964905, + 0.3626292943954468, + 0.38562148809432983, + 0.13662196695804596, + 0.15442071855068207, + 0.18618692457675934, + 0.2074802666902542, + 0.04909220710396767, + 0.0881529152393341, + 0.14954733848571777, + 0.23951733112335205, + 0.08217982947826385, + 0.14409078657627106, + 0.22605396807193756, + 0.6592875719070435, + 0.06389845162630081, + 0.09458807110786438, + 0.14630787074565887, + 0.21908092498779297, + 0.07773751765489578, + 0.10736925154924393, + 0.1424422264099121, + 0.17246152460575104, + 0.06572399288415909, + 0.10123347491025925, + 0.14551502466201782, + 0.19983991980552673, + 0.09023923426866531, + 0.15600618720054626, + 0.24885882437229156, + 0.3860291540622711 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2358778491616249, + "first_to_last_ratio": 1.3542132581102162, + "imbalance_abs_log_ratio": 0.30322066438495643, + "last_quartile_mean": 0.17418072652071714, + "mean": 0.18569613725412637, + "normalized": [ + 0.8154881047449846, + 1.6297007976288655, + 1.9528101109566227, + 2.076626330501449, + 0.7357286423846173, + 0.8315774406193291, + 1.0026429592445498, + 1.1173106223869165, + 0.26436848838048077, + 0.4747159340137284, + 0.8053335987331889, + 1.2898347518967022, + 0.442550021198342, + 0.7759492938675503, + 1.2173326349948854, + 3.5503569522547695, + 0.3441022122008675, + 0.5093701598026241, + 0.7878886061342009, + 1.1797818103667892, + 0.41862754284711634, + 0.5781986267291517, + 0.7670715638795383, + 0.9287297364173834, + 0.3539330104331431, + 0.5451566004936365, + 0.7836190176798323, + 1.0761662722797756, + 0.4859510574803844, + 0.8401154138551131, + 1.3401400161152877, + 2.078821669478174 + ], + "population_cv": 0.6621635595354842 + }, + "bits_per_byte": 1.9288425276664853, + "core_parameter_grad_rms_by_block": [ + 0.00017320581296885245, + 0.0001375362640366496, + 0.00014375813569971144, + 8.981648567965897e-05, + 0.00010459593200616139, + 6.156395218407552e-05, + 7.281931709149581e-05, + 7.667720183186952e-05, + 7.236475672264944e-05, + 7.890209128468536e-05, + 9.134838485444255e-05, + 0.00011250740948005864, + 7.70610699389613e-05, + 7.611216582142322e-05, + 8.146874247135892e-05, + 0.00017928549432252573, + 0.00013712162717798398, + 0.00013184310037104538, + 0.00022367504753246386, + 0.0002691717754589954, + 0.0008753648193242955, + 0.0006614972326966637, + 0.0006052862860379005, + 0.0005306195765578692, + 0.00037252876038501256, + 0.00029795485316373564, + 0.0002868510641817651, + 0.00023542884481748448, + 0.00018331496216173493, + 0.00013991664250432274, + 0.00012258431160994327, + 0.00012212563876750547 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00010749663768730933, + "first_to_last_ratio": 0.4884254111851857, + "imbalance_abs_log_ratio": 0.7165685086453754, + "last_quartile_mean": 0.00022008813469893802, + "mean": 0.0002132596174732282, + "normalized": [ + 0.812182892481255, + 0.6449240867362775, + 0.6740991914128248, + 0.42116030565859097, + 0.4904629073494984, + 0.2886807775119639, + 0.3414585375059921, + 0.3595486231306557, + 0.33932704925597945, + 0.3699813974138373, + 0.428343565166106, + 0.5275607769210333, + 0.36134862686150726, + 0.35689910130771973, + 0.3820167335786683, + 0.840691249692561, + 0.6429798046280267, + 0.6182281574597519, + 1.0488392044524941, + 1.262178834644052, + 4.104690938190328, + 3.101840097690814, + 2.838260207016858, + 2.4881390243724, + 1.746832170097925, + 1.3971461484082413, + 1.3450791461621905, + 1.1039541738230836, + 0.8595859091079332, + 0.6560859677143862, + 0.5748125831902143, + 0.5726618110568292 + ], + "population_cv": 0.9042592730039112 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2508508265018463, + "mean_weights": [ + 0.9251874089241028, + 0.07481260597705841 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5796181559562683, + "mean_weights": [ + 0.6163991689682007, + 0.38360080122947693 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3817020058631897, + "mean_weights": [ + 0.8651303648948669, + 0.13486966490745544 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6884468793869019, + "mean_weights": [ + 0.4932539463043213, + 0.5067461133003235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2797870337963104, + "mean_weights": [ + 0.9142746329307556, + 0.08572537451982498 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6642253994941711, + "mean_weights": [ + 0.605044960975647, + 0.39495497941970825 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5038842558860779, + "mean_weights": [ + 0.7756115198135376, + 0.2243884801864624 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6434851884841919, + "mean_weights": [ + 0.5374104976654053, + 0.4625895023345947 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0016276836395264, + "mean_weights": [ + 0.4353652000427246, + 0.18586981296539307, + 0.3787649869918823 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0655895471572876, + "mean_weights": [ + 0.2726097106933594, + 0.2790643572807312, + 0.4483259320259094 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0300546884536743, + "mean_weights": [ + 0.40803641080856323, + 0.2817816734313965, + 0.3101818859577179 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0874333381652832, + "mean_weights": [ + 0.36608952283859253, + 0.27405625581741333, + 0.35985422134399414 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9672750234603882, + "mean_weights": [ + 0.5343947410583496, + 0.1846013069152832, + 0.2810039520263672 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0731852054595947, + "mean_weights": [ + 0.3801807165145874, + 0.24373164772987366, + 0.37608763575553894 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9721636772155762, + "mean_weights": [ + 0.49137914180755615, + 0.2109004259109497, + 0.29772043228149414 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0864832401275635, + "mean_weights": [ + 0.39626431465148926, + 0.29088544845581055, + 0.3128502070903778 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3038380146026611, + "mean_weights": [ + 0.35696184635162354, + 0.1472446322441101, + 0.19980239868164062, + 0.29599112272262573 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3611116409301758, + "mean_weights": [ + 0.3264957070350647, + 0.22676822543144226, + 0.20138126611709595, + 0.2453548014163971 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3161137104034424, + "mean_weights": [ + 0.3259783387184143, + 0.17862647771835327, + 0.20532535016536713, + 0.2900698482990265 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568220138549805, + "mean_weights": [ + 0.3223523497581482, + 0.21851874887943268, + 0.1804703176021576, + 0.27865859866142273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2413711547851562, + "mean_weights": [ + 0.4272037148475647, + 0.16667884588241577, + 0.1365978717803955, + 0.269519567489624 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3540384769439697, + "mean_weights": [ + 0.32447606325149536, + 0.2030642032623291, + 0.18181107938289642, + 0.2906486392021179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1544246673583984, + "mean_weights": [ + 0.5128031969070435, + 0.13654224574565887, + 0.12138728052377701, + 0.22926729917526245 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3712997436523438, + "mean_weights": [ + 0.3103735148906708, + 0.24219870567321777, + 0.2247106432914734, + 0.22271710634231567 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5205519199371338, + "mean_weights": [ + 0.3307538628578186, + 0.1283297836780548, + 0.12636876106262207, + 0.17177441716194153, + 0.2427731454372406 + ], + "sources": 5 + }, + { + "entropy_mean": 1.518624186515808, + "mean_weights": [ + 0.2150207757949829, + 0.17106646299362183, + 0.13557285070419312, + 0.11898039281368256, + 0.3593595027923584 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4772696495056152, + "mean_weights": [ + 0.30324018001556396, + 0.13238273561000824, + 0.09796822816133499, + 0.15785877406597137, + 0.30855005979537964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4979676008224487, + "mean_weights": [ + 0.20932656526565552, + 0.16542977094650269, + 0.1272144764661789, + 0.1160244345664978, + 0.3820047676563263 + ], + "sources": 5 + }, + { + "entropy_mean": 1.49334716796875, + "mean_weights": [ + 0.28671571612358093, + 0.13947691023349762, + 0.11353637278079987, + 0.14176231622695923, + 0.31850868463516235 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5120060443878174, + "mean_weights": [ + 0.21571135520935059, + 0.28788846731185913, + 0.131511390209198, + 0.08343321830034256, + 0.2814555764198303 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5766459703445435, + "mean_weights": [ + 0.2565065026283264, + 0.22934341430664062, + 0.1593727320432663, + 0.1524350941181183, + 0.20234227180480957 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5796699523925781, + "mean_weights": [ + 0.2356114238500595, + 0.21223093569278717, + 0.1439642310142517, + 0.1658918261528015, + 0.2423015832901001 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7626371383666992, + "mean_weights": [ + 0.16389364004135132, + 0.20886825025081635, + 0.18027648329734802, + 0.1462823748588562, + 0.11240866780281067, + 0.18827059864997864 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6546597480773926, + "mean_weights": [ + 0.17577320337295532, + 0.14207448065280914, + 0.08111412078142166, + 0.0931762307882309, + 0.15011057257652283, + 0.35775139927864075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.430190086364746, + "mean_weights": [ + 0.1354856789112091, + 0.09479984641075134, + 0.07003438472747803, + 0.0834486186504364, + 0.09165525436401367, + 0.5245761871337891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7008471488952637, + "mean_weights": [ + 0.18653985857963562, + 0.149653822183609, + 0.09563066065311432, + 0.09960515052080154, + 0.15559062361717224, + 0.31297987699508667 + ], + "sources": 6 + }, + { + "entropy_mean": 1.514115333557129, + "mean_weights": [ + 0.14724519848823547, + 0.13274209201335907, + 0.096603162586689, + 0.11347334086894989, + 0.0527515783905983, + 0.4571846127510071 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7213332653045654, + "mean_weights": [ + 0.19527462124824524, + 0.16114139556884766, + 0.10406571626663208, + 0.10423287749290466, + 0.14977452158927917, + 0.2855108380317688 + ], + "sources": 6 + }, + { + "entropy_mean": 1.503009557723999, + "mean_weights": [ + 0.1677132397890091, + 0.12867671251296997, + 0.09363879263401031, + 0.12237933278083801, + 0.04204780235886574, + 0.44554415345191956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7531253099441528, + "mean_weights": [ + 0.21246549487113953, + 0.17559614777565002, + 0.11484666168689728, + 0.11058571934700012, + 0.1656094789505005, + 0.22089649736881256 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3981997966766357, + "mean_weights": [ + 0.08501995354890823, + 0.06276540458202362, + 0.0543542318046093, + 0.06455785036087036, + 0.03336101397871971, + 0.12979380786418915, + 0.5701477527618408 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6064693927764893, + "mean_weights": [ + 0.12975823879241943, + 0.09198254346847534, + 0.06163205951452255, + 0.06021754443645477, + 0.078399658203125, + 0.10503693670034409, + 0.4729730188846588 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3297276496887207, + "mean_weights": [ + 0.09028719365596771, + 0.057085148990154266, + 0.04790058359503746, + 0.05685827136039734, + 0.031231366097927094, + 0.11244906485080719, + 0.6041883826255798 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6772459745407104, + "mean_weights": [ + 0.1455874890089035, + 0.10527922213077545, + 0.06669703871011734, + 0.06069611385464668, + 0.09951996803283691, + 0.09685218334197998, + 0.4253680109977722 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3742375373840332, + "mean_weights": [ + 0.09774194657802582, + 0.06367622315883636, + 0.05300067365169525, + 0.05981069803237915, + 0.03241024166345596, + 0.10956928879022598, + 0.5837908983230591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.672610878944397, + "mean_weights": [ + 0.14811497926712036, + 0.10026681423187256, + 0.0711982250213623, + 0.06714795529842377, + 0.08556141704320908, + 0.09809904545545578, + 0.42961156368255615 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4325350522994995, + "mean_weights": [ + 0.11123399436473846, + 0.07182648032903671, + 0.05620439350605011, + 0.06410455703735352, + 0.03741488978266716, + 0.10516737401485443, + 0.5540482997894287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7746202945709229, + "mean_weights": [ + 0.15383490920066833, + 0.11435382068157196, + 0.09083531051874161, + 0.08329388499259949, + 0.08837151527404785, + 0.11093884706497192, + 0.3583717346191406 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5557539463043213, + "mean_weights": [ + 0.09889799356460571, + 0.05227670818567276, + 0.03886467218399048, + 0.039664674550294876, + 0.02741250954568386, + 0.06309205293655396, + 0.40426212549209595, + 0.27552926540374756 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7942585945129395, + "mean_weights": [ + 0.10892648994922638, + 0.08669497072696686, + 0.060229551047086716, + 0.05406671017408371, + 0.05429235100746155, + 0.07220236957073212, + 0.24074554443359375, + 0.322842001914978 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6406831741333008, + "mean_weights": [ + 0.08106359094381332, + 0.06122073531150818, + 0.04382575303316116, + 0.0495278537273407, + 0.031025834381580353, + 0.08371539413928986, + 0.3212447166442871, + 0.3283761143684387 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8771746158599854, + "mean_weights": [ + 0.11823929846286774, + 0.09187044948339462, + 0.06966835260391235, + 0.05918115749955177, + 0.06588791310787201, + 0.07802121341228485, + 0.22871823608875275, + 0.2884133756160736 + ], + "sources": 8 + }, + { + "entropy_mean": 1.612527847290039, + "mean_weights": [ + 0.09160974621772766, + 0.05898914486169815, + 0.039636120200157166, + 0.04581688344478607, + 0.032106831669807434, + 0.06671778112649918, + 0.3060194253921509, + 0.35910406708717346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8253357410430908, + "mean_weights": [ + 0.10355446487665176, + 0.08380813896656036, + 0.06609268486499786, + 0.07030215114355087, + 0.04850628599524498, + 0.0775601714849472, + 0.25825831294059753, + 0.2919177711009979 + ], + "sources": 8 + }, + { + "entropy_mean": 1.714170217514038, + "mean_weights": [ + 0.09696917235851288, + 0.07828262448310852, + 0.055068351328372955, + 0.0616651251912117, + 0.03598042577505112, + 0.07802210748195648, + 0.3351396918296814, + 0.25887250900268555 + ], + "sources": 8 + }, + { + "entropy_mean": 1.875918984413147, + "mean_weights": [ + 0.11156761646270752, + 0.0880264937877655, + 0.07785218954086304, + 0.07930342108011246, + 0.04990004748106003, + 0.08684437721967697, + 0.2776731252670288, + 0.22883272171020508 + ], + "sources": 8 + }, + { + "entropy_mean": 1.731968641281128, + "mean_weights": [ + 0.06617744266986847, + 0.05556279793381691, + 0.04319432005286217, + 0.045391663908958435, + 0.026374589651823044, + 0.054588086903095245, + 0.21148523688316345, + 0.10446193069219589, + 0.3927639424800873 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9649744033813477, + "mean_weights": [ + 0.07813189923763275, + 0.08751581609249115, + 0.05837181955575943, + 0.07608447968959808, + 0.03110998496413231, + 0.09630586206912994, + 0.27242881059646606, + 0.1154000535607338, + 0.18465125560760498 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8779311180114746, + "mean_weights": [ + 0.07922039180994034, + 0.064256951212883, + 0.04481552541255951, + 0.05368471518158913, + 0.03291679173707962, + 0.07845190167427063, + 0.26414477825164795, + 0.13897250592708588, + 0.24353644251823425 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9959204196929932, + "mean_weights": [ + 0.10073552280664444, + 0.0823616087436676, + 0.0686727911233902, + 0.0658436119556427, + 0.048376426100730896, + 0.07448098808526993, + 0.23861894011497498, + 0.14355987310409546, + 0.177350252866745 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9298079013824463, + "mean_weights": [ + 0.09570661187171936, + 0.07176163792610168, + 0.05499213933944702, + 0.057467732578516006, + 0.03367581591010094, + 0.06890923529863358, + 0.21427617967128754, + 0.18804281949996948, + 0.2151678204536438 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8635971546173096, + "mean_weights": [ + 0.07685244828462601, + 0.07133444398641586, + 0.05962139368057251, + 0.08040933310985565, + 0.033340416848659515, + 0.10067727416753769, + 0.36328762769699097, + 0.11175162345170975, + 0.10272547602653503 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0009379386901855, + "mean_weights": [ + 0.09139549732208252, + 0.08807491511106491, + 0.06457877159118652, + 0.07007958739995956, + 0.03462113067507744, + 0.09155915677547455, + 0.2692604660987854, + 0.15421991050243378, + 0.13621056079864502 + ], + "sources": 9 + } + ], + "loss_nats": 1.3369717597961426, + "output_weights": { + "entropy_mean": 1.7290174961090088, + "mean_weights": [ + 0.10267993807792664, + 0.05008202791213989, + 0.044380687177181244, + 0.0326685830950737, + 0.033161722123622894, + 0.043591417372226715, + 0.1060870811343193, + 0.3520195186138153, + 0.2353290319442749 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "population_cv": 0.7299683279122143 + }, + "activation_output_rms_by_block": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24231436662375927, + "first_to_last_ratio": 1.7382073612286646, + "imbalance_abs_log_ratio": 0.5528543299844475, + "last_quartile_mean": 0.139404752291739, + "mean": 0.17594548198394477, + "normalized": [ + 0.872176901111452, + 1.8466899321818973, + 2.1703463827261262, + 2.3075366266331874, + 0.767871429869389, + 0.870980627700565, + 1.0231460974138593, + 1.1589542538312023, + 0.24864957835709345, + 0.4666800200245053, + 0.7845885991457198, + 1.3049357734104974, + 0.46603991847126747, + 0.789568230136157, + 1.2326042132027004, + 4.183179000808161, + 0.32473565854469494, + 0.4690011715552792, + 0.7196215097851506, + 1.1150703717560293, + 0.38383209436703286, + 0.5425045292082467, + 0.7244651255474189, + 0.8882788682598759, + 0.3321309360788757, + 0.49982187502145003, + 0.6961071361611404, + 0.9322692927740504, + 0.42842289992260757, + 0.6985518108614562, + 1.0888090156129309, + 1.6624301195199809 + ], + "population_cv": 0.7619463065887785 + }, + "bits_per_byte": 1.877417995231997, + "core_parameter_grad_rms_by_block": [ + 0.00028365814152869, + 0.0002107335686261075, + 0.00020559059302825965, + 0.00013034830149957224, + 0.00015811506316286034, + 8.078727112532468e-05, + 9.745502139202278e-05, + 0.00010158534732968689, + 0.00010687310625858127, + 0.00011249947044062199, + 0.00014466041390513986, + 0.00018570148639523412, + 0.0001198924920321974, + 0.00012288623237884872, + 0.00013187077826681118, + 0.000321118955435273, + 0.0002173403860671073, + 0.00022706794561596516, + 0.0003846029635623724, + 0.00045310883243459936, + 0.001499135691470673, + 0.0011394398675334944, + 0.0010162826769661553, + 0.0008650696174683238, + 0.0006964490714203023, + 0.0005559793522399823, + 0.0005128319422021078, + 0.0003792784055626199, + 0.00037680605737877213, + 0.0002696585960614849, + 0.00021598580145167561, + 0.00019464643295656736 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015853416346156552, + "first_to_last_ratio": 0.39613292787359505, + "imbalance_abs_log_ratio": 0.9260054476059185, + "last_quartile_mean": 0.000400204457409189, + "mean": 0.0003599206213499198, + "normalized": [ + 0.7881130579981791, + 0.5855001245433764, + 0.5712109305023165, + 0.36215846986118033, + 0.439305373973387, + 0.2244585787341764, + 0.27076809610549, + 0.28224375404965807, + 0.29693521270813145, + 0.3125674489521634, + 0.4019231056074973, + 0.5159512275199496, + 0.3331081491872516, + 0.34142592863379456, + 0.36638850469921974, + 0.892193823823939, + 0.6038564427121444, + 0.6308834008018855, + 1.0685771827128963, + 1.258913231298522, + 4.165184217142125, + 3.165808792116179, + 2.8236300358520197, + 2.4035011226191765, + 1.9350074158246267, + 1.544727696220138, + 1.4248473462806275, + 1.0537834818691318, + 1.0469143334036313, + 0.7492168552335295, + 0.6000928778173319, + 0.540803781196325 + ], + "population_cv": 0.9317377718256953 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2200722098350525, + "mean_weights": [ + 0.9384047985076904, + 0.06159520894289017 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5651201605796814, + "mean_weights": [ + 0.6557347774505615, + 0.3442652225494385 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4018177092075348, + "mean_weights": [ + 0.856421947479248, + 0.14357808232307434 + ], + "sources": 2 + }, + { + "entropy_mean": 0.687358021736145, + "mean_weights": [ + 0.46872246265411377, + 0.5312775373458862 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2678314447402954, + "mean_weights": [ + 0.920531153678894, + 0.07946885377168655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.657238245010376, + "mean_weights": [ + 0.6222448348999023, + 0.37775513529777527 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044373869895935, + "mean_weights": [ + 0.7792346477508545, + 0.2207653522491455 + ], + "sources": 2 + }, + { + "entropy_mean": 0.650418758392334, + "mean_weights": [ + 0.5397791862487793, + 0.4602207541465759 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0032464265823364, + "mean_weights": [ + 0.4337552487850189, + 0.18956376612186432, + 0.37668097019195557 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0385558605194092, + "mean_weights": [ + 0.24478834867477417, + 0.2602521479129791, + 0.4949595034122467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.031773328781128, + "mean_weights": [ + 0.40370821952819824, + 0.2651250958442688, + 0.33116668462753296 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0838522911071777, + "mean_weights": [ + 0.35612422227859497, + 0.2625962495803833, + 0.38127952814102173 + ], + "sources": 3 + }, + { + "entropy_mean": 0.96053147315979, + "mean_weights": [ + 0.5271762013435364, + 0.1581989824771881, + 0.3146248161792755 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0704245567321777, + "mean_weights": [ + 0.3788670599460602, + 0.23852786421775818, + 0.38260507583618164 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9674104452133179, + "mean_weights": [ + 0.48829180002212524, + 0.18356263637542725, + 0.3281455934047699 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832805633544922, + "mean_weights": [ + 0.40388280153274536, + 0.2811351418495178, + 0.3149820566177368 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2917464971542358, + "mean_weights": [ + 0.3349481225013733, + 0.1244347095489502, + 0.20291922986507416, + 0.33769792318344116 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533791303634644, + "mean_weights": [ + 0.32115638256073, + 0.20602071285247803, + 0.18808895349502563, + 0.28473395109176636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308367371559143, + "mean_weights": [ + 0.31139707565307617, + 0.16179001331329346, + 0.2296154499053955, + 0.29719749093055725 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3437637090682983, + "mean_weights": [ + 0.3217787742614746, + 0.1920534074306488, + 0.1763250231742859, + 0.3098427951335907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2280659675598145, + "mean_weights": [ + 0.425346314907074, + 0.14848899841308594, + 0.13232898712158203, + 0.29383569955825806 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3462648391723633, + "mean_weights": [ + 0.3221249580383301, + 0.18557843565940857, + 0.18214309215545654, + 0.3101535141468048 + ], + "sources": 4 + }, + { + "entropy_mean": 1.147221326828003, + "mean_weights": [ + 0.505380392074585, + 0.12089012563228607, + 0.11514648050069809, + 0.2585829496383667 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3729398250579834, + "mean_weights": [ + 0.30785465240478516, + 0.23079943656921387, + 0.23862706124782562, + 0.22271886467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5232064723968506, + "mean_weights": [ + 0.3136816620826721, + 0.11285122483968735, + 0.13196846842765808, + 0.19688263535499573, + 0.24461600184440613 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4876129627227783, + "mean_weights": [ + 0.2035508155822754, + 0.15372204780578613, + 0.13167491555213928, + 0.11437506973743439, + 0.396677166223526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4587602615356445, + "mean_weights": [ + 0.28519001603126526, + 0.10532154142856598, + 0.09675675630569458, + 0.17811742424964905, + 0.33461427688598633 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4565696716308594, + "mean_weights": [ + 0.20232918858528137, + 0.14928825199604034, + 0.12029382586479187, + 0.1059027910232544, + 0.4221859574317932 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4788098335266113, + "mean_weights": [ + 0.2697942852973938, + 0.11948063224554062, + 0.11090938746929169, + 0.15228068828582764, + 0.34753501415252686 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5018762350082397, + "mean_weights": [ + 0.208971306681633, + 0.27331358194351196, + 0.12766750156879425, + 0.07916607707738876, + 0.31088152527809143 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5627843141555786, + "mean_weights": [ + 0.27452588081359863, + 0.21448183059692383, + 0.143874391913414, + 0.14059773087501526, + 0.22652018070220947 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580754280090332, + "mean_weights": [ + 0.2518634498119354, + 0.18758198618888855, + 0.13903988897800446, + 0.18044015765190125, + 0.24107450246810913 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7564777135849, + "mean_weights": [ + 0.15440437197685242, + 0.19411900639533997, + 0.17806550860404968, + 0.14696577191352844, + 0.10594475269317627, + 0.2205006331205368 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5899518728256226, + "mean_weights": [ + 0.1636897623538971, + 0.11789832264184952, + 0.07030131667852402, + 0.09238424897193909, + 0.1399868130683899, + 0.4157395362854004 + ], + "sources": 6 + }, + { + "entropy_mean": 1.2894535064697266, + "mean_weights": [ + 0.1212274506688118, + 0.06944876164197922, + 0.05726535990834236, + 0.09096578508615494, + 0.06335186958312988, + 0.5977407693862915 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6581614017486572, + "mean_weights": [ + 0.18132120370864868, + 0.12711825966835022, + 0.08560528606176376, + 0.09748884290456772, + 0.15072739124298096, + 0.35773903131484985 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3699336051940918, + "mean_weights": [ + 0.13604651391506195, + 0.10151475667953491, + 0.07715043425559998, + 0.11300846189260483, + 0.033717285841703415, + 0.538562536239624 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6926933526992798, + "mean_weights": [ + 0.19425281882286072, + 0.14001017808914185, + 0.09434761106967926, + 0.10331346094608307, + 0.14820793271064758, + 0.3198680281639099 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4148362874984741, + "mean_weights": [ + 0.16652247309684753, + 0.10329679399728775, + 0.08063912391662598, + 0.1165856122970581, + 0.033275946974754333, + 0.4996800422668457 + ], + "sources": 6 + }, + { + "entropy_mean": 1.740808129310608, + "mean_weights": [ + 0.21459883451461792, + 0.15640267729759216, + 0.10570268332958221, + 0.11097012460231781, + 0.17001523077487946, + 0.24231043457984924 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3179612159729004, + "mean_weights": [ + 0.08068332821130753, + 0.049232255667448044, + 0.045073021203279495, + 0.06111945956945419, + 0.028317857533693314, + 0.13192236423492432, + 0.6036517024040222 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5319744348526, + "mean_weights": [ + 0.12406684458255768, + 0.07462044060230255, + 0.05378449708223343, + 0.05575074627995491, + 0.07357081025838852, + 0.10496053099632263, + 0.5132461786270142 + ], + "sources": 7 + }, + { + "entropy_mean": 1.2461028099060059, + "mean_weights": [ + 0.08556011319160461, + 0.0446893684566021, + 0.04084867238998413, + 0.05386578291654587, + 0.025119207799434662, + 0.11240880191326141, + 0.6375080347061157 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6323832273483276, + "mean_weights": [ + 0.1418324112892151, + 0.08991365134716034, + 0.06188817322254181, + 0.05739111453294754, + 0.09848876297473907, + 0.0972195416688919, + 0.45326635241508484 + ], + "sources": 7 + }, + { + "entropy_mean": 1.301480770111084, + "mean_weights": [ + 0.09513165056705475, + 0.05029121786355972, + 0.045042067766189575, + 0.05902485549449921, + 0.02615261822938919, + 0.1109299436211586, + 0.6134276390075684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.629331111907959, + "mean_weights": [ + 0.14790503680706024, + 0.08658000826835632, + 0.06608416140079498, + 0.06427294760942459, + 0.0819322019815445, + 0.097825787961483, + 0.45539984107017517 + ], + "sources": 7 + }, + { + "entropy_mean": 1.338958501815796, + "mean_weights": [ + 0.10803166776895523, + 0.0559723861515522, + 0.047874726355075836, + 0.060293328016996384, + 0.029215168207883835, + 0.10392352938652039, + 0.5946891903877258 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7611865997314453, + "mean_weights": [ + 0.1573912799358368, + 0.10317619144916534, + 0.0853322446346283, + 0.08218386769294739, + 0.09052872657775879, + 0.11341015249490738, + 0.3679775595664978 + ], + "sources": 7 + }, + { + "entropy_mean": 1.476489782333374, + "mean_weights": [ + 0.09070298075675964, + 0.037138283252716064, + 0.030013620853424072, + 0.03533655032515526, + 0.020730752497911453, + 0.06278065592050552, + 0.3836256265640259, + 0.3396714925765991 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7213938236236572, + "mean_weights": [ + 0.10584281384944916, + 0.06887699663639069, + 0.05207539349794388, + 0.050678279250860214, + 0.04768219590187073, + 0.06809806823730469, + 0.2275032103061676, + 0.37924304604530334 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5405172109603882, + "mean_weights": [ + 0.07428755611181259, + 0.04491765797138214, + 0.03515764698386192, + 0.046252522617578506, + 0.023427214473485947, + 0.07749318331480026, + 0.29666566848754883, + 0.4017985761165619 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8063910007476807, + "mean_weights": [ + 0.11433833837509155, + 0.07374332845211029, + 0.05932536721229553, + 0.05389238893985748, + 0.061260174959897995, + 0.07349519431591034, + 0.22022446990013123, + 0.3437207341194153 + ], + "sources": 8 + }, + { + "entropy_mean": 1.4859801530838013, + "mean_weights": [ + 0.08387120068073273, + 0.041406162083148956, + 0.032444946467876434, + 0.03686721250414848, + 0.025052767246961594, + 0.060124777257442474, + 0.27774184942245483, + 0.4424910843372345 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7777256965637207, + "mean_weights": [ + 0.10586117208003998, + 0.07087038457393646, + 0.05850498005747795, + 0.06360965967178345, + 0.04672512412071228, + 0.07485505938529968, + 0.2385886013507843, + 0.3409850001335144 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6318418979644775, + "mean_weights": [ + 0.09244748204946518, + 0.05762210488319397, + 0.045099519193172455, + 0.056236449629068375, + 0.029373466968536377, + 0.0723743885755539, + 0.31761035323143005, + 0.3292362093925476 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8334553241729736, + "mean_weights": [ + 0.1157165914773941, + 0.07161639630794525, + 0.06577154994010925, + 0.07425849884748459, + 0.04881399869918823, + 0.08197253942489624, + 0.27155569195747375, + 0.270294725894928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6227548122406006, + "mean_weights": [ + 0.0605507455766201, + 0.040063511580228806, + 0.032619886100292206, + 0.03868011012673378, + 0.02029048651456833, + 0.04760073497891426, + 0.19551873207092285, + 0.11294309794902802, + 0.45173269510269165 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9787883758544922, + "mean_weights": [ + 0.0788881778717041, + 0.08108963817358017, + 0.057054273784160614, + 0.08211588114500046, + 0.026040803641080856, + 0.10537256300449371, + 0.24502354860305786, + 0.12593510746955872, + 0.1984800100326538 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7590525150299072, + "mean_weights": [ + 0.07131639868021011, + 0.044687941670417786, + 0.033290669322013855, + 0.043597716838121414, + 0.02658817172050476, + 0.06922741234302521, + 0.23554196953773499, + 0.14986532926559448, + 0.3258844017982483 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9736928939819336, + "mean_weights": [ + 0.09849467873573303, + 0.068942591547966, + 0.0599900484085083, + 0.06335455179214478, + 0.04671281576156616, + 0.0713595598936081, + 0.22560220956802368, + 0.1557777225971222, + 0.20976582169532776 + ], + "sources": 9 + }, + { + "entropy_mean": 1.809161901473999, + "mean_weights": [ + 0.08854075521230698, + 0.05356752127408981, + 0.04243364930152893, + 0.04702243581414223, + 0.02648795023560524, + 0.05820769816637039, + 0.18489578366279602, + 0.21069815754890442, + 0.2881460189819336 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8544633388519287, + "mean_weights": [ + 0.07977832853794098, + 0.05858254432678223, + 0.053744811564683914, + 0.07626348733901978, + 0.031090255826711655, + 0.0994817242026329, + 0.3525030016899109, + 0.1274903416633606, + 0.12106549739837646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9456019401550293, + "mean_weights": [ + 0.08823960274457932, + 0.06737187504768372, + 0.05197303742170334, + 0.0631948709487915, + 0.02739548310637474, + 0.08291902393102646, + 0.2524128556251526, + 0.186361163854599, + 0.18013209104537964 + ], + "sources": 9 + } + ], + "loss_nats": 1.3013269901275635, + "output_weights": { + "entropy_mean": 1.596132755279541, + "mean_weights": [ + 0.09668220579624176, + 0.03708431497216225, + 0.034579962491989136, + 0.026012204587459564, + 0.0258483849465847, + 0.037477150559425354, + 0.0845583826303482, + 0.3670029640197754, + 0.29075440764427185 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.034013546791773, + "cross_entropy_nats": 5.5687538385391235, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "cross_entropy_nats": 3.5575302243232727, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "cross_entropy_nats": 2.1426188945770264, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "cross_entropy_nats": 1.3501944988965988, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "cross_entropy_nats": 1.241536632180214, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "cross_entropy_nats": 1.1848131567239761, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d", + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "seed": 2026073002, + "timing": { + "mean_ms": 109.03141246394478, + "measured_steps": 7980, + "median_ms": 108.93342149211094, + "p95_ms": 109.53918776649516, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + } + }, + { + "architecture": "block", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "population_cv": 0.2498932502086342 + }, + "activation_output_rms_by_block": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005511203169589862, + "first_to_last_ratio": 0.6356203088456116, + "imbalance_abs_log_ratio": 0.45315389258709987, + "last_quartile_mean": 0.008670590119436383, + "mean": 0.007157071326219011, + "normalized": [ + 0.4829581625738996, + 0.6886426787339398, + 0.8458438602600115, + 0.9725589429115095, + 0.5050274053272774, + 0.7207530129760953, + 0.8901040788372985, + 1.0544005221525679, + 0.5528397766850007, + 0.7916459680002588, + 0.9438571997073384, + 1.1102770425361632, + 0.6125217730505899, + 0.9295491721602742, + 1.1083022471557418, + 1.3061042379491232, + 0.6819634955048963, + 0.9650194302374437, + 1.1362396940587196, + 1.3715945478756304, + 0.7619466772614072, + 1.0650300118366156, + 1.3201940438400879, + 1.490851576861243, + 0.7782743937317399, + 1.0812722353862223, + 1.3287334463206348, + 1.5181249897087195, + 0.8365475786723071, + 1.138987569259587, + 1.3797450031110483, + 1.6300892253166077 + ], + "population_cv": 0.3025375276561286 + }, + "bits_per_byte": 8.06433666787865, + "core_parameter_grad_rms_by_block": [ + 0.006159982634052032, + 0.005514663551817784, + 0.005191681274696802, + 0.00489831575692433, + 0.004666415075367917, + 0.004399813239791133, + 0.004483878567781538, + 0.004868281936291656, + 0.00443828064080646, + 0.004266304440803418, + 0.0039471188591477255, + 0.004494388758593749, + 0.0042334459038140095, + 0.00446434979681851, + 0.004322339302329385, + 0.004638471384339534, + 0.004165330760400444, + 0.004499937764754925, + 0.003981478974932938, + 0.004372386299982334, + 0.00458495381977442, + 0.00429501466057645, + 0.003963900484885007, + 0.004271473918508079, + 0.004041799650627023, + 0.004206875397956982, + 0.003955591472942362, + 0.003916464042690177, + 0.003989587235588314, + 0.003922309599766117, + 0.003679536678174494, + 0.003763244111925236 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005022879004590398, + "first_to_last_ratio": 1.2766484804448084, + "imbalance_abs_log_ratio": 0.24423826934058726, + "last_quartile_mean": 0.003934426023708838, + "mean": 0.0043936754999019155, + "normalized": [ + 1.4020112851278042, + 1.255136741878387, + 1.1816260155791434, + 1.1148560600421402, + 1.0620754936207943, + 1.0013969488391563, + 1.0205302070855338, + 1.108020366183241, + 1.0101521245493756, + 0.9710103627130996, + 0.8983637638318627, + 1.0229223252136126, + 0.9635317637610054, + 1.0160854612313022, + 0.9837638902613262, + 1.0557155130011497, + 0.9480287655502622, + 1.0241852783291305, + 0.9061841219320409, + 0.995154580733134, + 1.0435349219296635, + 0.9775448051801577, + 0.9021832597727114, + 0.9721869351988865, + 0.9199131002545027, + 0.9574843199164109, + 0.9002921296829196, + 0.891386731400079, + 0.9080295610536958, + 0.892717179467095, + 0.8374620925593245, + 0.8565138941210488 + ], + "population_cv": 0.11556590700625036 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "loss_nats": 5.5897722244262695, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0 + }, + { + "activation_grad_rms_by_block": [ + 6.886429764563218e-05, + 6.52737362543121e-05, + 6.186874816194177e-05, + 5.901425174670294e-05, + 5.411080928752199e-05, + 5.1034174248343334e-05, + 4.815354259335436e-05, + 4.550318044493906e-05, + 4.120725498069078e-05, + 3.8442405639216304e-05, + 3.604330777307041e-05, + 3.37670098815579e-05, + 3.108748569502495e-05, + 2.9203254598542117e-05, + 2.7565200070966966e-05, + 2.5956482204492204e-05, + 2.4566294086980633e-05, + 2.329334893147461e-05, + 2.2260202968027443e-05, + 2.14147730730474e-05, + 2.0902438336634077e-05, + 2.0279332602513023e-05, + 1.9777973648160696e-05, + 1.9328235794091597e-05, + 1.9035323930438608e-05, + 1.8655482563190162e-05, + 1.832095585996285e-05, + 1.8063024981529452e-05, + 1.7733544154907577e-05, + 1.752249227138236e-05, + 1.7320473489235155e-05, + 1.7174717868329026e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.6727842547843466e-05, + "first_to_last_ratio": 3.155359202626439, + "imbalance_abs_log_ratio": 1.1491023413751247, + "last_quartile_mean": 1.79782518898719e-05, + "mean": 3.227324236831919e-05, + "normalized": [ + 2.133789250541258, + 2.0225341944070556, + 1.9170292050567193, + 1.8285814320482991, + 1.67664620337743, + 1.581315371598383, + 1.4920577871848402, + 1.409935200362978, + 1.2768241415105412, + 1.1911541208190792, + 1.1168170635514478, + 1.0462850151897058, + 0.9632588303411923, + 0.904875136661487, + 0.8541193275958588, + 0.8042725273235084, + 0.7611969633115008, + 0.7217542218299198, + 0.6897417592562383, + 0.6635457580819046, + 0.647670850610065, + 0.628363657145899, + 0.6128288389014056, + 0.5988935221787641, + 0.5898175247840765, + 0.5780479801280578, + 0.5676825294116558, + 0.559690432569022, + 0.5494813304632692, + 0.5429417990112825, + 0.5366821620079203, + 0.5321658627392354 + ], + "population_cv": 0.49221598973047537 + }, + "activation_output_rms_by_block": [ + 0.0055193523876369, + 0.009546536952257156, + 0.013977172784507275, + 0.018744563683867455, + 0.00660830270498991, + 0.011765030212700367, + 0.017156247049570084, + 0.022267481312155724, + 0.007449587807059288, + 0.013175037689507008, + 0.019073888659477234, + 0.025712018832564354, + 0.008972279727458954, + 0.01624389924108982, + 0.023743286728858948, + 0.03214457631111145, + 0.00991375558078289, + 0.019281653687357903, + 0.02868141047656536, + 0.03889070823788643, + 0.0127727584913373, + 0.022841427475214005, + 0.034929387271404266, + 0.04634999856352806, + 0.014845813624560833, + 0.029589947313070297, + 0.046351123601198196, + 0.06254568696022034, + 0.01947367936372757, + 0.0370744951069355, + 0.05507569760084152, + 0.07403328269720078 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013198085885960609, + "first_to_last_ratio": 0.31146869331458016, + "imbalance_abs_log_ratio": 1.1664564488083473, + "last_quartile_mean": 0.04237371578346938, + "mean": 0.0251484402542701, + "normalized": [ + 0.21947096248642048, + 0.3796075166385794, + 0.5557868656341027, + 0.7453569085933552, + 0.26277187126417706, + 0.46782345520226504, + 0.6821992487847045, + 0.8854418439877121, + 0.2962246458125521, + 0.5238908479530829, + 0.7584521531604159, + 1.0224100808080359, + 0.35677281122575777, + 0.6459207440641045, + 0.9441256192748351, + 1.2781936369057099, + 0.39420956053525336, + 0.7667137004285568, + 1.140484665711838, + 1.5464461352144077, + 0.5078946591595691, + 0.9082641803733982, + 1.3889285744261377, + 1.8430565909811456, + 0.5903274109431131, + 1.1766116313335198, + 1.8431013268637195, + 2.487060283971303, + 0.7743493897368454, + 1.4742264224772512, + 2.190024392923927, + 2.9438518631242045 + ], + "population_cv": 0.6624204772463126 + }, + "bits_per_byte": 5.18286091123436, + "core_parameter_grad_rms_by_block": [ + 0.00023786681750428726, + 0.00020576573809540593, + 0.00019740566304489066, + 0.0001813902134209802, + 0.0001696969276866905, + 0.00015923444695871368, + 0.00017314685145041394, + 0.00015953319119459735, + 0.0001633125663305534, + 0.0001601758879769712, + 0.00014581270072009644, + 0.00016837778263142155, + 0.0001603117902727929, + 0.00016340123818181664, + 0.00015138579649161358, + 0.00015756725288262655, + 0.00013845240957374987, + 0.0001515626583813202, + 0.00014303549429743115, + 0.00016115390114098118, + 0.00015144047315466305, + 0.00013355245347473064, + 0.00014934284038847835, + 0.00014161248575260865, + 0.00014194691162247068, + 0.00015146794722523853, + 0.00015520640136475585, + 0.0001429086966261467, + 0.00014288517283478702, + 0.00013975759356060232, + 0.00013469418893153685, + 0.00013332122361590492 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018550498116949745, + "first_to_last_ratio": 1.2992954513055368, + "imbalance_abs_log_ratio": 0.26182215701959494, + "last_quartile_mean": 0.00014277351697268036, + "mean": 0.00015833517864966492, + "normalized": [ + 1.5022992333914333, + 1.2995579368416077, + 1.2467580782011416, + 1.1456090489045851, + 1.0717575786626974, + 1.0056795231275704, + 1.0935463169149642, + 1.0075663068381233, + 1.0314357663491922, + 1.0116253971038178, + 0.9209115874541316, + 1.0634262333071103, + 1.0124837173898131, + 1.0319957925671146, + 0.9561096768430238, + 0.9951499990489321, + 0.874426079880151, + 0.9572266862859976, + 0.9033715407863586, + 1.0178022503612605, + 0.9564549991113648, + 0.8434793494011273, + 0.9432069465681839, + 0.894384223141863, + 0.8964963619142705, + 0.9566285175348117, + 0.9802395316594056, + 0.9025707227220104, + 0.9024221531396832, + 0.8826692510944287, + 0.8506902261408595, + 0.8420189673129665 + ], + "population_cv": 0.13848454603612886 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6920576691627502, + "mean_weights": [ + 0.5229578018188477, + 0.47704219818115234 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922386288642883, + "mean_weights": [ + 0.5206724405288696, + 0.479327529668808 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930604577064514, + "mean_weights": [ + 0.5023424625396729, + 0.4976575970649719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924410462379456, + "mean_weights": [ + 0.5176653861999512, + 0.48233455419540405 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929901838302612, + "mean_weights": [ + 0.5013687014579773, + 0.4986313283443451 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929076313972473, + "mean_weights": [ + 0.5098068714141846, + 0.49019312858581543 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927430629730225, + "mean_weights": [ + 0.5127849578857422, + 0.4872150421142578 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928876638412476, + "mean_weights": [ + 0.5100651383399963, + 0.48993486166000366 + ], + "sources": 2 + }, + { + "entropy_mean": 1.098104476928711, + "mean_weights": [ + 0.33572155237197876, + 0.32083213329315186, + 0.34344637393951416 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098517656326294, + "mean_weights": [ + 0.3379058539867401, + 0.3309776186943054, + 0.33111652731895447 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984238386154175, + "mean_weights": [ + 0.34072375297546387, + 0.3312225341796875, + 0.3280537724494934 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985376834869385, + "mean_weights": [ + 0.3372243344783783, + 0.33110499382019043, + 0.3316706418991089 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983879566192627, + "mean_weights": [ + 0.34103795886039734, + 0.33302855491638184, + 0.3259335160255432 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984759330749512, + "mean_weights": [ + 0.3378061056137085, + 0.3331226110458374, + 0.3290712833404541 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982115268707275, + "mean_weights": [ + 0.3427826464176178, + 0.33246177434921265, + 0.32475557923316956 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098510980606079, + "mean_weights": [ + 0.336602121591568, + 0.33530712127685547, + 0.32809075713157654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860652446746826, + "mean_weights": [ + 0.25563088059425354, + 0.25276094675064087, + 0.24706901609897614, + 0.24453918635845184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861669301986694, + "mean_weights": [ + 0.25406360626220703, + 0.25270599126815796, + 0.24658669531345367, + 0.24664369225502014 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386000394821167, + "mean_weights": [ + 0.2551618814468384, + 0.2541123330593109, + 0.2483193278312683, + 0.24240648746490479 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386232852935791, + "mean_weights": [ + 0.25183945894241333, + 0.25246676802635193, + 0.24847325682640076, + 0.24722051620483398 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385960340499878, + "mean_weights": [ + 0.25454509258270264, + 0.25490614771842957, + 0.24972698092460632, + 0.24082177877426147 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386134147644043, + "mean_weights": [ + 0.25200384855270386, + 0.25437670946121216, + 0.24930089712142944, + 0.24431857466697693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859115839004517, + "mean_weights": [ + 0.2541099786758423, + 0.2563633620738983, + 0.2499358206987381, + 0.2395908385515213 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861732482910156, + "mean_weights": [ + 0.2492598295211792, + 0.25510817766189575, + 0.2504415512084961, + 0.24519044160842896 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6091704368591309, + "mean_weights": [ + 0.20294126868247986, + 0.20561827719211578, + 0.20076946914196014, + 0.19495637714862823, + 0.19571462273597717 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093817949295044, + "mean_weights": [ + 0.1999536156654358, + 0.20290037989616394, + 0.19905918836593628, + 0.1980593204498291, + 0.2000274658203125 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6089024543762207, + "mean_weights": [ + 0.20434537529945374, + 0.20689502358436584, + 0.20309439301490784, + 0.19485914707183838, + 0.1908060759305954 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609398365020752, + "mean_weights": [ + 0.20015040040016174, + 0.2025333195924759, + 0.2003234326839447, + 0.19931097328662872, + 0.19768190383911133 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085304021835327, + "mean_weights": [ + 0.20286625623703003, + 0.21047379076480865, + 0.20388861000537872, + 0.19463706016540527, + 0.18813428282737732 + ], + "sources": 5 + }, + { + "entropy_mean": 1.60928213596344, + "mean_weights": [ + 0.20026028156280518, + 0.20464842021465302, + 0.20178082585334778, + 0.1985689103603363, + 0.19474159181118011 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085617542266846, + "mean_weights": [ + 0.20295967161655426, + 0.2110326737165451, + 0.20332126319408417, + 0.194858580827713, + 0.18782781064510345 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093013286590576, + "mean_weights": [ + 0.20157885551452637, + 0.20404376089572906, + 0.20028473436832428, + 0.19877153635025024, + 0.19532111287117004 + ], + "sources": 5 + }, + { + "entropy_mean": 1.79081130027771, + "mean_weights": [ + 0.1695082187652588, + 0.17698298394680023, + 0.17063108086585999, + 0.16265572607517242, + 0.15738935768604279, + 0.16283263266086578 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917133569717407, + "mean_weights": [ + 0.16738522052764893, + 0.1689736694097519, + 0.16775797307491302, + 0.16576024889945984, + 0.16475272178649902, + 0.16537019610404968 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7902145385742188, + "mean_weights": [ + 0.17057637870311737, + 0.18038460612297058, + 0.17244961857795715, + 0.164292573928833, + 0.15751495957374573, + 0.15478186309337616 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917354106903076, + "mean_weights": [ + 0.16660897433757782, + 0.16850554943084717, + 0.1666441559791565, + 0.16635674238204956, + 0.16625551879405975, + 0.1656290590763092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898752689361572, + "mean_weights": [ + 0.17184747755527496, + 0.180228590965271, + 0.17470473051071167, + 0.16429948806762695, + 0.15632927417755127, + 0.15259043872356415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917178869247437, + "mean_weights": [ + 0.1659121811389923, + 0.16851314902305603, + 0.16693055629730225, + 0.16679996252059937, + 0.16575691103935242, + 0.16608723998069763 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7883799076080322, + "mean_weights": [ + 0.1728566288948059, + 0.18512636423110962, + 0.17585304379463196, + 0.16398008167743683, + 0.1537502557039261, + 0.1484336107969284 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917424440383911, + "mean_weights": [ + 0.1667865514755249, + 0.1680906116962433, + 0.16619735956192017, + 0.16709285974502563, + 0.16617436707019806, + 0.16565825045108795 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9426109790802002, + "mean_weights": [ + 0.14892546832561493, + 0.16000479459762573, + 0.15354622900485992, + 0.14328595995903015, + 0.13467344641685486, + 0.12974688410758972, + 0.12981724739074707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1419999748468399, + 0.1434222161769867, + 0.14279979467391968, + 0.14312142133712769, + 0.14279043674468994, + 0.1428302675485611, + 0.143035888671875 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415357112884521, + "mean_weights": [ + 0.1496729701757431, + 0.16313126683235168, + 0.15414540469646454, + 0.1440294086933136, + 0.1343868225812912, + 0.12868303060531616, + 0.1259511113166809 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458801746368408, + "mean_weights": [ + 0.1429852843284607, + 0.1429750919342041, + 0.14214137196540833, + 0.1414642632007599, + 0.1421438455581665, + 0.14385518431663513, + 0.14443498849868774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9424511194229126, + "mean_weights": [ + 0.14928653836250305, + 0.16045087575912476, + 0.15314346551895142, + 0.1440504789352417, + 0.1356543004512787, + 0.13001246750354767, + 0.1274019032716751 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458882808685303, + "mean_weights": [ + 0.14319148659706116, + 0.14391475915908813, + 0.14284171164035797, + 0.1420421153306961, + 0.14134715497493744, + 0.14342516660690308, + 0.1432376205921173 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395697116851807, + "mean_weights": [ + 0.15016257762908936, + 0.16719450056552887, + 0.15599417686462402, + 0.14574657380580902, + 0.13377557694911957, + 0.12622776627540588, + 0.12089881300926208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1428529918193817, + 0.14342239499092102, + 0.14335417747497559, + 0.14274317026138306, + 0.1419316530227661, + 0.14220726490020752, + 0.14348836243152618 + ], + "sources": 7 + }, + { + "entropy_mean": 2.074862480163574, + "mean_weights": [ + 0.13159659504890442, + 0.14486080408096313, + 0.13573890924453735, + 0.12842535972595215, + 0.12030278891324997, + 0.11495959758758545, + 0.10915482044219971, + 0.11496111750602722 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079418659210205, + "mean_weights": [ + 0.12459750473499298, + 0.12504151463508606, + 0.12489251047372818, + 0.125042125582695, + 0.12461285293102264, + 0.12601442635059357, + 0.1261361837387085, + 0.12366288155317307 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0741164684295654, + "mean_weights": [ + 0.1323997974395752, + 0.14626389741897583, + 0.1369163542985916, + 0.12836164236068726, + 0.12002508342266083, + 0.11502262949943542, + 0.10908686369657516, + 0.11192375421524048 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793213844299316, + "mean_weights": [ + 0.12429114431142807, + 0.12243181467056274, + 0.12357296049594879, + 0.12375843524932861, + 0.12469147145748138, + 0.12560917437076569, + 0.12766826152801514, + 0.12797673046588898 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0734808444976807, + "mean_weights": [ + 0.1330580711364746, + 0.14781729876995087, + 0.1374434530735016, + 0.12885303795337677, + 0.11949820816516876, + 0.11495119333267212, + 0.10982058197259903, + 0.10855817794799805 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07940673828125, + "mean_weights": [ + 0.12457560002803802, + 0.12431882321834564, + 0.12479400634765625, + 0.12437158077955246, + 0.12397614866495132, + 0.12488298863172531, + 0.12619814276695251, + 0.12688270211219788 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07299542427063, + "mean_weights": [ + 0.13274571299552917, + 0.14826728403568268, + 0.13701492547988892, + 0.12918588519096375, + 0.12125477194786072, + 0.1169566735625267, + 0.10889489948749542, + 0.10567984730005264 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079298734664917, + "mean_weights": [ + 0.12391229718923569, + 0.12222036719322205, + 0.12298265099525452, + 0.12416766583919525, + 0.125657856464386, + 0.12553520500659943, + 0.126623272895813, + 0.1289007067680359 + ], + "sources": 8 + }, + { + "entropy_mean": 2.189734935760498, + "mean_weights": [ + 0.1203557699918747, + 0.13466420769691467, + 0.12461012601852417, + 0.11655421555042267, + 0.10905998200178146, + 0.10487955063581467, + 0.09900732338428497, + 0.0950026735663414, + 0.0958661437034607 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197068691253662, + "mean_weights": [ + 0.1099812239408493, + 0.10853179544210434, + 0.10949578881263733, + 0.11001666635274887, + 0.1107073426246643, + 0.11118210852146149, + 0.11218173801898956, + 0.11490923166275024, + 0.11299411952495575 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907877922058105, + "mean_weights": [ + 0.11948458850383759, + 0.13228623569011688, + 0.12348943203687668, + 0.11669500172138214, + 0.11027230322360992, + 0.1054498627781868, + 0.10031206905841827, + 0.09697622060775757, + 0.09503427147865295 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971726417541504, + "mean_weights": [ + 0.11100216209888458, + 0.11012113094329834, + 0.1099993959069252, + 0.11000204086303711, + 0.10997181385755539, + 0.11260797083377838, + 0.11264219880104065, + 0.11185945570468903, + 0.11179381608963013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.190375804901123, + "mean_weights": [ + 0.11983220279216766, + 0.13351142406463623, + 0.12386792153120041, + 0.11615028977394104, + 0.10967587679624557, + 0.10584976524114609, + 0.10062196850776672, + 0.09596225619316101, + 0.09452830255031586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969356536865234, + "mean_weights": [ + 0.10960094630718231, + 0.10724944621324539, + 0.10892992466688156, + 0.1096237450838089, + 0.11032412201166153, + 0.11112386733293533, + 0.11386257410049438, + 0.11429048329591751, + 0.11499489098787308 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188795566558838, + "mean_weights": [ + 0.12045153975486755, + 0.13515985012054443, + 0.12505559623241425, + 0.11722971498966217, + 0.11039333790540695, + 0.10568177700042725, + 0.09951679408550262, + 0.09492656588554382, + 0.09158482402563095 + ], + "sources": 9 + } + ], + "loss_nats": 3.5924854278564453, + "output_weights": { + "entropy_mean": 2.1922647953033447, + "mean_weights": [ + 0.09991011023521423, + 0.0929517149925232, + 0.10029369592666626, + 0.10801893472671509, + 0.11530699580907822, + 0.11817377060651779, + 0.12114627659320831, + 0.12246126681566238, + 0.12173722684383392 + ], + "sources": 9 + }, + "step": 100 + }, + { + "activation_grad_rms_by_block": [ + 0.0001494410098530352, + 0.00015486938355024904, + 0.00014897517394274473, + 0.0001431136333849281, + 0.0001401594199705869, + 0.00012453079398255795, + 0.00011206007184227929, + 0.00010208225285168737, + 9.998067253036425e-05, + 9.082697215490043e-05, + 8.336143946507946e-05, + 7.669178739888594e-05, + 7.405834912788123e-05, + 6.847811891930178e-05, + 6.22319639660418e-05, + 5.739524931414053e-05, + 5.947135286987759e-05, + 5.3890027629677206e-05, + 4.890635682386346e-05, + 4.4437034375732765e-05, + 4.31341104558669e-05, + 3.886142803821713e-05, + 3.5836677852785215e-05, + 3.340865805512294e-05, + 3.223997919121757e-05, + 3.0436871384154074e-05, + 2.9292581530171447e-05, + 2.8587397537194192e-05, + 2.2913789507583715e-05, + 2.2530648493557237e-05, + 2.244284223706927e-05, + 2.2519136109622195e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013440396742225857, + "first_to_last_ratio": 5.096772825661455, + "imbalance_abs_log_ratio": 1.6286075601589056, + "last_quartile_mean": 2.6370405748821213e-05, + "mean": 7.053641201082428e-05, + "normalized": [ + 2.1186363977530145, + 2.195594858531836, + 2.112032207136982, + 2.0289327073081966, + 1.9870506023056362, + 1.7654824002594274, + 1.5886840377574483, + 1.4472277500593902, + 1.4174334883240383, + 1.287660792002894, + 1.1818213754945048, + 1.0872652182409996, + 1.0499307664886113, + 0.9708194245660434, + 0.8822672176250175, + 0.8136967514782698, + 0.8431298271983457, + 0.7640029609304118, + 0.6933490863748281, + 0.6299871670381194, + 0.6115155169679979, + 0.5509413780821966, + 0.5080592679889337, + 0.47363704932987183, + 0.4570686014801849, + 0.43150580695093105, + 0.4152831239229507, + 0.4052856775987878, + 0.32485051130850606, + 0.319418692435047, + 0.31817385655546626, + 0.3192554805051111 + ], + "population_cv": 0.606636113722199 + }, + "activation_output_rms_by_block": [ + 0.021243074908852577, + 0.03624089062213898, + 0.04951142147183418, + 0.05851924046874046, + 0.017167888581752777, + 0.02674153819680214, + 0.03352004662156105, + 0.03946679085493088, + 0.020600929856300354, + 0.027865095064044, + 0.035815127193927765, + 0.04293550178408623, + 0.021564343944191933, + 0.029616132378578186, + 0.035480253398418427, + 0.04259813204407692, + 0.021436138078570366, + 0.03586751967668533, + 0.04331304505467415, + 0.051551222801208496, + 0.025608548894524574, + 0.04091007262468338, + 0.05466190353035927, + 0.07478146255016327, + 0.02793189324438572, + 0.05498877540230751, + 0.08125203847885132, + 0.10916247963905334, + 0.026290444657206535, + 0.05761713907122612, + 0.08842677623033524, + 0.12436132878065109 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03530136146582663, + "first_to_last_ratio": 0.495430868506741, + "imbalance_abs_log_ratio": 0.7023274536001348, + "last_quartile_mean": 0.07125385943800211, + "mean": 0.04553272487828508, + "normalized": [ + 0.4665452148018396, + 0.7959306349228079, + 1.0873810342821493, + 1.2852127920121195, + 0.37704505117242126, + 0.5873037088881845, + 0.7361748437231577, + 0.8667785853016872, + 0.452442279950724, + 0.6119795188741951, + 0.7865799222354093, + 0.9429591991003927, + 0.4736009980038805, + 0.6504361963345259, + 0.7792253482140983, + 0.9355498085815704, + 0.47078531179216626, + 0.7877305777891374, + 0.9512508897821419, + 1.1321796123340224, + 0.5624207416309823, + 0.8984762658953829, + 1.2004970859195814, + 1.6423673907077576, + 0.6134465556157976, + 1.2076759198861848, + 1.784475642431869, + 2.397451062523908, + 0.5773966905667149, + 1.2654006371295428, + 1.9420488553389141, + 2.7312516242567333 + ], + "population_cv": 0.5563959277170655 + }, + "bits_per_byte": 3.0745070201808953, + "core_parameter_grad_rms_by_block": [ + 0.0004975869912568777, + 0.0004115293940014644, + 0.00043817390286628833, + 0.00039988528646951293, + 0.0003968514715139995, + 0.00031053290415162903, + 0.00024154094413850278, + 0.00022869177850848996, + 0.0002476899080691786, + 0.00021028907646624228, + 0.0002010354933691209, + 0.0001845208571901889, + 0.0002273114134138899, + 0.0001715437435350543, + 0.00015668460984383524, + 0.00015894955615086492, + 0.00016556491904516564, + 0.0001408361881569501, + 0.00014198814555526783, + 0.00012254378127881199, + 0.00011450446233020633, + 0.0001100879077446283, + 0.00010549256620556044, + 9.975908855450905e-05, + 0.00010071115476761671, + 9.794352566848755e-05, + 9.525064309078406e-05, + 8.226123370955787e-05, + 6.340888944803813e-05, + 6.739220497758678e-05, + 6.650164980556211e-05, + 7.228417264196778e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003655990841133456, + "first_to_last_ratio": 4.5292712934136725, + "imbalance_abs_log_ratio": 1.5105610641741143, + "last_quartile_mean": 8.071918426370012e-05, + "mean": 0.0001915421207476825, + "normalized": [ + 2.5977941004023166, + 2.148505991241321, + 2.28761121134026, + 2.0877146233348847, + 2.071875731379053, + 1.621225153712757, + 1.2610330469123472, + 1.1939503312158923, + 1.293135458155751, + 1.0978738026152224, + 1.0495628459389565, + 0.963343501000792, + 1.186743743499249, + 0.8955927963282188, + 0.8180164719499744, + 0.8298412669255573, + 0.8643786463201141, + 0.7352752888356756, + 0.7412894093529857, + 0.6397745874404216, + 0.5978030413531993, + 0.574745164744451, + 0.5507538801062211, + 0.5208206328983963, + 0.5257911647552604, + 0.5113419716048152, + 0.4972830138821386, + 0.4294681158820424, + 0.33104410246958865, + 0.3518401316353921, + 0.3471907356250086, + 0.3773800371417385 + ], + "population_cv": 0.6210714087578444 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6506600379943848, + "mean_weights": [ + 0.3747701942920685, + 0.6252298355102539 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6806463003158569, + "mean_weights": [ + 0.45339685678482056, + 0.5466030836105347 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6869370937347412, + "mean_weights": [ + 0.4668571949005127, + 0.5331428050994873 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6791321635246277, + "mean_weights": [ + 0.4727538824081421, + 0.5272461175918579 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6804035902023315, + "mean_weights": [ + 0.4487338662147522, + 0.5512661933898926 + ], + "sources": 2 + }, + { + "entropy_mean": 0.682766318321228, + "mean_weights": [ + 0.4646179676055908, + 0.5353820323944092 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873732805252075, + "mean_weights": [ + 0.4883750379085541, + 0.5116249322891235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6835889220237732, + "mean_weights": [ + 0.46665582060813904, + 0.5333442091941833 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0856965780258179, + "mean_weights": [ + 0.27463093400001526, + 0.3524433374404907, + 0.372925728559494 + ], + "sources": 3 + }, + { + "entropy_mean": 1.077350378036499, + "mean_weights": [ + 0.2904614210128784, + 0.3131614923477173, + 0.3963770568370819 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0850789546966553, + "mean_weights": [ + 0.27783387899398804, + 0.34068456292152405, + 0.3814815878868103 + ], + "sources": 3 + }, + { + "entropy_mean": 1.062077522277832, + "mean_weights": [ + 0.28390786051750183, + 0.29115602374076843, + 0.42493611574172974 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0936429500579834, + "mean_weights": [ + 0.30466002225875854, + 0.3333379328250885, + 0.36200207471847534 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0627415180206299, + "mean_weights": [ + 0.2850727438926697, + 0.29934102296829224, + 0.4155862331390381 + ], + "sources": 3 + }, + { + "entropy_mean": 1.087097406387329, + "mean_weights": [ + 0.28865373134613037, + 0.3320052921772003, + 0.3793409764766693 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0500720739364624, + "mean_weights": [ + 0.2745887339115143, + 0.286259263753891, + 0.4391520023345947 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3743348121643066, + "mean_weights": [ + 0.2166871875524521, + 0.23293715715408325, + 0.2495419979095459, + 0.30083364248275757 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3609715700149536, + "mean_weights": [ + 0.23144879937171936, + 0.21839870512485504, + 0.24986028671264648, + 0.3002921938896179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3810515403747559, + "mean_weights": [ + 0.2385442852973938, + 0.23454180359840393, + 0.24874086678028107, + 0.27817302942276 + ], + "sources": 4 + }, + { + "entropy_mean": 1.348030686378479, + "mean_weights": [ + 0.23874622583389282, + 0.21300309896469116, + 0.22721335291862488, + 0.32103732228279114 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3825427293777466, + "mean_weights": [ + 0.24534732103347778, + 0.2375141680240631, + 0.25072187185287476, + 0.26641663908958435 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3499886989593506, + "mean_weights": [ + 0.24629786610603333, + 0.22150886058807373, + 0.2235339879989624, + 0.30865928530693054 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3807358741760254, + "mean_weights": [ + 0.24433431029319763, + 0.23110222816467285, + 0.23992134630680084, + 0.2846421003341675 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3399771451950073, + "mean_weights": [ + 0.24593281745910645, + 0.21928201615810394, + 0.21277041733264923, + 0.3220147490501404 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6075894832611084, + "mean_weights": [ + 0.19650746881961823, + 0.19567455351352692, + 0.18970930576324463, + 0.21128275990486145, + 0.20682592689990997 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5706448554992676, + "mean_weights": [ + 0.1975565403699875, + 0.1772933304309845, + 0.16386017203330994, + 0.21728253364562988, + 0.2440074384212494 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6042816638946533, + "mean_weights": [ + 0.21478912234306335, + 0.18552109599113464, + 0.17930489778518677, + 0.20310312509536743, + 0.2172817587852478 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5764570236206055, + "mean_weights": [ + 0.20157726109027863, + 0.17631688714027405, + 0.1703747808933258, + 0.19828173518180847, + 0.25344932079315186 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6011263132095337, + "mean_weights": [ + 0.20813238620758057, + 0.18490281701087952, + 0.17333751916885376, + 0.20533671975135803, + 0.22829055786132812 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5817476511001587, + "mean_weights": [ + 0.20727968215942383, + 0.18177060782909393, + 0.17236655950546265, + 0.18925222754478455, + 0.24933092296123505 + ], + "sources": 5 + }, + { + "entropy_mean": 1.603710412979126, + "mean_weights": [ + 0.2016504406929016, + 0.1979413628578186, + 0.17889465391635895, + 0.19759604334831238, + 0.22391751408576965 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5885388851165771, + "mean_weights": [ + 0.20988406240940094, + 0.18461555242538452, + 0.17252585291862488, + 0.19482356309890747, + 0.2381509691476822 + ], + "sources": 5 + }, + { + "entropy_mean": 1.783994197845459, + "mean_weights": [ + 0.16140900552272797, + 0.16211673617362976, + 0.15125815570354462, + 0.1585075855255127, + 0.17266258597373962, + 0.19404590129852295 + ], + "sources": 6 + }, + { + "entropy_mean": 1.765566349029541, + "mean_weights": [ + 0.1630604863166809, + 0.1509958952665329, + 0.14303800463676453, + 0.14991191029548645, + 0.16761469841003418, + 0.22537899017333984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7848252058029175, + "mean_weights": [ + 0.16126418113708496, + 0.1664336919784546, + 0.15361233055591583, + 0.1546509563922882, + 0.16693615913391113, + 0.19710269570350647 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7731435298919678, + "mean_weights": [ + 0.16123875975608826, + 0.15981441736221313, + 0.1509810984134674, + 0.14913737773895264, + 0.16297483444213867, + 0.2158534824848175 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7815005779266357, + "mean_weights": [ + 0.15527218580245972, + 0.17254948616027832, + 0.15193141996860504, + 0.15522462129592896, + 0.1650869995355606, + 0.19993525743484497 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7801456451416016, + "mean_weights": [ + 0.1661754697561264, + 0.16477231681346893, + 0.15300315618515015, + 0.15292593836784363, + 0.1582789272069931, + 0.2048441767692566 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7743862867355347, + "mean_weights": [ + 0.15205824375152588, + 0.1771804541349411, + 0.15399661660194397, + 0.15122173726558685, + 0.15813913941383362, + 0.20740382373332977 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781898856163025, + "mean_weights": [ + 0.17058727145195007, + 0.1644459068775177, + 0.15141303837299347, + 0.15486885607242584, + 0.15909278392791748, + 0.19959214329719543 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9389331340789795, + "mean_weights": [ + 0.12895603477954865, + 0.15703952312469482, + 0.13832643628120422, + 0.13419194519519806, + 0.1312965601682663, + 0.15331263840198517, + 0.15687686204910278 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941878080368042, + "mean_weights": [ + 0.14007402956485748, + 0.15248695015907288, + 0.14026451110839844, + 0.13737857341766357, + 0.1305495798587799, + 0.14631187915802002, + 0.1529344916343689 + ], + "sources": 7 + }, + { + "entropy_mean": 1.936349630355835, + "mean_weights": [ + 0.12599323689937592, + 0.16029544174671173, + 0.1399340182542801, + 0.13338115811347961, + 0.12978437542915344, + 0.14873966574668884, + 0.16187208890914917 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9385986328125, + "mean_weights": [ + 0.1397915780544281, + 0.15300613641738892, + 0.14345945417881012, + 0.13018865883350372, + 0.1257915198802948, + 0.14249205589294434, + 0.1652706116437912 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939380407333374, + "mean_weights": [ + 0.12855647504329681, + 0.15473899245262146, + 0.1372847855091095, + 0.13797816634178162, + 0.13512183725833893, + 0.15090687572956085, + 0.15541282296180725 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9388402700424194, + "mean_weights": [ + 0.14282521605491638, + 0.15391701459884644, + 0.1462559849023819, + 0.13105326890945435, + 0.12675246596336365, + 0.13615548610687256, + 0.16304057836532593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9313902854919434, + "mean_weights": [ + 0.11694864928722382, + 0.17515113949775696, + 0.15024805068969727, + 0.1394161432981491, + 0.13194763660430908, + 0.1451682150363922, + 0.14112016558647156 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9366493225097656, + "mean_weights": [ + 0.14114168286323547, + 0.16076743602752686, + 0.14226865768432617, + 0.12737974524497986, + 0.12711969017982483, + 0.1412842869758606, + 0.16003848612308502 + ], + "sources": 7 + }, + { + "entropy_mean": 2.064785957336426, + "mean_weights": [ + 0.106104776263237, + 0.15484434366226196, + 0.13487765192985535, + 0.12357068061828613, + 0.12023378908634186, + 0.12704278528690338, + 0.11963208019733429, + 0.11369390785694122 + ], + "sources": 8 + }, + { + "entropy_mean": 2.062044143676758, + "mean_weights": [ + 0.11719764769077301, + 0.14862585067749023, + 0.12573552131652832, + 0.12179906666278839, + 0.11081404983997345, + 0.1260838657617569, + 0.12823458015918732, + 0.12150939553976059 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054962158203125, + "mean_weights": [ + 0.10514847934246063, + 0.16456037759780884, + 0.14191412925720215, + 0.1262327879667282, + 0.1181878000497818, + 0.12522782385349274, + 0.1187765821814537, + 0.09995201230049133 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0610406398773193, + "mean_weights": [ + 0.11198848485946655, + 0.15103809535503387, + 0.13588255643844604, + 0.12388211488723755, + 0.11551118642091751, + 0.12874530255794525, + 0.12429477274417877, + 0.10865749418735504 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0338308811187744, + "mean_weights": [ + 0.0961676836013794, + 0.17635446786880493, + 0.14848342537879944, + 0.13749772310256958, + 0.12027012556791306, + 0.12292921543121338, + 0.10873906314373016, + 0.08955828845500946 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068699598312378, + "mean_weights": [ + 0.11418519914150238, + 0.14292436838150024, + 0.1359846293926239, + 0.13056153059005737, + 0.12100572884082794, + 0.11985942721366882, + 0.12221725285053253, + 0.1132618635892868 + ], + "sources": 8 + }, + { + "entropy_mean": 2.036879539489746, + "mean_weights": [ + 0.09746269136667252, + 0.17786666750907898, + 0.15074899792671204, + 0.13200224936008453, + 0.11842608451843262, + 0.12164927273988724, + 0.1129462867975235, + 0.08889776468276978 + ], + "sources": 8 + }, + { + "entropy_mean": 2.06752347946167, + "mean_weights": [ + 0.11505836993455887, + 0.14705273509025574, + 0.1377231776714325, + 0.12483140081167221, + 0.11869609355926514, + 0.1215740442276001, + 0.12386028468608856, + 0.1112038791179657 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1517844200134277, + "mean_weights": [ + 0.08786984533071518, + 0.16287443041801453, + 0.13886377215385437, + 0.12039706110954285, + 0.10935483872890472, + 0.1119438111782074, + 0.10067413747310638, + 0.07631799578666687, + 0.0917041003704071 + ], + "sources": 9 + }, + { + "entropy_mean": 2.177518129348755, + "mean_weights": [ + 0.10418819636106491, + 0.13366681337356567, + 0.12014994025230408, + 0.11398349702358246, + 0.10732515156269073, + 0.12029910087585449, + 0.11035183072090149, + 0.09321349114179611, + 0.09682197868824005 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1535391807556152, + "mean_weights": [ + 0.08976081013679504, + 0.163266122341156, + 0.13892531394958496, + 0.12143255770206451, + 0.10986581444740295, + 0.11029069125652313, + 0.10418520867824554, + 0.08120252937078476, + 0.0810709297657013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191593647003174, + "mean_weights": [ + 0.10739251971244812, + 0.12338980287313461, + 0.11759916692972183, + 0.1066875234246254, + 0.1060015857219696, + 0.1124890148639679, + 0.11740898340940475, + 0.10738331079483032, + 0.10164809972047806 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1465134620666504, + "mean_weights": [ + 0.08895281702280045, + 0.1655423492193222, + 0.14156755805015564, + 0.12290289252996445, + 0.11169517785310745, + 0.11284246295690536, + 0.1037452220916748, + 0.0791342705488205, + 0.07361726462841034 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1857833862304688, + "mean_weights": [ + 0.10998444259166718, + 0.12038750946521759, + 0.11211743205785751, + 0.11047163605690002, + 0.10957512259483337, + 0.12139561027288437, + 0.11392621695995331, + 0.10294321179389954, + 0.09919881820678711 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1400556564331055, + "mean_weights": [ + 0.08778931945562363, + 0.16802933812141418, + 0.1466498225927353, + 0.12668846547603607, + 0.11206841468811035, + 0.11208955943584442, + 0.10068599879741669, + 0.07686632871627808, + 0.0691327303647995 + ], + "sources": 9 + } + ], + "loss_nats": 2.1310858726501465, + "output_weights": { + "entropy_mean": 2.1613364219665527, + "mean_weights": [ + 0.1255333125591278, + 0.08712290227413177, + 0.09080196917057037, + 0.08838342130184174, + 0.09530800580978394, + 0.09807588160037994, + 0.13817133009433746, + 0.15537576377391815, + 0.12122740596532822 + ], + "sources": 9 + }, + "step": 500 + }, + { + "activation_grad_rms_by_block": [ + 6.294298509601504e-05, + 6.491570093203336e-05, + 6.380436389008537e-05, + 6.04347005719319e-05, + 7.516580808442086e-05, + 7.751413795631379e-05, + 6.965622014831752e-05, + 6.359309190884233e-05, + 6.543538620462641e-05, + 6.0226317145861685e-05, + 5.608194260275923e-05, + 5.3055387979839e-05, + 4.572631951305084e-05, + 4.3727712181862444e-05, + 4.314268517191522e-05, + 4.164052370470017e-05, + 7.694400119362399e-05, + 7.326623017434031e-05, + 6.749018211849034e-05, + 6.173704605316743e-05, + 0.00010233377543045208, + 8.934178913477808e-05, + 7.613583147758618e-05, + 6.664563989033923e-05, + 5.163387322681956e-05, + 4.2875533836195245e-05, + 3.8466609112219885e-05, + 3.59200312232133e-05, + 2.308324837940745e-05, + 2.0679148292401806e-05, + 1.9936938770115376e-05, + 1.9895105651812628e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.725337607349502e-05, + "first_to_last_ratio": 2.130880302861834, + "imbalance_abs_log_ratio": 0.7565351820700446, + "last_quartile_mean": 3.1561311061523156e-05, + "mean": 5.6670258345548064e-05, + "normalized": [ + 1.1106881622493918, + 1.145498588275503, + 1.1258880010928651, + 1.0664271230850946, + 1.3263713679598177, + 1.367809856868302, + 1.2291495077292092, + 1.122159908308202, + 1.154668923611267, + 1.062750001572791, + 0.9896186154790125, + 0.9362122130506743, + 0.8068839078557516, + 0.7716166020495587, + 0.7612932503122151, + 0.7347861986228512, + 1.357749257546284, + 1.2928515294142118, + 1.190927729797305, + 1.089408233799192, + 1.8057756999530474, + 1.5765198843812338, + 1.3434883429213678, + 1.176024987992221, + 0.9111282484717285, + 0.7565791137700626, + 0.678779490957478, + 0.6338427293588494, + 0.4073256246452393, + 0.3649030178459797, + 0.3518060328673549, + 0.35106784815593767 + ], + "population_cv": 0.34914284904214865 + }, + "activation_output_rms_by_block": [ + 0.10241243988275528, + 0.1819411665201187, + 0.24157075583934784, + 0.26444777846336365, + 0.10636615753173828, + 0.13034582138061523, + 0.14029134809970856, + 0.15042904019355774, + 0.04812335968017578, + 0.07954440265893936, + 0.10673297941684723, + 0.14089107513427734, + 0.06885477155447006, + 0.10946234315633774, + 0.13876035809516907, + 0.1687854677438736, + 0.05059739947319031, + 0.07094386219978333, + 0.09166282415390015, + 0.12053482979536057, + 0.048727843910455704, + 0.07109228521585464, + 0.09690473228693008, + 0.11853098124265671, + 0.05259029567241669, + 0.08889466524124146, + 0.12455403059720993, + 0.1728687882423401, + 0.06489895284175873, + 0.14679722487926483, + 0.23576539754867554, + 0.35208994150161743 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16472556348890066, + "first_to_last_ratio": 1.0640676779683806, + "imbalance_abs_log_ratio": 0.0620989960097457, + "last_quartile_mean": 0.15480741206556559, + "mean": 0.127700416254811, + "normalized": [ + 0.8019742056157706, + 1.4247499879489567, + 1.8916990478505735, + 2.0708450780276983, + 0.8329350886335192, + 1.020715712629516, + 1.098597422108428, + 1.177983945591802, + 0.3768457542389787, + 0.6228985385624584, + 0.8358076077361742, + 1.1032937813855357, + 0.5391898755997675, + 0.8571807858317286, + 1.086608502655851, + 1.3217299731199175, + 0.396219534415841, + 0.5555491851978249, + 0.7177958133746243, + 0.9438875246486875, + 0.3815793662976406, + 0.5567114603134765, + 0.7588442945524017, + 0.9281957312194052, + 0.411825561848436, + 0.6961188369493074, + 0.9753611949758815, + 1.3537057537651325, + 0.5082125395132666, + 1.1495438244018596, + 1.8462382853806345, + 2.7571557856089037 + ], + "population_cv": 0.532159231456062 + }, + "bits_per_byte": 2.0533380169036586, + "core_parameter_grad_rms_by_block": [ + 0.00034319970120439075, + 0.0002207216513437987, + 0.00019628105547348242, + 0.00016658832628724645, + 0.00021517433675904697, + 0.00019244240293185932, + 0.0001402257608174488, + 0.00015224450647613049, + 0.00018347185792801386, + 0.00016427150761043105, + 0.00015319316983403424, + 0.00015483500137268905, + 0.00013970482137555046, + 0.00012521709576159806, + 0.00014171058058500498, + 0.00014304258084243227, + 0.00024403444477986432, + 0.0002443248488541129, + 0.0002486173108247806, + 0.0002822915928542698, + 0.00039666535280576766, + 0.0003607137256748422, + 0.0003593407909791253, + 0.00030791631238161, + 0.0002432000733486066, + 0.00020051667741228585, + 0.0001766506293272091, + 0.00017705575210645458, + 0.00010859706957547057, + 0.0001081942080930384, + 0.00010514694748959741, + 0.00011321633644894032 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002033597176616755, + "first_to_last_ratio": 1.3198987369921267, + "imbalance_abs_log_ratio": 0.2775550192556754, + "last_quartile_mean": 0.00015407221172520036, + "mean": 0.00020340020092372293, + "normalized": [ + 1.6873124984428802, + 1.085159455799021, + 0.9649993194799731, + 0.8190175109498476, + 1.0578865496781857, + 0.9461269067478802, + 0.6894081725614166, + 0.7484973258862397, + 0.9020239758603661, + 0.8076270665633928, + 0.7531613495872759, + 0.7612332764152662, + 0.6868470175599342, + 0.6156193317063423, + 0.6967081642075066, + 0.703256831570558, + 1.1997748461978148, + 1.2012025934317396, + 1.2223061220967748, + 1.3878629006867693, + 1.9501718828415566, + 1.7734187283822467, + 1.7666688102922847, + 1.5138446814862372, + 1.1956727291523621, + 0.985823398903532, + 0.8684879785023163, + 0.8704797306117324, + 0.5339083692262209, + 0.5319277345926137, + 0.516946133839015, + 0.5566186067406962 + ], + "population_cv": 0.38898494230064556 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.39672940969467163, + "mean_weights": [ + 0.854300856590271, + 0.1456991732120514 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6258258819580078, + "mean_weights": [ + 0.5948886871337891, + 0.40511131286621094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5053832530975342, + "mean_weights": [ + 0.778947651386261, + 0.221052348613739 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6737473011016846, + "mean_weights": [ + 0.5406605005264282, + 0.4593394994735718 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6529731750488281, + "mean_weights": [ + 0.5619670748710632, + 0.43803298473358154 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6891955733299255, + "mean_weights": [ + 0.5099825263023376, + 0.49001747369766235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48565131425857544, + "mean_weights": [ + 0.7992967367172241, + 0.20070326328277588 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6875144243240356, + "mean_weights": [ + 0.5191382169723511, + 0.4808618128299713 + ], + "sources": 2 + }, + { + "entropy_mean": 1.048142671585083, + "mean_weights": [ + 0.3059995770454407, + 0.2586204409599304, + 0.4353799819946289 + ], + "sources": 3 + }, + { + "entropy_mean": 1.078475832939148, + "mean_weights": [ + 0.2919672429561615, + 0.28997206687927246, + 0.4180607199668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0040717124938965, + "mean_weights": [ + 0.27878478169441223, + 0.21860124170780182, + 0.5026139616966248 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0727062225341797, + "mean_weights": [ + 0.3306647837162018, + 0.2443503737449646, + 0.42498481273651123 + ], + "sources": 3 + }, + { + "entropy_mean": 1.036304235458374, + "mean_weights": [ + 0.3365095853805542, + 0.20819777250289917, + 0.455292671918869 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071068525314331, + "mean_weights": [ + 0.3291535973548889, + 0.24366843700408936, + 0.42717796564102173 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0272725820541382, + "mean_weights": [ + 0.29237720370292664, + 0.2188395857810974, + 0.48878318071365356 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856674909591675, + "mean_weights": [ + 0.3748946189880371, + 0.26831528544425964, + 0.35679009556770325 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3326265811920166, + "mean_weights": [ + 0.26391589641571045, + 0.14978256821632385, + 0.2545396089553833, + 0.3317619264125824 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3771977424621582, + "mean_weights": [ + 0.27525436878204346, + 0.2067744880914688, + 0.23979949951171875, + 0.2781716585159302 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3576545715332031, + "mean_weights": [ + 0.2848266065120697, + 0.17341844737529755, + 0.23106196522712708, + 0.3106929659843445 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772028684616089, + "mean_weights": [ + 0.2702856659889221, + 0.21318203210830688, + 0.23082655668258667, + 0.2857057452201843 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3277652263641357, + "mean_weights": [ + 0.3119839131832123, + 0.16646917164325714, + 0.1912841796875, + 0.3302627205848694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3750126361846924, + "mean_weights": [ + 0.2621780037879944, + 0.21051618456840515, + 0.23754313588142395, + 0.2897626757621765 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568847179412842, + "mean_weights": [ + 0.2628105878829956, + 0.1954234540462494, + 0.21842113137245178, + 0.3233448266983032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3733241558074951, + "mean_weights": [ + 0.25842776894569397, + 0.22964225709438324, + 0.21348324418067932, + 0.2984467148780823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5855276584625244, + "mean_weights": [ + 0.18060272932052612, + 0.1563272476196289, + 0.18859915435314178, + 0.24991817772388458, + 0.2245527058839798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5906484127044678, + "mean_weights": [ + 0.19773373007774353, + 0.18790484964847565, + 0.15935488045215607, + 0.18813443183898926, + 0.2668721079826355 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5916568040847778, + "mean_weights": [ + 0.2148856818675995, + 0.14808836579322815, + 0.18413493037223816, + 0.22754491865634918, + 0.22534608840942383 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5474352836608887, + "mean_weights": [ + 0.19825294613838196, + 0.1729484647512436, + 0.11353601515293121, + 0.19613605737686157, + 0.3191264867782593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5919160842895508, + "mean_weights": [ + 0.21136541664600372, + 0.1572093367576599, + 0.18091553449630737, + 0.2257765829563141, + 0.2247331142425537 + ], + "sources": 5 + }, + { + "entropy_mean": 1.542572021484375, + "mean_weights": [ + 0.2053474485874176, + 0.19789552688598633, + 0.10896679759025574, + 0.1649853140115738, + 0.32280486822128296 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5774884223937988, + "mean_weights": [ + 0.1874420940876007, + 0.14777487516403198, + 0.18392466008663177, + 0.23523849248886108, + 0.24561989307403564 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585869312286377, + "mean_weights": [ + 0.2123720645904541, + 0.19153764843940735, + 0.14032581448554993, + 0.1918477714061737, + 0.26391667127609253 + ], + "sources": 5 + }, + { + "entropy_mean": 1.747511625289917, + "mean_weights": [ + 0.13209694623947144, + 0.11971477419137955, + 0.14703533053398132, + 0.17972922325134277, + 0.15732964873313904, + 0.2640940546989441 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6791677474975586, + "mean_weights": [ + 0.1429731547832489, + 0.128341943025589, + 0.08682706952095032, + 0.12974600493907928, + 0.16802877187728882, + 0.3440830111503601 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7079570293426514, + "mean_weights": [ + 0.12207645177841187, + 0.11454501748085022, + 0.14774824678897858, + 0.1675407588481903, + 0.1342848241329193, + 0.3138047158718109 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689455270767212, + "mean_weights": [ + 0.14791087806224823, + 0.15263086557388306, + 0.0942135751247406, + 0.12123478949069977, + 0.14674627780914307, + 0.3372636139392853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6673545837402344, + "mean_weights": [ + 0.119766965508461, + 0.10584045946598053, + 0.1391725242137909, + 0.15867865085601807, + 0.1261359602212906, + 0.3504054546356201 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7276721000671387, + "mean_weights": [ + 0.16197484731674194, + 0.16130125522613525, + 0.10335841774940491, + 0.12991490960121155, + 0.15279719233512878, + 0.2906533479690552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.655259609222412, + "mean_weights": [ + 0.12291313707828522, + 0.10535868257284164, + 0.14431551098823547, + 0.1644165962934494, + 0.11736815422773361, + 0.34562790393829346 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7456457614898682, + "mean_weights": [ + 0.17144063115119934, + 0.17272350192070007, + 0.09372985363006592, + 0.13482090830802917, + 0.1837429702281952, + 0.2435421198606491 + ], + "sources": 6 + }, + { + "entropy_mean": 1.8196223974227905, + "mean_weights": [ + 0.097611203789711, + 0.08662202209234238, + 0.10460364818572998, + 0.12231925129890442, + 0.09176328778266907, + 0.22007866203784943, + 0.27700191736221313 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8201665878295898, + "mean_weights": [ + 0.12032124400138855, + 0.12977322936058044, + 0.07435770332813263, + 0.09230788052082062, + 0.1133352667093277, + 0.15118572115898132, + 0.31871896982192993 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7483060359954834, + "mean_weights": [ + 0.09602338075637817, + 0.07951871305704117, + 0.09892410784959793, + 0.11023538559675217, + 0.08284324407577515, + 0.17450374364852905, + 0.35795143246650696 + ], + "sources": 7 + }, + { + "entropy_mean": 1.851926565170288, + "mean_weights": [ + 0.14006076753139496, + 0.14811867475509644, + 0.08932580053806305, + 0.09198777377605438, + 0.11175547540187836, + 0.12843111157417297, + 0.29032039642333984 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7523608207702637, + "mean_weights": [ + 0.10785886645317078, + 0.08607444912195206, + 0.09595336019992828, + 0.10649852454662323, + 0.0838143527507782, + 0.14955362677574158, + 0.3702468276023865 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8693451881408691, + "mean_weights": [ + 0.1378144919872284, + 0.1244172528386116, + 0.09968579560518265, + 0.10974414646625519, + 0.11655184626579285, + 0.130243182182312, + 0.2815432548522949 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7898914813995361, + "mean_weights": [ + 0.10115306824445724, + 0.09615937620401382, + 0.11218561232089996, + 0.1163020133972168, + 0.09238721430301666, + 0.14564841985702515, + 0.33616429567337036 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9094455242156982, + "mean_weights": [ + 0.14588291943073273, + 0.14179158210754395, + 0.1023377850651741, + 0.11511939018964767, + 0.13270339369773865, + 0.13753530383110046, + 0.22462961077690125 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9082129001617432, + "mean_weights": [ + 0.09145341068506241, + 0.07698438316583633, + 0.08525429666042328, + 0.08438773453235626, + 0.06885559856891632, + 0.11482041329145432, + 0.2519546449184418, + 0.2262895107269287 + ], + "sources": 8 + }, + { + "entropy_mean": 2.025789499282837, + "mean_weights": [ + 0.11047068238258362, + 0.12069687247276306, + 0.09073689579963684, + 0.09690164029598236, + 0.10987945646047592, + 0.11353722959756851, + 0.15697816014289856, + 0.20079906284809113 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9302079677581787, + "mean_weights": [ + 0.0971018448472023, + 0.08240298926830292, + 0.0830029845237732, + 0.08926018327474594, + 0.07480385154485703, + 0.10335960984230042, + 0.22211775183677673, + 0.24795079231262207 + ], + "sources": 8 + }, + { + "entropy_mean": 2.010532855987549, + "mean_weights": [ + 0.10700424760580063, + 0.09611141681671143, + 0.09769981354475021, + 0.10596007108688354, + 0.09634587168693542, + 0.11705268919467926, + 0.17551329731941223, + 0.20431260764598846 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9408204555511475, + "mean_weights": [ + 0.08590779453516006, + 0.09530991315841675, + 0.09476494789123535, + 0.1030702069401741, + 0.08048167079687119, + 0.12287819385528564, + 0.22555866837501526, + 0.19202858209609985 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0024678707122803, + "mean_weights": [ + 0.10720442980527878, + 0.09982497990131378, + 0.10896851122379303, + 0.1345594823360443, + 0.09260416775941849, + 0.1016254797577858, + 0.1643495261669159, + 0.19086343050003052 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9389225244522095, + "mean_weights": [ + 0.08632777631282806, + 0.09041796624660492, + 0.09941554069519043, + 0.10805264860391617, + 0.07865813374519348, + 0.11747924983501434, + 0.2661469578742981, + 0.1535017192363739 + ], + "sources": 8 + }, + { + "entropy_mean": 2.008256673812866, + "mean_weights": [ + 0.1016545370221138, + 0.10481878370046616, + 0.10823093354701996, + 0.1270347535610199, + 0.0919865146279335, + 0.14368976652622223, + 0.20522482693195343, + 0.11735989153385162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.094825267791748, + "mean_weights": [ + 0.08519881963729858, + 0.09903685003519058, + 0.08900875598192215, + 0.09697237610816956, + 0.071251779794693, + 0.09201781451702118, + 0.19253811240196228, + 0.1091965064406395, + 0.16477900743484497 + ], + "sources": 9 + }, + { + "entropy_mean": 2.140371322631836, + "mean_weights": [ + 0.10001853853464127, + 0.10924500226974487, + 0.10232416540384293, + 0.11669869720935822, + 0.10123330354690552, + 0.1287832260131836, + 0.14476822316646576, + 0.09069216251373291, + 0.10623672604560852 + ], + "sources": 9 + }, + { + "entropy_mean": 2.117004632949829, + "mean_weights": [ + 0.09184888750314713, + 0.09620077908039093, + 0.09620209038257599, + 0.10222946852445602, + 0.06924223154783249, + 0.0821654349565506, + 0.17978665232658386, + 0.14873988926410675, + 0.13358455896377563 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1423683166503906, + "mean_weights": [ + 0.09596475213766098, + 0.07681103050708771, + 0.11225531250238419, + 0.11592596024274826, + 0.0810609757900238, + 0.11778091639280319, + 0.1649385243654251, + 0.12366649508476257, + 0.11159604042768478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.093705415725708, + "mean_weights": [ + 0.08838655799627304, + 0.09557235240936279, + 0.09992480278015137, + 0.10721968859434128, + 0.07486072182655334, + 0.09290309250354767, + 0.21980023384094238, + 0.12490785866975784, + 0.09642469882965088 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1627297401428223, + "mean_weights": [ + 0.0994885265827179, + 0.09216944873332977, + 0.11588040739297867, + 0.12231601774692535, + 0.09873615205287933, + 0.13436099886894226, + 0.13960200548171997, + 0.1002899631857872, + 0.09715649485588074 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1300058364868164, + "mean_weights": [ + 0.089942067861557, + 0.1065598577260971, + 0.11013364791870117, + 0.12377548217773438, + 0.08726659417152405, + 0.11411209404468536, + 0.1931532621383667, + 0.10132836550474167, + 0.07372863590717316 + ], + "sources": 9 + } + ], + "loss_nats": 1.4232654571533203, + "output_weights": { + "entropy_mean": 1.9258099794387817, + "mean_weights": [ + 0.11242504417896271, + 0.0684729665517807, + 0.07014584541320801, + 0.059657223522663116, + 0.05741160362958908, + 0.05127572640776634, + 0.12547241151332855, + 0.2706507742404938, + 0.18448840081691742 + ], + "sources": 9 + }, + "step": 2000 + }, + { + "activation_grad_rms_by_block": [ + 4.705962055595592e-05, + 4.8540012357989326e-05, + 4.8112680815393105e-05, + 4.5881148253101856e-05, + 6.218662019819021e-05, + 6.19510465185158e-05, + 5.5873108067316934e-05, + 5.11586076754611e-05, + 5.377902198233642e-05, + 4.923442247672938e-05, + 4.591418110067025e-05, + 4.340316809248179e-05, + 3.853498856187798e-05, + 3.637185727711767e-05, + 3.548145105014555e-05, + 3.440591899561696e-05, + 7.517015910707414e-05, + 7.005458610365167e-05, + 6.350460171233863e-05, + 5.674128988175653e-05, + 0.00010789759107865393, + 9.314908675150946e-05, + 7.818391168257222e-05, + 6.73785325489007e-05, + 5.735654485761188e-05, + 4.582547626341693e-05, + 3.993835343862884e-05, + 3.59240839316044e-05, + 2.40640074480325e-05, + 2.1051846488262527e-05, + 1.9940822312491946e-05, + 1.993459591176361e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.259535555524053e-05, + "first_to_last_ratio": 1.59358297228639, + "imbalance_abs_log_ratio": 0.46598492272985975, + "last_quartile_mean": 3.300446633147658e-05, + "mean": 5.106260448428657e-05, + "normalized": [ + 0.921606350307445, + 0.9505980521014451, + 0.9422292752458163, + 0.8985273805847644, + 1.2178505351666271, + 1.2132371065714032, + 1.0942079557361923, + 1.0018801076079868, + 1.0531977858451338, + 0.9641972432463806, + 0.8991742893725557, + 0.8499991046449304, + 0.7546616344988103, + 0.712299296999438, + 0.6948617566317875, + 0.6737987484795189, + 1.4721176067352022, + 1.3719352314903852, + 1.2436616258358075, + 1.111210257581308, + 2.1130451955668876, + 1.8242133884920444, + 1.531138344238427, + 1.319527925169485, + 1.1232592899812257, + 0.8974371112918603, + 0.7821448561426008, + 0.7035301918972675, + 0.4712647874324093, + 0.41227521981846393, + 0.3905171409466258, + 0.3903952043397641 + ], + "population_cv": 0.3859968358313239 + }, + "activation_output_rms_by_block": [ + 0.16142085194587708, + 0.2773628532886505, + 0.3768865168094635, + 0.40868353843688965, + 0.14338424801826477, + 0.16827327013015747, + 0.18227216601371765, + 0.19825012981891632, + 0.06985136866569519, + 0.1086883395910263, + 0.14860449731349945, + 0.21192558109760284, + 0.08672910183668137, + 0.13778631389141083, + 0.18114051222801208, + 0.23228809237480164, + 0.062035780400037766, + 0.0909910649061203, + 0.12021689862012863, + 0.1606491208076477, + 0.061457011848688126, + 0.092374786734581, + 0.12395651638507843, + 0.15162576735019684, + 0.06645520776510239, + 0.10725662857294083, + 0.14660172164440155, + 0.20186874270439148, + 0.07922796159982681, + 0.17203646898269653, + 0.2743971645832062, + 0.41855114698410034 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23956669680774212, + "first_to_last_ratio": 1.306969485354029, + "imbalance_abs_log_ratio": 0.26771108728004156, + "last_quartile_mean": 0.18329938035458326, + "mean": 0.1694765428546816, + "normalized": [ + 0.9524672218755847, + 1.6365855039093904, + 2.223827029163714, + 2.4114460417529116, + 0.846041851002414, + 0.9929000633110867, + 1.0755008506988941, + 1.1697791710850909, + 0.41215950885657104, + 0.6413178942658843, + 0.8768440446706599, + 1.250471466598887, + 0.5117469378110197, + 0.8130111198312341, + 1.068823503104685, + 1.370620904003087, + 0.36604346156170187, + 0.5368947429151949, + 0.7093424057110318, + 0.9479136056332998, + 0.36262842522923483, + 0.5450594234376621, + 0.7314081010689808, + 0.8946711137494054, + 0.392120388141766, + 0.6328700524939813, + 0.8650266235965518, + 1.1911308745392848, + 0.46748629789882584, + 1.0151048993854561, + 1.6190863936756676, + 2.4696700790208403 + ], + "population_cv": 0.5508327874312803 + }, + "bits_per_byte": 1.9147359946947653, + "core_parameter_grad_rms_by_block": [ + 0.00023209204329478853, + 0.00014705811075659293, + 0.0001443553349415038, + 0.0001409677954299832, + 0.0001547598070491459, + 0.00010679063100428474, + 0.00010173131038502872, + 0.00012885752112210694, + 0.00018269174364617834, + 0.00015061935227176197, + 0.00015101151039961197, + 0.0001756105532961918, + 0.00014068815971577485, + 0.0001326364304799064, + 0.00015121016067746042, + 0.00016411832907359437, + 0.0002779928432077, + 0.00028472923376674154, + 0.0003007215693300081, + 0.0003372879732191828, + 0.0005019822813973469, + 0.0004655562424498461, + 0.00043909856222474334, + 0.00038874448014086744, + 0.0003187708239967095, + 0.00026126643200496464, + 0.0002213996139509037, + 0.00021927053392099255, + 0.00013957865743180746, + 0.00013181726813679266, + 0.00012444014246083002, + 0.00012714374135915476 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00014457656924792933, + "first_to_last_ratio": 0.74925318033778, + "imbalance_abs_log_ratio": 0.28867832809930477, + "last_quartile_mean": 0.0001929609016577694, + "mean": 0.00021703122476695333, + "normalized": [ + 1.069394708268395, + 0.6775896459806783, + 0.6651362498484334, + 0.6495277146472977, + 0.7130762276963877, + 0.49205192072687143, + 0.46874043352179906, + 0.5937280281234798, + 0.8417763104933472, + 0.6939985360792947, + 0.6958054563889003, + 0.8091487917683791, + 0.6482392562030871, + 0.6111398515228881, + 0.6967207637510633, + 0.7561968525488604, + 1.2808886993390323, + 1.311927507539442, + 1.3856143034391522, + 1.554098833388423, + 2.3129495856477433, + 2.145111805684907, + 2.0232045536131698, + 1.7911914774397033, + 1.468778625467388, + 1.2038195530873992, + 1.0201279294656385, + 1.0103179123485286, + 0.6431270780584042, + 0.6073654529588411, + 0.573374373178113, + 0.5858315617749513 + ], + "population_cv": 0.5013169471957857 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22879935801029205, + "mean_weights": [ + 0.9326952695846558, + 0.06730473041534424 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5610548257827759, + "mean_weights": [ + 0.6759846210479736, + 0.3240153193473816 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43457120656967163, + "mean_weights": [ + 0.8295697569847107, + 0.1704302579164505 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6696063280105591, + "mean_weights": [ + 0.545992374420166, + 0.45400765538215637 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6690812110900879, + "mean_weights": [ + 0.5202739238739014, + 0.47972607612609863 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865538358688354, + "mean_weights": [ + 0.5361902713775635, + 0.4638097584247589 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4152516722679138, + "mean_weights": [ + 0.8452726602554321, + 0.15472736954689026 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6853918433189392, + "mean_weights": [ + 0.5220736265182495, + 0.4779263138771057 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0265861749649048, + "mean_weights": [ + 0.3228098750114441, + 0.23137259483337402, + 0.4458175599575043 + ], + "sources": 3 + }, + { + "entropy_mean": 1.066486120223999, + "mean_weights": [ + 0.2909930944442749, + 0.26484501361846924, + 0.44416186213493347 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8915231227874756, + "mean_weights": [ + 0.21873247623443604, + 0.16816270351409912, + 0.6131048202514648 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0651111602783203, + "mean_weights": [ + 0.3639928698539734, + 0.22285741567611694, + 0.41314971446990967 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0005524158477783, + "mean_weights": [ + 0.3166183531284332, + 0.18387392163276672, + 0.49950775504112244 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0648009777069092, + "mean_weights": [ + 0.35599708557128906, + 0.22703218460083008, + 0.41697072982788086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0006463527679443, + "mean_weights": [ + 0.28650256991386414, + 0.19727063179016113, + 0.5162267684936523 + ], + "sources": 3 + }, + { + "entropy_mean": 1.072684407234192, + "mean_weights": [ + 0.41332390904426575, + 0.2456190586090088, + 0.34105703234672546 + ], + "sources": 3 + }, + { + "entropy_mean": 1.304518699645996, + "mean_weights": [ + 0.2721603810787201, + 0.12663133442401886, + 0.2481587529182434, + 0.35304954648017883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3646502494812012, + "mean_weights": [ + 0.29890745878219604, + 0.18950369954109192, + 0.2219487726688385, + 0.2896400988101959 + ], + "sources": 4 + }, + { + "entropy_mean": 1.331923484802246, + "mean_weights": [ + 0.31035515666007996, + 0.14580613374710083, + 0.22454026341438293, + 0.31929847598075867 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3639233112335205, + "mean_weights": [ + 0.2952677309513092, + 0.19548878073692322, + 0.21280673146247864, + 0.29643672704696655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2764989137649536, + "mean_weights": [ + 0.3656603693962097, + 0.14041098952293396, + 0.17602922022342682, + 0.3178994059562683 + ], + "sources": 4 + }, + { + "entropy_mean": 1.364103078842163, + "mean_weights": [ + 0.27470695972442627, + 0.18989428877830505, + 0.23374854028224945, + 0.30165016651153564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3429455757141113, + "mean_weights": [ + 0.2830207943916321, + 0.17677831649780273, + 0.23265977203845978, + 0.3075411319732666 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3732160329818726, + "mean_weights": [ + 0.2642192244529724, + 0.22012627124786377, + 0.22637531161308289, + 0.28927919268608093 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5599325895309448, + "mean_weights": [ + 0.18314200639724731, + 0.13039523363113403, + 0.18218961358070374, + 0.26679670810699463, + 0.2374764084815979 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5772080421447754, + "mean_weights": [ + 0.19838765263557434, + 0.18937477469444275, + 0.15325719118118286, + 0.16343195736408234, + 0.2955484390258789 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5662866830825806, + "mean_weights": [ + 0.22973918914794922, + 0.12454323470592499, + 0.16623356938362122, + 0.21617932617664337, + 0.2633046507835388 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5421115159988403, + "mean_weights": [ + 0.2021060585975647, + 0.1677371859550476, + 0.11891372501850128, + 0.17452098429203033, + 0.3367220163345337 + ], + "sources": 5 + }, + { + "entropy_mean": 1.564535140991211, + "mean_weights": [ + 0.22419840097427368, + 0.12987476587295532, + 0.1717071682214737, + 0.21590615808963776, + 0.25831347703933716 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5267750024795532, + "mean_weights": [ + 0.215010404586792, + 0.19819971919059753, + 0.10917140543460846, + 0.13804513216018677, + 0.33957335352897644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.556459903717041, + "mean_weights": [ + 0.2037975788116455, + 0.12750469148159027, + 0.1812792420387268, + 0.22193393111228943, + 0.2654845714569092 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5851528644561768, + "mean_weights": [ + 0.21840542554855347, + 0.1923592984676361, + 0.1442098617553711, + 0.1790001392364502, + 0.26602524518966675 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7158764600753784, + "mean_weights": [ + 0.13390135765075684, + 0.10310791432857513, + 0.14190614223480225, + 0.16341811418533325, + 0.15331929922103882, + 0.3043471872806549 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6410574913024902, + "mean_weights": [ + 0.14665581285953522, + 0.12231245636940002, + 0.08100371062755585, + 0.11105543375015259, + 0.16015371680259705, + 0.3788188695907593 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6537573337554932, + "mean_weights": [ + 0.12217442691326141, + 0.09171081334352493, + 0.1348741054534912, + 0.16064059734344482, + 0.1268058568239212, + 0.363794207572937 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6571812629699707, + "mean_weights": [ + 0.15173450112342834, + 0.14253422617912292, + 0.09320533275604248, + 0.10436245799064636, + 0.14149092137813568, + 0.3666725754737854 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6046829223632812, + "mean_weights": [ + 0.12253664433956146, + 0.08343374729156494, + 0.1302124410867691, + 0.14439499378204346, + 0.11938183009624481, + 0.4000403583049774 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7041404247283936, + "mean_weights": [ + 0.1659325659275055, + 0.15505465865135193, + 0.10219550132751465, + 0.11347830295562744, + 0.1465306580066681, + 0.3168083429336548 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5808616876602173, + "mean_weights": [ + 0.12713032960891724, + 0.08526259660720825, + 0.1294248104095459, + 0.1489948332309723, + 0.10688205063343048, + 0.40230536460876465 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7305450439453125, + "mean_weights": [ + 0.17891491949558258, + 0.17673785984516144, + 0.09075747430324554, + 0.11627361923456192, + 0.17774534225463867, + 0.25957077741622925 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7513642311096191, + "mean_weights": [ + 0.09819091856479645, + 0.06520214676856995, + 0.091708242893219, + 0.11103156208992004, + 0.08388154953718185, + 0.22660189867019653, + 0.3233836889266968 + ], + "sources": 7 + }, + { + "entropy_mean": 1.766061782836914, + "mean_weights": [ + 0.12103402614593506, + 0.1159343495965004, + 0.06956179440021515, + 0.07680170238018036, + 0.10650800168514252, + 0.1487376093864441, + 0.36142250895500183 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6586570739746094, + "mean_weights": [ + 0.09605097770690918, + 0.06012085825204849, + 0.08562932163476944, + 0.10102558881044388, + 0.07161714881658554, + 0.17364460229873657, + 0.4119114875793457 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7821578979492188, + "mean_weights": [ + 0.13399386405944824, + 0.12986589968204498, + 0.0802837461233139, + 0.07629825174808502, + 0.10134232044219971, + 0.12680892646312714, + 0.351406991481781 + ], + "sources": 7 + }, + { + "entropy_mean": 1.655133843421936, + "mean_weights": [ + 0.10968073457479477, + 0.0657510757446289, + 0.08727138489484787, + 0.09466101229190826, + 0.0703636109828949, + 0.14186590909957886, + 0.4304063022136688 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8160101175308228, + "mean_weights": [ + 0.13769462704658508, + 0.11569900065660477, + 0.08787183463573456, + 0.0929793268442154, + 0.10649852454662323, + 0.1315194070339203, + 0.3277372717857361 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6999847888946533, + "mean_weights": [ + 0.10220035165548325, + 0.07546016573905945, + 0.1028895378112793, + 0.10639216750860214, + 0.07665040343999863, + 0.13976991176605225, + 0.3966374397277832 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8706753253936768, + "mean_weights": [ + 0.14694823324680328, + 0.12422746419906616, + 0.09348422288894653, + 0.1027352511882782, + 0.12104703485965729, + 0.14027650654315948, + 0.27128130197525024 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7911043167114258, + "mean_weights": [ + 0.09078209102153778, + 0.05932023376226425, + 0.06744872033596039, + 0.0717800185084343, + 0.05460817739367485, + 0.10924961417913437, + 0.271850049495697, + 0.27496111392974854 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9753689765930176, + "mean_weights": [ + 0.10981250554323196, + 0.10515635460615158, + 0.07847881317138672, + 0.0806870236992836, + 0.10027817636728287, + 0.1099884957075119, + 0.1757785975933075, + 0.23982004821300507 + ], + "sources": 8 + }, + { + "entropy_mean": 1.81705904006958, + "mean_weights": [ + 0.0980612188577652, + 0.060419049113988876, + 0.06808654963970184, + 0.07581339031457901, + 0.06074263155460358, + 0.09064903855323792, + 0.23993313312530518, + 0.3062949776649475 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9814136028289795, + "mean_weights": [ + 0.11045937240123749, + 0.08757394552230835, + 0.08714327216148376, + 0.09700852632522583, + 0.09435248374938965, + 0.114869624376297, + 0.1777447760105133, + 0.23084799945354462 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8373757600784302, + "mean_weights": [ + 0.08649781346321106, + 0.07379771769046783, + 0.0769633948802948, + 0.0880836620926857, + 0.066677525639534, + 0.1187550276517868, + 0.23661984503269196, + 0.25260502099990845 + ], + "sources": 8 + }, + { + "entropy_mean": 1.968585729598999, + "mean_weights": [ + 0.10324696451425552, + 0.0860632136464119, + 0.09797552973031998, + 0.1271316111087799, + 0.08284065127372742, + 0.10640716552734375, + 0.17243653535842896, + 0.22389833629131317 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8529754877090454, + "mean_weights": [ + 0.08385773003101349, + 0.07188580930233002, + 0.08174896240234375, + 0.09558320790529251, + 0.06307300925254822, + 0.108722984790802, + 0.2897399365901947, + 0.20538833737373352 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9637342691421509, + "mean_weights": [ + 0.10303820669651031, + 0.08717727661132812, + 0.09220535308122635, + 0.11336663365364075, + 0.08865514397621155, + 0.14618384838104248, + 0.2349606156349182, + 0.13441291451454163 + ], + "sources": 8 + }, + { + "entropy_mean": 1.962258219718933, + "mean_weights": [ + 0.07841141521930695, + 0.07437640428543091, + 0.07400830090045929, + 0.07925958186388016, + 0.05560165271162987, + 0.08937598764896393, + 0.1956467628479004, + 0.1118645966053009, + 0.24145527184009552 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0993216037750244, + "mean_weights": [ + 0.1043272316455841, + 0.09323035180568695, + 0.08778752386569977, + 0.10311444103717804, + 0.09694567322731018, + 0.13869863748550415, + 0.1680138111114502, + 0.0874539315700531, + 0.12042839825153351 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0489909648895264, + "mean_weights": [ + 0.0919051468372345, + 0.07878251373767853, + 0.07574650645256042, + 0.09311886876821518, + 0.05773407965898514, + 0.07325181365013123, + 0.17863741517066956, + 0.18801960349082947, + 0.1628040373325348 + ], + "sources": 9 + }, + { + "entropy_mean": 2.126072406768799, + "mean_weights": [ + 0.09415344893932343, + 0.07414553314447403, + 0.10936354845762253, + 0.11765885353088379, + 0.07923045754432678, + 0.12059831619262695, + 0.17183929681777954, + 0.12057285010814667, + 0.11243770271539688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.028747081756592, + "mean_weights": [ + 0.08796507865190506, + 0.0757753849029541, + 0.0847562700510025, + 0.09340266138315201, + 0.06450396031141281, + 0.08635804057121277, + 0.24226921796798706, + 0.14922890067100525, + 0.11574049293994904 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1297550201416016, + "mean_weights": [ + 0.10085826367139816, + 0.08009681105613708, + 0.0982624888420105, + 0.11403533071279526, + 0.09301646053791046, + 0.12771156430244446, + 0.17309343814849854, + 0.10872717201709747, + 0.10419845581054688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1051812171936035, + "mean_weights": [ + 0.09203394502401352, + 0.08722756803035736, + 0.10295312106609344, + 0.11847955733537674, + 0.07401065528392792, + 0.11382973194122314, + 0.21416787803173065, + 0.11479790508747101, + 0.08249962329864502 + ], + "sources": 9 + } + ], + "loss_nats": 1.3271938562393188, + "output_weights": { + "entropy_mean": 1.8321013450622559, + "mean_weights": [ + 0.09867185354232788, + 0.05898427963256836, + 0.057430200278759, + 0.05067210644483566, + 0.04944504797458649, + 0.047245003283023834, + 0.1162109300494194, + 0.30178403854370117, + 0.219556525349617 + ], + "sources": 9 + }, + "step": 4000 + }, + { + "activation_grad_rms_by_block": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "population_cv": 0.42684708327825066 + }, + "activation_output_rms_by_block": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2482638694345951, + "first_to_last_ratio": 1.6504997944655762, + "imbalance_abs_log_ratio": 0.5010781477825214, + "last_quartile_mean": 0.150417388882488, + "mean": 0.1577255482552573, + "normalized": [ + 1.0413313772202522, + 1.8759762186094133, + 2.6842751238870792, + 2.86288304745295, + 0.8807369558111258, + 0.9863695480754758, + 1.0723915340184837, + 1.1882317995492147, + 0.3938410687519575, + 0.6240548714989927, + 0.8686355260833801, + 1.2755653854399962, + 0.532526584341831, + 0.7695868670150526, + 1.006314124418815, + 1.3133324349770026, + 0.3684251942759779, + 0.5299682889456079, + 0.6940544684956076, + 0.9337557124120812, + 0.3540539965769534, + 0.5274131112327584, + 0.7125108234042923, + 0.874443203356988, + 0.3801289540418877, + 0.5954785163594047, + 0.7985322424957776, + 1.083277634082024, + 0.4248333923179985, + 0.8857998846594436, + 1.4024216545081702, + 2.0588504556840044 + ], + "population_cv": 0.6067899489376947 + }, + "bits_per_byte": 1.8565786854442996, + "core_parameter_grad_rms_by_block": [ + 0.0003637111411518744, + 0.0002328437345071155, + 0.00026754486121072013, + 0.00020961054010746675, + 0.00023324217449373482, + 0.00013226952769200274, + 0.0001490970084679736, + 0.0001891597343986376, + 0.0002918746014559555, + 0.00023307226647352922, + 0.00024024223572494834, + 0.00029360747277183056, + 0.00025626387819923106, + 0.00024114885933974834, + 0.0002716298155375107, + 0.0003086967264992385, + 0.0005417594428612147, + 0.0005350465304351142, + 0.0005646380221663597, + 0.0006164043223392351, + 0.0009244274084097353, + 0.0008484364345428017, + 0.0007892136159088089, + 0.0006791712867767833, + 0.0005903268079560613, + 0.00047055631494833115, + 0.0003773406829883918, + 0.0003589161190616393, + 0.00026391753197443733, + 0.00022775345188039026, + 0.00019334088037647755, + 0.0001778215115139631 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022218484025369068, + "first_to_last_ratio": 0.6682317907333766, + "imbalance_abs_log_ratio": 0.4031201735174996, + "last_quartile_mean": 0.00033249666258746146, + "mean": 0.00037728390444285195, + "normalized": [ + 0.9640250667172751, + 0.6171578796900011, + 0.7091340448403511, + 0.5555777430182339, + 0.6182139543911144, + 0.3505835423520907, + 0.3951851820664077, + 0.5013723940277184, + 0.7736206024664017, + 0.6177636091253748, + 0.6367677838780912, + 0.7782136192780627, + 0.6792335299266601, + 0.6391708114234587, + 0.7199613138509996, + 0.81820804668331, + 1.4359463430099129, + 1.4181536082893218, + 1.496586563903859, + 1.633794378929312, + 2.4502169255666204, + 2.2488010343184843, + 2.0918295390158974, + 1.8001597173347186, + 1.5646753042057733, + 1.2472207518187604, + 1.000150492891086, + 0.9513157461399342, + 0.6995197220622846, + 0.6036659640085139, + 0.5124546213069721, + 0.471320163462998 + ], + "population_cv": 0.5587385684613161 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.21598131954669952, + "mean_weights": [ + 0.9387255907058716, + 0.06127442046999931 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5377621054649353, + "mean_weights": [ + 0.7082071900367737, + 0.2917928099632263 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43774232268333435, + "mean_weights": [ + 0.829714834690094, + 0.1702851504087448 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6749181151390076, + "mean_weights": [ + 0.5407299399375916, + 0.45927006006240845 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6663151383399963, + "mean_weights": [ + 0.5011438131332397, + 0.49885618686676025 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865031719207764, + "mean_weights": [ + 0.5308656692504883, + 0.4691343307495117 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4133813679218292, + "mean_weights": [ + 0.8479139804840088, + 0.15208598971366882 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6850745677947998, + "mean_weights": [ + 0.5398513674736023, + 0.4601486027240753 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0074119567871094, + "mean_weights": [ + 0.32744842767715454, + 0.20981493592262268, + 0.4627366065979004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489801168441772, + "mean_weights": [ + 0.28095993399620056, + 0.24264614284038544, + 0.4763939380645752 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8527575731277466, + "mean_weights": [ + 0.20522087812423706, + 0.1493145227432251, + 0.6454645991325378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.049942135810852, + "mean_weights": [ + 0.35788866877555847, + 0.20253834128379822, + 0.4395729899406433 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9791408181190491, + "mean_weights": [ + 0.31465935707092285, + 0.1638563871383667, + 0.5214842557907104 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0485806465148926, + "mean_weights": [ + 0.346283882856369, + 0.20800183713436127, + 0.4457142949104309 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9801348447799683, + "mean_weights": [ + 0.2748759686946869, + 0.1817527413368225, + 0.5433712601661682 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0688585042953491, + "mean_weights": [ + 0.4051227569580078, + 0.23231396079063416, + 0.36256328225135803 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2915732860565186, + "mean_weights": [ + 0.26621368527412415, + 0.1108347550034523, + 0.2563389539718628, + 0.36661261320114136 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568191528320312, + "mean_weights": [ + 0.294538676738739, + 0.1710844486951828, + 0.2252884805202484, + 0.30908840894699097 + ], + "sources": 4 + }, + { + "entropy_mean": 1.320021152496338, + "mean_weights": [ + 0.304654598236084, + 0.12911874055862427, + 0.23493197560310364, + 0.3312946557998657 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3578683137893677, + "mean_weights": [ + 0.29558902978897095, + 0.17812995612621307, + 0.21904321014881134, + 0.30723777413368225 + ], + "sources": 4 + }, + { + "entropy_mean": 1.258887767791748, + "mean_weights": [ + 0.35744374990463257, + 0.12067858874797821, + 0.17883820831775665, + 0.34303945302963257 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3539760112762451, + "mean_weights": [ + 0.2736307978630066, + 0.1722734123468399, + 0.2365075945854187, + 0.3175881803035736 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3203014135360718, + "mean_weights": [ + 0.28207293152809143, + 0.15420743823051453, + 0.22419029474258423, + 0.3395293354988098 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3734662532806396, + "mean_weights": [ + 0.2766023576259613, + 0.21176086366176605, + 0.22839435935020447, + 0.283242404460907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5325813293457031, + "mean_weights": [ + 0.17451074719429016, + 0.10643941164016724, + 0.1691833734512329, + 0.27394938468933105, + 0.27591708302497864 + ], + "sources": 5 + }, + { + "entropy_mean": 1.560360074043274, + "mean_weights": [ + 0.19377154111862183, + 0.17211592197418213, + 0.14605137705802917, + 0.16254930198192596, + 0.3255118727684021 + ], + "sources": 5 + }, + { + "entropy_mean": 1.540909767150879, + "mean_weights": [ + 0.21921303868293762, + 0.10205383598804474, + 0.16381604969501495, + 0.215368390083313, + 0.2995486855506897 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5142419338226318, + "mean_weights": [ + 0.2000826597213745, + 0.15480534732341766, + 0.10827909409999847, + 0.16766759753227234, + 0.3691653311252594 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5333527326583862, + "mean_weights": [ + 0.21770668029785156, + 0.10525082051753998, + 0.15970060229301453, + 0.217601016163826, + 0.29974091053009033 + ], + "sources": 5 + }, + { + "entropy_mean": 1.512608289718628, + "mean_weights": [ + 0.21392285823822021, + 0.1860550343990326, + 0.10291537642478943, + 0.1385909616947174, + 0.35851576924324036 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5262645483016968, + "mean_weights": [ + 0.20246261358261108, + 0.10530520975589752, + 0.16650918126106262, + 0.21146778762340546, + 0.3142552077770233 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5794764757156372, + "mean_weights": [ + 0.2210189402103424, + 0.18839940428733826, + 0.14016607403755188, + 0.17440536618232727, + 0.2760102152824402 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6602650880813599, + "mean_weights": [ + 0.12695768475532532, + 0.08036886155605316, + 0.12672865390777588, + 0.15391480922698975, + 0.1544383019208908, + 0.3575916886329651 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5925545692443848, + "mean_weights": [ + 0.14075040817260742, + 0.11028306186199188, + 0.07537610828876495, + 0.10244421660900116, + 0.1520608514547348, + 0.41908538341522217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5918848514556885, + "mean_weights": [ + 0.11773697286844254, + 0.07344591617584229, + 0.11976177245378494, + 0.14735516905784607, + 0.1265447735786438, + 0.4151553511619568 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6208878755569458, + "mean_weights": [ + 0.14785674214363098, + 0.12492769956588745, + 0.08781817555427551, + 0.10019385814666748, + 0.14005999267101288, + 0.3991435170173645 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5470075607299805, + "mean_weights": [ + 0.12043031305074692, + 0.06855648756027222, + 0.11753211170434952, + 0.13396470248699188, + 0.1162077933549881, + 0.44330859184265137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.683492660522461, + "mean_weights": [ + 0.1658952832221985, + 0.1435280442237854, + 0.09461991488933563, + 0.10784181207418442, + 0.15157341957092285, + 0.3365415334701538 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5540567636489868, + "mean_weights": [ + 0.12987136840820312, + 0.07178180664777756, + 0.12132491171360016, + 0.1441602110862732, + 0.11020088195800781, + 0.42266082763671875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7208337783813477, + "mean_weights": [ + 0.18048501014709473, + 0.1685236394405365, + 0.08726631104946136, + 0.11160479485988617, + 0.18141084909439087, + 0.27070939540863037 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6729483604431152, + "mean_weights": [ + 0.09229261428117752, + 0.051420144736766815, + 0.08079522103071213, + 0.09550285339355469, + 0.07590493559837341, + 0.22372649610042572, + 0.3803577423095703 + ], + "sources": 7 + }, + { + "entropy_mean": 1.710235595703125, + "mean_weights": [ + 0.11657393723726273, + 0.10617929697036743, + 0.061112720519304276, + 0.06796726584434509, + 0.10333786904811859, + 0.14365722239017487, + 0.4011717140674591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5640671253204346, + "mean_weights": [ + 0.0904226303100586, + 0.04561381787061691, + 0.07565715163946152, + 0.08878682553768158, + 0.06555216014385223, + 0.16504976153373718, + 0.4689176678657532 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7556228637695312, + "mean_weights": [ + 0.13281463086605072, + 0.12175846099853516, + 0.07235272228717804, + 0.0707377940416336, + 0.10551950335502625, + 0.12620802223682404, + 0.3706088662147522 + ], + "sources": 7 + }, + { + "entropy_mean": 1.594892978668213, + "mean_weights": [ + 0.10721130669116974, + 0.053925663232803345, + 0.07673212885856628, + 0.08865121006965637, + 0.06888366490602493, + 0.14094024896621704, + 0.4636557698249817 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7775511741638184, + "mean_weights": [ + 0.13637228310108185, + 0.10478455573320389, + 0.07710614800453186, + 0.08548048138618469, + 0.11000139266252518, + 0.12931162118911743, + 0.3569435477256775 + ], + "sources": 7 + }, + { + "entropy_mean": 1.654036045074463, + "mean_weights": [ + 0.10232305526733398, + 0.061968062072992325, + 0.08900615572929382, + 0.1005433052778244, + 0.07864195853471756, + 0.14099234342575073, + 0.4265251159667969 + ], + "sources": 7 + }, + { + "entropy_mean": 1.84956693649292, + "mean_weights": [ + 0.14851760864257812, + 0.11630287766456604, + 0.08512166142463684, + 0.09215595573186874, + 0.1263951063156128, + 0.14375528693199158, + 0.28775155544281006 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7161309719085693, + "mean_weights": [ + 0.0837959572672844, + 0.04434927552938461, + 0.0567089319229126, + 0.06495873630046844, + 0.04885464161634445, + 0.09997115284204483, + 0.2760006785392761, + 0.32536062598228455 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9285151958465576, + "mean_weights": [ + 0.10564712435007095, + 0.09173665940761566, + 0.07135222852230072, + 0.07045470178127289, + 0.09785081446170807, + 0.10425601899623871, + 0.1782122403383255, + 0.2804902195930481 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7219383716583252, + "mean_weights": [ + 0.0923302173614502, + 0.04646994173526764, + 0.05750556290149689, + 0.06549686193466187, + 0.054404959082603455, + 0.08427833020687103, + 0.2333412766456604, + 0.3661728501319885 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9470824003219604, + "mean_weights": [ + 0.10804469138383865, + 0.07850410044193268, + 0.07298290729522705, + 0.08511960506439209, + 0.09968686103820801, + 0.11502514779567719, + 0.18013443052768707, + 0.26050227880477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7855088710784912, + "mean_weights": [ + 0.08493785560131073, + 0.05972691997885704, + 0.07017223536968231, + 0.07963316142559052, + 0.0645163506269455, + 0.11586237698793411, + 0.23710276186466217, + 0.28804832696914673 + ], + "sources": 8 + }, + { + "entropy_mean": 1.973182201385498, + "mean_weights": [ + 0.1049819216132164, + 0.0840599462389946, + 0.09310521930456161, + 0.11566175520420074, + 0.0963074192404747, + 0.1096896231174469, + 0.16087493300437927, + 0.23531918227672577 + ], + "sources": 8 + }, + { + "entropy_mean": 1.802741289138794, + "mean_weights": [ + 0.08190082013607025, + 0.05632811039686203, + 0.07102417945861816, + 0.08421063423156738, + 0.06341388821601868, + 0.10201875865459442, + 0.2913346588611603, + 0.24976897239685059 + ], + "sources": 8 + }, + { + "entropy_mean": 1.945815086364746, + "mean_weights": [ + 0.1064736619591713, + 0.07925166189670563, + 0.07823627442121506, + 0.09447507560253143, + 0.09724453091621399, + 0.1459730863571167, + 0.2451319396495819, + 0.153213769197464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8717057704925537, + "mean_weights": [ + 0.07336360216140747, + 0.05720400810241699, + 0.057025715708732605, + 0.06655597686767578, + 0.05032068490982056, + 0.08329878002405167, + 0.1901058554649353, + 0.11727118492126465, + 0.3048541843891144 + ], + "sources": 9 + }, + { + "entropy_mean": 2.09096622467041, + "mean_weights": [ + 0.10326676070690155, + 0.08232800662517548, + 0.07164021581411362, + 0.08822821080684662, + 0.10803373903036118, + 0.13578416407108307, + 0.17320571839809418, + 0.09955950826406479, + 0.13795366883277893 + ], + "sources": 9 + }, + { + "entropy_mean": 1.96881103515625, + "mean_weights": [ + 0.08843819797039032, + 0.06052382290363312, + 0.0642135739326477, + 0.07744920998811722, + 0.05161681026220322, + 0.06631232053041458, + 0.16417551040649414, + 0.2115624099969864, + 0.21570813655853271 + ], + "sources": 9 + }, + { + "entropy_mean": 2.124063014984131, + "mean_weights": [ + 0.09220977127552032, + 0.06839151680469513, + 0.10039852559566498, + 0.10902463644742966, + 0.0849665105342865, + 0.12137697637081146, + 0.17017139494419098, + 0.12761826813220978, + 0.1258423924446106 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9633058309555054, + "mean_weights": [ + 0.0846441313624382, + 0.05872901901602745, + 0.07210009545087814, + 0.07792744785547256, + 0.05918677896261215, + 0.07445863634347916, + 0.23511825501918793, + 0.18020221590995789, + 0.15763342380523682 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1238560676574707, + "mean_weights": [ + 0.10231245309114456, + 0.07575537264347076, + 0.08897507935762405, + 0.0967075377702713, + 0.09801244735717773, + 0.12438583374023438, + 0.1766587197780609, + 0.12203006446361542, + 0.11516247689723969 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0835556983947754, + "mean_weights": [ + 0.09133295714855194, + 0.07054752856492996, + 0.09000501036643982, + 0.10956106334924698, + 0.07114207744598389, + 0.11565886437892914, + 0.22511571645736694, + 0.13003571331501007, + 0.09660108387470245 + ], + "sources": 9 + } + ], + "loss_nats": 1.2868822813034058, + "output_weights": { + "entropy_mean": 1.679484486579895, + "mean_weights": [ + 0.08993907272815704, + 0.043493133038282394, + 0.04922422021627426, + 0.037924446165561676, + 0.03939434140920639, + 0.03898489475250244, + 0.09112586826086044, + 0.32717907428741455, + 0.2827349305152893 + ], + "sources": 9 + }, + "step": 8000 + } + ], + "evaluations": [ + { + "bits_per_byte": 8.061194200882085, + "cross_entropy_nats": 5.587594032287598, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "cross_entropy_nats": 3.598757028579712, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "cross_entropy_nats": 2.1405738294124603, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "cross_entropy_nats": 1.3509249985218048, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "cross_entropy_nats": 1.2388448417186737, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "cross_entropy_nats": 1.1804966628551483, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a", + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "seed": 2026073003, + "timing": { + "mean_ms": 109.96596761305909, + "measured_steps": 7980, + "median_ms": 109.86474549281411, + "p95_ms": 110.51533734425902, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + } + } + ], + "depth_summaries": { + "16": { + "by_seed": [ + { + "baseline_activation_grad_mean": 0.00019925835545109294, + "baseline_bpc": 1.7488000388579794, + "baseline_cv": 0.41719159337744666, + "baseline_first_to_last_ratio": 3.6494630846841103, + "baseline_imbalance": 1.2945800566875763, + "baseline_parameter_grad_cv": 0.44619098566843096, + "block_activation_grad_mean": 0.0001172052661786438, + "block_bpc": 1.737390666804078, + "block_cv": 0.46767509062432555, + "block_first_to_last_ratio": 0.6782682211585493, + "block_imbalance": 0.3882124628447938, + "block_minus_baseline_bpc": -0.011409372053901468, + "block_parameter_grad_cv": 0.6710654904340576, + "block_to_baseline_activation_grad_mean": 0.5882075354546992, + "relative_cv_reduction": -0.12100794466681607, + "relative_imbalance_reduction": 0.700124792716097, + "seed": 2026073001 + }, + { + "baseline_activation_grad_mean": 0.00020270596132831997, + "baseline_bpc": 1.7368251448523782, + "baseline_cv": 0.4374351028998013, + "baseline_first_to_last_ratio": 3.8958554703706607, + "baseline_imbalance": 1.3599132881647322, + "baseline_parameter_grad_cv": 0.37363268494405877, + "block_activation_grad_mean": 0.00011620417808444472, + "block_bpc": 1.7311772990915377, + "block_cv": 0.4517546138899352, + "block_first_to_last_ratio": 0.5545581271583456, + "block_imbalance": 0.5895836496274593, + "block_minus_baseline_bpc": -0.005647845760840431, + "block_parameter_grad_cv": 0.675897465620864, + "block_to_baseline_activation_grad_mean": 0.573264729477938, + "relative_cv_reduction": -0.032735166645768525, + "relative_imbalance_reduction": 0.5664549682993902, + "seed": 2026073002 + }, + { + "baseline_activation_grad_mean": 0.00020541975504784205, + "baseline_bpc": 1.7363699927668699, + "baseline_cv": 0.41926708997335715, + "baseline_first_to_last_ratio": 3.662935915109688, + "baseline_imbalance": 1.2982649884510844, + "baseline_parameter_grad_cv": 0.42806100897327076, + "block_activation_grad_mean": 0.00011485184586490504, + "block_bpc": 1.72661337398776, + "block_cv": 0.4839100882418841, + "block_first_to_last_ratio": 0.5672633797074002, + "block_imbalance": 0.5669315686327029, + "block_minus_baseline_bpc": -0.009756618779109827, + "block_parameter_grad_cv": 0.7016607196207993, + "block_to_baseline_activation_grad_mean": 0.5591080849948252, + "relative_cv_reduction": -0.15418095007799146, + "relative_imbalance_reduction": 0.5633159842744511, + "seed": 2026073003 + } + ], + "depth": 16, + "means": { + "baseline_bpc": 1.7406650588257424, + "baseline_cv": 0.4246312620835351, + "baseline_imbalance": 1.317586111101131, + "baseline_parameter_grad_cv": 0.4159615598619202, + "block_bpc": 1.7317271132944585, + "block_cv": 0.46777993091871495, + "block_imbalance": 0.5149092270349853, + "block_minus_baseline_bpc": -0.008937945531283908, + "block_parameter_grad_cv": 0.6828745585585736, + "block_to_baseline_activation_grad_mean": 0.5735267833091542, + "relative_cv_reduction": -0.10264135379685868, + "relative_imbalance_reduction": 0.6099652484299795 + }, + "verdict": { + "all_cv_improve": false, + "all_imbalance_improve": true, + "cv_reductions": [ + -0.12100794466681607, + -0.032735166645768525, + -0.15418095007799146 + ], + "imbalance_reductions": [ + 0.700124792716097, + 0.5664549682993902, + 0.5633159842744511 + ], + "label": "mixed / inconclusive at this depth", + "mean_cv_reduction": -0.10264135379685868, + "mean_imbalance_reduction": 0.6099652484299795, + "threshold_relative": 0.2 + } + }, + "32": { + "by_seed": [ + { + "baseline_activation_grad_mean": 0.00014728142508602104, + "baseline_bpc": 1.7178972072507075, + "baseline_cv": 0.361085575415688, + "baseline_first_to_last_ratio": 3.0050767723956184, + "baseline_imbalance": 1.1003031158789967, + "baseline_parameter_grad_cv": 0.43653500885793567, + "block_activation_grad_mean": 7.614457786075945e-05, + "block_bpc": 1.7123525940760473, + "block_cv": 0.6420341358971389, + "block_first_to_last_ratio": 0.8123633197064085, + "block_imbalance": 0.2078076008289402, + "block_minus_baseline_bpc": -0.005544613174660151, + "block_parameter_grad_cv": 0.8248093894360681, + "block_to_baseline_activation_grad_mean": 0.5170005505873299, + "relative_cv_reduction": -0.7780664186267149, + "relative_imbalance_reduction": 0.8111360425777496, + "seed": 2026073001 + }, + { + "baseline_activation_grad_mean": 0.00014731888438745955, + "baseline_bpc": 1.7269767303333703, + "baseline_cv": 0.3716245052237534, + "baseline_first_to_last_ratio": 3.129490087873211, + "baseline_imbalance": 1.140870080057971, + "baseline_parameter_grad_cv": 0.4280332727708177, + "block_activation_grad_mean": 7.03343731061068e-05, + "block_bpc": 1.7093240655856785, + "block_cv": 0.7299683279122143, + "block_first_to_last_ratio": 0.6452634668607965, + "block_imbalance": 0.43809656983670076, + "block_minus_baseline_bpc": -0.017652664747691782, + "block_parameter_grad_cv": 0.9317377718256953, + "block_to_baseline_activation_grad_mean": 0.4774294442871438, + "relative_cv_reduction": -0.9642631679326523, + "relative_imbalance_reduction": 0.6159978445447182, + "seed": 2026073002 + }, + { + "baseline_activation_grad_mean": 0.00013714665936959136, + "baseline_bpc": 1.709514213904039, + "baseline_cv": 0.4031953558435787, + "baseline_first_to_last_ratio": 3.49624785701179, + "baseline_imbalance": 1.2516903525947785, + "baseline_parameter_grad_cv": 0.3252608102570666, + "block_activation_grad_mean": 8.757388354752038e-05, + "block_bpc": 1.7030966812870931, + "block_cv": 0.42684708327825066, + "block_first_to_last_ratio": 1.3967870279562358, + "block_imbalance": 0.33417461908716395, + "block_minus_baseline_bpc": -0.006417532616945776, + "block_parameter_grad_cv": 0.5587385684613161, + "block_to_baseline_activation_grad_mean": 0.63854186423543, + "relative_cv_reduction": -0.05866071394891702, + "relative_imbalance_reduction": 0.733021335193314, + "seed": 2026073003 + } + ], + "depth": 32, + "means": { + "baseline_bpc": 1.7181293838293723, + "baseline_cv": 0.37863514549434, + "baseline_imbalance": 1.164287849510582, + "baseline_parameter_grad_cv": 0.3966096972952733, + "block_bpc": 1.7082577803162728, + "block_cv": 0.5996165156958679, + "block_imbalance": 0.3266929299176016, + "block_minus_baseline_bpc": -0.009871603513099236, + "block_parameter_grad_cv": 0.7717619099076932, + "block_to_baseline_activation_grad_mean": 0.5443239530366345, + "relative_cv_reduction": -0.600330100169428, + "relative_imbalance_reduction": 0.7200517407719272 + }, + "verdict": { + "all_cv_improve": false, + "all_imbalance_improve": true, + "cv_reductions": [ + -0.7780664186267149, + -0.9642631679326523, + -0.05866071394891702 + ], + "imbalance_reductions": [ + 0.8111360425777496, + 0.6159978445447182, + 0.733021335193314 + ], + "label": "mixed / inconclusive at this depth", + "mean_cv_reduction": -0.600330100169428, + "mean_imbalance_reduction": 0.7200517407719272, + "threshold_relative": 0.2 + } + } + }, + "manifest_summary": { + "dataset_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "train_bytes_sha256": "0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "overall_verdict": "depth-dependent or inconclusive", + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "replay_exact": true, + "schema_version": 1, + "study": { + "architectures": [ + "baseline", + "block" + ], + "depths": [ + 16, + 32 + ], + "diagnostic_steps": [ + 0, + 100, + 500, + 2000, + 4000, + 8000 + ], + "formal_runs": 12, + "formal_target_bytes": 786432000, + "gradient_object": "RMS of d(mean token CE)/d(post-MLP Transformer-block output) over batch×time×channel", + "replay_target_bytes": 65536000, + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ] + } +} diff --git a/src/data/k3-attnres-gradient.json b/src/data/k3-attnres-gradient.json new file mode 100644 index 0000000..a17b207 --- /dev/null +++ b/src/data/k3-attnres-gradient.json @@ -0,0 +1,116733 @@ +{ + "canonical_sha256_without_self": "69be133c8f5b11f8a58e831bc03f5432e737ed21efe89bf186784ead7c7f7b51", + "depth_summaries": { + "16": { + "by_seed": [ + { + "baseline_activation_grad_mean": 0.00019925835545109294, + "baseline_bpc": 1.7488000388579794, + "baseline_cv": 0.41719159337744666, + "baseline_first_to_last_ratio": 3.6494630846841103, + "baseline_imbalance": 1.2945800566875763, + "baseline_parameter_grad_cv": 0.44619098566843096, + "block_activation_grad_mean": 0.0001172052661786438, + "block_bpc": 1.737390666804078, + "block_cv": 0.46767509062432555, + "block_first_to_last_ratio": 0.6782682211585493, + "block_imbalance": 0.3882124628447938, + "block_minus_baseline_bpc": -0.011409372053901468, + "block_parameter_grad_cv": 0.6710654904340576, + "block_to_baseline_activation_grad_mean": 0.5882075354546992, + "relative_cv_reduction": -0.12100794466681607, + "relative_imbalance_reduction": 0.700124792716097, + "seed": 2026073001 + }, + { + "baseline_activation_grad_mean": 0.00020270596132831997, + "baseline_bpc": 1.7368251448523782, + "baseline_cv": 0.4374351028998013, + "baseline_first_to_last_ratio": 3.8958554703706607, + "baseline_imbalance": 1.3599132881647322, + "baseline_parameter_grad_cv": 0.37363268494405877, + "block_activation_grad_mean": 0.00011620417808444472, + "block_bpc": 1.7311772990915377, + "block_cv": 0.4517546138899352, + "block_first_to_last_ratio": 0.5545581271583456, + "block_imbalance": 0.5895836496274593, + "block_minus_baseline_bpc": -0.005647845760840431, + "block_parameter_grad_cv": 0.675897465620864, + "block_to_baseline_activation_grad_mean": 0.573264729477938, + "relative_cv_reduction": -0.032735166645768525, + "relative_imbalance_reduction": 0.5664549682993902, + "seed": 2026073002 + }, + { + "baseline_activation_grad_mean": 0.00020541975504784205, + "baseline_bpc": 1.7363699927668699, + "baseline_cv": 0.41926708997335715, + "baseline_first_to_last_ratio": 3.662935915109688, + "baseline_imbalance": 1.2982649884510844, + "baseline_parameter_grad_cv": 0.42806100897327076, + "block_activation_grad_mean": 0.00011485184586490504, + "block_bpc": 1.72661337398776, + "block_cv": 0.4839100882418841, + "block_first_to_last_ratio": 0.5672633797074002, + "block_imbalance": 0.5669315686327029, + "block_minus_baseline_bpc": -0.009756618779109827, + "block_parameter_grad_cv": 0.7016607196207993, + "block_to_baseline_activation_grad_mean": 0.5591080849948252, + "relative_cv_reduction": -0.15418095007799146, + "relative_imbalance_reduction": 0.5633159842744511, + "seed": 2026073003 + } + ], + "depth": 16, + "means": { + "baseline_bpc": 1.7406650588257424, + "baseline_cv": 0.4246312620835351, + "baseline_imbalance": 1.317586111101131, + "baseline_parameter_grad_cv": 0.4159615598619202, + "block_bpc": 1.7317271132944585, + "block_cv": 0.46777993091871495, + "block_imbalance": 0.5149092270349853, + "block_minus_baseline_bpc": -0.008937945531283908, + "block_parameter_grad_cv": 0.6828745585585736, + "block_to_baseline_activation_grad_mean": 0.5735267833091542, + "relative_cv_reduction": -0.10264135379685868, + "relative_imbalance_reduction": 0.6099652484299795 + }, + "verdict": { + "all_cv_improve": false, + "all_imbalance_improve": true, + "cv_reductions": [ + -0.12100794466681607, + -0.032735166645768525, + -0.15418095007799146 + ], + "imbalance_reductions": [ + 0.700124792716097, + 0.5664549682993902, + 0.5633159842744511 + ], + "label": "mixed / inconclusive at this depth", + "mean_cv_reduction": -0.10264135379685868, + "mean_imbalance_reduction": 0.6099652484299795, + "threshold_relative": 0.2 + } + }, + "32": { + "by_seed": [ + { + "baseline_activation_grad_mean": 0.00014728142508602104, + "baseline_bpc": 1.7178972072507075, + "baseline_cv": 0.361085575415688, + "baseline_first_to_last_ratio": 3.0050767723956184, + "baseline_imbalance": 1.1003031158789967, + "baseline_parameter_grad_cv": 0.43653500885793567, + "block_activation_grad_mean": 7.614457786075945e-05, + "block_bpc": 1.7123525940760473, + "block_cv": 0.6420341358971389, + "block_first_to_last_ratio": 0.8123633197064085, + "block_imbalance": 0.2078076008289402, + "block_minus_baseline_bpc": -0.005544613174660151, + "block_parameter_grad_cv": 0.8248093894360681, + "block_to_baseline_activation_grad_mean": 0.5170005505873299, + "relative_cv_reduction": -0.7780664186267149, + "relative_imbalance_reduction": 0.8111360425777496, + "seed": 2026073001 + }, + { + "baseline_activation_grad_mean": 0.00014731888438745955, + "baseline_bpc": 1.7269767303333703, + "baseline_cv": 0.3716245052237534, + "baseline_first_to_last_ratio": 3.129490087873211, + "baseline_imbalance": 1.140870080057971, + "baseline_parameter_grad_cv": 0.4280332727708177, + "block_activation_grad_mean": 7.03343731061068e-05, + "block_bpc": 1.7093240655856785, + "block_cv": 0.7299683279122143, + "block_first_to_last_ratio": 0.6452634668607965, + "block_imbalance": 0.43809656983670076, + "block_minus_baseline_bpc": -0.017652664747691782, + "block_parameter_grad_cv": 0.9317377718256953, + "block_to_baseline_activation_grad_mean": 0.4774294442871438, + "relative_cv_reduction": -0.9642631679326523, + "relative_imbalance_reduction": 0.6159978445447182, + "seed": 2026073002 + }, + { + "baseline_activation_grad_mean": 0.00013714665936959136, + "baseline_bpc": 1.709514213904039, + "baseline_cv": 0.4031953558435787, + "baseline_first_to_last_ratio": 3.49624785701179, + "baseline_imbalance": 1.2516903525947785, + "baseline_parameter_grad_cv": 0.3252608102570666, + "block_activation_grad_mean": 8.757388354752038e-05, + "block_bpc": 1.7030966812870931, + "block_cv": 0.42684708327825066, + "block_first_to_last_ratio": 1.3967870279562358, + "block_imbalance": 0.33417461908716395, + "block_minus_baseline_bpc": -0.006417532616945776, + "block_parameter_grad_cv": 0.5587385684613161, + "block_to_baseline_activation_grad_mean": 0.63854186423543, + "relative_cv_reduction": -0.05866071394891702, + "relative_imbalance_reduction": 0.733021335193314, + "seed": 2026073003 + } + ], + "depth": 32, + "means": { + "baseline_bpc": 1.7181293838293723, + "baseline_cv": 0.37863514549434, + "baseline_imbalance": 1.164287849510582, + "baseline_parameter_grad_cv": 0.3966096972952733, + "block_bpc": 1.7082577803162728, + "block_cv": 0.5996165156958679, + "block_imbalance": 0.3266929299176016, + "block_minus_baseline_bpc": -0.009871603513099236, + "block_parameter_grad_cv": 0.7717619099076932, + "block_to_baseline_activation_grad_mean": 0.5443239530366345, + "relative_cv_reduction": -0.600330100169428, + "relative_imbalance_reduction": 0.7200517407719272 + }, + "verdict": { + "all_cv_improve": false, + "all_imbalance_improve": true, + "cv_reductions": [ + -0.7780664186267149, + -0.9642631679326523, + -0.05866071394891702 + ], + "imbalance_reductions": [ + 0.8111360425777496, + 0.6159978445447182, + 0.733021335193314 + ], + "label": "mixed / inconclusive at this depth", + "mean_cv_reduction": -0.600330100169428, + "mean_imbalance_reduction": 0.7200517407719272, + "threshold_relative": 0.2 + } + } + }, + "gates": { + "common_initial_parameters": { + "depth-16-seed-2026073001": { + "baseline": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "block": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "exact": true + }, + "depth-16-seed-2026073002": { + "baseline": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "block": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "exact": true + }, + "depth-16-seed-2026073003": { + "baseline": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "block": { + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "exact": true + }, + "depth-32-seed-2026073001": { + "baseline": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "block": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "exact": true + }, + "depth-32-seed-2026073002": { + "baseline": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "block": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "exact": true + }, + "depth-32-seed-2026073003": { + "baseline": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "block": { + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "exact": true + } + }, + "paired_input_tensors": { + "depth-16-seed-2026073001": { + "exact": true, + "hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + } + }, + "depth-16-seed-2026073002": { + "exact": true, + "hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + } + }, + "depth-16-seed-2026073003": { + "exact": true, + "hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + } + }, + "depth-32-seed-2026073001": { + "exact": true, + "hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + } + }, + "depth-32-seed-2026073002": { + "exact": true, + "hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + } + }, + "depth-32-seed-2026073003": { + "exact": true, + "hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + } + } + }, + "replay": { + "compare_fields": [ + "protocol_id", + "architecture", + "depth", + "seed", + "steps", + "batch_size", + "target_bytes_seen", + "manifest", + "model", + "optimizer", + "hashes", + "evaluations", + "diagnostics", + "training_history", + "gradient_gate", + "environment" + ], + "exact": true, + "formal_compare_sha256": "46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817", + "formal_final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "formal_final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "replay_compare_sha256": "46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817", + "replay_final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "replay_final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637" + }, + "smoke_exact": { + "depth-16-baseline": { + "compare_sha256": "525cdcd9a79ec61096ecf64fedcbf93144d3168b434e2f8fd384c4374cbebd2a", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-16-block": { + "compare_sha256": "e4330a0d878d10b474b9aeb0be58131f801ebe2e5dd9553d4713c3d16b6590cc", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-32-baseline": { + "compare_sha256": "8ee0dc37e88742b6705968f5f2845a3a1df1250f5e0fd77be6b909236ea28e56", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + }, + "depth-32-block": { + "compare_sha256": "289073b1940d4747b7e049f8d767f9c22a49867d37403ff6ef7746a55bd1734c", + "exact": true, + "gradient_gate": { + "first_to_last_ratio_abs_delta": 0.0, + "max_abs_scale_ratio_error": 0.0, + "normalized_spectrum_max_abs_delta": 0.0, + "passed": true, + "per_block_scale_ratios": [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + "population_cv_abs_delta": 0.0, + "thresholds": { + "scale_ratio_abs": 1e-05, + "shape_abs": 1e-06 + } + } + } + } + }, + "manifest": { + "dataset": { + "preprocessing": "parquet row order; (text or empty string) + LF; UTF-8; no normalization; vocabulary is raw bytes 0..255", + "repository": "Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "splits": { + "test": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-gradient-scale-v1/test.bin", + "binary_sha256": "bfe9eb16ab9987fb88bde4ea9a30a00f2a45db01dfc14bad78d05325789c4f12", + "concatenated_bytes": 1292014, + "concatenated_sha256": "bfe9eb16ab9987fb88bde4ea9a30a00f2a45db01dfc14bad78d05325789c4f12", + "parquet_bytes": 732610, + "parquet_sha256": "5f1bea067869d04849c0f975a2b29c4ff47d867f484f5010ea5e861eab246d91", + "rows": 4358, + "source_path": "wikitext-2-raw-v1/test-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/test-00000-of-00001.parquet" + }, + "train": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-gradient-scale-v1/train.bin", + "binary_sha256": "0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4", + "concatenated_bytes": 10951563, + "concatenated_sha256": "0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4", + "parquet_bytes": 6357543, + "parquet_sha256": "e83889baabc497075506f91975be5fac0d45c5290b6b20582c8cd1e853d0c9f7", + "rows": 36718, + "source_path": "wikitext-2-raw-v1/train-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/train-00000-of-00001.parquet" + }, + "validation": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-gradient-scale-v1/validation.bin", + "binary_sha256": "a42356f6a8ff1d25daf25ec9db49e10a537c265581b61c74604bb63231dee719", + "concatenated_bytes": 1148008, + "concatenated_sha256": "a42356f6a8ff1d25daf25ec9db49e10a537c265581b61c74604bb63231dee719", + "parquet_bytes": 657209, + "parquet_sha256": "204929b7ff9d6184953f867dedb860e40aa69c078fc1e54b3baaa8fb28511c4c", + "rows": 3760, + "source_path": "wikitext-2-raw-v1/validation-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/validation-00000-of-00001.parquet" + } + }, + "variant": "wikitext-2-raw-v1" + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "schema_version": 1, + "status": "frozen-before-model-output", + "windows": { + "context": 256, + "diagnostic_starts": [ + 399861, + 210983, + 449025, + 1098024, + 323754, + 152932, + 1091551, + 1078021, + 415985, + 612288, + 910624, + 530272, + 827285, + 765798, + 1086876, + 1035447 + ], + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "formal_batch": 32, + "formal_schedule_cells": 768000, + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "formal_steps": 8000, + "gate_steps": [ + 0, + 1, + 7999 + ], + "gate_training_tensor_sha256": { + "2026073001": { + "0": "52fdd6885cc2bef8e29cedec8c293e1ea71f63fe3cd0b83f627fe640e19a95c5", + "1": "9d0a960595a3f57cd18834d880bde56fa8dcbb0b1cbed26bcc9bbf773a67949c", + "7999": "8f4f04a889d1f8c9eb75e917dc7cd6ddef1196bdea94e87466270db7be1289ee" + }, + "2026073002": { + "0": "156813ff7ab93736c8dba340711a9633b6c952d0f54df30f277e254705cc8b82", + "1": "9ee5a434bdf417d658a1485135d48749e98c86f03239eef86dbd81a9d1309da0", + "7999": "bca78ffefa000dc3693a790d65933251646c70facc64b42007d500c19bb90977" + }, + "2026073003": { + "0": "8ba13ad55eb54501ec443430446793ef41dea21fd067b2a5b0c84eda202ed0ba", + "1": "326a12f07637fe70fd39fa2758b389e91dacc7f2f2ede708b80e09c9d4d1c39d", + "7999": "7d6b10c3febfb697444909f695d3a45297789022714f8b71dadfec4d6236aea6" + } + }, + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ], + "target_bytes_per_window": 256, + "validation_starts": [ + 19785, + 600807, + 1029500, + 319878, + 652204, + 1072662, + 679625, + 1027264, + 500250, + 944134, + 187313, + 834295, + 968556, + 550645, + 239009, + 452519, + 356354, + 134015, + 64555, + 397632, + 203140, + 346032, + 314812, + 10817, + 1141274, + 807645, + 417975, + 870687, + 377265, + 635426, + 597238, + 805324, + 12300, + 264343, + 84743, + 596894, + 188690, + 992517, + 854512, + 427504, + 94167, + 296670, + 760313, + 912279, + 1054297, + 81970, + 419690, + 971472, + 1041491, + 669963, + 735537, + 434513, + 169153, + 6229, + 136413, + 1098303, + 400950, + 457810, + 659776, + 911665, + 909832, + 532969, + 555820, + 1019138 + ], + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + } + }, + "overall_verdict": "depth-dependent or inconclusive", + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "runs": { + "depth-16-baseline-seed-2026073001": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "09d1bf3e3857ce9c1fd57a3596a5acf1e980ac95e31a3ef6ad060395c15d99ae", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002865509013645351, + 0.00024785567075014114, + 0.00021551651298068464, + 0.00019683866412378848, + 0.0001823508064262569, + 0.0001679604029050097, + 0.00015584587526973337, + 0.00014737028686795384, + 0.00014130656199995428, + 0.00013612695329356939, + 0.00013129493163432926, + 0.00012807638267986476, + 0.00012550120300147682, + 0.00012324050476308912, + 0.00012108208466088399, + 0.00011888353037647903 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023669043730478734, + "first_to_last_ratio": 1.9372775995887173, + "imbalance_abs_log_ratio": 0.6612836883477485, + "last_quartile_mean": 0.00012217683070048224, + "mean": 0.00016411257956860936, + "normalized": [ + 1.7460629899168627, + 1.5102783187106137, + 1.313223602646409, + 1.1994124072706904, + 1.1111324123086055, + 1.0234462424910682, + 0.9496278449793059, + 0.8979828801383491, + 0.8610343117596252, + 0.8294729974472175, + 0.8000296624393728, + 0.7804178266926869, + 0.7647262832098098, + 0.7509509940495869, + 0.7377989242455608, + 0.7244023016942358 + ], + "population_cv": 0.29361872380036635 + }, + "activation_output_rms_by_block": [ + 0.02866268903017044, + 0.029099803417921066, + 0.02953805774450302, + 0.030088091269135475, + 0.030644793063402176, + 0.03131929785013199, + 0.03213750571012497, + 0.03286394104361534, + 0.033869802951812744, + 0.03499744459986687, + 0.03588006645441055, + 0.037365153431892395, + 0.03789033368229866, + 0.039445169270038605, + 0.04050002992153168, + 0.04121527820825577 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0293471603654325, + "first_to_last_ratio": 0.7380574840395957, + "imbalance_abs_log_ratio": 0.3037335657624931, + "last_quartile_mean": 0.03976270277053118, + "mean": 0.034094841103069484, + "normalized": [ + 0.8406752488894867, + 0.8534957922212247, + 0.8663497699023965, + 0.8824822259232266, + 0.8988102619619861, + 0.9185934539320196, + 0.942591449919727, + 0.9638977622528551, + 0.9933996421752943, + 1.0264733158329964, + 1.0523605710888722, + 1.0959180985456622, + 1.1113216092650298, + 1.1569248600043318, + 1.1878638706395246, + 1.2088420674453664 + ], + "population_cv": 0.11952987076282337 + }, + "bits_per_byte": 8.096463027059821, + "branch_output_rms_by_sublayer": [ + 0.0033055038657039404, + 0.0037518907338380814, + 0.0033751516602933407, + 0.003911525942385197, + 0.004229962360113859, + 0.0038461871445178986, + 0.004243654198944569, + 0.0038366704247891903, + 0.004372371360659599, + 0.003851204412057996, + 0.005078981164842844, + 0.003818925702944398, + 0.0054668826051056385, + 0.0038749484810978174, + 0.005823117680847645, + 0.0037566579412668943, + 0.0065947300754487514, + 0.003876061411574483, + 0.006763988174498081, + 0.003795720636844635, + 0.006863070651888847, + 0.0038706199266016483, + 0.007856340147554874, + 0.003839300014078617, + 0.00830968376249075, + 0.003927029203623533, + 0.009655521251261234, + 0.0038513424806296825, + 0.009004893712699413, + 0.0039484030567109585, + 0.008608299307525158, + 0.003791053779423237 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00809059897248305, + 0.007313812062277777, + 0.007090710224412361, + 0.0067033738367094624, + 0.006535407867454384, + 0.006785021795736036, + 0.006295993569919972, + 0.005509300326230647, + 0.006231736048910802, + 0.006047390980174007, + 0.005629405359532731, + 0.005784042737956458, + 0.0059591736113607996, + 0.0061983173180850636, + 0.006571690039252182, + 0.005060205437758869 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007299623773970663, + "first_to_last_ratio": 1.227374872012571, + "imbalance_abs_log_ratio": 0.2048776382299483, + "last_quartile_mean": 0.005947346601614228, + "mean": 0.006362886261765913, + "normalized": [ + 1.271529717747562, + 1.1494488132258316, + 1.1143858200043408, + 1.05351149791715, + 1.0271137340180259, + 1.066343404015675, + 0.9894870520870547, + 0.8658492545019393, + 0.9793882512652816, + 0.9504163254515969, + 0.8847251275509024, + 0.909028151691524, + 0.9365519618304368, + 0.9741361173356609, + 1.0328158902888074, + 0.7952688810682109 + ], + "population_cv": 0.11384977604868386 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02823694422841072, + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679 + ], + "loss_nats": 5.6120405197143555, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028404271230101585, + 0.02866268903017044, + 0.02886480651795864, + 0.029099803417921066, + 0.029365191236138344, + 0.02953805774450302, + 0.029881296679377556, + 0.030088091269135475, + 0.030441828072071075, + 0.030644793063402176, + 0.03106885403394699, + 0.03131929785013199, + 0.031892918050289154, + 0.03213750571012497, + 0.03266792371869087, + 0.03286394104361534, + 0.03364725783467293, + 0.033869802951812744, + 0.03485054895281792, + 0.03499744459986687, + 0.03562505170702934, + 0.03588006645441055, + 0.03719272464513779, + 0.037365153431892395, + 0.03768136352300644, + 0.03789033368229866, + 0.03927159309387207, + 0.039445169270038605, + 0.04041972756385803, + 0.04050002992153168, + 0.04110744968056679, + 0.04121527820825577 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.509291365044191e-05, + 7.616560469614342e-05, + 6.756903894711286e-05, + 5.824641266372055e-05, + 5.0209364417241886e-05, + 4.2265295633114874e-05, + 3.587255923775956e-05, + 3.076014036196284e-05, + 2.70738746621646e-05, + 2.470596882631071e-05, + 2.293860052304808e-05, + 2.172161293856334e-05, + 2.083007166220341e-05, + 2.0244739062036388e-05, + 1.9782984963967465e-05, + 1.9472558051347733e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.176849248935468e-05, + "first_to_last_ratio": 3.5736674444153773, + "imbalance_abs_log_ratio": 1.2735923638392248, + "last_quartile_mean": 2.008258843488875e-05, + "mean": 3.8934483768571226e-05, + "normalized": [ + 2.185541078603713, + 1.9562505348440236, + 1.7354548566444863, + 1.496010914384804, + 1.28958598027623, + 1.0855491467240759, + 0.921357001957136, + 0.7900487533057547, + 0.6953700689366589, + 0.6345523668212576, + 0.5891589743271394, + 0.5579016551928063, + 0.5350031551983206, + 0.519969371685323, + 0.5081095997460412, + 0.5001365413522295 + ], + "population_cv": 0.5464287537448471 + }, + "activation_output_rms_by_block": [ + 0.030502017587423325, + 0.03408988192677498, + 0.038947492837905884, + 0.04481571167707443, + 0.0512830950319767, + 0.05888458341360092, + 0.06758055835962296, + 0.07802816480398178, + 0.08916938304901123, + 0.10193294286727905, + 0.11700897663831711, + 0.13432858884334564, + 0.15294352173805237, + 0.17351281642913818, + 0.19949108362197876, + 0.2260059416294098 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037088776007294655, + "first_to_last_ratio": 0.19729295890734105, + "imbalance_abs_log_ratio": 1.623065553831421, + "last_quartile_mean": 0.18798834085464478, + "mean": 0.09990779752843082, + "normalized": [ + 0.3053016715611547, + 0.3412134264802905, + 0.38983436529889043, + 0.4485707100521478, + 0.5133042295060407, + 0.5893892656060614, + 0.676429268099835, + 0.7810017509571989, + 0.8925167529955432, + 1.0202701429612833, + 1.1711696137132819, + 1.344525574243789, + 1.530846695870051, + 1.7367294717888415, + 1.9967518908142226, + 2.262145170051368 + ], + "population_cv": 0.5980263271119333 + }, + "bits_per_byte": 5.097173307164379, + "branch_output_rms_by_sublayer": [ + 0.004186291713267565, + 0.008577318862080574, + 0.004227571655064821, + 0.009763575159013271, + 0.005347618833184242, + 0.009802278131246567, + 0.006270815618336201, + 0.009791693650186062, + 0.005826776847243309, + 0.010311179794371128, + 0.006844967603683472, + 0.010455452837049961, + 0.00757947750389576, + 0.01094198040664196, + 0.009920897893607616, + 0.011405051685869694, + 0.010475707240402699, + 0.012199299409985542, + 0.012102399952709675, + 0.012131479568779469, + 0.013124288059771061, + 0.013151587918400764, + 0.01721765287220478, + 0.012752228416502476, + 0.016962353140115738, + 0.012886312790215015, + 0.01880411058664322, + 0.012915017083287239, + 0.024245616048574448, + 0.014021328650414944, + 0.02384282648563385, + 0.013684137724339962 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005223696653307753, + 0.0004106838611462913, + 0.0004143731432945227, + 0.00042601318321813184, + 0.0003557105812490162, + 0.00034956609676611394, + 0.00031105173709675404, + 0.00028609388990673116, + 0.0002585596876336936, + 0.00022586454081316873, + 0.00020762781370074637, + 0.0002072362198298983, + 0.00019288470561650055, + 0.00019339001384203696, + 0.00020597582961175812, + 0.0001909336964292271 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004433599632474303, + "first_to_last_ratio": 2.2643967408442993, + "imbalance_abs_log_ratio": 0.8173083838772207, + "last_quartile_mean": 0.0001957960613748807, + "mean": 0.00029739591659283537, + "normalized": [ + 1.7564789433405417, + 1.3809330869480578, + 1.3933383754620974, + 1.4324782535645448, + 1.1960842816010129, + 1.175423323800053, + 1.0459179825344234, + 0.9619966984900542, + 0.8694123665043039, + 0.7594742503556288, + 0.6981528733799311, + 0.696836130785297, + 0.6485788635779385, + 0.6502779730725327, + 0.6925980422715741, + 0.6420185543120094 + ], + "population_cv": 0.3407962002760142 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028665566816926003, + 0.0289339367300272, + 0.030502017587423325, + 0.030956681817770004, + 0.03408988192677498, + 0.03459245339035988, + 0.038947492837905884, + 0.03989224135875702, + 0.04481571167707443, + 0.04556844383478165, + 0.0512830950319767, + 0.052534833550453186, + 0.05888458341360092, + 0.060703132301568985, + 0.06758055835962296, + 0.07060512155294418, + 0.07802816480398178, + 0.08117377012968063, + 0.08916938304901123, + 0.09386278688907623, + 0.10193294286727905, + 0.10801172256469727, + 0.11700897663831711, + 0.12567132711410522, + 0.13432858884334564, + 0.14358282089233398, + 0.15294352173805237, + 0.16399860382080078, + 0.17351281642913818, + 0.18877196311950684, + 0.19949108362197876, + 0.21516621112823486 + ], + "loss_nats": 3.5330913066864014, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.0289339367300272, + 0.030502017587423325, + 0.030956681817770004, + 0.03408988192677498, + 0.03459245339035988, + 0.038947492837905884, + 0.03989224135875702, + 0.04481571167707443, + 0.04556844383478165, + 0.0512830950319767, + 0.052534833550453186, + 0.05888458341360092, + 0.060703132301568985, + 0.06758055835962296, + 0.07060512155294418, + 0.07802816480398178, + 0.08117377012968063, + 0.08916938304901123, + 0.09386278688907623, + 0.10193294286727905, + 0.10801172256469727, + 0.11700897663831711, + 0.12567132711410522, + 0.13432858884334564, + 0.14358282089233398, + 0.15294352173805237, + 0.16399860382080078, + 0.17351281642913818, + 0.18877196311950684, + 0.19949108362197876, + 0.21516621112823486, + 0.2260059416294098 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014349650882650167, + 0.00014052700134925544, + 0.00012983661144971848, + 0.0001149297459051013, + 0.0001033403241308406, + 9.356468945043162e-05, + 8.457523654215038e-05, + 7.733132952125743e-05, + 7.047204417176545e-05, + 6.370909250108525e-05, + 5.659156886395067e-05, + 4.938157871947624e-05, + 4.27361810579896e-05, + 3.8237969420151785e-05, + 3.594611553126015e-05, + 3.5643912269733846e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013219746688264422, + "first_to_last_ratio": 3.4660158989817993, + "imbalance_abs_log_ratio": 1.243005778409783, + "last_quartile_mean": 3.8141044569783844e-05, + "mean": 8.001999435691687e-05, + "normalized": [ + 1.7932581722820122, + 1.7561486035909524, + 1.622552119544051, + 1.4362628594107976, + 1.2914312849099563, + 1.1692663840127346, + 1.0569262997559778, + 0.9664000871623777, + 0.8806804441579406, + 0.7961646712560445, + 0.7072178562214424, + 0.6171154986492164, + 0.5340687837013764, + 0.47785518766218865, + 0.44921417228459215, + 0.44543757539833934 + ], + "population_cv": 0.4531561582313658 + }, + "activation_output_rms_by_block": [ + 0.05229421332478523, + 0.06407254934310913, + 0.07844720035791397, + 0.08364550769329071, + 0.0890495702624321, + 0.09326893091201782, + 0.09662611037492752, + 0.09771094471216202, + 0.09944888204336166, + 0.10062506794929504, + 0.10494095832109451, + 0.11238908767700195, + 0.12405165284872055, + 0.14480136334896088, + 0.17238570749759674, + 0.2138574868440628 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.06961486767977476, + "first_to_last_ratio": 0.4250665264722616, + "imbalance_abs_log_ratio": 0.8555095894318028, + "last_quartile_mean": 0.16377405263483524, + "mean": 0.10797595209442079, + "normalized": [ + 0.48431351898667185, + 0.5933964748658124, + 0.7265247384835739, + 0.7746679336538376, + 0.8247166941816979, + 0.8637935494234656, + 0.8948854675569957, + 0.9049324670619027, + 0.9210280633265217, + 0.9319210990730817, + 0.9718919471006628, + 1.040871467183008, + 1.1488822320385021, + 1.3410519707419453, + 1.596519448579181, + 1.9806029277431396 + ], + "population_cv": 0.361888909970992 + }, + "bits_per_byte": 3.1304997557876018, + "branch_output_rms_by_sublayer": [ + 0.014489351771771908, + 0.02032402716577053, + 0.0192408449947834, + 0.013354538939893246, + 0.02438616193830967, + 0.010533644817769527, + 0.026007508859038353, + 0.009820356033742428, + 0.019688289612531662, + 0.009847180917859077, + 0.015725189819931984, + 0.00970252975821495, + 0.017999378964304924, + 0.012247176840901375, + 0.019808221608400345, + 0.012050271965563297, + 0.019276488572359085, + 0.012983737513422966, + 0.01952224411070347, + 0.015011198818683624, + 0.01836247928440571, + 0.01764002814888954, + 0.020674988627433777, + 0.02150883898139, + 0.01928989216685295, + 0.025364212691783905, + 0.025591954588890076, + 0.031167680397629738, + 0.011463184840977192, + 0.039293259382247925, + 0.019522786140441895, + 0.04468192160129547 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006452065099731355, + 0.00042477980025779674, + 0.0003406169302843733, + 0.00031739507901072504, + 0.0002899614562735046, + 0.0003022750042151422, + 0.0002946052417003999, + 0.0002490739832611916, + 0.00022953214745651343, + 0.00020477988838642542, + 0.00019411347501299613, + 0.00019138854173966402, + 0.00017481673923870362, + 0.00016715970609089626, + 0.0001418296786517743, + 0.00014407196058940572 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00043199957988150763, + "first_to_last_ratio": 2.752123958438998, + "imbalance_abs_log_ratio": 1.0123729620941166, + "last_quartile_mean": 0.00015696952114269496, + "mean": 0.00026947538388391546, + "normalized": [ + 2.394305931301975, + 1.576321347558719, + 1.2640001671956211, + 1.177825872018991, + 1.0760220547581225, + 1.121716573360022, + 1.0932547435475957, + 0.9242921548948887, + 0.8517740809876394, + 0.7599205739498657, + 0.720338430231595, + 0.7102264369427908, + 0.6487299014815064, + 0.6203153092562447, + 0.526317753434843, + 0.5346386690795809 + ], + "population_cv": 0.4584519745606109 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.043568700551986694, + 0.04671650007367134, + 0.05229421332478523, + 0.059547748416662216, + 0.06407254934310913, + 0.07493934780359268, + 0.07844720035791397, + 0.08121047168970108, + 0.08364550769329071, + 0.0867149755358696, + 0.0890495702624321, + 0.09094811230897903, + 0.09326893091201782, + 0.09377976506948471, + 0.09662611037492752, + 0.09500574320554733, + 0.09771094471216202, + 0.0959710031747818, + 0.09944888204336166, + 0.09544596821069717, + 0.10062506794929504, + 0.097707599401474, + 0.10494095832109451, + 0.10151325911283493, + 0.11238908767700195, + 0.10858181118965149, + 0.12405165284872055, + 0.1238408088684082, + 0.14480136334896088, + 0.1446269154548645, + 0.17238570749759674, + 0.18076260387897491 + ], + "loss_nats": 2.1698970794677734, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04671650007367134, + 0.05229421332478523, + 0.059547748416662216, + 0.06407254934310913, + 0.07493934780359268, + 0.07844720035791397, + 0.08121047168970108, + 0.08364550769329071, + 0.0867149755358696, + 0.0890495702624321, + 0.09094811230897903, + 0.09326893091201782, + 0.09377976506948471, + 0.09662611037492752, + 0.09500574320554733, + 0.09771094471216202, + 0.0959710031747818, + 0.09944888204336166, + 0.09544596821069717, + 0.10062506794929504, + 0.097707599401474, + 0.10494095832109451, + 0.10151325911283493, + 0.11238908767700195, + 0.10858181118965149, + 0.12405165284872055, + 0.1238408088684082, + 0.14480136334896088, + 0.1446269154548645, + 0.17238570749759674, + 0.18076260387897491, + 0.2138574868440628 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00016704612062312663, + 0.00018574431305751204, + 0.0002023741544689983, + 0.00018453448137734085, + 0.00017654163821134716, + 0.00017114127695094794, + 0.00016412502736784518, + 0.00015690893633291125, + 0.0001434983714716509, + 0.00012688931019511074, + 0.00010475149611011147, + 9.039731958182529e-05, + 7.724175520706922e-05, + 6.240545917535201e-05, + 4.393732160679065e-05, + 2.913598837039899e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018492476738174446, + "first_to_last_ratio": 3.4773281598182453, + "imbalance_abs_log_ratio": 1.246264228732751, + "last_quartile_mean": 5.3180131089902716e-05, + "mean": 0.00013041706063177116, + "normalized": [ + 1.2808609534206308, + 1.4242332418605552, + 1.551746017650221, + 1.4149566052433022, + 1.3536698140269194, + 1.312261418267664, + 1.2584628619352762, + 1.2031319796106978, + 1.100303677881642, + 0.9729502381086403, + 0.8032039336162798, + 0.6931402927187534, + 0.5922672603790627, + 0.47850686768314793, + 0.33689857288569336, + 0.22340626471151362 + ], + "population_cv": 0.41009229518876916 + }, + "activation_output_rms_by_block": [ + 0.08545628190040588, + 0.1388079822063446, + 0.16856427490711212, + 0.17068639397621155, + 0.17051894962787628, + 0.1691380739212036, + 0.16795755922794342, + 0.15964734554290771, + 0.14745007455348969, + 0.13513104617595673, + 0.13549363613128662, + 0.14635984599590302, + 0.16386321187019348, + 0.19344064593315125, + 0.2505952715873718, + 0.40341031551361084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14087873324751854, + "first_to_last_ratio": 0.5572131614407935, + "imbalance_abs_log_ratio": 0.5848074166792235, + "last_quartile_mean": 0.25282736122608185, + "mean": 0.17540755681693554, + "normalized": [ + 0.4871870029498929, + 0.7913455082850954, + 0.9609863905865502, + 0.9730846097716802, + 0.9721300080921755, + 0.9642576237335369, + 0.9575274992469833, + 0.9101508990831222, + 0.8406141515748735, + 0.77038326414287, + 0.7724503926173194, + 0.8343987491294447, + 0.9341855895137383, + 1.1028067971725755, + 1.4286458128420665, + 2.2998457012580755 + ], + "population_cv": 0.3845661956400748 + }, + "bits_per_byte": 2.092252187349085, + "branch_output_rms_by_sublayer": [ + 0.036218997091054916, + 0.037682514637708664, + 0.07264158874750137, + 0.009124485775828362, + 0.041704896837472916, + 0.003249978879466653, + 0.02913317084312439, + 0.0056674908846616745, + 0.02289137803018093, + 0.008465219289064407, + 0.019867170602083206, + 0.010350172407925129, + 0.024333713576197624, + 0.014746740460395813, + 0.03023042529821396, + 0.01600552536547184, + 0.0432618223130703, + 0.020560188218951225, + 0.04687124863266945, + 0.025769658386707306, + 0.04757922887802124, + 0.04326478764414787, + 0.029664678499102592, + 0.051030874252319336, + 0.02047067880630493, + 0.05886564403772354, + 0.0222918838262558, + 0.07155895233154297, + 0.011497496627271175, + 0.10460206121206284, + 0.015142823569476604, + 0.212371364235878 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040658806478613896, + 0.0003235049151816101, + 0.00024397059654808256, + 0.0002149925859972515, + 0.00020566351114887415, + 0.00024487046739718384, + 0.00031061648932541417, + 0.0003911240721720923, + 0.000499392430316199, + 0.0005175204124242609, + 0.0005752549257371158, + 0.00046811732551685614, + 0.00041118767134459974, + 0.0003618888130405475, + 0.0003117946917671641, + 0.00025214502653225866 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029726404062827076, + "first_to_last_ratio": 0.8893356416516115, + "imbalance_abs_log_ratio": 0.11728056505781807, + "last_quartile_mean": 0.0003342540506711425, + "mean": 0.0003586644999522281, + "normalized": [ + 1.1336166942652368, + 0.9019708257290558, + 0.6802195271084202, + 0.5994253293144072, + 0.5734147404503854, + 0.6827284758591918, + 0.8660363358146301, + 1.0905012127606375, + 1.3923664884110778, + 1.4429094947874448, + 1.6038802998728234, + 1.3051677140592575, + 1.1464409535913576, + 1.0089897748139245, + 0.8693213067049939, + 0.7030108264571566 + ], + "population_cv": 0.30960458504189897 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05539870634675026, + 0.07199405878782272, + 0.08545628190040588, + 0.1365138292312622, + 0.1388079822063446, + 0.16841135919094086, + 0.16856427490711212, + 0.17091451585292816, + 0.17068639397621155, + 0.17101584374904633, + 0.17051894962787628, + 0.16970102488994598, + 0.1691380739212036, + 0.16952496767044067, + 0.16795755922794342, + 0.16297046840190887, + 0.15964734554290771, + 0.14957723021507263, + 0.14745007455348969, + 0.1322014182806015, + 0.13513104617595673, + 0.12475068122148514, + 0.13549363613128662, + 0.1319173276424408, + 0.14635984599590302, + 0.14501839876174927, + 0.16386321187019348, + 0.16489192843437195, + 0.19344064593315125, + 0.19336768984794617, + 0.2505952715873718, + 0.2517329454421997 + ], + "loss_nats": 1.4502387046813965, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.07199405878782272, + 0.08545628190040588, + 0.1365138292312622, + 0.1388079822063446, + 0.16841135919094086, + 0.16856427490711212, + 0.17091451585292816, + 0.17068639397621155, + 0.17101584374904633, + 0.17051894962787628, + 0.16970102488994598, + 0.1691380739212036, + 0.16952496767044067, + 0.16795755922794342, + 0.16297046840190887, + 0.15964734554290771, + 0.14957723021507263, + 0.14745007455348969, + 0.1322014182806015, + 0.13513104617595673, + 0.12475068122148514, + 0.13549363613128662, + 0.1319173276424408, + 0.14635984599590302, + 0.14501839876174927, + 0.16386321187019348, + 0.16489192843437195, + 0.19344064593315125, + 0.19336768984794617, + 0.2505952715873718, + 0.2517329454421997, + 0.40341031551361084 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00017181786824949086, + 0.0001916729233926162, + 0.00020346585370134562, + 0.00018768427253235132, + 0.00018128244846593589, + 0.00017597789701540023, + 0.0001705518807284534, + 0.00016394301201216877, + 0.0001481794024584815, + 0.0001306164776906371, + 0.00010858976020244882, + 9.323586709797382e-05, + 7.920334610389546e-05, + 6.374791701091453e-05, + 4.397464726935141e-05, + 2.655091702763457e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000188660229468951, + "first_to_last_ratio": 3.535001559771659, + "imbalance_abs_log_ratio": 1.2627137405851174, + "last_quartile_mean": 5.336920685294899e-05, + "mean": 0.00013378090568494372, + "normalized": [ + 1.2843228065305883, + 1.4327375226776322, + 1.5208885951221702, + 1.4029227233245896, + 1.3550696755847886, + 1.315418640010041, + 1.2748596659235207, + 1.2254589784154797, + 1.107627442794015, + 0.9763461909746755, + 0.8116984979768304, + 0.6969295552352281, + 0.5920377478264399, + 0.476509832883341, + 0.3287064551118561, + 0.19846566960880369 + ], + "population_cv": 0.41282075642614385 + }, + "activation_output_rms_by_block": [ + 0.10104851424694061, + 0.1665971875190735, + 0.18799307942390442, + 0.18892255425453186, + 0.18902969360351562, + 0.1888171136379242, + 0.18645423650741577, + 0.1778823733329773, + 0.16393223404884338, + 0.15616974234580994, + 0.16218091547489166, + 0.17675508558750153, + 0.19963829219341278, + 0.23325374722480774, + 0.2989467680454254, + 0.5114414095878601 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1611403338611126, + "first_to_last_ratio": 0.5184360907576058, + "imbalance_abs_log_ratio": 0.6569385168504118, + "last_quartile_mean": 0.3108200542628765, + "mean": 0.20556643418967724, + "normalized": [ + 0.49156135166358245, + 0.8104299136957027, + 0.9145125281029207, + 0.9190340582558922, + 0.9195552491273791, + 0.9185211310505179, + 0.907026661441106, + 0.8653279122837938, + 0.797465961284235, + 0.7597044865880744, + 0.7889464839636536, + 0.8598440999585013, + 0.9711619164887857, + 1.1346879082875139, + 1.4542586644743063, + 2.4879616733340346 + ], + "population_cv": 0.4294257518805603 + }, + "bits_per_byte": 1.9625077876075785, + "branch_output_rms_by_sublayer": [ + 0.053652405738830566, + 0.041928116232156754, + 0.08492356538772583, + 0.010198036208748817, + 0.03331374749541283, + 0.00243964372202754, + 0.028503542765975, + 0.005333597306162119, + 0.022150175645947456, + 0.008892077021300793, + 0.02107803337275982, + 0.011626621708273888, + 0.028634268790483475, + 0.017805112525820732, + 0.03554495424032211, + 0.0204806849360466, + 0.0531805157661438, + 0.028403647243976593, + 0.05312826856970787, + 0.03512146323919296, + 0.05336543545126915, + 0.05582646653056145, + 0.03752146288752556, + 0.06727498769760132, + 0.032218869775533676, + 0.07892175018787384, + 0.028152117505669594, + 0.09350594878196716, + 0.016922952607274055, + 0.13253848254680634, + 0.02406182885169983, + 0.29775944352149963 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00044260282991257096, + 0.0003095538604202773, + 0.00019879384700301542, + 0.00020176150294426366, + 0.00022494965081173505, + 0.00026863158839826584, + 0.00037546716364749414, + 0.0004673111939829261, + 0.0006383031641084419, + 0.0006422551721113508, + 0.0007081700332195422, + 0.0005856936655792496, + 0.0005168621280586129, + 0.00045203145589075384, + 0.0003829553066507503, + 0.0003084257693651058 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002881780100700318, + "first_to_last_ratio": 0.6942899678444003, + "imbalance_abs_log_ratio": 0.36486558463682156, + "last_quartile_mean": 0.0004150686649913057, + "mean": 0.00042023552075652225, + "normalized": [ + 1.0532256509772957, + 0.7366199312780793, + 0.47305341215597385, + 0.48011529958497023, + 0.5352942331166415, + 0.6392405570920472, + 0.8934684125976915, + 1.1120221183151215, + 1.5189176844435874, + 1.5283219537347563, + 1.6851741422159423, + 1.3937271759532939, + 1.2299344118463382, + 1.075662178858635, + 0.9112873323067531, + 0.7339355055228726 + ], + "population_cv": 0.3791677394455083 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.058993492275476456, + 0.08599413186311722, + 0.10104851424694061, + 0.16386346518993378, + 0.1665971875190735, + 0.18777909874916077, + 0.18799307942390442, + 0.18934063613414764, + 0.18892255425453186, + 0.18942749500274658, + 0.18902969360351562, + 0.18932336568832397, + 0.1888171136379242, + 0.18826909363269806, + 0.18645423650741577, + 0.18273916840553284, + 0.1778823733329773, + 0.16554005444049835, + 0.16393223404884338, + 0.15082815289497375, + 0.15616974234580994, + 0.1484535187482834, + 0.16218091547489166, + 0.15933844447135925, + 0.17675508558750153, + 0.177049919962883, + 0.19963829219341278, + 0.20120804011821747, + 0.23325374722480774, + 0.23333875834941864, + 0.2989467680454254, + 0.3003007471561432 + ], + "loss_nats": 1.360306739807129, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08599413186311722, + 0.10104851424694061, + 0.16386346518993378, + 0.1665971875190735, + 0.18777909874916077, + 0.18799307942390442, + 0.18934063613414764, + 0.18892255425453186, + 0.18942749500274658, + 0.18902969360351562, + 0.18932336568832397, + 0.1888171136379242, + 0.18826909363269806, + 0.18645423650741577, + 0.18273916840553284, + 0.1778823733329773, + 0.16554005444049835, + 0.16393223404884338, + 0.15082815289497375, + 0.15616974234580994, + 0.1484535187482834, + 0.16218091547489166, + 0.15933844447135925, + 0.17675508558750153, + 0.177049919962883, + 0.19963829219341278, + 0.20120804011821747, + 0.23325374722480774, + 0.23333875834941864, + 0.2989467680454254, + 0.3003007471561432, + 0.5114414095878601 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0002529843186493963, + 0.0002798262285068631, + 0.0002946711319964379, + 0.00027859603869728744, + 0.0002704691141843796, + 0.0002655670396052301, + 0.0002578931744210422, + 0.00024818466044962406, + 0.00022750988136976957, + 0.00020225343178026378, + 0.00016641257388982922, + 0.0001406865194439888, + 0.000117213734483812, + 9.274082549382001e-05, + 6.181560456752777e-05, + 3.130940967821516e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002765194294624962, + "first_to_last_ratio": 3.6494630846841103, + "imbalance_abs_log_ratio": 1.2945800566875763, + "last_quartile_mean": 7.576989355584374e-05, + "mean": 0.00019925835545109294, + "normalized": [ + 1.269629662839861, + 1.4043387434036372, + 1.478839526349316, + 1.3981649003706025, + 1.3573790347314445, + 1.3327774336188996, + 1.2942652961137227, + 1.2455420496057434, + 1.1417833939998108, + 1.0150311203883542, + 0.8351598281190995, + 0.7060507908212645, + 0.5882500345767512, + 0.4654300457507431, + 0.31022841891666686, + 0.15712972039408363 + ], + "population_cv": 0.41719159337744666 + }, + "activation_output_rms_by_block": [ + 0.10425227880477905, + 0.15637855231761932, + 0.1697961539030075, + 0.17037443816661835, + 0.16992402076721191, + 0.16921496391296387, + 0.1664763242006302, + 0.1570158749818802, + 0.1424521654844284, + 0.13391907513141632, + 0.13948261737823486, + 0.15248820185661316, + 0.17248840630054474, + 0.20097237825393677, + 0.2573379576206207, + 0.4688045382499695 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15020035579800606, + "first_to_last_ratio": 0.5463801662721245, + "imbalance_abs_log_ratio": 0.6044402703006951, + "last_quartile_mean": 0.27490082010626793, + "mean": 0.18321112170815468, + "normalized": [ + 0.5690281126647281, + 0.853542900996599, + 0.9267786383267906, + 0.9299350202004414, + 0.927476559189958, + 0.9236063964638243, + 0.9086583972004597, + 0.8570215253198324, + 0.7775301202038757, + 0.7309549435800197, + 0.7613217804562272, + 0.8323086526347376, + 0.9414734470940544, + 1.0969442050252538, + 1.4045979044358783, + 2.5588213962073194 + ], + "population_cv": 0.43808900348348184 + }, + "bits_per_byte": 1.8689156889212923, + "branch_output_rms_by_sublayer": [ + 0.06537462025880814, + 0.033620789647102356, + 0.06649091839790344, + 0.009441502392292023, + 0.024681083858013153, + 0.00226230570115149, + 0.0236189141869545, + 0.004977936390787363, + 0.01721731387078762, + 0.008134713396430016, + 0.01816825568675995, + 0.010567971505224705, + 0.02614794671535492, + 0.015832560136914253, + 0.03134627267718315, + 0.01896953023970127, + 0.04721185564994812, + 0.02692008763551712, + 0.04645893722772598, + 0.03406013175845146, + 0.04632203280925751, + 0.053266141563653946, + 0.032039228826761246, + 0.06378299742937088, + 0.028799353167414665, + 0.07541537284851074, + 0.025642387568950653, + 0.08898185938596725, + 0.017618922516703606, + 0.12298691272735596, + 0.026708398014307022, + 0.2926487922668457 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005010958566464473, + 0.000367257410216879, + 0.0002469638962710481, + 0.00025453790022962814, + 0.00027094925891000524, + 0.0003539210655471446, + 0.0005067460397887177, + 0.0006525825825832493, + 0.0009467684204760593, + 0.0009621097966915796, + 0.0010586988551551607, + 0.0008532929973946807, + 0.0007592856290288629, + 0.0006522457278840908, + 0.0005350698764559804, + 0.0004007556744895193 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034246376584100063, + "first_to_last_ratio": 0.5835734049551724, + "imbalance_abs_log_ratio": 0.5385850340449921, + "last_quartile_mean": 0.0005868392269646134, + "mean": 0.0005826425617355658, + "normalized": [ + 0.8600399104963966, + 0.6303305565643863, + 0.4238686160094385, + 0.43686801642402323, + 0.4650351290899624, + 0.6074411462370523, + 0.8697374223387168, + 1.1200393267518036, + 1.6249558179475279, + 1.6512865002955899, + 1.8170640538197664, + 1.4645222533226991, + 1.3031757013547307, + 1.1194611769198464, + 0.9183501371100032, + 0.6878242353180568 + ], + "population_cv": 0.44619098566843096 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060730163007974625, + 0.09550311416387558, + 0.10425227880477905, + 0.15394315123558044, + 0.15637855231761932, + 0.1696780025959015, + 0.1697961539030075, + 0.17087572813034058, + 0.17037443816661835, + 0.1706146001815796, + 0.16992402076721191, + 0.17012864351272583, + 0.16921496391296387, + 0.16865167021751404, + 0.1664763242006302, + 0.16138805449008942, + 0.1570158749818802, + 0.14425092935562134, + 0.1424521654844284, + 0.1296483427286148, + 0.13391907513141632, + 0.1270018070936203, + 0.13948261737823486, + 0.13754454255104065, + 0.15248820185661316, + 0.1527755856513977, + 0.17248840630054474, + 0.17395897209644318, + 0.20097237825393677, + 0.2011977583169937, + 0.2573379576206207, + 0.25867754220962524 + ], + "loss_nats": 1.2954336404800415, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.09550311416387558, + 0.10425227880477905, + 0.15394315123558044, + 0.15637855231761932, + 0.1696780025959015, + 0.1697961539030075, + 0.17087572813034058, + 0.17037443816661835, + 0.1706146001815796, + 0.16992402076721191, + 0.17012864351272583, + 0.16921496391296387, + 0.16865167021751404, + 0.1664763242006302, + 0.16138805449008942, + 0.1570158749818802, + 0.14425092935562134, + 0.1424521654844284, + 0.1296483427286148, + 0.13391907513141632, + 0.1270018070936203, + 0.13948261737823486, + 0.13754454255104065, + 0.15248820185661316, + 0.1527755856513977, + 0.17248840630054474, + 0.17395897209644318, + 0.20097237825393677, + 0.2011977583169937, + 0.2573379576206207, + 0.25867754220962524, + 0.4688045382499695 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.076786148009306, + "cross_entropy_nats": 5.5984015464782715, + "step": 0 + }, + { + "bits_per_byte": 5.114718375266125, + "cross_entropy_nats": 3.5452526211738586, + "step": 100 + }, + { + "bits_per_byte": 3.1628078576350735, + "cross_entropy_nats": 2.192291349172592, + "step": 500 + }, + { + "bits_per_byte": 1.9765894690862489, + "cross_entropy_nats": 1.3700674176216125, + "step": 2000 + }, + { + "bits_per_byte": 1.840425322451167, + "cross_entropy_nats": 1.2756856232881546, + "step": 4000 + }, + { + "bits_per_byte": 1.7488000388579794, + "cross_entropy_nats": 1.2121758162975311, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "final_optimizer_state": "e9f9db84e99e09e299da28a6a358f3b99d039dde4de0eceec7d22aff7014508e", + "final_public_parameters": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.257796359896474, + "measured_steps": 7980, + "median_ms": 21.201465002377518, + "p95_ms": 21.46604766312521, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.088097790921855, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.6062421798706055, + "step": 1, + "unclipped_grad_norm": 19.475919723510742 + }, + { + "bits_per_byte": 7.474302716882146, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.180791854858398, + "step": 10, + "unclipped_grad_norm": 12.938376426696777 + }, + { + "bits_per_byte": 6.789615018295581, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.706202507019043, + "step": 20, + "unclipped_grad_norm": 4.482712745666504 + }, + { + "bits_per_byte": 6.558995914517002, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.54634952545166, + "step": 30, + "unclipped_grad_norm": 2.908937454223633 + }, + { + "bits_per_byte": 6.417572516920704, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.448322296142578, + "step": 40, + "unclipped_grad_norm": 2.8319714069366455 + }, + { + "bits_per_byte": 6.245190178377291, + "learning_rate": 3.75e-05, + "loss_nats": 4.328835964202881, + "step": 50, + "unclipped_grad_norm": 2.816138982772827 + }, + { + "bits_per_byte": 5.943838049951856, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.119954586029053, + "step": 60, + "unclipped_grad_norm": 2.6310009956359863 + }, + { + "bits_per_byte": 5.699140446373649, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.950343132019043, + "step": 70, + "unclipped_grad_norm": 2.5801262855529785 + }, + { + "bits_per_byte": 5.518637431021795, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.825227975845337, + "step": 80, + "unclipped_grad_norm": 2.382185459136963 + }, + { + "bits_per_byte": 5.304690325417858, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.676931142807007, + "step": 90, + "unclipped_grad_norm": 2.2607486248016357 + }, + { + "bits_per_byte": 5.142740110510481, + "learning_rate": 7.5e-05, + "loss_nats": 3.564675807952881, + "step": 100, + "unclipped_grad_norm": 2.145101547241211 + }, + { + "bits_per_byte": 4.946587017895412, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.428712844848633, + "step": 110, + "unclipped_grad_norm": 1.954435110092163 + }, + { + "bits_per_byte": 4.732811894942338, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2805352210998535, + "step": 120, + "unclipped_grad_norm": 1.7519731521606445 + }, + { + "bits_per_byte": 4.552989930887901, + "learning_rate": 9.75e-05, + "loss_nats": 3.1558921337127686, + "step": 130, + "unclipped_grad_norm": 1.5486056804656982 + }, + { + "bits_per_byte": 4.412413719829753, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0584521293640137, + "step": 140, + "unclipped_grad_norm": 1.3235344886779785 + }, + { + "bits_per_byte": 4.26477418130067, + "learning_rate": 0.0001125, + "loss_nats": 2.956116199493408, + "step": 150, + "unclipped_grad_norm": 1.2421116828918457 + }, + { + "bits_per_byte": 4.169433535003061, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.890031099319458, + "step": 160, + "unclipped_grad_norm": 0.9132174849510193 + }, + { + "bits_per_byte": 4.040492638206386, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.8006560802459717, + "step": 170, + "unclipped_grad_norm": 0.901505708694458 + }, + { + "bits_per_byte": 3.967424777091417, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.750009298324585, + "step": 180, + "unclipped_grad_norm": 0.677867591381073 + }, + { + "bits_per_byte": 3.8797366708266, + "learning_rate": 0.0001425, + "loss_nats": 2.6892285346984863, + "step": 190, + "unclipped_grad_norm": 0.7871851325035095 + }, + { + "bits_per_byte": 3.825796032209989, + "learning_rate": 0.00015, + "loss_nats": 2.6518397331237793, + "step": 200, + "unclipped_grad_norm": 0.6514210104942322 + }, + { + "bits_per_byte": 3.7775081193619364, + "learning_rate": 0.0001575, + "loss_nats": 2.6183691024780273, + "step": 210, + "unclipped_grad_norm": 0.6251815557479858 + }, + { + "bits_per_byte": 3.6875825188093945, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.556037425994873, + "step": 220, + "unclipped_grad_norm": 0.7170947790145874 + }, + { + "bits_per_byte": 3.6778121844138782, + "learning_rate": 0.0001725, + "loss_nats": 2.549265146255493, + "step": 230, + "unclipped_grad_norm": 0.6953222155570984 + }, + { + "bits_per_byte": 3.5779222530356467, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4800267219543457, + "step": 240, + "unclipped_grad_norm": 0.6264667510986328 + }, + { + "bits_per_byte": 3.617201026700902, + "learning_rate": 0.0001875, + "loss_nats": 2.5072526931762695, + "step": 250, + "unclipped_grad_norm": 0.8498449921607971 + }, + { + "bits_per_byte": 3.6087914190390125, + "learning_rate": 0.000195, + "loss_nats": 2.5014235973358154, + "step": 260, + "unclipped_grad_norm": 0.6882923245429993 + }, + { + "bits_per_byte": 3.6099880743237156, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.5022530555725098, + "step": 270, + "unclipped_grad_norm": 0.7451158165931702 + }, + { + "bits_per_byte": 3.535934064688699, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.450922727584839, + "step": 280, + "unclipped_grad_norm": 0.7390120029449463 + }, + { + "bits_per_byte": 3.452133454263985, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.392836570739746, + "step": 290, + "unclipped_grad_norm": 0.601737380027771 + }, + { + "bits_per_byte": 3.4977253670771122, + "learning_rate": 0.000225, + "loss_nats": 2.4244384765625, + "step": 300, + "unclipped_grad_norm": 0.8893438577651978 + }, + { + "bits_per_byte": 3.4549257645833937, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.3947720527648926, + "step": 310, + "unclipped_grad_norm": 1.0071029663085938 + }, + { + "bits_per_byte": 3.428612419001384, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.376533031463623, + "step": 320, + "unclipped_grad_norm": 0.7961780428886414 + }, + { + "bits_per_byte": 3.4202605975101843, + "learning_rate": 0.0002475, + "loss_nats": 2.370743989944458, + "step": 330, + "unclipped_grad_norm": 1.010308027267456 + }, + { + "bits_per_byte": 3.507607834160991, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.431288480758667, + "step": 340, + "unclipped_grad_norm": 0.7371622920036316 + }, + { + "bits_per_byte": 3.4475772898773274, + "learning_rate": 0.0002625, + "loss_nats": 2.389678478240967, + "step": 350, + "unclipped_grad_norm": 0.8701758980751038 + }, + { + "bits_per_byte": 3.4091412312013003, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.363036632537842, + "step": 360, + "unclipped_grad_norm": 0.7921563386917114 + }, + { + "bits_per_byte": 3.4048547356111953, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.360065460205078, + "step": 370, + "unclipped_grad_norm": 0.8983097076416016 + }, + { + "bits_per_byte": 3.4312740225061367, + "learning_rate": 0.000285, + "loss_nats": 2.378377914428711, + "step": 380, + "unclipped_grad_norm": 0.6835634708404541 + }, + { + "bits_per_byte": 3.36810135519121, + "learning_rate": 0.0002925, + "loss_nats": 2.334589958190918, + "step": 390, + "unclipped_grad_norm": 0.7815158367156982 + }, + { + "bits_per_byte": 3.32671476215698, + "learning_rate": 0.0003, + "loss_nats": 2.3059029579162598, + "step": 400, + "unclipped_grad_norm": 0.9281390309333801 + }, + { + "bits_per_byte": 3.3045228088202467, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.290520668029785, + "step": 410, + "unclipped_grad_norm": 0.8773083090782166 + }, + { + "bits_per_byte": 3.346598020388531, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3196849822998047, + "step": 420, + "unclipped_grad_norm": 0.9256104230880737 + }, + { + "bits_per_byte": 3.314825601502834, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2976620197296143, + "step": 430, + "unclipped_grad_norm": 0.7776153087615967 + }, + { + "bits_per_byte": 3.382362156600754, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.3444747924804688, + "step": 440, + "unclipped_grad_norm": 1.105894923210144 + }, + { + "bits_per_byte": 3.28705625079862, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.278413772583008, + "step": 450, + "unclipped_grad_norm": 1.029218077659607 + }, + { + "bits_per_byte": 3.3030692114551545, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.289513111114502, + "step": 460, + "unclipped_grad_norm": 0.986072301864624 + }, + { + "bits_per_byte": 3.2452720338755956, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.249451160430908, + "step": 470, + "unclipped_grad_norm": 0.9585361480712891 + }, + { + "bits_per_byte": 3.221687708997477, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2331037521362305, + "step": 480, + "unclipped_grad_norm": 1.0470387935638428 + }, + { + "bits_per_byte": 3.155786020978659, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1874241828918457, + "step": 490, + "unclipped_grad_norm": 1.4241349697113037 + }, + { + "bits_per_byte": 3.107888164782763, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.154223918914795, + "step": 500, + "unclipped_grad_norm": 1.0626088380813599 + }, + { + "bits_per_byte": 3.1122186879314886, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1572256088256836, + "step": 510, + "unclipped_grad_norm": 0.9446885585784912 + }, + { + "bits_per_byte": 3.0380233085574626, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.105797290802002, + "step": 520, + "unclipped_grad_norm": 1.1227535009384155 + }, + { + "bits_per_byte": 3.076978410873794, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.132798910140991, + "step": 530, + "unclipped_grad_norm": 0.9776996374130249 + }, + { + "bits_per_byte": 3.01776100056341, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.091752529144287, + "step": 540, + "unclipped_grad_norm": 1.1720482110977173 + }, + { + "bits_per_byte": 2.99280700585382, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.074455738067627, + "step": 550, + "unclipped_grad_norm": 1.1986867189407349 + }, + { + "bits_per_byte": 2.977761275625738, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0640268325805664, + "step": 560, + "unclipped_grad_norm": 1.1189862489700317 + }, + { + "bits_per_byte": 2.9082912915668184, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.015873908996582, + "step": 570, + "unclipped_grad_norm": 0.9214995503425598 + }, + { + "bits_per_byte": 2.96765970264466, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0570249557495117, + "step": 580, + "unclipped_grad_norm": 0.9309345483779907 + }, + { + "bits_per_byte": 2.863993376218783, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9851689338684082, + "step": 590, + "unclipped_grad_norm": 0.9733412861824036 + }, + { + "bits_per_byte": 2.9319891249945065, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0322999954223633, + "step": 600, + "unclipped_grad_norm": 1.4851497411727905 + }, + { + "bits_per_byte": 2.842428815592397, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9702215194702148, + "step": 610, + "unclipped_grad_norm": 1.0783556699752808 + }, + { + "bits_per_byte": 2.8677622040297908, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.987781286239624, + "step": 620, + "unclipped_grad_norm": 0.9511895179748535 + }, + { + "bits_per_byte": 2.802268974806772, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9423848390579224, + "step": 630, + "unclipped_grad_norm": 1.0907046794891357 + }, + { + "bits_per_byte": 2.773638163004386, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.922539472579956, + "step": 640, + "unclipped_grad_norm": 1.5626765489578247 + }, + { + "bits_per_byte": 2.718988439918267, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8846591711044312, + "step": 650, + "unclipped_grad_norm": 1.6085642576217651 + }, + { + "bits_per_byte": 2.628252113149558, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8217655420303345, + "step": 660, + "unclipped_grad_norm": 1.223048448562622 + }, + { + "bits_per_byte": 2.676602111734572, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8552792072296143, + "step": 670, + "unclipped_grad_norm": 1.3098845481872559 + }, + { + "bits_per_byte": 2.620622790774633, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8164772987365723, + "step": 680, + "unclipped_grad_norm": 1.1348049640655518 + }, + { + "bits_per_byte": 2.6956237368853, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8684639930725098, + "step": 690, + "unclipped_grad_norm": 0.956748902797699 + }, + { + "bits_per_byte": 2.6070459643675705, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.807066559791565, + "step": 700, + "unclipped_grad_norm": 1.1346901655197144 + }, + { + "bits_per_byte": 2.6253973731278863, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.819786787033081, + "step": 710, + "unclipped_grad_norm": 1.3684734106063843 + }, + { + "bits_per_byte": 2.6636225810739558, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8462824821472168, + "step": 720, + "unclipped_grad_norm": 1.6031205654144287 + }, + { + "bits_per_byte": 2.6068867084328713, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.806956171989441, + "step": 730, + "unclipped_grad_norm": 1.1380469799041748 + }, + { + "bits_per_byte": 2.5424343182128744, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7622811794281006, + "step": 740, + "unclipped_grad_norm": 1.2064875364303589 + }, + { + "bits_per_byte": 2.5150987077541362, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7433335781097412, + "step": 750, + "unclipped_grad_norm": 1.2007335424423218 + }, + { + "bits_per_byte": 2.45541642833847, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7019649744033813, + "step": 760, + "unclipped_grad_norm": 1.2129080295562744 + }, + { + "bits_per_byte": 2.6149697210407723, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.812558889389038, + "step": 770, + "unclipped_grad_norm": 1.221384048461914 + }, + { + "bits_per_byte": 2.484800524151635, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.72233247756958, + "step": 780, + "unclipped_grad_norm": 1.1399351358413696 + }, + { + "bits_per_byte": 2.4467844471090148, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.695981740951538, + "step": 790, + "unclipped_grad_norm": 1.4305745363235474 + }, + { + "bits_per_byte": 2.3889862376335507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.655919075012207, + "step": 800, + "unclipped_grad_norm": 0.9679463505744934 + }, + { + "bits_per_byte": 2.474951249719373, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7155054807662964, + "step": 810, + "unclipped_grad_norm": 1.4494884014129639 + }, + { + "bits_per_byte": 2.4123399337768623, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6721066236495972, + "step": 820, + "unclipped_grad_norm": 1.4326691627502441 + }, + { + "bits_per_byte": 2.4471743317785206, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6962519884109497, + "step": 830, + "unclipped_grad_norm": 1.0609163045883179 + }, + { + "bits_per_byte": 2.528181256022619, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7524017095565796, + "step": 840, + "unclipped_grad_norm": 1.2206887006759644 + }, + { + "bits_per_byte": 2.4185156587866947, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6763873100280762, + "step": 850, + "unclipped_grad_norm": 1.1635538339614868 + }, + { + "bits_per_byte": 2.438352137737211, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6901369094848633, + "step": 860, + "unclipped_grad_norm": 1.1800516843795776 + }, + { + "bits_per_byte": 2.364430554743359, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6388983726501465, + "step": 870, + "unclipped_grad_norm": 1.172108769416809 + }, + { + "bits_per_byte": 2.3856504621374155, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.65360689163208, + "step": 880, + "unclipped_grad_norm": 1.2126058340072632 + }, + { + "bits_per_byte": 2.2982358082874708, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5930156707763672, + "step": 890, + "unclipped_grad_norm": 0.9879708886146545 + }, + { + "bits_per_byte": 2.330757727565628, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.61555814743042, + "step": 900, + "unclipped_grad_norm": 1.5637768507003784 + }, + { + "bits_per_byte": 2.3422774694857202, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6235430240631104, + "step": 910, + "unclipped_grad_norm": 1.2122199535369873 + }, + { + "bits_per_byte": 2.3285455147275793, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6140247583389282, + "step": 920, + "unclipped_grad_norm": 1.228180170059204 + }, + { + "bits_per_byte": 2.3433181365060913, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6242643594741821, + "step": 930, + "unclipped_grad_norm": 1.113643765449524 + }, + { + "bits_per_byte": 2.3376019491393643, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6203022003173828, + "step": 940, + "unclipped_grad_norm": 1.0455681085586548 + }, + { + "bits_per_byte": 2.3211645353005004, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6089086532592773, + "step": 950, + "unclipped_grad_norm": 1.2410560846328735 + }, + { + "bits_per_byte": 2.362830772125033, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6377894878387451, + "step": 960, + "unclipped_grad_norm": 1.1456577777862549 + }, + { + "bits_per_byte": 2.268770708645749, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.57259202003479, + "step": 970, + "unclipped_grad_norm": 1.1831417083740234 + }, + { + "bits_per_byte": 2.35130105521119, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6297976970672607, + "step": 980, + "unclipped_grad_norm": 1.1095668077468872 + }, + { + "bits_per_byte": 2.3073481370607856, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5993318557739258, + "step": 990, + "unclipped_grad_norm": 1.1233346462249756 + }, + { + "bits_per_byte": 2.2862505093314924, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5847080945968628, + "step": 1000, + "unclipped_grad_norm": 1.2164878845214844 + }, + { + "bits_per_byte": 2.265617922690132, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5704066753387451, + "step": 1010, + "unclipped_grad_norm": 1.195701241493225 + }, + { + "bits_per_byte": 2.236535656429231, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.550248384475708, + "step": 1020, + "unclipped_grad_norm": 1.3137857913970947 + }, + { + "bits_per_byte": 2.2352426908600447, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5493521690368652, + "step": 1030, + "unclipped_grad_norm": 1.0306543111801147 + }, + { + "bits_per_byte": 2.288523947993248, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5862839221954346, + "step": 1040, + "unclipped_grad_norm": 1.111948847770691 + }, + { + "bits_per_byte": 2.267572505517188, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5717614889144897, + "step": 1050, + "unclipped_grad_norm": 1.4793521165847778 + }, + { + "bits_per_byte": 2.1784870402141108, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.510012149810791, + "step": 1060, + "unclipped_grad_norm": 1.2200031280517578 + }, + { + "bits_per_byte": 2.2691612812458586, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5728627443313599, + "step": 1070, + "unclipped_grad_norm": 1.243715524673462 + }, + { + "bits_per_byte": 2.2409759045092073, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.55332612991333, + "step": 1080, + "unclipped_grad_norm": 1.1007108688354492 + }, + { + "bits_per_byte": 2.2116298168618824, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.532984972000122, + "step": 1090, + "unclipped_grad_norm": 1.183898687362671 + }, + { + "bits_per_byte": 2.265987857372138, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5706630945205688, + "step": 1100, + "unclipped_grad_norm": 0.9869716167449951 + }, + { + "bits_per_byte": 2.1294642454989217, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4760321378707886, + "step": 1110, + "unclipped_grad_norm": 1.0796256065368652 + }, + { + "bits_per_byte": 2.1685739602183784, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.503140926361084, + "step": 1120, + "unclipped_grad_norm": 1.0218125581741333 + }, + { + "bits_per_byte": 2.111904300897876, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4638605117797852, + "step": 1130, + "unclipped_grad_norm": 1.0956017971038818 + }, + { + "bits_per_byte": 2.24492118652, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.556060791015625, + "step": 1140, + "unclipped_grad_norm": 1.0603337287902832 + }, + { + "bits_per_byte": 2.107949387858635, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4611191749572754, + "step": 1150, + "unclipped_grad_norm": 1.118997573852539 + }, + { + "bits_per_byte": 2.133223098316179, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4786375761032104, + "step": 1160, + "unclipped_grad_norm": 1.0214279890060425 + }, + { + "bits_per_byte": 2.1914696665624422, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5190110206604004, + "step": 1170, + "unclipped_grad_norm": 1.0747911930084229 + }, + { + "bits_per_byte": 2.168808888519457, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.5033037662506104, + "step": 1180, + "unclipped_grad_norm": 1.0325632095336914 + }, + { + "bits_per_byte": 2.099094345131011, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4549813270568848, + "step": 1190, + "unclipped_grad_norm": 1.1188828945159912 + }, + { + "bits_per_byte": 2.0751268429067697, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.438368320465088, + "step": 1200, + "unclipped_grad_norm": 1.219267725944519 + }, + { + "bits_per_byte": 2.1310088216863203, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4771027565002441, + "step": 1210, + "unclipped_grad_norm": 1.0049350261688232 + }, + { + "bits_per_byte": 2.151060622915717, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4910016059875488, + "step": 1220, + "unclipped_grad_norm": 1.0629826784133911 + }, + { + "bits_per_byte": 2.098358087402667, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4544709920883179, + "step": 1230, + "unclipped_grad_norm": 1.041517734527588 + }, + { + "bits_per_byte": 2.0484189691236807, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4198558330535889, + "step": 1240, + "unclipped_grad_norm": 1.308692455291748 + }, + { + "bits_per_byte": 2.1357799643865563, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.480409860610962, + "step": 1250, + "unclipped_grad_norm": 1.1052577495574951 + }, + { + "bits_per_byte": 2.115429601275262, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.466304063796997, + "step": 1260, + "unclipped_grad_norm": 1.1282013654708862 + }, + { + "bits_per_byte": 2.1198928950304534, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.469397783279419, + "step": 1270, + "unclipped_grad_norm": 1.2154173851013184 + }, + { + "bits_per_byte": 2.0054133314142284, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.3900465965270996, + "step": 1280, + "unclipped_grad_norm": 1.0942060947418213 + }, + { + "bits_per_byte": 2.1170947373009157, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4674582481384277, + "step": 1290, + "unclipped_grad_norm": 1.0796113014221191 + }, + { + "bits_per_byte": 2.0596614750105826, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4276485443115234, + "step": 1300, + "unclipped_grad_norm": 1.0578112602233887 + }, + { + "bits_per_byte": 2.0645378710931466, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4310286045074463, + "step": 1310, + "unclipped_grad_norm": 1.0866352319717407 + }, + { + "bits_per_byte": 2.0503159377626976, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.421170711517334, + "step": 1320, + "unclipped_grad_norm": 0.958570659160614 + }, + { + "bits_per_byte": 2.123195993822925, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4716873168945312, + "step": 1330, + "unclipped_grad_norm": 1.0188450813293457 + }, + { + "bits_per_byte": 2.0677759604435915, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4332730770111084, + "step": 1340, + "unclipped_grad_norm": 1.1106895208358765 + }, + { + "bits_per_byte": 2.0615133841950732, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4289321899414062, + "step": 1350, + "unclipped_grad_norm": 1.2472416162490845 + }, + { + "bits_per_byte": 2.122740239798138, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4713714122772217, + "step": 1360, + "unclipped_grad_norm": 1.2623285055160522 + }, + { + "bits_per_byte": 2.1191595610071747, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4688894748687744, + "step": 1370, + "unclipped_grad_norm": 0.9515148401260376 + }, + { + "bits_per_byte": 2.0626532852049917, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4297223091125488, + "step": 1380, + "unclipped_grad_norm": 0.9549561142921448 + }, + { + "bits_per_byte": 2.085573929033431, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.44560968875885, + "step": 1390, + "unclipped_grad_norm": 1.1485545635223389 + }, + { + "bits_per_byte": 2.0049195692236013, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3897043466567993, + "step": 1400, + "unclipped_grad_norm": 0.9498034715652466 + }, + { + "bits_per_byte": 2.044304112219137, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4170036315917969, + "step": 1410, + "unclipped_grad_norm": 1.0090101957321167 + }, + { + "bits_per_byte": 2.0692653302000634, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4343054294586182, + "step": 1420, + "unclipped_grad_norm": 1.0467784404754639 + }, + { + "bits_per_byte": 2.1090376940732947, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4618735313415527, + "step": 1430, + "unclipped_grad_norm": 1.0122960805892944 + }, + { + "bits_per_byte": 2.054903402991812, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4243505001068115, + "step": 1440, + "unclipped_grad_norm": 1.1181519031524658 + }, + { + "bits_per_byte": 1.9969074134678557, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.384150743484497, + "step": 1450, + "unclipped_grad_norm": 0.900530219078064 + }, + { + "bits_per_byte": 2.037356701054883, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4121880531311035, + "step": 1460, + "unclipped_grad_norm": 1.1993578672409058 + }, + { + "bits_per_byte": 1.9743343465768104, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.368504285812378, + "step": 1470, + "unclipped_grad_norm": 0.9825910329818726 + }, + { + "bits_per_byte": 2.0340613414817006, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.409903883934021, + "step": 1480, + "unclipped_grad_norm": 0.9332759380340576 + }, + { + "bits_per_byte": 2.041156141777744, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4148216247558594, + "step": 1490, + "unclipped_grad_norm": 0.9170629978179932 + }, + { + "bits_per_byte": 2.0456156519146167, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.4179127216339111, + "step": 1500, + "unclipped_grad_norm": 1.007337212562561 + }, + { + "bits_per_byte": 2.0521109206897528, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4224148988723755, + "step": 1510, + "unclipped_grad_norm": 0.9886259436607361 + }, + { + "bits_per_byte": 2.0030100458510716, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.388380765914917, + "step": 1520, + "unclipped_grad_norm": 0.9661282896995544 + }, + { + "bits_per_byte": 2.0761377569285413, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4390690326690674, + "step": 1530, + "unclipped_grad_norm": 0.9895579218864441 + }, + { + "bits_per_byte": 2.0208730238829373, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4007624387741089, + "step": 1540, + "unclipped_grad_norm": 0.9900161623954773 + }, + { + "bits_per_byte": 2.0817916865756563, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4429880380630493, + "step": 1550, + "unclipped_grad_norm": 1.0090205669403076 + }, + { + "bits_per_byte": 1.98651725359863, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3769488334655762, + "step": 1560, + "unclipped_grad_norm": 0.9516891837120056 + }, + { + "bits_per_byte": 2.0695575287238794, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.434507966041565, + "step": 1570, + "unclipped_grad_norm": 0.897994339466095 + }, + { + "bits_per_byte": 2.0129038637899073, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3952386379241943, + "step": 1580, + "unclipped_grad_norm": 0.9900956749916077 + }, + { + "bits_per_byte": 2.0172801343237623, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3982720375061035, + "step": 1590, + "unclipped_grad_norm": 0.9180708527565002 + }, + { + "bits_per_byte": 2.0575405849601425, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4261784553527832, + "step": 1600, + "unclipped_grad_norm": 1.0603415966033936 + }, + { + "bits_per_byte": 1.9872597027024046, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.377463459968567, + "step": 1610, + "unclipped_grad_norm": 1.0459290742874146 + }, + { + "bits_per_byte": 1.943624092542842, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3472175598144531, + "step": 1620, + "unclipped_grad_norm": 0.9182554483413696 + }, + { + "bits_per_byte": 1.9757394448343595, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3694782257080078, + "step": 1630, + "unclipped_grad_norm": 0.9694951772689819 + }, + { + "bits_per_byte": 2.015004803852847, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3966948986053467, + "step": 1640, + "unclipped_grad_norm": 0.9169139266014099 + }, + { + "bits_per_byte": 2.0755808771050477, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.438683032989502, + "step": 1650, + "unclipped_grad_norm": 0.9065545797348022 + }, + { + "bits_per_byte": 1.9985508796795006, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3852899074554443, + "step": 1660, + "unclipped_grad_norm": 0.9444847702980042 + }, + { + "bits_per_byte": 2.017943299425489, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3987317085266113, + "step": 1670, + "unclipped_grad_norm": 0.9302135109901428 + }, + { + "bits_per_byte": 2.035893988609295, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4111741781234741, + "step": 1680, + "unclipped_grad_norm": 0.9957410097122192 + }, + { + "bits_per_byte": 2.1006239588277844, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4560415744781494, + "step": 1690, + "unclipped_grad_norm": 1.046242117881775 + }, + { + "bits_per_byte": 1.9860793857695331, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3766453266143799, + "step": 1700, + "unclipped_grad_norm": 0.9597991108894348 + }, + { + "bits_per_byte": 1.902240595196328, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3185327053070068, + "step": 1710, + "unclipped_grad_norm": 0.8989817500114441 + }, + { + "bits_per_byte": 1.90692127502221, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.321777105331421, + "step": 1720, + "unclipped_grad_norm": 0.8322402238845825 + }, + { + "bits_per_byte": 1.8988062736412483, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3161522150039673, + "step": 1730, + "unclipped_grad_norm": 0.9716639518737793 + }, + { + "bits_per_byte": 1.9978667326943684, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3848156929016113, + "step": 1740, + "unclipped_grad_norm": 0.8983874917030334 + }, + { + "bits_per_byte": 2.0305991587371817, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4075040817260742, + "step": 1750, + "unclipped_grad_norm": 1.0045808553695679 + }, + { + "bits_per_byte": 2.0290275812735974, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.4064147472381592, + "step": 1760, + "unclipped_grad_norm": 1.0337824821472168 + }, + { + "bits_per_byte": 2.0294850551248924, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.4067318439483643, + "step": 1770, + "unclipped_grad_norm": 0.985150933265686 + }, + { + "bits_per_byte": 1.8961565369394051, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3143155574798584, + "step": 1780, + "unclipped_grad_norm": 0.9629419445991516 + }, + { + "bits_per_byte": 1.9830617781774944, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3745536804199219, + "step": 1790, + "unclipped_grad_norm": 0.8722974061965942 + }, + { + "bits_per_byte": 1.903928776897197, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3197028636932373, + "step": 1800, + "unclipped_grad_norm": 0.8771339058876038 + }, + { + "bits_per_byte": 1.9579086275582052, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.357118844985962, + "step": 1810, + "unclipped_grad_norm": 0.8671345710754395 + }, + { + "bits_per_byte": 1.9298429507465542, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3376652002334595, + "step": 1820, + "unclipped_grad_norm": 0.9717899560928345 + }, + { + "bits_per_byte": 1.9258892415858693, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3349246978759766, + "step": 1830, + "unclipped_grad_norm": 0.8506329655647278 + }, + { + "bits_per_byte": 1.920128166747267, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3309314250946045, + "step": 1840, + "unclipped_grad_norm": 0.9434375166893005 + }, + { + "bits_per_byte": 1.9401304369732149, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3447959423065186, + "step": 1850, + "unclipped_grad_norm": 0.9371633529663086 + }, + { + "bits_per_byte": 1.9244834553977166, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3339502811431885, + "step": 1860, + "unclipped_grad_norm": 0.9993670582771301 + }, + { + "bits_per_byte": 1.923443476307949, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.333229422569275, + "step": 1870, + "unclipped_grad_norm": 0.8980090618133545 + }, + { + "bits_per_byte": 1.9331135448180123, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3399322032928467, + "step": 1880, + "unclipped_grad_norm": 0.8608688116073608 + }, + { + "bits_per_byte": 1.937305793915444, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3428380489349365, + "step": 1890, + "unclipped_grad_norm": 0.9540092945098877 + }, + { + "bits_per_byte": 1.9529099518108777, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3536540269851685, + "step": 1900, + "unclipped_grad_norm": 0.9091746211051941 + }, + { + "bits_per_byte": 1.8773674323326432, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3012919425964355, + "step": 1910, + "unclipped_grad_norm": 0.8313417434692383 + }, + { + "bits_per_byte": 1.8889088440667445, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3092918395996094, + "step": 1920, + "unclipped_grad_norm": 0.9861888289451599 + }, + { + "bits_per_byte": 1.9783957168769355, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3713194131851196, + "step": 1930, + "unclipped_grad_norm": 0.9654703140258789 + }, + { + "bits_per_byte": 1.8474522540936138, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.280556321144104, + "step": 1940, + "unclipped_grad_norm": 0.8361129760742188 + }, + { + "bits_per_byte": 1.9440165569521108, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.347489595413208, + "step": 1950, + "unclipped_grad_norm": 0.7723413705825806 + }, + { + "bits_per_byte": 1.8547983210425678, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.285648226737976, + "step": 1960, + "unclipped_grad_norm": 0.8670309782028198 + }, + { + "bits_per_byte": 1.9366295581322517, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3423693180084229, + "step": 1970, + "unclipped_grad_norm": 0.8607841730117798 + }, + { + "bits_per_byte": 1.973360924772927, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3678295612335205, + "step": 1980, + "unclipped_grad_norm": 0.8964365124702454 + }, + { + "bits_per_byte": 1.9723059831925343, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.367098331451416, + "step": 1990, + "unclipped_grad_norm": 0.9170846343040466 + }, + { + "bits_per_byte": 1.931895563684602, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.339087963104248, + "step": 2000, + "unclipped_grad_norm": 0.8662557601928711 + }, + { + "bits_per_byte": 1.9008518352906107, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3175700902938843, + "step": 2010, + "unclipped_grad_norm": 0.8140403628349304 + }, + { + "bits_per_byte": 1.945781786880567, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3487131595611572, + "step": 2020, + "unclipped_grad_norm": 0.8827610015869141 + }, + { + "bits_per_byte": 2.023320680970017, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.4024590253829956, + "step": 2030, + "unclipped_grad_norm": 0.9354773759841919 + }, + { + "bits_per_byte": 1.9386018551723463, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3437364101409912, + "step": 2040, + "unclipped_grad_norm": 0.8753958940505981 + }, + { + "bits_per_byte": 1.9926081911615872, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3811707496643066, + "step": 2050, + "unclipped_grad_norm": 0.8750029802322388 + }, + { + "bits_per_byte": 1.948962090060322, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3509175777435303, + "step": 2060, + "unclipped_grad_norm": 0.9496069550514221 + }, + { + "bits_per_byte": 1.9059292790920332, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.321089506149292, + "step": 2070, + "unclipped_grad_norm": 0.8713699579238892 + }, + { + "bits_per_byte": 1.8954827089133246, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3138484954833984, + "step": 2080, + "unclipped_grad_norm": 0.8487783074378967 + }, + { + "bits_per_byte": 1.892261817827966, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3116159439086914, + "step": 2090, + "unclipped_grad_norm": 0.859647810459137 + }, + { + "bits_per_byte": 1.8838277886296537, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.305769920349121, + "step": 2100, + "unclipped_grad_norm": 0.8237807750701904 + }, + { + "bits_per_byte": 1.9342706440930173, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3407342433929443, + "step": 2110, + "unclipped_grad_norm": 0.885201632976532 + }, + { + "bits_per_byte": 1.89223670836094, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.311598539352417, + "step": 2120, + "unclipped_grad_norm": 0.8643916249275208 + }, + { + "bits_per_byte": 1.8591027028283622, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.288631796836853, + "step": 2130, + "unclipped_grad_norm": 0.9507161378860474 + }, + { + "bits_per_byte": 1.991344806608349, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3802950382232666, + "step": 2140, + "unclipped_grad_norm": 0.967012882232666 + }, + { + "bits_per_byte": 1.8634151677987472, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2916209697723389, + "step": 2150, + "unclipped_grad_norm": 0.8769172430038452 + }, + { + "bits_per_byte": 1.9079485273958137, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3224891424179077, + "step": 2160, + "unclipped_grad_norm": 0.8441570401191711 + }, + { + "bits_per_byte": 1.982388982047319, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3740873336791992, + "step": 2170, + "unclipped_grad_norm": 0.8798146843910217 + }, + { + "bits_per_byte": 1.855854810466818, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2863805294036865, + "step": 2180, + "unclipped_grad_norm": 0.9269494414329529 + }, + { + "bits_per_byte": 1.881138839883414, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3039060831069946, + "step": 2190, + "unclipped_grad_norm": 0.9328587055206299 + }, + { + "bits_per_byte": 1.909031158182995, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3232395648956299, + "step": 2200, + "unclipped_grad_norm": 0.8771402835845947 + }, + { + "bits_per_byte": 1.8627256893514381, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2911430597305298, + "step": 2210, + "unclipped_grad_norm": 0.9345882534980774 + }, + { + "bits_per_byte": 1.85186464098415, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2836147546768188, + "step": 2220, + "unclipped_grad_norm": 0.8119871020317078 + }, + { + "bits_per_byte": 1.929916043373171, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3377158641815186, + "step": 2230, + "unclipped_grad_norm": 0.8476319909095764 + }, + { + "bits_per_byte": 1.8588350978236197, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.288446307182312, + "step": 2240, + "unclipped_grad_norm": 0.9140865206718445 + }, + { + "bits_per_byte": 1.8803798804451561, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.303380012512207, + "step": 2250, + "unclipped_grad_norm": 0.891239583492279 + }, + { + "bits_per_byte": 1.8744335802915746, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2992583513259888, + "step": 2260, + "unclipped_grad_norm": 0.8316894769668579 + }, + { + "bits_per_byte": 1.8524942694689588, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2840511798858643, + "step": 2270, + "unclipped_grad_norm": 0.8655860424041748 + }, + { + "bits_per_byte": 1.8715530428722735, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2972617149353027, + "step": 2280, + "unclipped_grad_norm": 0.8277263045310974 + }, + { + "bits_per_byte": 1.9110787836241676, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.3246588706970215, + "step": 2290, + "unclipped_grad_norm": 0.9131273031234741 + }, + { + "bits_per_byte": 1.8344360911283637, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2715342044830322, + "step": 2300, + "unclipped_grad_norm": 0.882969856262207 + }, + { + "bits_per_byte": 1.851766782855809, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2835469245910645, + "step": 2310, + "unclipped_grad_norm": 0.854379415512085 + }, + { + "bits_per_byte": 1.8345169229742693, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.271590232849121, + "step": 2320, + "unclipped_grad_norm": 0.8358980417251587 + }, + { + "bits_per_byte": 1.902897396939973, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.3189879655838013, + "step": 2330, + "unclipped_grad_norm": 0.8988526463508606 + }, + { + "bits_per_byte": 1.820362922790911, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2617794275283813, + "step": 2340, + "unclipped_grad_norm": 0.8210547566413879 + }, + { + "bits_per_byte": 1.8788371960669168, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3023107051849365, + "step": 2350, + "unclipped_grad_norm": 0.8696999549865723 + }, + { + "bits_per_byte": 1.8997241450489033, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.3167884349822998, + "step": 2360, + "unclipped_grad_norm": 0.8768331408500671 + }, + { + "bits_per_byte": 1.9060586100454822, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3211791515350342, + "step": 2370, + "unclipped_grad_norm": 0.8498597741127014 + }, + { + "bits_per_byte": 1.8700977256806723, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.296252965927124, + "step": 2380, + "unclipped_grad_norm": 0.9666873216629028 + }, + { + "bits_per_byte": 1.9181192374025384, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3295389413833618, + "step": 2390, + "unclipped_grad_norm": 0.8190925717353821 + }, + { + "bits_per_byte": 1.8665703615114762, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2938079833984375, + "step": 2400, + "unclipped_grad_norm": 0.8776637315750122 + }, + { + "bits_per_byte": 1.8564054989148806, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2867622375488281, + "step": 2410, + "unclipped_grad_norm": 0.8437303900718689 + }, + { + "bits_per_byte": 1.8440313471853016, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2781851291656494, + "step": 2420, + "unclipped_grad_norm": 0.8381853103637695 + }, + { + "bits_per_byte": 1.8694933786455403, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2958340644836426, + "step": 2430, + "unclipped_grad_norm": 0.8620160818099976 + }, + { + "bits_per_byte": 1.9125165585853807, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.325655460357666, + "step": 2440, + "unclipped_grad_norm": 0.8893632292747498 + }, + { + "bits_per_byte": 1.8473590394968462, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2804917097091675, + "step": 2450, + "unclipped_grad_norm": 0.8294656872749329 + }, + { + "bits_per_byte": 1.8569149115267363, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2871153354644775, + "step": 2460, + "unclipped_grad_norm": 0.8894950151443481 + }, + { + "bits_per_byte": 1.8524935815383552, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.284050703048706, + "step": 2470, + "unclipped_grad_norm": 0.857410728931427 + }, + { + "bits_per_byte": 1.8769517502655078, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.3010038137435913, + "step": 2480, + "unclipped_grad_norm": 0.8339816331863403 + }, + { + "bits_per_byte": 1.8457628685141885, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2793853282928467, + "step": 2490, + "unclipped_grad_norm": 0.8252822756767273 + }, + { + "bits_per_byte": 1.8621884155501427, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2907706499099731, + "step": 2500, + "unclipped_grad_norm": 0.8138288259506226 + }, + { + "bits_per_byte": 1.815880023013521, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2586721181869507, + "step": 2510, + "unclipped_grad_norm": 0.8689138889312744 + }, + { + "bits_per_byte": 1.8820847444631592, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.304561734199524, + "step": 2520, + "unclipped_grad_norm": 0.8937636613845825 + }, + { + "bits_per_byte": 1.878350485164975, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.3019733428955078, + "step": 2530, + "unclipped_grad_norm": 0.9091646671295166 + }, + { + "bits_per_byte": 1.8572593927764145, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2873541116714478, + "step": 2540, + "unclipped_grad_norm": 0.8881837129592896 + }, + { + "bits_per_byte": 1.8423865051124497, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2770450115203857, + "step": 2550, + "unclipped_grad_norm": 1.0689516067504883 + }, + { + "bits_per_byte": 1.8515612635880279, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2834044694900513, + "step": 2560, + "unclipped_grad_norm": 0.829948365688324 + }, + { + "bits_per_byte": 1.8581186181001252, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2879496812820435, + "step": 2570, + "unclipped_grad_norm": 0.8880435824394226 + }, + { + "bits_per_byte": 1.880620656156364, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.3035469055175781, + "step": 2580, + "unclipped_grad_norm": 0.9021074175834656 + }, + { + "bits_per_byte": 1.8318979711669305, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2697749137878418, + "step": 2590, + "unclipped_grad_norm": 0.7873226404190063 + }, + { + "bits_per_byte": 1.8382764637221303, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2741961479187012, + "step": 2600, + "unclipped_grad_norm": 0.8448187708854675 + }, + { + "bits_per_byte": 1.7796977969770509, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2335925102233887, + "step": 2610, + "unclipped_grad_norm": 0.8379483222961426 + }, + { + "bits_per_byte": 1.8338702682070251, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2711420059204102, + "step": 2620, + "unclipped_grad_norm": 0.8167812824249268 + }, + { + "bits_per_byte": 1.8538771819645465, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.285009741783142, + "step": 2630, + "unclipped_grad_norm": 0.8943092226982117 + }, + { + "bits_per_byte": 1.7228497916476038, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1941884756088257, + "step": 2640, + "unclipped_grad_norm": 0.8208516240119934 + }, + { + "bits_per_byte": 1.79524330878854, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2443678379058838, + "step": 2650, + "unclipped_grad_norm": 0.8445570468902588 + }, + { + "bits_per_byte": 1.8489555544448055, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2815983295440674, + "step": 2660, + "unclipped_grad_norm": 0.8848552703857422 + }, + { + "bits_per_byte": 1.8531913151529056, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2845343351364136, + "step": 2670, + "unclipped_grad_norm": 0.8651367425918579 + }, + { + "bits_per_byte": 1.762816667898961, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2218914031982422, + "step": 2680, + "unclipped_grad_norm": 0.7578476071357727 + }, + { + "bits_per_byte": 1.9435059404616992, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3471356630325317, + "step": 2690, + "unclipped_grad_norm": 0.8484299778938293 + }, + { + "bits_per_byte": 1.8394798263302174, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.275030255317688, + "step": 2700, + "unclipped_grad_norm": 0.8363795876502991 + }, + { + "bits_per_byte": 1.7593267959476528, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2194724082946777, + "step": 2710, + "unclipped_grad_norm": 0.7895286083221436 + }, + { + "bits_per_byte": 1.7812189835239323, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2346469163894653, + "step": 2720, + "unclipped_grad_norm": 0.8936913013458252 + }, + { + "bits_per_byte": 1.806379013449256, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2520865201950073, + "step": 2730, + "unclipped_grad_norm": 0.8484287858009338 + }, + { + "bits_per_byte": 1.8120035340630734, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.255985140800476, + "step": 2740, + "unclipped_grad_norm": 0.8566655516624451 + }, + { + "bits_per_byte": 1.8444568322635362, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.278480052947998, + "step": 2750, + "unclipped_grad_norm": 0.8961714506149292 + }, + { + "bits_per_byte": 1.855966255224577, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2864577770233154, + "step": 2760, + "unclipped_grad_norm": 0.9054936766624451 + }, + { + "bits_per_byte": 1.7360374213157639, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2033294439315796, + "step": 2770, + "unclipped_grad_norm": 0.8361671566963196 + }, + { + "bits_per_byte": 1.75377691580431, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.215625524520874, + "step": 2780, + "unclipped_grad_norm": 0.7793561220169067 + }, + { + "bits_per_byte": 1.8377969760915247, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2738637924194336, + "step": 2790, + "unclipped_grad_norm": 0.7846143245697021 + }, + { + "bits_per_byte": 1.9117343814892567, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.325113296508789, + "step": 2800, + "unclipped_grad_norm": 0.872995913028717 + }, + { + "bits_per_byte": 1.8341784611173713, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2713556289672852, + "step": 2810, + "unclipped_grad_norm": 0.8531777858734131 + }, + { + "bits_per_byte": 1.8258434939259562, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.265578269958496, + "step": 2820, + "unclipped_grad_norm": 0.8563913702964783 + }, + { + "bits_per_byte": 1.8998995673527832, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.3169100284576416, + "step": 2830, + "unclipped_grad_norm": 0.864924430847168 + }, + { + "bits_per_byte": 1.8091240285396772, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2539892196655273, + "step": 2840, + "unclipped_grad_norm": 0.8685922622680664 + }, + { + "bits_per_byte": 1.8212324670736735, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.26238214969635, + "step": 2850, + "unclipped_grad_norm": 0.8504610061645508 + }, + { + "bits_per_byte": 1.7804541766755455, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.234116792678833, + "step": 2860, + "unclipped_grad_norm": 0.8544119000434875 + }, + { + "bits_per_byte": 1.8702564656674188, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2963629961013794, + "step": 2870, + "unclipped_grad_norm": 0.855370044708252 + }, + { + "bits_per_byte": 1.8717095470845586, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.297370195388794, + "step": 2880, + "unclipped_grad_norm": 0.8495773673057556 + }, + { + "bits_per_byte": 1.7666914370229, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2245771884918213, + "step": 2890, + "unclipped_grad_norm": 0.8828970789909363 + }, + { + "bits_per_byte": 1.7353923143923775, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2028822898864746, + "step": 2900, + "unclipped_grad_norm": 0.8020629286766052 + }, + { + "bits_per_byte": 1.7995504422967483, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2473533153533936, + "step": 2910, + "unclipped_grad_norm": 0.8652002811431885 + }, + { + "bits_per_byte": 1.7761088630188457, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.231104850769043, + "step": 2920, + "unclipped_grad_norm": 0.8863427042961121 + }, + { + "bits_per_byte": 1.7940502651395047, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.24354088306427, + "step": 2930, + "unclipped_grad_norm": 0.8493190407752991 + }, + { + "bits_per_byte": 1.8227453984533135, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2634308338165283, + "step": 2940, + "unclipped_grad_norm": 0.9217897653579712 + }, + { + "bits_per_byte": 1.8002454241888848, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2478350400924683, + "step": 2950, + "unclipped_grad_norm": 0.9446085095405579 + }, + { + "bits_per_byte": 1.8533994141604497, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.28467857837677, + "step": 2960, + "unclipped_grad_norm": 0.9335201382637024 + }, + { + "bits_per_byte": 1.7928699482066333, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.242722749710083, + "step": 2970, + "unclipped_grad_norm": 0.9046677947044373 + }, + { + "bits_per_byte": 1.784769565350995, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2371079921722412, + "step": 2980, + "unclipped_grad_norm": 0.8317873477935791 + }, + { + "bits_per_byte": 1.8006910312372704, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2481439113616943, + "step": 2990, + "unclipped_grad_norm": 0.8293972015380859 + }, + { + "bits_per_byte": 1.8134308180825838, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.256974458694458, + "step": 3000, + "unclipped_grad_norm": 0.8042416572570801 + }, + { + "bits_per_byte": 1.811533849443567, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.255659580230713, + "step": 3010, + "unclipped_grad_norm": 0.7810729146003723 + }, + { + "bits_per_byte": 1.7608942458276164, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2205588817596436, + "step": 3020, + "unclipped_grad_norm": 0.846473753452301 + }, + { + "bits_per_byte": 1.842709660513421, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2772690057754517, + "step": 3030, + "unclipped_grad_norm": 0.8461940884590149 + }, + { + "bits_per_byte": 1.775488005649231, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2306745052337646, + "step": 3040, + "unclipped_grad_norm": 0.8409631252288818 + }, + { + "bits_per_byte": 1.7848693152884954, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2371771335601807, + "step": 3050, + "unclipped_grad_norm": 0.8528071641921997 + }, + { + "bits_per_byte": 1.8093759831231913, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2541638612747192, + "step": 3060, + "unclipped_grad_norm": 0.9029015898704529 + }, + { + "bits_per_byte": 1.7287833650850215, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.1983013153076172, + "step": 3070, + "unclipped_grad_norm": 0.802097737789154 + }, + { + "bits_per_byte": 1.755199900257549, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2166118621826172, + "step": 3080, + "unclipped_grad_norm": 0.8510171175003052 + }, + { + "bits_per_byte": 1.8287252352238135, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.267575740814209, + "step": 3090, + "unclipped_grad_norm": 0.9035980105400085 + }, + { + "bits_per_byte": 1.7486468453117232, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2120696306228638, + "step": 3100, + "unclipped_grad_norm": 0.8420509099960327 + }, + { + "bits_per_byte": 1.8143894493784931, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.257638931274414, + "step": 3110, + "unclipped_grad_norm": 0.8073800206184387 + }, + { + "bits_per_byte": 1.7807735484581977, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2343381643295288, + "step": 3120, + "unclipped_grad_norm": 0.7948007583618164 + }, + { + "bits_per_byte": 1.8404088766101783, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2756742238998413, + "step": 3130, + "unclipped_grad_norm": 0.7875926494598389 + }, + { + "bits_per_byte": 1.8127681689288093, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2565151453018188, + "step": 3140, + "unclipped_grad_norm": 0.8293087482452393 + }, + { + "bits_per_byte": 1.7580950562021733, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.218618631362915, + "step": 3150, + "unclipped_grad_norm": 0.831396222114563 + }, + { + "bits_per_byte": 1.7956681059361712, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2446622848510742, + "step": 3160, + "unclipped_grad_norm": 0.8769922256469727 + }, + { + "bits_per_byte": 1.6987359321874784, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.177474021911621, + "step": 3170, + "unclipped_grad_norm": 0.8662227988243103 + }, + { + "bits_per_byte": 1.8090672742648926, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2539498805999756, + "step": 3180, + "unclipped_grad_norm": 0.87493896484375 + }, + { + "bits_per_byte": 1.77672765659665, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2315337657928467, + "step": 3190, + "unclipped_grad_norm": 0.8155474066734314 + }, + { + "bits_per_byte": 1.7352984118650063, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2028172016143799, + "step": 3200, + "unclipped_grad_norm": 0.8111788630485535 + }, + { + "bits_per_byte": 1.7460088034301386, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2102410793304443, + "step": 3210, + "unclipped_grad_norm": 0.8163402080535889 + }, + { + "bits_per_byte": 1.7462268774314327, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2103922367095947, + "step": 3220, + "unclipped_grad_norm": 0.8070465326309204 + }, + { + "bits_per_byte": 1.8407530138945547, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2759127616882324, + "step": 3230, + "unclipped_grad_norm": 0.9377122521400452 + }, + { + "bits_per_byte": 1.717580415207818, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1905360221862793, + "step": 3240, + "unclipped_grad_norm": 0.817809522151947 + }, + { + "bits_per_byte": 1.8646075235171788, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2924474477767944, + "step": 3250, + "unclipped_grad_norm": 0.8675336837768555 + }, + { + "bits_per_byte": 1.7240638171800442, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1950299739837646, + "step": 3260, + "unclipped_grad_norm": 0.8383887410163879 + }, + { + "bits_per_byte": 1.7558293567597067, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.217048168182373, + "step": 3270, + "unclipped_grad_norm": 0.7917745113372803 + }, + { + "bits_per_byte": 1.8139120255396979, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.257308006286621, + "step": 3280, + "unclipped_grad_norm": 0.819724440574646 + }, + { + "bits_per_byte": 1.8132829130028418, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2568719387054443, + "step": 3290, + "unclipped_grad_norm": 0.8676843047142029 + }, + { + "bits_per_byte": 1.7244280764345716, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1952824592590332, + "step": 3300, + "unclipped_grad_norm": 0.7852486968040466 + }, + { + "bits_per_byte": 1.7655542877353951, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2237889766693115, + "step": 3310, + "unclipped_grad_norm": 0.8482234477996826 + }, + { + "bits_per_byte": 1.7764401016044076, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2313344478607178, + "step": 3320, + "unclipped_grad_norm": 0.80937659740448 + }, + { + "bits_per_byte": 1.7828375122512021, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2357687950134277, + "step": 3330, + "unclipped_grad_norm": 0.8368547558784485 + }, + { + "bits_per_byte": 1.7427524839187023, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2079839706420898, + "step": 3340, + "unclipped_grad_norm": 0.8288648128509521 + }, + { + "bits_per_byte": 1.7505973005551585, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2134215831756592, + "step": 3350, + "unclipped_grad_norm": 0.7933014631271362 + }, + { + "bits_per_byte": 1.7744660847378042, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.229966163635254, + "step": 3360, + "unclipped_grad_norm": 0.8237564563751221 + }, + { + "bits_per_byte": 1.725253937124015, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.195854902267456, + "step": 3370, + "unclipped_grad_norm": 0.8081465363502502 + }, + { + "bits_per_byte": 1.7597923529835384, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2197951078414917, + "step": 3380, + "unclipped_grad_norm": 0.816970944404602 + }, + { + "bits_per_byte": 1.7815524578839552, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2348780632019043, + "step": 3390, + "unclipped_grad_norm": 0.8336570858955383 + }, + { + "bits_per_byte": 1.834212685664893, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2713793516159058, + "step": 3400, + "unclipped_grad_norm": 0.8331590890884399 + }, + { + "bits_per_byte": 1.8815787715043208, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.3042110204696655, + "step": 3410, + "unclipped_grad_norm": 0.8871732354164124 + }, + { + "bits_per_byte": 1.7373591079876445, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2042455673217773, + "step": 3420, + "unclipped_grad_norm": 0.8121070265769958 + }, + { + "bits_per_byte": 1.6796587566405214, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1642507314682007, + "step": 3430, + "unclipped_grad_norm": 0.9302552938461304 + }, + { + "bits_per_byte": 1.7533135945428855, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2153043746948242, + "step": 3440, + "unclipped_grad_norm": 0.8166707754135132 + }, + { + "bits_per_byte": 1.8162779908676177, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2589479684829712, + "step": 3450, + "unclipped_grad_norm": 0.8299344182014465 + }, + { + "bits_per_byte": 1.7445694806250678, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2092434167861938, + "step": 3460, + "unclipped_grad_norm": 0.8381974101066589 + }, + { + "bits_per_byte": 1.7494488004126965, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.212625503540039, + "step": 3470, + "unclipped_grad_norm": 0.8255388140678406 + }, + { + "bits_per_byte": 1.6905657243755898, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1718108654022217, + "step": 3480, + "unclipped_grad_norm": 0.7999947667121887 + }, + { + "bits_per_byte": 1.9018883747273607, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.3182885646820068, + "step": 3490, + "unclipped_grad_norm": 0.8489032983779907 + }, + { + "bits_per_byte": 1.7449648687394015, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.209517478942871, + "step": 3500, + "unclipped_grad_norm": 0.8267634510993958 + }, + { + "bits_per_byte": 1.784236763098622, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.236738681793213, + "step": 3510, + "unclipped_grad_norm": 0.8314446806907654 + }, + { + "bits_per_byte": 1.7790934499419189, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2331736087799072, + "step": 3520, + "unclipped_grad_norm": 0.8323742151260376 + }, + { + "bits_per_byte": 1.7501370749814495, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2131025791168213, + "step": 3530, + "unclipped_grad_norm": 0.8194237947463989 + }, + { + "bits_per_byte": 1.7620009541859187, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2213259935379028, + "step": 3540, + "unclipped_grad_norm": 0.8233645558357239 + }, + { + "bits_per_byte": 1.7622630557458336, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2215076684951782, + "step": 3550, + "unclipped_grad_norm": 0.8949882984161377 + }, + { + "bits_per_byte": 1.7110605529136091, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1860167980194092, + "step": 3560, + "unclipped_grad_norm": 0.862553060054779 + }, + { + "bits_per_byte": 1.73054515536046, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1995224952697754, + "step": 3570, + "unclipped_grad_norm": 0.8900191783905029 + }, + { + "bits_per_byte": 1.763418263211679, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2223083972930908, + "step": 3580, + "unclipped_grad_norm": 0.9058765769004822 + }, + { + "bits_per_byte": 1.7831680629061606, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2359979152679443, + "step": 3590, + "unclipped_grad_norm": 0.8453392386436462 + }, + { + "bits_per_byte": 1.7970675287662419, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.245632290840149, + "step": 3600, + "unclipped_grad_norm": 0.8932130932807922 + }, + { + "bits_per_byte": 1.7267670404863058, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1969037055969238, + "step": 3610, + "unclipped_grad_norm": 0.8229808211326599 + }, + { + "bits_per_byte": 1.6908183668897072, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1719859838485718, + "step": 3620, + "unclipped_grad_norm": 0.8186308741569519 + }, + { + "bits_per_byte": 1.7541439267812513, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2158799171447754, + "step": 3630, + "unclipped_grad_norm": 0.8367486596107483 + }, + { + "bits_per_byte": 1.7116509693540212, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.186426043510437, + "step": 3640, + "unclipped_grad_norm": 0.8415036797523499 + }, + { + "bits_per_byte": 1.7837473004742666, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2363994121551514, + "step": 3650, + "unclipped_grad_norm": 0.8471388816833496 + }, + { + "bits_per_byte": 1.6866421121788058, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1690912246704102, + "step": 3660, + "unclipped_grad_norm": 0.823422908782959 + }, + { + "bits_per_byte": 1.7654146378228945, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2236921787261963, + "step": 3670, + "unclipped_grad_norm": 0.819604218006134 + }, + { + "bits_per_byte": 1.6976106497028831, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1766940355300903, + "step": 3680, + "unclipped_grad_norm": 0.8133332133293152 + }, + { + "bits_per_byte": 1.6802502049768386, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1646606922149658, + "step": 3690, + "unclipped_grad_norm": 0.7951709032058716 + }, + { + "bits_per_byte": 1.6953377269890801, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1751185655593872, + "step": 3700, + "unclipped_grad_norm": 0.8330065011978149 + }, + { + "bits_per_byte": 1.747265652642644, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.2111122608184814, + "step": 3710, + "unclipped_grad_norm": 0.8084886074066162 + }, + { + "bits_per_byte": 1.7300491573953718, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.199178695678711, + "step": 3720, + "unclipped_grad_norm": 0.8567172288894653 + }, + { + "bits_per_byte": 1.6577672569948454, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1490767002105713, + "step": 3730, + "unclipped_grad_norm": 0.8171049952507019 + }, + { + "bits_per_byte": 1.720541440507723, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.192588448524475, + "step": 3740, + "unclipped_grad_norm": 0.8777816295623779 + }, + { + "bits_per_byte": 1.7468836791750777, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.2108474969863892, + "step": 3750, + "unclipped_grad_norm": 0.8143476843833923 + }, + { + "bits_per_byte": 1.8090201510185562, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2539172172546387, + "step": 3760, + "unclipped_grad_norm": 0.8902925252914429 + }, + { + "bits_per_byte": 1.7778630860576463, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.232320785522461, + "step": 3770, + "unclipped_grad_norm": 0.8659905791282654 + }, + { + "bits_per_byte": 1.6107073002739534, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.116457223892212, + "step": 3780, + "unclipped_grad_norm": 0.7876954674720764 + }, + { + "bits_per_byte": 1.7097620838995948, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1851167678833008, + "step": 3790, + "unclipped_grad_norm": 0.8166313767433167 + }, + { + "bits_per_byte": 1.703528744701723, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1807961463928223, + "step": 3800, + "unclipped_grad_norm": 0.7970600128173828 + }, + { + "bits_per_byte": 1.7306136044555034, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1995699405670166, + "step": 3810, + "unclipped_grad_norm": 0.9229923486709595 + }, + { + "bits_per_byte": 1.7223408949837007, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.193835735321045, + "step": 3820, + "unclipped_grad_norm": 0.88306725025177 + }, + { + "bits_per_byte": 1.7747577673136674, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.230168342590332, + "step": 3830, + "unclipped_grad_norm": 0.9060350060462952 + }, + { + "bits_per_byte": 1.6914633018304428, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1724330186843872, + "step": 3840, + "unclipped_grad_norm": 0.8194422721862793 + }, + { + "bits_per_byte": 1.648618811864849, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.142735481262207, + "step": 3850, + "unclipped_grad_norm": 0.8284996747970581 + }, + { + "bits_per_byte": 1.7050731489064712, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1818666458129883, + "step": 3860, + "unclipped_grad_norm": 0.8395226001739502 + }, + { + "bits_per_byte": 1.7881865166583373, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2394764423370361, + "step": 3870, + "unclipped_grad_norm": 0.8676077127456665 + }, + { + "bits_per_byte": 1.6805537543556115, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.164871096611023, + "step": 3880, + "unclipped_grad_norm": 0.8120661377906799 + }, + { + "bits_per_byte": 1.6503448297489083, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1439318656921387, + "step": 3890, + "unclipped_grad_norm": 0.8249826431274414 + }, + { + "bits_per_byte": 1.7267109741421247, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1968648433685303, + "step": 3900, + "unclipped_grad_norm": 0.9262217879295349 + }, + { + "bits_per_byte": 1.7407927416121205, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.2066255807876587, + "step": 3910, + "unclipped_grad_norm": 0.894594669342041 + }, + { + "bits_per_byte": 1.6851732083577864, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.168073058128357, + "step": 3920, + "unclipped_grad_norm": 0.7890235185623169 + }, + { + "bits_per_byte": 1.7178566193451037, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1907274723052979, + "step": 3930, + "unclipped_grad_norm": 0.8578466773033142 + }, + { + "bits_per_byte": 1.7552539028099199, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.2166492938995361, + "step": 3940, + "unclipped_grad_norm": 0.8101285696029663 + }, + { + "bits_per_byte": 1.7865160491705068, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2383185625076294, + "step": 3950, + "unclipped_grad_norm": 0.8577295541763306 + }, + { + "bits_per_byte": 1.7671413436375571, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2248890399932861, + "step": 3960, + "unclipped_grad_norm": 0.9096070528030396 + }, + { + "bits_per_byte": 1.6949598811051345, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1748566627502441, + "step": 3970, + "unclipped_grad_norm": 0.8878244757652283 + }, + { + "bits_per_byte": 1.7218173797944742, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1934728622436523, + "step": 3980, + "unclipped_grad_norm": 0.8564468026161194 + }, + { + "bits_per_byte": 1.7444124604648301, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.209134578704834, + "step": 3990, + "unclipped_grad_norm": 0.8601830005645752 + }, + { + "bits_per_byte": 1.6364575746570382, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1343059539794922, + "step": 4000, + "unclipped_grad_norm": 0.8214763402938843 + }, + { + "bits_per_byte": 1.7331732222482947, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2013441324234009, + "step": 4010, + "unclipped_grad_norm": 0.8387731909751892 + }, + { + "bits_per_byte": 1.7078159282224312, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1837677955627441, + "step": 4020, + "unclipped_grad_norm": 0.8044644594192505 + }, + { + "bits_per_byte": 1.7505291954254167, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.213374376296997, + "step": 4030, + "unclipped_grad_norm": 0.8809406161308289 + }, + { + "bits_per_byte": 1.6714938802908095, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1585912704467773, + "step": 4040, + "unclipped_grad_norm": 0.877934455871582 + }, + { + "bits_per_byte": 1.7160740191689108, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.189491868019104, + "step": 4050, + "unclipped_grad_norm": 0.8732415437698364 + }, + { + "bits_per_byte": 1.7110662283410876, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1860207319259644, + "step": 4060, + "unclipped_grad_norm": 0.8489102721214294 + }, + { + "bits_per_byte": 1.7342386547703899, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.202082633972168, + "step": 4070, + "unclipped_grad_norm": 0.8639312386512756 + }, + { + "bits_per_byte": 1.7989254573435127, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.246920108795166, + "step": 4080, + "unclipped_grad_norm": 0.8819956183433533 + }, + { + "bits_per_byte": 1.705004355846126, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.181818962097168, + "step": 4090, + "unclipped_grad_norm": 0.8767969012260437 + }, + { + "bits_per_byte": 1.6958074116085866, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1754441261291504, + "step": 4100, + "unclipped_grad_norm": 0.8374283313751221 + }, + { + "bits_per_byte": 1.693619276341659, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1739274263381958, + "step": 4110, + "unclipped_grad_norm": 0.8304628729820251 + }, + { + "bits_per_byte": 1.744371184628623, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.2091059684753418, + "step": 4120, + "unclipped_grad_norm": 0.9000032544136047 + }, + { + "bits_per_byte": 1.6641087732801096, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1534723043441772, + "step": 4130, + "unclipped_grad_norm": 0.8784226775169373 + }, + { + "bits_per_byte": 1.7338391390724355, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2018057107925415, + "step": 4140, + "unclipped_grad_norm": 0.864092230796814 + }, + { + "bits_per_byte": 1.6882804189109248, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.170226812362671, + "step": 4150, + "unclipped_grad_norm": 0.8413121104240417 + }, + { + "bits_per_byte": 1.698938527750195, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.177614450454712, + "step": 4160, + "unclipped_grad_norm": 0.9109639525413513 + }, + { + "bits_per_byte": 1.7387922394172843, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.205238938331604, + "step": 4170, + "unclipped_grad_norm": 0.8913268446922302 + }, + { + "bits_per_byte": 1.7814171075377263, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.2347842454910278, + "step": 4180, + "unclipped_grad_norm": 0.9075717926025391 + }, + { + "bits_per_byte": 1.694894183732505, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1748111248016357, + "step": 4190, + "unclipped_grad_norm": 0.8244585990905762 + }, + { + "bits_per_byte": 1.7220003693349923, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1935997009277344, + "step": 4200, + "unclipped_grad_norm": 0.8734019994735718 + }, + { + "bits_per_byte": 1.725182564323907, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1958054304122925, + "step": 4210, + "unclipped_grad_norm": 0.834058403968811 + }, + { + "bits_per_byte": 1.7486925926968528, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.2121013402938843, + "step": 4220, + "unclipped_grad_norm": 0.8368670344352722 + }, + { + "bits_per_byte": 1.720319754870761, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1924347877502441, + "step": 4230, + "unclipped_grad_norm": 0.8471981287002563 + }, + { + "bits_per_byte": 1.6660473617206353, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1548160314559937, + "step": 4240, + "unclipped_grad_norm": 0.8083959221839905 + }, + { + "bits_per_byte": 1.6911604403822735, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1722230911254883, + "step": 4250, + "unclipped_grad_norm": 0.8626519441604614 + }, + { + "bits_per_byte": 1.6717248529909183, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.158751368522644, + "step": 4260, + "unclipped_grad_norm": 0.8391480445861816 + }, + { + "bits_per_byte": 1.6638891514349579, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.153320074081421, + "step": 4270, + "unclipped_grad_norm": 0.8006730079650879 + }, + { + "bits_per_byte": 1.6802054894876142, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1646296977996826, + "step": 4280, + "unclipped_grad_norm": 0.8485363125801086 + }, + { + "bits_per_byte": 1.732838028061763, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.2011117935180664, + "step": 4290, + "unclipped_grad_norm": 0.8599663972854614 + }, + { + "bits_per_byte": 1.6415455094001634, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1378326416015625, + "step": 4300, + "unclipped_grad_norm": 0.8445435166358948 + }, + { + "bits_per_byte": 1.7360513519104837, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2033390998840332, + "step": 4310, + "unclipped_grad_norm": 0.9501782655715942 + }, + { + "bits_per_byte": 1.7070877536786782, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1832630634307861, + "step": 4320, + "unclipped_grad_norm": 0.8405798673629761 + }, + { + "bits_per_byte": 1.6698505860618154, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1574522256851196, + "step": 4330, + "unclipped_grad_norm": 0.8460624814033508 + }, + { + "bits_per_byte": 1.709274341101748, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1847786903381348, + "step": 4340, + "unclipped_grad_norm": 0.8663898706436157 + }, + { + "bits_per_byte": 1.6889219141986431, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1706714630126953, + "step": 4350, + "unclipped_grad_norm": 0.9172630310058594 + }, + { + "bits_per_byte": 1.675118930605696, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1611039638519287, + "step": 4360, + "unclipped_grad_norm": 0.8827696442604065 + }, + { + "bits_per_byte": 1.690404060683779, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.171698808670044, + "step": 4370, + "unclipped_grad_norm": 0.8694136738777161 + }, + { + "bits_per_byte": 1.7489961420756257, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.2123117446899414, + "step": 4380, + "unclipped_grad_norm": 0.8883715867996216 + }, + { + "bits_per_byte": 1.7051840777162777, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1819435358047485, + "step": 4390, + "unclipped_grad_norm": 0.8627059459686279 + }, + { + "bits_per_byte": 1.6575398959304046, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1489191055297852, + "step": 4400, + "unclipped_grad_norm": 0.8316671848297119 + }, + { + "bits_per_byte": 1.7285451691135765, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1981362104415894, + "step": 4410, + "unclipped_grad_norm": 0.8469527959823608 + }, + { + "bits_per_byte": 1.6753765606166884, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1612825393676758, + "step": 4420, + "unclipped_grad_norm": 0.829068124294281 + }, + { + "bits_per_byte": 1.7212594680750752, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1930861473083496, + "step": 4430, + "unclipped_grad_norm": 0.8506612777709961 + }, + { + "bits_per_byte": 1.6578714784812683, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.149148941040039, + "step": 4440, + "unclipped_grad_norm": 0.8394047021865845 + }, + { + "bits_per_byte": 1.6499258800114065, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.143641471862793, + "step": 4450, + "unclipped_grad_norm": 0.8416959047317505 + }, + { + "bits_per_byte": 1.64743299148715, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1419135332107544, + "step": 4460, + "unclipped_grad_norm": 0.8612979054450989 + }, + { + "bits_per_byte": 1.6939761403421993, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1741747856140137, + "step": 4470, + "unclipped_grad_norm": 0.9045028686523438 + }, + { + "bits_per_byte": 1.7161144350918636, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1895198822021484, + "step": 4480, + "unclipped_grad_norm": 0.9105556607246399 + }, + { + "bits_per_byte": 1.6800321309755446, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1645095348358154, + "step": 4490, + "unclipped_grad_norm": 0.8570636510848999 + }, + { + "bits_per_byte": 1.7013357939205715, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1792761087417603, + "step": 4500, + "unclipped_grad_norm": 0.8411341905593872 + }, + { + "bits_per_byte": 1.6660826181640622, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1548404693603516, + "step": 4510, + "unclipped_grad_norm": 0.8592174053192139 + }, + { + "bits_per_byte": 1.7057920363870778, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1823649406433105, + "step": 4520, + "unclipped_grad_norm": 0.8456743955612183 + }, + { + "bits_per_byte": 1.7124625554834427, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1869885921478271, + "step": 4530, + "unclipped_grad_norm": 0.8667335510253906 + }, + { + "bits_per_byte": 1.761984099886134, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.2213143110275269, + "step": 4540, + "unclipped_grad_norm": 0.8827010989189148 + }, + { + "bits_per_byte": 1.608659158884828, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1150375604629517, + "step": 4550, + "unclipped_grad_norm": 0.8445418477058411 + }, + { + "bits_per_byte": 1.6846809940110172, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1677318811416626, + "step": 4560, + "unclipped_grad_norm": 0.9384724497795105 + }, + { + "bits_per_byte": 1.6632275341770886, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.152861475944519, + "step": 4570, + "unclipped_grad_norm": 0.836455762386322 + }, + { + "bits_per_byte": 1.6185414539860559, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.121887445449829, + "step": 4580, + "unclipped_grad_norm": 0.8520491123199463 + }, + { + "bits_per_byte": 1.6763542819868433, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.161960244178772, + "step": 4590, + "unclipped_grad_norm": 0.9324198961257935 + }, + { + "bits_per_byte": 1.667847676109867, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1560639142990112, + "step": 4600, + "unclipped_grad_norm": 0.8337410092353821 + }, + { + "bits_per_byte": 1.7332526782329933, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.2013992071151733, + "step": 4610, + "unclipped_grad_norm": 0.934492290019989 + }, + { + "bits_per_byte": 1.7177494741536163, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1906532049179077, + "step": 4620, + "unclipped_grad_norm": 0.8968923091888428 + }, + { + "bits_per_byte": 1.707787379102388, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1837480068206787, + "step": 4630, + "unclipped_grad_norm": 0.9312987327575684 + }, + { + "bits_per_byte": 1.7495194852822011, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.2126744985580444, + "step": 4640, + "unclipped_grad_norm": 0.8537502884864807 + }, + { + "bits_per_byte": 1.6591016703828898, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1500016450881958, + "step": 4650, + "unclipped_grad_norm": 0.8534259796142578 + }, + { + "bits_per_byte": 1.63647786860984, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1343200206756592, + "step": 4660, + "unclipped_grad_norm": 0.862847626209259 + }, + { + "bits_per_byte": 1.758119477738596, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.2186355590820312, + "step": 4670, + "unclipped_grad_norm": 0.8828908205032349 + }, + { + "bits_per_byte": 1.7750497938548324, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.2303707599639893, + "step": 4680, + "unclipped_grad_norm": 0.9205049872398376 + }, + { + "bits_per_byte": 1.7426339878722579, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.2079018354415894, + "step": 4690, + "unclipped_grad_norm": 0.92873615026474 + }, + { + "bits_per_byte": 1.659708425175134, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.150422215461731, + "step": 4700, + "unclipped_grad_norm": 0.8619311451911926 + }, + { + "bits_per_byte": 1.7262008735996655, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1965112686157227, + "step": 4710, + "unclipped_grad_norm": 0.9467828869819641 + }, + { + "bits_per_byte": 1.6608278602495998, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.151198148727417, + "step": 4720, + "unclipped_grad_norm": 0.8912813067436218 + }, + { + "bits_per_byte": 1.5957210760430698, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.106069564819336, + "step": 4730, + "unclipped_grad_norm": 0.8848060369491577 + }, + { + "bits_per_byte": 1.6368768683598418, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.134596586227417, + "step": 4740, + "unclipped_grad_norm": 0.9091616868972778 + }, + { + "bits_per_byte": 1.6949358035340139, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.174839973449707, + "step": 4750, + "unclipped_grad_norm": 0.9046142101287842 + }, + { + "bits_per_byte": 1.665315747523865, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1543089151382446, + "step": 4760, + "unclipped_grad_norm": 0.8868388533592224 + }, + { + "bits_per_byte": 1.6325195159175816, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1315762996673584, + "step": 4770, + "unclipped_grad_norm": 0.839006781578064 + }, + { + "bits_per_byte": 1.6782909786182094, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1633026599884033, + "step": 4780, + "unclipped_grad_norm": 0.930033266544342 + }, + { + "bits_per_byte": 1.6275290953374955, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1281172037124634, + "step": 4790, + "unclipped_grad_norm": 0.8176500201225281 + }, + { + "bits_per_byte": 1.6333247386889214, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1321344375610352, + "step": 4800, + "unclipped_grad_norm": 0.855272650718689 + }, + { + "bits_per_byte": 1.6820415762682257, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1659023761749268, + "step": 4810, + "unclipped_grad_norm": 0.9120509624481201 + }, + { + "bits_per_byte": 1.6350118884938853, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1333038806915283, + "step": 4820, + "unclipped_grad_norm": 0.89643794298172 + }, + { + "bits_per_byte": 1.6668641073295827, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1553821563720703, + "step": 4830, + "unclipped_grad_norm": 0.9077607989311218 + }, + { + "bits_per_byte": 1.667841656717087, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.156059741973877, + "step": 4840, + "unclipped_grad_norm": 0.9171648621559143 + }, + { + "bits_per_byte": 1.6750915853642085, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1610850095748901, + "step": 4850, + "unclipped_grad_norm": 0.8878066539764404 + }, + { + "bits_per_byte": 1.6131449823672828, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1181468963623047, + "step": 4860, + "unclipped_grad_norm": 0.8896912932395935 + }, + { + "bits_per_byte": 1.6943018754829335, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1744005680084229, + "step": 4870, + "unclipped_grad_norm": 0.8964843153953552 + }, + { + "bits_per_byte": 1.5440334101527613, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0702424049377441, + "step": 4880, + "unclipped_grad_norm": 0.8637375235557556 + }, + { + "bits_per_byte": 1.609086879737524, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1153340339660645, + "step": 4890, + "unclipped_grad_norm": 0.8875832557678223 + }, + { + "bits_per_byte": 1.651299333461197, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1445934772491455, + "step": 4900, + "unclipped_grad_norm": 0.9340603351593018 + }, + { + "bits_per_byte": 1.7515653908968651, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.2140926122665405, + "step": 4910, + "unclipped_grad_norm": 0.9391474723815918 + }, + { + "bits_per_byte": 1.6798804422774836, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1644043922424316, + "step": 4920, + "unclipped_grad_norm": 0.8976443409919739 + }, + { + "bits_per_byte": 1.6454846000355252, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1405630111694336, + "step": 4930, + "unclipped_grad_norm": 0.8620614409446716 + }, + { + "bits_per_byte": 1.6197019929140781, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1226918697357178, + "step": 4940, + "unclipped_grad_norm": 0.8644847869873047 + }, + { + "bits_per_byte": 1.5870352642438945, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1000490188598633, + "step": 4950, + "unclipped_grad_norm": 0.8775791525840759 + }, + { + "bits_per_byte": 1.5784220291233835, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.094078779220581, + "step": 4960, + "unclipped_grad_norm": 0.8890455961227417 + }, + { + "bits_per_byte": 1.6649912162616867, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1540839672088623, + "step": 4970, + "unclipped_grad_norm": 0.9286753535270691 + }, + { + "bits_per_byte": 1.7156910138054393, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1892263889312744, + "step": 4980, + "unclipped_grad_norm": 0.9152454137802124 + }, + { + "bits_per_byte": 1.6570270436655319, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1485636234283447, + "step": 4990, + "unclipped_grad_norm": 0.9534220099449158 + }, + { + "bits_per_byte": 1.6136648859208411, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1185072660446167, + "step": 5000, + "unclipped_grad_norm": 0.8542637825012207 + }, + { + "bits_per_byte": 1.5871047452348432, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1000971794128418, + "step": 5010, + "unclipped_grad_norm": 0.8614861965179443 + }, + { + "bits_per_byte": 1.6463324745042789, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.141150712966919, + "step": 5020, + "unclipped_grad_norm": 0.8538332581520081 + }, + { + "bits_per_byte": 1.630251408718003, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1300041675567627, + "step": 5030, + "unclipped_grad_norm": 0.8974499702453613 + }, + { + "bits_per_byte": 1.6081984173631667, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1147181987762451, + "step": 5040, + "unclipped_grad_norm": 0.8664147853851318 + }, + { + "bits_per_byte": 1.566623675308893, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0859007835388184, + "step": 5050, + "unclipped_grad_norm": 0.8855953216552734 + }, + { + "bits_per_byte": 1.5903796388725728, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1023671627044678, + "step": 5060, + "unclipped_grad_norm": 0.8359810709953308 + }, + { + "bits_per_byte": 1.6634062241513352, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1529853343963623, + "step": 5070, + "unclipped_grad_norm": 0.8875454068183899 + }, + { + "bits_per_byte": 1.6210933325595582, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1236562728881836, + "step": 5080, + "unclipped_grad_norm": 0.9011101722717285 + }, + { + "bits_per_byte": 1.6245405528134524, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1260457038879395, + "step": 5090, + "unclipped_grad_norm": 0.8935633301734924 + }, + { + "bits_per_byte": 1.6269426344980533, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1277107000350952, + "step": 5100, + "unclipped_grad_norm": 0.8533574342727661 + }, + { + "bits_per_byte": 1.565103176692615, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0848468542099, + "step": 5110, + "unclipped_grad_norm": 0.9117326736450195 + }, + { + "bits_per_byte": 1.6439374441083634, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1394906044006348, + "step": 5120, + "unclipped_grad_norm": 0.8894507884979248 + }, + { + "bits_per_byte": 1.6476570848812242, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.142068862915039, + "step": 5130, + "unclipped_grad_norm": 0.8907402753829956 + }, + { + "bits_per_byte": 1.615579052824944, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.119834065437317, + "step": 5140, + "unclipped_grad_norm": 0.8878148794174194 + }, + { + "bits_per_byte": 1.6886897376199783, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1705105304718018, + "step": 5150, + "unclipped_grad_norm": 0.906775712966919 + }, + { + "bits_per_byte": 1.5749478075933039, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0916706323623657, + "step": 5160, + "unclipped_grad_norm": 0.8668916821479797 + }, + { + "bits_per_byte": 1.7387628303839868, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.2052185535430908, + "step": 5170, + "unclipped_grad_norm": 0.951082706451416 + }, + { + "bits_per_byte": 1.633963310271575, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1325770616531372, + "step": 5180, + "unclipped_grad_norm": 0.9125628471374512 + }, + { + "bits_per_byte": 1.571220427601154, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0890870094299316, + "step": 5190, + "unclipped_grad_norm": 0.837740421295166 + }, + { + "bits_per_byte": 1.600543469573263, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1094121932983398, + "step": 5200, + "unclipped_grad_norm": 0.8841894865036011 + }, + { + "bits_per_byte": 1.630884304873178, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1304428577423096, + "step": 5210, + "unclipped_grad_norm": 0.8948630690574646 + }, + { + "bits_per_byte": 1.7074470254363305, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1835120916366577, + "step": 5220, + "unclipped_grad_norm": 0.9275010228157043 + }, + { + "bits_per_byte": 1.6155556631844268, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.119817852973938, + "step": 5230, + "unclipped_grad_norm": 0.8827324509620667 + }, + { + "bits_per_byte": 1.615808649663846, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1199932098388672, + "step": 5240, + "unclipped_grad_norm": 0.8995877504348755 + }, + { + "bits_per_byte": 1.592627108154048, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1039249897003174, + "step": 5250, + "unclipped_grad_norm": 0.8981198668479919 + }, + { + "bits_per_byte": 1.60489222288298, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.112426519393921, + "step": 5260, + "unclipped_grad_norm": 0.8831214308738708 + }, + { + "bits_per_byte": 1.5994845723919007, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1086782217025757, + "step": 5270, + "unclipped_grad_norm": 0.8702658414840698 + }, + { + "bits_per_byte": 1.6097115207254575, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.115767002105713, + "step": 5280, + "unclipped_grad_norm": 0.9307021498680115 + }, + { + "bits_per_byte": 1.6134614304448702, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1183662414550781, + "step": 5290, + "unclipped_grad_norm": 0.9075614213943481 + }, + { + "bits_per_byte": 1.6247061721062332, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.1261605024337769, + "step": 5300, + "unclipped_grad_norm": 0.8755994439125061 + }, + { + "bits_per_byte": 1.6182275856482313, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.121669888496399, + "step": 5310, + "unclipped_grad_norm": 0.914459764957428 + }, + { + "bits_per_byte": 1.6139610400456268, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1187125444412231, + "step": 5320, + "unclipped_grad_norm": 0.8940036296844482 + }, + { + "bits_per_byte": 1.6078936641058377, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1145069599151611, + "step": 5330, + "unclipped_grad_norm": 0.9279354214668274 + }, + { + "bits_per_byte": 1.569059465593063, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.087589144706726, + "step": 5340, + "unclipped_grad_norm": 0.8639222979545593 + }, + { + "bits_per_byte": 1.6467650108711989, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1414505243301392, + "step": 5350, + "unclipped_grad_norm": 0.9308397769927979 + }, + { + "bits_per_byte": 1.565089418080546, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0848373174667358, + "step": 5360, + "unclipped_grad_norm": 0.9050061702728271 + }, + { + "bits_per_byte": 1.5389489150626532, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0667181015014648, + "step": 5370, + "unclipped_grad_norm": 0.8974517583847046 + }, + { + "bits_per_byte": 1.6578491207366561, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1491334438323975, + "step": 5380, + "unclipped_grad_norm": 0.9271295070648193 + }, + { + "bits_per_byte": 1.5921135679585716, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.1035690307617188, + "step": 5390, + "unclipped_grad_norm": 0.884811282157898 + }, + { + "bits_per_byte": 1.6126496723327979, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1178035736083984, + "step": 5400, + "unclipped_grad_norm": 0.9159505367279053 + }, + { + "bits_per_byte": 1.6339560870002385, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.132572054862976, + "step": 5410, + "unclipped_grad_norm": 0.9119471907615662 + }, + { + "bits_per_byte": 1.595336178870439, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1058027744293213, + "step": 5420, + "unclipped_grad_norm": 0.9820973873138428 + }, + { + "bits_per_byte": 1.649176723584248, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1431221961975098, + "step": 5430, + "unclipped_grad_norm": 0.9529035091400146 + }, + { + "bits_per_byte": 1.5866154545931386, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0997580289840698, + "step": 5440, + "unclipped_grad_norm": 0.9037122130393982 + }, + { + "bits_per_byte": 1.5834752233710345, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.097581386566162, + "step": 5450, + "unclipped_grad_norm": 0.9287867546081543 + }, + { + "bits_per_byte": 1.5614380544200777, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0823063850402832, + "step": 5460, + "unclipped_grad_norm": 0.8947009444236755 + }, + { + "bits_per_byte": 1.607795462012195, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1144388914108276, + "step": 5470, + "unclipped_grad_norm": 0.9344596862792969 + }, + { + "bits_per_byte": 1.6195384374131077, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.122578501701355, + "step": 5480, + "unclipped_grad_norm": 0.8986119031906128 + }, + { + "bits_per_byte": 1.630767700635893, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1303620338439941, + "step": 5490, + "unclipped_grad_norm": 0.9683650135993958 + }, + { + "bits_per_byte": 1.6068012303075572, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1137497425079346, + "step": 5500, + "unclipped_grad_norm": 0.913102924823761 + }, + { + "bits_per_byte": 1.7072599083121918, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1833823919296265, + "step": 5510, + "unclipped_grad_norm": 1.0086424350738525 + }, + { + "bits_per_byte": 1.6241427569420068, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1257699728012085, + "step": 5520, + "unclipped_grad_norm": 0.9219017624855042 + }, + { + "bits_per_byte": 1.5865065895751422, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0996825695037842, + "step": 5530, + "unclipped_grad_norm": 0.8849281668663025 + }, + { + "bits_per_byte": 1.663044200671269, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.152734398841858, + "step": 5540, + "unclipped_grad_norm": 1.0463494062423706 + }, + { + "bits_per_byte": 1.588076447212218, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.1007707118988037, + "step": 5550, + "unclipped_grad_norm": 0.8995247483253479 + }, + { + "bits_per_byte": 1.5728786843207734, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0902364253997803, + "step": 5560, + "unclipped_grad_norm": 0.9297628998756409 + }, + { + "bits_per_byte": 1.553519629209052, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0768177509307861, + "step": 5570, + "unclipped_grad_norm": 0.9231326580047607 + }, + { + "bits_per_byte": 1.6281497807244594, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1285474300384521, + "step": 5580, + "unclipped_grad_norm": 0.932182788848877 + }, + { + "bits_per_byte": 1.5666377778862637, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0859105587005615, + "step": 5590, + "unclipped_grad_norm": 0.9885890483856201 + }, + { + "bits_per_byte": 1.6130727496539203, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1180968284606934, + "step": 5600, + "unclipped_grad_norm": 0.9734658598899841 + }, + { + "bits_per_byte": 1.6005983320388883, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1094502210617065, + "step": 5610, + "unclipped_grad_norm": 0.8992385864257812 + }, + { + "bits_per_byte": 1.607953514068338, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1145484447479248, + "step": 5620, + "unclipped_grad_norm": 0.9051816463470459 + }, + { + "bits_per_byte": 1.594605252604272, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.1052961349487305, + "step": 5630, + "unclipped_grad_norm": 0.9290134906768799 + }, + { + "bits_per_byte": 1.5677630603708592, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0866905450820923, + "step": 5640, + "unclipped_grad_norm": 0.9029556512832642 + }, + { + "bits_per_byte": 1.5944298303003919, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1051745414733887, + "step": 5650, + "unclipped_grad_norm": 0.9501181244850159 + }, + { + "bits_per_byte": 1.676524028863245, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1620779037475586, + "step": 5660, + "unclipped_grad_norm": 0.9740756750106812 + }, + { + "bits_per_byte": 1.5742370032972879, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0911779403686523, + "step": 5670, + "unclipped_grad_norm": 0.8980613350868225 + }, + { + "bits_per_byte": 1.5945870224432805, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.105283498764038, + "step": 5680, + "unclipped_grad_norm": 0.9082752466201782 + }, + { + "bits_per_byte": 1.6598031876157593, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1504878997802734, + "step": 5690, + "unclipped_grad_norm": 0.9417368769645691 + }, + { + "bits_per_byte": 1.5763857545371678, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0926673412322998, + "step": 5700, + "unclipped_grad_norm": 0.9450368881225586 + }, + { + "bits_per_byte": 1.6005926566114097, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1094462871551514, + "step": 5710, + "unclipped_grad_norm": 0.9505442976951599 + }, + { + "bits_per_byte": 1.6062563892696238, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1133720874786377, + "step": 5720, + "unclipped_grad_norm": 0.8568877577781677 + }, + { + "bits_per_byte": 1.6007586198694923, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.1095613241195679, + "step": 5730, + "unclipped_grad_norm": 0.885662853717804 + }, + { + "bits_per_byte": 1.596347780822814, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.106503963470459, + "step": 5740, + "unclipped_grad_norm": 0.8908458948135376 + }, + { + "bits_per_byte": 1.6113288455741714, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.1168880462646484, + "step": 5750, + "unclipped_grad_norm": 0.9814403057098389 + }, + { + "bits_per_byte": 1.6378140018243932, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.1352461576461792, + "step": 5760, + "unclipped_grad_norm": 1.000325322151184 + }, + { + "bits_per_byte": 1.57939407506606, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.094752550125122, + "step": 5770, + "unclipped_grad_norm": 0.9007365703582764 + }, + { + "bits_per_byte": 1.5515046804715436, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0754210948944092, + "step": 5780, + "unclipped_grad_norm": 0.9067476987838745 + }, + { + "bits_per_byte": 1.586475632697987, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.099661111831665, + "step": 5790, + "unclipped_grad_norm": 0.9168080687522888 + }, + { + "bits_per_byte": 1.544505330546729, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0705695152282715, + "step": 5800, + "unclipped_grad_norm": 0.9089300036430359 + }, + { + "bits_per_byte": 1.6342659997370934, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.1327868700027466, + "step": 5810, + "unclipped_grad_norm": 0.9730345010757446 + }, + { + "bits_per_byte": 1.5628627586998252, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0832939147949219, + "step": 5820, + "unclipped_grad_norm": 0.9135822057723999 + }, + { + "bits_per_byte": 1.5753394120893185, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0919420719146729, + "step": 5830, + "unclipped_grad_norm": 0.9162984490394592 + }, + { + "bits_per_byte": 1.5812992988723182, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0960731506347656, + "step": 5840, + "unclipped_grad_norm": 0.946280300617218 + }, + { + "bits_per_byte": 1.5446322537430657, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.07065749168396, + "step": 5850, + "unclipped_grad_norm": 0.9933390021324158 + }, + { + "bits_per_byte": 1.596009318965916, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.106269359588623, + "step": 5860, + "unclipped_grad_norm": 1.0133196115493774 + }, + { + "bits_per_byte": 1.5986716104012721, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1081147193908691, + "step": 5870, + "unclipped_grad_norm": 0.9077220559120178 + }, + { + "bits_per_byte": 1.616603209510832, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1205439567565918, + "step": 5880, + "unclipped_grad_norm": 0.9455923438072205 + }, + { + "bits_per_byte": 1.6185034458202152, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1218611001968384, + "step": 5890, + "unclipped_grad_norm": 0.9350056648254395 + }, + { + "bits_per_byte": 1.6333419369540076, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1321463584899902, + "step": 5900, + "unclipped_grad_norm": 1.0195506811141968 + }, + { + "bits_per_byte": 1.6111236702716922, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.1167458295822144, + "step": 5910, + "unclipped_grad_norm": 0.9529072642326355 + }, + { + "bits_per_byte": 1.5460519705259381, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0716415643692017, + "step": 5920, + "unclipped_grad_norm": 0.8979566097259521 + }, + { + "bits_per_byte": 1.6630168554297817, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1527154445648193, + "step": 5930, + "unclipped_grad_norm": 0.9750632047653198 + }, + { + "bits_per_byte": 1.5225558727477122, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0553553104400635, + "step": 5940, + "unclipped_grad_norm": 0.8854946494102478 + }, + { + "bits_per_byte": 1.5768313615855534, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0929762125015259, + "step": 5950, + "unclipped_grad_norm": 0.8891376852989197 + }, + { + "bits_per_byte": 1.6173250206965033, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1210442781448364, + "step": 5960, + "unclipped_grad_norm": 1.0107448101043701 + }, + { + "bits_per_byte": 1.598686744874548, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.1081252098083496, + "step": 5970, + "unclipped_grad_norm": 1.0125014781951904 + }, + { + "bits_per_byte": 1.623099338199222, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.125046730041504, + "step": 5980, + "unclipped_grad_norm": 0.9798391461372375 + }, + { + "bits_per_byte": 1.6025701131310304, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.1108169555664062, + "step": 5990, + "unclipped_grad_norm": 0.9963014125823975 + }, + { + "bits_per_byte": 1.561719762002191, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0825016498565674, + "step": 6000, + "unclipped_grad_norm": 0.9239053130149841 + }, + { + "bits_per_byte": 1.5905753551292547, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1025028228759766, + "step": 6010, + "unclipped_grad_norm": 0.9512903094291687 + }, + { + "bits_per_byte": 1.5463665267943663, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.07185959815979, + "step": 6020, + "unclipped_grad_norm": 0.931431770324707 + }, + { + "bits_per_byte": 1.5669568057036143, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0861316919326782, + "step": 6030, + "unclipped_grad_norm": 0.9379295110702515 + }, + { + "bits_per_byte": 1.6392313108501535, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.1362285614013672, + "step": 6040, + "unclipped_grad_norm": 1.0010037422180176 + }, + { + "bits_per_byte": 1.547184132316568, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0724263191223145, + "step": 6050, + "unclipped_grad_norm": 0.9731131792068481 + }, + { + "bits_per_byte": 1.550029757257744, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0743987560272217, + "step": 6060, + "unclipped_grad_norm": 0.9350199699401855 + }, + { + "bits_per_byte": 1.6229744787946958, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.12496018409729, + "step": 6070, + "unclipped_grad_norm": 1.0221160650253296 + }, + { + "bits_per_byte": 1.5381166910151283, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0661412477493286, + "step": 6080, + "unclipped_grad_norm": 0.9525545239448547 + }, + { + "bits_per_byte": 1.54977178328145, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0742199420928955, + "step": 6090, + "unclipped_grad_norm": 0.9721336364746094 + }, + { + "bits_per_byte": 1.5613978104797757, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0822784900665283, + "step": 6100, + "unclipped_grad_norm": 0.9061561822891235 + }, + { + "bits_per_byte": 1.6307178256671429, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1303274631500244, + "step": 6110, + "unclipped_grad_norm": 0.9887846112251282 + }, + { + "bits_per_byte": 1.6160071176429416, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.1201307773590088, + "step": 6120, + "unclipped_grad_norm": 1.008078694343567 + }, + { + "bits_per_byte": 1.5519858879286577, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0757546424865723, + "step": 6130, + "unclipped_grad_norm": 0.958514928817749 + }, + { + "bits_per_byte": 1.5641512527200894, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0841870307922363, + "step": 6140, + "unclipped_grad_norm": 0.9408689141273499 + }, + { + "bits_per_byte": 1.6176376851557719, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.1212610006332397, + "step": 6150, + "unclipped_grad_norm": 0.9648155570030212 + }, + { + "bits_per_byte": 1.4733855169180825, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0212730169296265, + "step": 6160, + "unclipped_grad_norm": 0.9190199375152588 + }, + { + "bits_per_byte": 1.587719411229027, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.1005232334136963, + "step": 6170, + "unclipped_grad_norm": 0.9420804977416992 + }, + { + "bits_per_byte": 1.561453876823957, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0823173522949219, + "step": 6180, + "unclipped_grad_norm": 0.9441719055175781 + }, + { + "bits_per_byte": 1.573854341899118, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0909126996994019, + "step": 6190, + "unclipped_grad_norm": 0.9590060114860535 + }, + { + "bits_per_byte": 1.5911162405662183, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.1028777360916138, + "step": 6200, + "unclipped_grad_norm": 0.9321414232254028 + }, + { + "bits_per_byte": 1.586496958546694, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0996758937835693, + "step": 6210, + "unclipped_grad_norm": 1.0076017379760742 + }, + { + "bits_per_byte": 1.574202778749766, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0911542177200317, + "step": 6220, + "unclipped_grad_norm": 0.9875207543373108 + }, + { + "bits_per_byte": 1.5328827430014202, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0625133514404297, + "step": 6230, + "unclipped_grad_norm": 0.9433984756469727 + }, + { + "bits_per_byte": 1.5747416003949193, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0915277004241943, + "step": 6240, + "unclipped_grad_norm": 0.9963345527648926 + }, + { + "bits_per_byte": 1.6134170589209478, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.118335485458374, + "step": 6250, + "unclipped_grad_norm": 1.0228519439697266 + }, + { + "bits_per_byte": 1.4956729206260448, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0367214679718018, + "step": 6260, + "unclipped_grad_norm": 0.9954123497009277 + }, + { + "bits_per_byte": 1.574694133183281, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0914947986602783, + "step": 6270, + "unclipped_grad_norm": 0.9892840385437012 + }, + { + "bits_per_byte": 1.5724050441002972, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0899081230163574, + "step": 6280, + "unclipped_grad_norm": 0.976605236530304 + }, + { + "bits_per_byte": 1.528724890434161, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.05963134765625, + "step": 6290, + "unclipped_grad_norm": 0.9648171067237854 + }, + { + "bits_per_byte": 1.589885188751342, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.1020244359970093, + "step": 6300, + "unclipped_grad_norm": 1.012677550315857 + }, + { + "bits_per_byte": 1.549742718213454, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0741997957229614, + "step": 6310, + "unclipped_grad_norm": 0.9430392384529114 + }, + { + "bits_per_byte": 1.5003391539092545, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0399558544158936, + "step": 6320, + "unclipped_grad_norm": 0.9601491689682007 + }, + { + "bits_per_byte": 1.5786977173127166, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.094269871711731, + "step": 6330, + "unclipped_grad_norm": 0.9872254133224487 + }, + { + "bits_per_byte": 1.5486566477732553, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0734469890594482, + "step": 6340, + "unclipped_grad_norm": 0.9915545582771301 + }, + { + "bits_per_byte": 1.643350295338318, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1390836238861084, + "step": 6350, + "unclipped_grad_norm": 1.0489985942840576 + }, + { + "bits_per_byte": 1.5515856843001, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0754772424697876, + "step": 6360, + "unclipped_grad_norm": 1.0075738430023193 + }, + { + "bits_per_byte": 1.515523502153932, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.050480842590332, + "step": 6370, + "unclipped_grad_norm": 0.9415894746780396 + }, + { + "bits_per_byte": 1.5423135836441333, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0690503120422363, + "step": 6380, + "unclipped_grad_norm": 0.9571269154548645 + }, + { + "bits_per_byte": 1.6015189551689568, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1100883483886719, + "step": 6390, + "unclipped_grad_norm": 0.962506115436554 + }, + { + "bits_per_byte": 1.5875768376114616, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1004244089126587, + "step": 6400, + "unclipped_grad_norm": 0.9905776977539062 + }, + { + "bits_per_byte": 1.5643576319011245, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0843300819396973, + "step": 6410, + "unclipped_grad_norm": 1.0040770769119263 + }, + { + "bits_per_byte": 1.5604245606585432, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0816038846969604, + "step": 6420, + "unclipped_grad_norm": 0.9999781250953674 + }, + { + "bits_per_byte": 1.592441538873767, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.103796362876892, + "step": 6430, + "unclipped_grad_norm": 0.9464544653892517 + }, + { + "bits_per_byte": 1.5425199628251687, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0691933631896973, + "step": 6440, + "unclipped_grad_norm": 0.9930383563041687 + }, + { + "bits_per_byte": 1.6124689185667411, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.1176782846450806, + "step": 6450, + "unclipped_grad_norm": 1.0360738039016724 + }, + { + "bits_per_byte": 1.6512981295826408, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.1445926427841187, + "step": 6460, + "unclipped_grad_norm": 1.0094163417816162 + }, + { + "bits_per_byte": 1.6379674103489628, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.1353524923324585, + "step": 6470, + "unclipped_grad_norm": 1.026789665222168 + }, + { + "bits_per_byte": 1.5269778906666966, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0584204196929932, + "step": 6480, + "unclipped_grad_norm": 0.9595211744308472 + }, + { + "bits_per_byte": 1.6320837118802953, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1312742233276367, + "step": 6490, + "unclipped_grad_norm": 0.9658059477806091 + }, + { + "bits_per_byte": 1.5804063649490385, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.095454216003418, + "step": 6500, + "unclipped_grad_norm": 1.007287621498108 + }, + { + "bits_per_byte": 1.5282990613906247, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0593361854553223, + "step": 6510, + "unclipped_grad_norm": 0.9717181921005249 + }, + { + "bits_per_byte": 1.518525631307393, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0525617599487305, + "step": 6520, + "unclipped_grad_norm": 0.9680256843566895 + }, + { + "bits_per_byte": 1.5436008737858413, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.069942593574524, + "step": 6530, + "unclipped_grad_norm": 0.9684496521949768 + }, + { + "bits_per_byte": 1.5757463230412598, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.09222412109375, + "step": 6540, + "unclipped_grad_norm": 0.9695699214935303 + }, + { + "bits_per_byte": 1.5133627121284916, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.048983097076416, + "step": 6550, + "unclipped_grad_norm": 0.9651139974594116 + }, + { + "bits_per_byte": 1.5985231893735776, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.1080118417739868, + "step": 6560, + "unclipped_grad_norm": 0.9865891337394714 + }, + { + "bits_per_byte": 1.5411050615565203, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.068212628364563, + "step": 6570, + "unclipped_grad_norm": 1.0009613037109375 + }, + { + "bits_per_byte": 1.585936467087532, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0992873907089233, + "step": 6580, + "unclipped_grad_norm": 1.0042976140975952 + }, + { + "bits_per_byte": 1.5806860087393415, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0956480503082275, + "step": 6590, + "unclipped_grad_norm": 1.0104939937591553 + }, + { + "bits_per_byte": 1.4917766536707477, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0340207815170288, + "step": 6600, + "unclipped_grad_norm": 0.9221757054328918 + }, + { + "bits_per_byte": 1.5136690132296782, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.049195408821106, + "step": 6610, + "unclipped_grad_norm": 0.9053012132644653 + }, + { + "bits_per_byte": 1.5109461839012182, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.047308087348938, + "step": 6620, + "unclipped_grad_norm": 0.9886469841003418 + }, + { + "bits_per_byte": 1.5485409034492248, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0733667612075806, + "step": 6630, + "unclipped_grad_norm": 1.005003809928894 + }, + { + "bits_per_byte": 1.5964243131024478, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.106557011604309, + "step": 6640, + "unclipped_grad_norm": 1.033184289932251 + }, + { + "bits_per_byte": 1.579020356765735, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0944935083389282, + "step": 6650, + "unclipped_grad_norm": 1.0069372653961182 + }, + { + "bits_per_byte": 1.596521827265487, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1066246032714844, + "step": 6660, + "unclipped_grad_norm": 0.9888476729393005 + }, + { + "bits_per_byte": 1.5643971879108232, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.084357500076294, + "step": 6670, + "unclipped_grad_norm": 0.9964956045150757 + }, + { + "bits_per_byte": 1.541306109275379, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0683519840240479, + "step": 6680, + "unclipped_grad_norm": 0.9876991510391235 + }, + { + "bits_per_byte": 1.57202289865008, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0896432399749756, + "step": 6690, + "unclipped_grad_norm": 1.0232517719268799 + }, + { + "bits_per_byte": 1.634657604233108, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.1330583095550537, + "step": 6700, + "unclipped_grad_norm": 1.028996467590332 + }, + { + "bits_per_byte": 1.582099534146783, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0966278314590454, + "step": 6710, + "unclipped_grad_norm": 0.982172429561615 + }, + { + "bits_per_byte": 1.5951110535804596, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1056467294692993, + "step": 6720, + "unclipped_grad_norm": 0.9665309190750122 + }, + { + "bits_per_byte": 1.5713714283686118, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0891916751861572, + "step": 6730, + "unclipped_grad_norm": 1.033214807510376 + }, + { + "bits_per_byte": 1.5070635035753395, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0446168184280396, + "step": 6740, + "unclipped_grad_norm": 0.9829891920089722 + }, + { + "bits_per_byte": 1.5742997769648528, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0912214517593384, + "step": 6750, + "unclipped_grad_norm": 0.9734475612640381 + }, + { + "bits_per_byte": 1.5504435475157199, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0746855735778809, + "step": 6760, + "unclipped_grad_norm": 0.947333574295044 + }, + { + "bits_per_byte": 1.4574869247243718, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0102529525756836, + "step": 6770, + "unclipped_grad_norm": 0.9365482330322266 + }, + { + "bits_per_byte": 1.5535751796052808, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.076856255531311, + "step": 6780, + "unclipped_grad_norm": 1.000013828277588 + }, + { + "bits_per_byte": 1.5272709491037668, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0586235523223877, + "step": 6790, + "unclipped_grad_norm": 1.0473871231079102 + }, + { + "bits_per_byte": 1.588691973119656, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.101197361946106, + "step": 6800, + "unclipped_grad_norm": 1.0225850343704224 + }, + { + "bits_per_byte": 1.5961993597951194, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.1064010858535767, + "step": 6810, + "unclipped_grad_norm": 1.0318177938461304 + }, + { + "bits_per_byte": 1.5861701915100546, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0994493961334229, + "step": 6820, + "unclipped_grad_norm": 1.0743839740753174 + }, + { + "bits_per_byte": 1.5382866098741805, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0662590265274048, + "step": 6830, + "unclipped_grad_norm": 0.9585179090499878 + }, + { + "bits_per_byte": 1.521101759434667, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0543473958969116, + "step": 6840, + "unclipped_grad_norm": 1.0282230377197266 + }, + { + "bits_per_byte": 1.5590901472704985, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.080678939819336, + "step": 6850, + "unclipped_grad_norm": 1.064353346824646 + }, + { + "bits_per_byte": 1.5486480486407121, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0734410285949707, + "step": 6860, + "unclipped_grad_norm": 1.054447889328003 + }, + { + "bits_per_byte": 1.486493862584195, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0303590297698975, + "step": 6870, + "unclipped_grad_norm": 0.992324709892273 + }, + { + "bits_per_byte": 1.501989499426934, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0410997867584229, + "step": 6880, + "unclipped_grad_norm": 0.9187514781951904 + }, + { + "bits_per_byte": 1.4984072727921125, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0386167764663696, + "step": 6890, + "unclipped_grad_norm": 0.9562593102455139 + }, + { + "bits_per_byte": 1.6084883801125214, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.1149191856384277, + "step": 6900, + "unclipped_grad_norm": 1.046886920928955 + }, + { + "bits_per_byte": 1.5851809473022918, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0987637042999268, + "step": 6910, + "unclipped_grad_norm": 1.0160750150680542 + }, + { + "bits_per_byte": 1.5136987662282775, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0492160320281982, + "step": 6920, + "unclipped_grad_norm": 0.9592021107673645 + }, + { + "bits_per_byte": 1.5198106856746398, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.053452491760254, + "step": 6930, + "unclipped_grad_norm": 1.0287657976150513 + }, + { + "bits_per_byte": 1.494010192357503, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0355689525604248, + "step": 6940, + "unclipped_grad_norm": 0.972965657711029 + }, + { + "bits_per_byte": 1.513553956836251, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0491156578063965, + "step": 6950, + "unclipped_grad_norm": 1.0125614404678345 + }, + { + "bits_per_byte": 1.583024456843123, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0972689390182495, + "step": 6960, + "unclipped_grad_norm": 0.9876657724380493 + }, + { + "bits_per_byte": 1.5602042508827878, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.081451177597046, + "step": 6970, + "unclipped_grad_norm": 1.0233051776885986 + }, + { + "bits_per_byte": 1.5255848311947078, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0574548244476318, + "step": 6980, + "unclipped_grad_norm": 0.9819255471229553 + }, + { + "bits_per_byte": 1.5587054220805185, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0804122686386108, + "step": 6990, + "unclipped_grad_norm": 1.0355700254440308 + }, + { + "bits_per_byte": 1.6012561656784385, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.1099061965942383, + "step": 7000, + "unclipped_grad_norm": 1.0185670852661133 + }, + { + "bits_per_byte": 1.585546754400677, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0990172624588013, + "step": 7010, + "unclipped_grad_norm": 1.014304757118225 + }, + { + "bits_per_byte": 1.542434143482388, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0691338777542114, + "step": 7020, + "unclipped_grad_norm": 1.027226448059082 + }, + { + "bits_per_byte": 1.5806457647990395, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0956201553344727, + "step": 7030, + "unclipped_grad_norm": 1.0283117294311523 + }, + { + "bits_per_byte": 1.5888914729946568, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.1013356447219849, + "step": 7040, + "unclipped_grad_norm": 1.0418747663497925 + }, + { + "bits_per_byte": 1.5596535624347252, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0810694694519043, + "step": 7050, + "unclipped_grad_norm": 1.007202386856079 + }, + { + "bits_per_byte": 1.5766827685752078, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.092873215675354, + "step": 7060, + "unclipped_grad_norm": 1.057618498802185 + }, + { + "bits_per_byte": 1.5601829250340808, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0814363956451416, + "step": 7070, + "unclipped_grad_norm": 1.0584709644317627 + }, + { + "bits_per_byte": 1.5180239579148262, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0522140264511108, + "step": 7080, + "unclipped_grad_norm": 1.0032520294189453 + }, + { + "bits_per_byte": 1.5539740073726318, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0771327018737793, + "step": 7090, + "unclipped_grad_norm": 1.0915608406066895 + }, + { + "bits_per_byte": 1.5426179929361605, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0692613124847412, + "step": 7100, + "unclipped_grad_norm": 1.0186309814453125 + }, + { + "bits_per_byte": 1.5505398578002032, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0747523307800293, + "step": 7110, + "unclipped_grad_norm": 1.0057092905044556 + }, + { + "bits_per_byte": 1.58839977459584, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.1009948253631592, + "step": 7120, + "unclipped_grad_norm": 1.0634574890136719 + }, + { + "bits_per_byte": 1.5622621952830122, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0828776359558105, + "step": 7130, + "unclipped_grad_norm": 1.0760761499404907 + }, + { + "bits_per_byte": 1.5468659644124718, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0722057819366455, + "step": 7140, + "unclipped_grad_norm": 1.0254970788955688 + }, + { + "bits_per_byte": 1.5017244741619544, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.040916085243225, + "step": 7150, + "unclipped_grad_norm": 0.9808151125907898 + }, + { + "bits_per_byte": 1.5967245948108544, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.1067651510238647, + "step": 7160, + "unclipped_grad_norm": 1.0412861108779907 + }, + { + "bits_per_byte": 1.5266782968888937, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0582127571105957, + "step": 7170, + "unclipped_grad_norm": 0.993831217288971 + }, + { + "bits_per_byte": 1.5582240426407534, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0800786018371582, + "step": 7180, + "unclipped_grad_norm": 1.0170890092849731 + }, + { + "bits_per_byte": 1.5191605912543786, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.053001880645752, + "step": 7190, + "unclipped_grad_norm": 1.0318375825881958 + }, + { + "bits_per_byte": 1.5658617921655709, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0853726863861084, + "step": 7200, + "unclipped_grad_norm": 1.0288457870483398 + }, + { + "bits_per_byte": 1.6065680218329872, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.1135880947113037, + "step": 7210, + "unclipped_grad_norm": 1.079895257949829 + }, + { + "bits_per_byte": 1.5943379915648312, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.1051108837127686, + "step": 7220, + "unclipped_grad_norm": 1.0698199272155762 + }, + { + "bits_per_byte": 1.5270222621906193, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0584511756896973, + "step": 7230, + "unclipped_grad_norm": 1.0323591232299805 + }, + { + "bits_per_byte": 1.5416120664112638, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0685640573501587, + "step": 7240, + "unclipped_grad_norm": 1.0111463069915771 + }, + { + "bits_per_byte": 1.5732295289285334, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0904796123504639, + "step": 7250, + "unclipped_grad_norm": 1.0152740478515625 + }, + { + "bits_per_byte": 1.467654711026032, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0173007249832153, + "step": 7260, + "unclipped_grad_norm": 0.9989895820617676 + }, + { + "bits_per_byte": 1.5079978853174771, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.045264482498169, + "step": 7270, + "unclipped_grad_norm": 1.015647053718567 + }, + { + "bits_per_byte": 1.5841817281007788, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0980710983276367, + "step": 7280, + "unclipped_grad_norm": 1.0212509632110596 + }, + { + "bits_per_byte": 1.50208838945118, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0411683320999146, + "step": 7290, + "unclipped_grad_norm": 1.0158213376998901 + }, + { + "bits_per_byte": 1.529626423489984, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0602562427520752, + "step": 7300, + "unclipped_grad_norm": 1.0379971265792847 + }, + { + "bits_per_byte": 1.530171608493219, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0606341361999512, + "step": 7310, + "unclipped_grad_norm": 0.9976150393486023 + }, + { + "bits_per_byte": 1.4838797262910803, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0285470485687256, + "step": 7320, + "unclipped_grad_norm": 1.0280481576919556 + }, + { + "bits_per_byte": 1.510586568178264, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0470588207244873, + "step": 7330, + "unclipped_grad_norm": 1.0713517665863037 + }, + { + "bits_per_byte": 1.4810440763436543, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0265815258026123, + "step": 7340, + "unclipped_grad_norm": 1.0121647119522095 + }, + { + "bits_per_byte": 1.5558935057389116, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0784631967544556, + "step": 7350, + "unclipped_grad_norm": 1.046762228012085 + }, + { + "bits_per_byte": 1.5163350882833535, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0510433912277222, + "step": 7360, + "unclipped_grad_norm": 1.0365986824035645 + }, + { + "bits_per_byte": 1.536221614185271, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0648276805877686, + "step": 7370, + "unclipped_grad_norm": 1.0269999504089355 + }, + { + "bits_per_byte": 1.4635044256954106, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0144239664077759, + "step": 7380, + "unclipped_grad_norm": 1.0192774534225464 + }, + { + "bits_per_byte": 1.5685326827334702, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.087224006652832, + "step": 7390, + "unclipped_grad_norm": 1.030569076538086 + }, + { + "bits_per_byte": 1.49844063742638, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0386399030685425, + "step": 7400, + "unclipped_grad_norm": 1.0422602891921997 + }, + { + "bits_per_byte": 1.48524905215725, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.029496192932129, + "step": 7410, + "unclipped_grad_norm": 0.9939574003219604 + }, + { + "bits_per_byte": 1.556964785671136, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0792057514190674, + "step": 7420, + "unclipped_grad_norm": 1.0415644645690918 + }, + { + "bits_per_byte": 1.5795878995135824, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0948868989944458, + "step": 7430, + "unclipped_grad_norm": 1.0722742080688477 + }, + { + "bits_per_byte": 1.591624105334216, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.1032297611236572, + "step": 7440, + "unclipped_grad_norm": 1.0902073383331299 + }, + { + "bits_per_byte": 1.5369473809719119, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0653307437896729, + "step": 7450, + "unclipped_grad_norm": 1.0743108987808228 + }, + { + "bits_per_byte": 1.5231069051610766, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0557372570037842, + "step": 7460, + "unclipped_grad_norm": 0.9955658912658691 + }, + { + "bits_per_byte": 1.5091140527215767, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0460381507873535, + "step": 7470, + "unclipped_grad_norm": 1.048521876335144 + }, + { + "bits_per_byte": 1.5543542610136893, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.077396273612976, + "step": 7480, + "unclipped_grad_norm": 1.0729283094406128 + }, + { + "bits_per_byte": 1.6078190236353633, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.114455223083496, + "step": 7490, + "unclipped_grad_norm": 1.0465319156646729 + }, + { + "bits_per_byte": 1.5529930183321103, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0764527320861816, + "step": 7500, + "unclipped_grad_norm": 1.0308523178100586 + }, + { + "bits_per_byte": 1.5173444544612673, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0517430305480957, + "step": 7510, + "unclipped_grad_norm": 1.008348822593689 + }, + { + "bits_per_byte": 1.5327847128904286, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0624454021453857, + "step": 7520, + "unclipped_grad_norm": 1.0445557832717896 + }, + { + "bits_per_byte": 1.5854733178087586, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.098966360092163, + "step": 7530, + "unclipped_grad_norm": 1.1247494220733643 + }, + { + "bits_per_byte": 1.5176457680655788, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0519518852233887, + "step": 7540, + "unclipped_grad_norm": 0.9896391034126282 + }, + { + "bits_per_byte": 1.5701451920679599, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0883417129516602, + "step": 7550, + "unclipped_grad_norm": 1.0925025939941406 + }, + { + "bits_per_byte": 1.5354839805957203, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0643163919448853, + "step": 7560, + "unclipped_grad_norm": 1.032513976097107 + }, + { + "bits_per_byte": 1.4933951823980176, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0351426601409912, + "step": 7570, + "unclipped_grad_norm": 1.0785413980484009 + }, + { + "bits_per_byte": 1.6648412473901344, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.153980016708374, + "step": 7580, + "unclipped_grad_norm": 1.1147191524505615 + }, + { + "bits_per_byte": 1.5229436936254077, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0556241273880005, + "step": 7590, + "unclipped_grad_norm": 1.0130250453948975 + }, + { + "bits_per_byte": 1.506402230282772, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0441584587097168, + "step": 7600, + "unclipped_grad_norm": 1.0460206270217896 + }, + { + "bits_per_byte": 1.5204523529450091, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0538972616195679, + "step": 7610, + "unclipped_grad_norm": 1.0537638664245605 + }, + { + "bits_per_byte": 1.4833304137042245, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0281662940979004, + "step": 7620, + "unclipped_grad_norm": 1.0038108825683594 + }, + { + "bits_per_byte": 1.4865265392878588, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.030381679534912, + "step": 7630, + "unclipped_grad_norm": 1.040962815284729 + }, + { + "bits_per_byte": 1.4985199214284277, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0386948585510254, + "step": 7640, + "unclipped_grad_norm": 1.037416696548462 + }, + { + "bits_per_byte": 1.6218830768923203, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1242036819458008, + "step": 7650, + "unclipped_grad_norm": 1.0509169101715088 + }, + { + "bits_per_byte": 1.5436070651612726, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0699468851089478, + "step": 7660, + "unclipped_grad_norm": 1.0892055034637451 + }, + { + "bits_per_byte": 1.5646802713541432, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0845537185668945, + "step": 7670, + "unclipped_grad_norm": 1.0592975616455078 + }, + { + "bits_per_byte": 1.5230315767599987, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.055685043334961, + "step": 7680, + "unclipped_grad_norm": 1.0794378519058228 + }, + { + "bits_per_byte": 1.5199706295399424, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0535633563995361, + "step": 7690, + "unclipped_grad_norm": 1.030026912689209 + }, + { + "bits_per_byte": 1.486342001903483, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0302537679672241, + "step": 7700, + "unclipped_grad_norm": 1.021731972694397 + }, + { + "bits_per_byte": 1.565274987360827, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0849659442901611, + "step": 7710, + "unclipped_grad_norm": 1.054081678390503 + }, + { + "bits_per_byte": 1.5620626954080112, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0827393531799316, + "step": 7720, + "unclipped_grad_norm": 1.0361971855163574 + }, + { + "bits_per_byte": 1.5781238112067872, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0938720703125, + "step": 7730, + "unclipped_grad_norm": 1.0852437019348145 + }, + { + "bits_per_byte": 1.5098900384422698, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0465760231018066, + "step": 7740, + "unclipped_grad_norm": 1.0850290060043335 + }, + { + "bits_per_byte": 1.5288445903591616, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0597143173217773, + "step": 7750, + "unclipped_grad_norm": 1.0632920265197754 + }, + { + "bits_per_byte": 1.5461653070928567, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0717201232910156, + "step": 7760, + "unclipped_grad_norm": 1.0411477088928223 + }, + { + "bits_per_byte": 1.4760610510175551, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.023127555847168, + "step": 7770, + "unclipped_grad_norm": 1.0089881420135498 + }, + { + "bits_per_byte": 1.5534400012417027, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0767625570297241, + "step": 7780, + "unclipped_grad_norm": 1.0804110765457153 + }, + { + "bits_per_byte": 1.4972104455247583, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0377871990203857, + "step": 7790, + "unclipped_grad_norm": 1.0377298593521118 + }, + { + "bits_per_byte": 1.5201942069860641, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0537183284759521, + "step": 7800, + "unclipped_grad_norm": 1.0282031297683716 + }, + { + "bits_per_byte": 1.4946203868027643, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.035991907119751, + "step": 7810, + "unclipped_grad_norm": 1.041825771331787 + }, + { + "bits_per_byte": 1.4985438270168976, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.038711428642273, + "step": 7820, + "unclipped_grad_norm": 1.0459965467453003 + }, + { + "bits_per_byte": 1.5248091894393165, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0569171905517578, + "step": 7830, + "unclipped_grad_norm": 1.093446969985962 + }, + { + "bits_per_byte": 1.5216629388244325, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0547363758087158, + "step": 7840, + "unclipped_grad_norm": 1.0666427612304688 + }, + { + "bits_per_byte": 1.4598855667559556, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0119155645370483, + "step": 7850, + "unclipped_grad_norm": 1.0127599239349365 + }, + { + "bits_per_byte": 1.522777730367325, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.055509090423584, + "step": 7860, + "unclipped_grad_norm": 1.0349164009094238 + }, + { + "bits_per_byte": 1.5484055531029957, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.073272943496704, + "step": 7870, + "unclipped_grad_norm": 1.0840545892715454 + }, + { + "bits_per_byte": 1.4683410937856254, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0177764892578125, + "step": 7880, + "unclipped_grad_norm": 1.0721038579940796 + }, + { + "bits_per_byte": 1.5719193650942607, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.089571475982666, + "step": 7890, + "unclipped_grad_norm": 1.1230442523956299 + }, + { + "bits_per_byte": 1.5165904825198848, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.051220417022705, + "step": 7900, + "unclipped_grad_norm": 1.0280076265335083 + }, + { + "bits_per_byte": 1.5320310849143477, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0619230270385742, + "step": 7910, + "unclipped_grad_norm": 1.0358502864837646 + }, + { + "bits_per_byte": 1.5396064047369018, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0671738386154175, + "step": 7920, + "unclipped_grad_norm": 1.0826281309127808 + }, + { + "bits_per_byte": 1.4911463372553357, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0335838794708252, + "step": 7930, + "unclipped_grad_norm": 1.0188233852386475 + }, + { + "bits_per_byte": 1.500506321045893, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.040071725845337, + "step": 7940, + "unclipped_grad_norm": 1.0400398969650269 + }, + { + "bits_per_byte": 1.561352235077297, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0822468996047974, + "step": 7950, + "unclipped_grad_norm": 1.0635569095611572 + }, + { + "bits_per_byte": 1.4866159702663075, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0304436683654785, + "step": 7960, + "unclipped_grad_norm": 1.0361822843551636 + }, + { + "bits_per_byte": 1.5208446453716271, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0541691780090332, + "step": 7970, + "unclipped_grad_norm": 1.0433354377746582 + }, + { + "bits_per_byte": 1.555866504462726, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.078444480895996, + "step": 7980, + "unclipped_grad_norm": 1.1136473417282104 + }, + { + "bits_per_byte": 1.613144638401981, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.1181466579437256, + "step": 7990, + "unclipped_grad_norm": 1.1455442905426025 + }, + { + "bits_per_byte": 1.4895616891102856, + "learning_rate": 3e-05, + "loss_nats": 1.0324854850769043, + "step": 8000, + "unclipped_grad_norm": 1.039122223854065 + } + ] + }, + "depth-16-baseline-seed-2026073002": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "68d80bba3d8494c2df7d715607540a0248ed186d1f5b4bbcc53645c92dfa1a84", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027575084823183715, + 0.00024360731185879558, + 0.00021477397240232676, + 0.00019141066877637058, + 0.00017414781905245036, + 0.00016276225505862385, + 0.00015418927068822086, + 0.00014831383305136114, + 0.0001412670244462788, + 0.00013702295836992562, + 0.0001326367782894522, + 0.00012906579650007188, + 0.00012653179874178022, + 0.00012431222421582788, + 0.000122369994642213, + 0.000120684111607261 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00023138570031733252, + "first_to_last_ratio": 1.8739548634355478, + "imbalance_abs_log_ratio": 0.6280510978460953, + "last_quartile_mean": 0.00012347453230177052, + "mean": 0.0001624279166207998, + "normalized": [ + 1.6976813713346965, + 1.4997872097782008, + 1.3222725309204868, + 1.1784345496669344, + 1.072154483511671, + 1.0020583803866954, + 0.9492781407040226, + 0.9131055556023103, + 0.8697213347633911, + 0.8435924145343563, + 0.8165885584748509, + 0.79460353358706, + 0.77900278089005, + 0.7653377990807093, + 0.7533803128676146, + 0.74300104389695 + ], + "population_cv": 0.2778958958835334 + }, + "activation_output_rms_by_block": [ + 0.028587408363819122, + 0.028957132250070572, + 0.029405318200588226, + 0.029973436146974564, + 0.030426617711782455, + 0.030960483476519585, + 0.03149084374308586, + 0.03222988545894623, + 0.03307616710662842, + 0.03391994163393974, + 0.034928515553474426, + 0.035913772881031036, + 0.03729887306690216, + 0.038532089442014694, + 0.03956005349755287, + 0.0403735376894474 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02923082374036312, + "first_to_last_ratio": 0.7506412221981493, + "imbalance_abs_log_ratio": 0.2868274747935402, + "last_quartile_mean": 0.03894113842397928, + "mean": 0.033477129763923585, + "normalized": [ + 0.8539384518748725, + 0.8649825255113729, + 0.8783703503840009, + 0.8953406805883115, + 0.9088777301503161, + 0.9248249086719481, + 0.9406673739700877, + 0.962743391868635, + 0.9880227886881968, + 1.0132272949664087, + 1.0433545468140735, + 1.0727853054993168, + 1.114159825825243, + 1.1509974037122668, + 1.1817038610097486, + 1.206003560465201 + ], + "population_cv": 0.11262664842931464 + }, + "bits_per_byte": 8.031398550585404, + "branch_output_rms_by_sublayer": [ + 0.0029695050325244665, + 0.003912033978849649, + 0.0030677192844450474, + 0.0038352999836206436, + 0.00318979867734015, + 0.0038548391312360764, + 0.003840027377009392, + 0.003801720915362239, + 0.00428530341014266, + 0.00395593885332346, + 0.004158117808401585, + 0.003868808038532734, + 0.004507514648139477, + 0.0038615495432168245, + 0.0055657969787716866, + 0.003801694605499506, + 0.005897593684494495, + 0.0038540444802492857, + 0.00619681179523468, + 0.003802293911576271, + 0.006646857596933842, + 0.0038319535087794065, + 0.00662674754858017, + 0.003906778059899807, + 0.007378070615231991, + 0.003958481829613447, + 0.007767242379486561, + 0.0038770362734794617, + 0.00770326005294919, + 0.0037339557893574238, + 0.007914260029792786, + 0.003793976968154311 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007112265943118693, + 0.0061890396223896905, + 0.005352992918648162, + 0.005122911821075447, + 0.005146431820875539, + 0.004279647792521989, + 0.004250462252416264, + 0.004421502222176574, + 0.004575182382709001, + 0.004269268868925463, + 0.0040238830908244155, + 0.003973260546139901, + 0.0038964174441780434, + 0.004243476176619415, + 0.00401474940360252, + 0.003997243047861786 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005944302576307998, + "first_to_last_ratio": 1.472101165080992, + "imbalance_abs_log_ratio": 0.38669074422298993, + "last_quartile_mean": 0.004037971518065441, + "mean": 0.004679295959630182, + "normalized": [ + 1.5199435993103534, + 1.3226433369003716, + 1.1439740005398642, + 1.0948039759127195, + 1.0998303730466061, + 0.9145922440990935, + 0.9083550792867974, + 0.944907580183498, + 0.9777501620287747, + 0.9123741917070097, + 0.8599334441633468, + 0.8491150336329483, + 0.8326930969517022, + 0.9068621034508766, + 0.857981507953136, + 0.854240270832901 + ], + "population_cv": 0.1873316910212935 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.02822216972708702, + 0.02833663299679756, + 0.028587408363819122, + 0.02871040627360344, + 0.028957132250070572, + 0.029123734682798386, + 0.029405318200588226, + 0.0297223012894392, + 0.029973436146974564, + 0.03016654960811138, + 0.030426617711782455, + 0.030734822154045105, + 0.030960483476519585, + 0.03127134218811989, + 0.03149084374308586, + 0.0320059098303318, + 0.03222988545894623, + 0.03279566392302513, + 0.03307616710662842, + 0.033656880259513855, + 0.03391994163393974, + 0.03468930721282959, + 0.034928515553474426, + 0.03560047596693039, + 0.035913772881031036, + 0.03701025992631912, + 0.03729887306690216, + 0.038355886936187744, + 0.038532089442014694, + 0.0393846333026886, + 0.03956005349755287, + 0.04016231372952461 + ], + "loss_nats": 5.566941261291504, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.02833663299679756, + 0.028587408363819122, + 0.02871040627360344, + 0.028957132250070572, + 0.029123734682798386, + 0.029405318200588226, + 0.0297223012894392, + 0.029973436146974564, + 0.03016654960811138, + 0.030426617711782455, + 0.030734822154045105, + 0.030960483476519585, + 0.03127134218811989, + 0.03149084374308586, + 0.0320059098303318, + 0.03222988545894623, + 0.03279566392302513, + 0.03307616710662842, + 0.033656880259513855, + 0.03391994163393974, + 0.03468930721282959, + 0.034928515553474426, + 0.03560047596693039, + 0.035913772881031036, + 0.03701025992631912, + 0.03729887306690216, + 0.038355886936187744, + 0.038532089442014694, + 0.0393846333026886, + 0.03956005349755287, + 0.04016231372952461, + 0.0403735376894474 + ] + }, + { + "activation_grad_rms_by_block": [ + 9.082402539206669e-05, + 8.28480115160346e-05, + 7.297925185412169e-05, + 6.382335413945839e-05, + 5.4197575082071126e-05, + 4.6132205170579255e-05, + 3.93990776501596e-05, + 3.41209233738482e-05, + 2.9922650355729274e-05, + 2.6855812393478118e-05, + 2.4530303562642075e-05, + 2.2887908926350065e-05, + 2.182140451623127e-05, + 2.1062218365841545e-05, + 2.0474079065024853e-05, + 2.0058148948010057e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.761866072542034e-05, + "first_to_last_ratio": 3.722010140399952, + "imbalance_abs_log_ratio": 1.3142638826655557, + "last_quartile_mean": 2.085396272377693e-05, + "mean": 4.1996059394477925e-05, + "normalized": [ + 2.1626797061823653, + 1.972756794579834, + 1.7377642785150873, + 1.5197462585704062, + 1.2905395378404856, + 1.098488901952672, + 0.938161299375155, + 0.8124791674700501, + 0.7125109066700628, + 0.6394841035254226, + 0.584109650198932, + 0.5450013467063437, + 0.5196060018693224, + 0.5015284450381319, + 0.4875238143823798, + 0.47761978712334874 + ], + "population_cv": 0.55073751796069 + }, + "activation_output_rms_by_block": [ + 0.03037329390645027, + 0.03339420258998871, + 0.03790459781885147, + 0.04353494197130203, + 0.04983566701412201, + 0.057033151388168335, + 0.06467092037200928, + 0.07435981929302216, + 0.08538646996021271, + 0.09887228161096573, + 0.11526436358690262, + 0.13244163990020752, + 0.15128590166568756, + 0.17302390933036804, + 0.19715538620948792, + 0.22326189279556274 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03630175907164812, + "first_to_last_ratio": 0.194980199104046, + "imbalance_abs_log_ratio": 1.63485726863041, + "last_quartile_mean": 0.18618177250027657, + "mean": 0.09798740246333182, + "normalized": [ + 0.30997141615031953, + 0.34080097798781084, + 0.38683133613053855, + 0.4442912137363105, + 0.5085925921219432, + 0.5820457523558795, + 0.65999219028395, + 0.7588712163366979, + 0.8714025253620276, + 1.0090305398999126, + 1.1763181867182986, + 1.351619050722045, + 1.543932157220932, + 1.7657770793048206, + 2.012048296547774, + 2.2784754691207403 + ], + "population_cv": 0.6071319453340859 + }, + "bits_per_byte": 5.218263195949167, + "branch_output_rms_by_sublayer": [ + 0.004278926644474268, + 0.007860537618398666, + 0.004119343124330044, + 0.008471532724797726, + 0.004720318131148815, + 0.009602016769349575, + 0.0057782139629125595, + 0.009444202296435833, + 0.005580307450145483, + 0.010230599902570248, + 0.00646675331518054, + 0.010376903228461742, + 0.00653326790779829, + 0.010733475908637047, + 0.008962317369878292, + 0.01155572384595871, + 0.010035770013928413, + 0.012197046540677547, + 0.011996372602880001, + 0.012902887538075447, + 0.015837425366044044, + 0.013319648802280426, + 0.01564962975680828, + 0.013529929332435131, + 0.01653609797358513, + 0.013358287513256073, + 0.01997774839401245, + 0.013715346343815327, + 0.020573746412992477, + 0.01530109066516161, + 0.024493996053934097, + 0.014329549856483936 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003576967965622096, + 0.00031529422707157425, + 0.00030800051934586363, + 0.00027582768007438345, + 0.00025927860434436177, + 0.0002499710324331713, + 0.00022865411268768897, + 0.00021433881461782036, + 0.00020517591210118428, + 0.00017936399546700514, + 0.000174300235585937, + 0.00015047843844280973, + 0.00014083116496318749, + 0.00013997136590409212, + 0.0001362051425885804, + 0.00013592122810918616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031420480576350773, + "first_to_last_ratio": 2.2730213947880955, + "imbalance_abs_log_ratio": 0.8211099574033609, + "last_quartile_mean": 0.00013823222539126155, + "mean": 0.000216956829393691, + "normalized": [ + 1.6487003315905355, + 1.4532579036700417, + 1.4196396592197809, + 1.271348225567332, + 1.19507002876537, + 1.1521694575448123, + 1.0539152573656578, + 0.9879330151386018, + 0.9456992558131047, + 0.8267266624805357, + 0.8033867201739516, + 0.6935870092835418, + 0.6491206815510496, + 0.6451576854955755, + 0.6277983641686699, + 0.6264897421714382 + ], + "population_cv": 0.3194559569879116 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028607700020074844, + 0.028983348980545998, + 0.03037329390645027, + 0.030827771872282028, + 0.03339420258998871, + 0.03403528779745102, + 0.03790459781885147, + 0.03889948129653931, + 0.04353494197130203, + 0.04432978481054306, + 0.04983566701412201, + 0.050834495574235916, + 0.057033151388168335, + 0.0581708587706089, + 0.06467092037200928, + 0.06689786911010742, + 0.07435981929302216, + 0.07719690352678299, + 0.08538646996021271, + 0.0897849053144455, + 0.09887228161096573, + 0.10603746026754379, + 0.11526436358690262, + 0.12274596095085144, + 0.13244163990020752, + 0.14162033796310425, + 0.15128590166568756, + 0.16265419125556946, + 0.17302390933036804, + 0.18535073101520538, + 0.19715538620948792, + 0.21195416152477264 + ], + "loss_nats": 3.6170244216918945, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028983348980545998, + 0.03037329390645027, + 0.030827771872282028, + 0.03339420258998871, + 0.03403528779745102, + 0.03790459781885147, + 0.03889948129653931, + 0.04353494197130203, + 0.04432978481054306, + 0.04983566701412201, + 0.050834495574235916, + 0.057033151388168335, + 0.0581708587706089, + 0.06467092037200928, + 0.06689786911010742, + 0.07435981929302216, + 0.07719690352678299, + 0.08538646996021271, + 0.0897849053144455, + 0.09887228161096573, + 0.10603746026754379, + 0.11526436358690262, + 0.12274596095085144, + 0.13244163990020752, + 0.14162033796310425, + 0.15128590166568756, + 0.16265419125556946, + 0.17302390933036804, + 0.18535073101520538, + 0.19715538620948792, + 0.21195416152477264, + 0.22326189279556274 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015646869724150747, + 0.0001555694470880553, + 0.00014404315152205527, + 0.00012938148574903607, + 0.00011473113409010693, + 0.00010209716856479645, + 9.211692668031901e-05, + 8.257795707322657e-05, + 7.45681973057799e-05, + 6.577855674549937e-05, + 5.7623117754701525e-05, + 5.006180435884744e-05, + 4.3452899262774736e-05, + 3.91966023016721e-05, + 3.658098648884334e-05, + 3.536837175488472e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014636569540016353, + "first_to_last_ratio": 3.786979944917394, + "imbalance_abs_log_ratio": 1.3315688531352086, + "last_quartile_mean": 3.864971495204372e-05, + "mean": 8.622603149888164e-05, + "normalized": [ + 1.8146341020407148, + 1.8042051151347847, + 1.670529757871595, + 1.5004921773619393, + 1.330585811450629, + 1.1840643340534658, + 1.068319219602653, + 0.9576917276344513, + 0.8647991332727294, + 0.7628619293044065, + 0.6682798309632156, + 0.5805880601091649, + 0.5039417738173225, + 0.4545796857435157, + 0.42424527550381114, + 0.41018206613560143 + ], + "population_cv": 0.48370287188714023 + }, + "activation_output_rms_by_block": [ + 0.05259466543793678, + 0.067360520362854, + 0.0797014907002449, + 0.08837522566318512, + 0.09281065315008163, + 0.09652531147003174, + 0.09793278574943542, + 0.09766053408384323, + 0.09857666492462158, + 0.10091027617454529, + 0.10140916705131531, + 0.11094733327627182, + 0.12530675530433655, + 0.14505283534526825, + 0.17291401326656342, + 0.21681295335292816 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0720079755410552, + "first_to_last_ratio": 0.43635474619550435, + "imbalance_abs_log_ratio": 0.829299728394033, + "last_quartile_mean": 0.1650216393172741, + "mean": 0.10905569908209145, + "normalized": [ + 0.4822734243200464, + 0.6176707951057974, + 0.7308328805471209, + 0.8103677882680927, + 0.8510390005406192, + 0.8851010289464333, + 0.8980070420319504, + 0.8955105960150644, + 0.9039111734125715, + 0.9253095163654427, + 0.9298841592403141, + 1.0173455785447438, + 1.1490161115744364, + 1.33008028526854, + 1.5855568734321968, + 1.98809374638663 + ], + "population_cv": 0.3579222353559594 + }, + "bits_per_byte": 3.109639980064452, + "branch_output_rms_by_sublayer": [ + 0.015709737315773964, + 0.018998395651578903, + 0.02321256324648857, + 0.011548326350748539, + 0.022954270243644714, + 0.009733532555401325, + 0.020237641409039497, + 0.00895718764513731, + 0.01714940369129181, + 0.010070553049445152, + 0.016518093645572662, + 0.010933448560535908, + 0.01985226757824421, + 0.011251374147832394, + 0.022037845104932785, + 0.012155328877270222, + 0.027303192764520645, + 0.013005394488573074, + 0.03429069370031357, + 0.01562904566526413, + 0.035662416368722916, + 0.018316252157092094, + 0.019603708758950233, + 0.021821532398462296, + 0.018347281962633133, + 0.02647775411605835, + 0.01771368645131588, + 0.02924429066479206, + 0.014302654191851616, + 0.03788068890571594, + 0.024004697799682617, + 0.04663809388875961 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0010353898057018447, + 0.0006549276400495494, + 0.000583207798204784, + 0.0004740072073606198, + 0.00043399488807089015, + 0.0004525702167816539, + 0.0003859790612774441, + 0.0004183888521327983, + 0.00036657723102634496, + 0.00038828410964056754, + 0.0003961803157723344, + 0.00030130391836939404, + 0.00026972366751228246, + 0.0002437005745447571, + 0.00023517814034756533, + 0.0002654411672498285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006868831128291994, + "first_to_last_ratio": 2.7094817103793067, + "imbalance_abs_log_ratio": 0.9967573658183029, + "last_quartile_mean": 0.00025351088741360834, + "mean": 0.0004315534121276662, + "normalized": [ + 2.3992158944987, + 1.5176050557000407, + 1.3514151013878533, + 1.0983743704484823, + 1.0056574131373146, + 1.0487003556532426, + 0.8943946460172122, + 0.9694949463382451, + 0.8494365256412354, + 0.8997359277643753, + 0.9180330977319039, + 0.6981845349892853, + 0.6250064532741785, + 0.5647054749103996, + 0.5449572028363264, + 0.6150829996712046 + ], + "population_cv": 0.44646086778633726 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04346943646669388, + 0.04716380685567856, + 0.05259466543793678, + 0.06381504237651825, + 0.067360520362854, + 0.076678566634655, + 0.0797014907002449, + 0.08585279434919357, + 0.08837522566318512, + 0.09026700258255005, + 0.09281065315008163, + 0.09405135363340378, + 0.09652531147003174, + 0.09571083635091782, + 0.09793278574943542, + 0.09508897364139557, + 0.09766053408384323, + 0.09508920460939407, + 0.09857666492462158, + 0.09573151916265488, + 0.10091027617454529, + 0.09348288178443909, + 0.10140916705131531, + 0.09920287877321243, + 0.11094733327627182, + 0.10943257063627243, + 0.12530675530433655, + 0.12546418607234955, + 0.14505283534526825, + 0.1458611935377121, + 0.17291401326656342, + 0.1839449107646942 + ], + "loss_nats": 2.155438184738159, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04716380685567856, + 0.05259466543793678, + 0.06381504237651825, + 0.067360520362854, + 0.076678566634655, + 0.0797014907002449, + 0.08585279434919357, + 0.08837522566318512, + 0.09026700258255005, + 0.09281065315008163, + 0.09405135363340378, + 0.09652531147003174, + 0.09571083635091782, + 0.09793278574943542, + 0.09508897364139557, + 0.09766053408384323, + 0.09508920460939407, + 0.09857666492462158, + 0.09573151916265488, + 0.10091027617454529, + 0.09348288178443909, + 0.10140916705131531, + 0.09920287877321243, + 0.11094733327627182, + 0.10943257063627243, + 0.12530675530433655, + 0.12546418607234955, + 0.14505283534526825, + 0.1458611935377121, + 0.17291401326656342, + 0.1839449107646942, + 0.21681295335292816 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001656424137763679, + 0.00018305814592167735, + 0.00020359577320050448, + 0.00018629668920766562, + 0.0001775996934156865, + 0.00016671977937221527, + 0.00015271134907379746, + 0.0001390485413139686, + 0.00012298233923502266, + 0.00011299804464215413, + 9.434259118279442e-05, + 8.304810035042465e-05, + 7.153192564146593e-05, + 5.851028618053533e-05, + 4.2740048229461536e-05, + 2.919154394476209e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018464825552655384, + "first_to_last_ratio": 3.6568753347836163, + "imbalance_abs_log_ratio": 1.2966090491094597, + "last_quartile_mean": 5.049345099905622e-05, + "mean": 0.0001243760790430315, + "normalized": [ + 1.3317867475068026, + 1.4718115197886492, + 1.6369367386960691, + 1.4978498328702814, + 1.4279248452127258, + 1.3404489083028075, + 1.2278192900819984, + 1.1179685224347742, + 0.9887941490137615, + 0.9085191100377042, + 0.7585268156761388, + 0.6677176269698272, + 0.5751260707994935, + 0.4704303804294394, + 0.34363559744212857, + 0.23470384473739866 + ], + "population_cv": 0.4334270573824917 + }, + "activation_output_rms_by_block": [ + 0.08618219196796417, + 0.145728200674057, + 0.16391748189926147, + 0.16589416563510895, + 0.16637063026428223, + 0.1656494140625, + 0.16352342069149017, + 0.1510169357061386, + 0.14240477979183197, + 0.14064612984657288, + 0.14404520392417908, + 0.15505897998809814, + 0.1711372584104538, + 0.19663043320178986, + 0.24908031523227692, + 0.4027986526489258 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1404305100440979, + "first_to_last_ratio": 0.5508987205974385, + "imbalance_abs_log_ratio": 0.5962042968923701, + "last_quartile_mean": 0.2549116648733616, + "mean": 0.1756302621215582, + "normalized": [ + 0.49070240473885574, + 0.8297442531469593, + 0.9333100111517798, + 0.9445648126419658, + 0.9472776972178796, + 0.9431712511358084, + 0.9310663134939208, + 0.859857144673712, + 0.8108214271938511, + 0.8008080620481458, + 0.820161640620235, + 0.8828716538655387, + 0.9744178272193509, + 1.1195703452614385, + 1.4182084125108352, + 2.293446743079724 + ], + "population_cv": 0.38002251655845937 + }, + "bits_per_byte": 2.081392170877702, + "branch_output_rms_by_sublayer": [ + 0.03403226286172867, + 0.04189683124423027, + 0.08070491254329681, + 0.007666230201721191, + 0.03132990375161171, + 0.004042334854602814, + 0.023764250800013542, + 0.004978273529559374, + 0.022670205682516098, + 0.009758620522916317, + 0.027493907138705254, + 0.012959085404872894, + 0.036309871822595596, + 0.017362121492624283, + 0.039587654173374176, + 0.02089655213057995, + 0.04225468263030052, + 0.02445071190595627, + 0.032471224665641785, + 0.02915186993777752, + 0.04851555824279785, + 0.047949694097042084, + 0.01816859096288681, + 0.05110293999314308, + 0.02429918199777603, + 0.05613848567008972, + 0.01771482080221176, + 0.06749960780143738, + 0.015686625614762306, + 0.09889734536409378, + 0.016557788476347923, + 0.2156895101070404 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00041094985323999886, + 0.0003483840396349268, + 0.0002459421498427246, + 0.00023334166050981232, + 0.000254581953755271, + 0.0003388594313501643, + 0.0004326353023247584, + 0.0004560987158898983, + 0.0004819816052452772, + 0.0004606041111206281, + 0.00053396017856909, + 0.0004147477851776299, + 0.00037952749197896316, + 0.00033346538787636917, + 0.0002981284512971557, + 0.0002588526279031963 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030965442580686565, + "first_to_last_ratio": 0.9753095285106969, + "imbalance_abs_log_ratio": 0.0250003932265083, + "last_quartile_mean": 0.0003174934897639211, + "mean": 0.0003676287966072415, + "normalized": [ + 1.117839127490983, + 0.9476516607242959, + 0.6689958787572303, + 0.6347208452201429, + 0.6924973127914547, + 0.9217434392447414, + 1.176826479093712, + 1.2406501343178897, + 1.3110550906059877, + 1.2529054181049841, + 1.4524438332820528, + 1.1281700155298995, + 1.0323660591377277, + 0.9070709121642312, + 0.8109496700163646, + 0.7041141235183029 + ], + "population_cv": 0.2454972418327454 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05505011975765228, + 0.06996718049049377, + 0.08618219196796417, + 0.1434549242258072, + 0.145728200674057, + 0.16383236646652222, + 0.16391748189926147, + 0.16620540618896484, + 0.16589416563510895, + 0.16680952906608582, + 0.16637063026428223, + 0.16661377251148224, + 0.1656494140625, + 0.1662609726190567, + 0.16352342069149017, + 0.15300041437149048, + 0.1510169357061386, + 0.14022879302501678, + 0.14240477979183197, + 0.13479726016521454, + 0.14064612984657288, + 0.13225789368152618, + 0.14404520392417908, + 0.1422654092311859, + 0.15505897998809814, + 0.1554868519306183, + 0.1711372584104538, + 0.17103315889835358, + 0.19663043320178986, + 0.19691547751426697, + 0.24908031523227692, + 0.25020092725753784 + ], + "loss_nats": 1.4427111148834229, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06996718049049377, + 0.08618219196796417, + 0.1434549242258072, + 0.145728200674057, + 0.16383236646652222, + 0.16391748189926147, + 0.16620540618896484, + 0.16589416563510895, + 0.16680952906608582, + 0.16637063026428223, + 0.16661377251148224, + 0.1656494140625, + 0.1662609726190567, + 0.16352342069149017, + 0.15300041437149048, + 0.1510169357061386, + 0.14022879302501678, + 0.14240477979183197, + 0.13479726016521454, + 0.14064612984657288, + 0.13225789368152618, + 0.14404520392417908, + 0.1422654092311859, + 0.15505897998809814, + 0.1554868519306183, + 0.1711372584104538, + 0.17103315889835358, + 0.19663043320178986, + 0.19691547751426697, + 0.24908031523227692, + 0.25020092725753784, + 0.4027986526489258 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001670715573709458, + 0.0001843688660301268, + 0.00019890567637048662, + 0.00018651749996934086, + 0.00017948838649317622, + 0.0001703005691524595, + 0.00015747325960546732, + 0.0001445972011424601, + 0.00012834250810556114, + 0.00011769140110118315, + 9.774974023457617e-05, + 8.55058096931316e-05, + 7.324934267671779e-05, + 5.983562004985288e-05, + 4.2463150748517364e-05, + 2.6351406631874852e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018421589993522502, + "first_to_last_ratio": 3.6496550331101454, + "imbalance_abs_log_ratio": 1.2946326516512365, + "last_quartile_mean": 5.047488002674072e-05, + "mean": 0.00012624449971099239, + "normalized": [ + 1.323396724240799, + 1.4604110789158873, + 1.5755591477318631, + 1.477430703090666, + 1.42175213101619, + 1.348974169506975, + 1.2473672909787434, + 1.145374265599547, + 1.016618612291004, + 0.9322497326268504, + 0.7742891013735379, + 0.6773032479741881, + 0.5802180914368962, + 0.47396615446084944, + 0.336356442028975, + 0.2087331067270282 + ], + "population_cv": 0.42845315545725604 + }, + "activation_output_rms_by_block": [ + 0.10210617631673813, + 0.1711120903491974, + 0.18304166197776794, + 0.1841333508491516, + 0.18528582155704498, + 0.18522554636001587, + 0.18429037928581238, + 0.1677425056695938, + 0.15884697437286377, + 0.1616142988204956, + 0.1705332249403, + 0.18593955039978027, + 0.20567089319229126, + 0.23390550911426544, + 0.2939614951610565, + 0.5084910988807678 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16009831987321377, + "first_to_last_ratio": 0.5156025192452343, + "imbalance_abs_log_ratio": 0.6624191218832031, + "last_quartile_mean": 0.31050724908709526, + "mean": 0.20511878607794642, + "normalized": [ + 0.4977904670220559, + 0.8342097455869972, + 0.8923690778289365, + 0.8976913054622885, + 0.9033098581552409, + 0.9030160030765246, + 0.8984568542433792, + 0.8177822659590556, + 0.7744145595347903, + 0.7879058857099571, + 0.8313876471338725, + 0.9064969326072458, + 1.002691645777072, + 1.1403417189948652, + 1.4331280950997307, + 2.479007937807988 + ], + "population_cv": 0.4251348773789681 + }, + "bits_per_byte": 1.9581638498120857, + "branch_output_rms_by_sublayer": [ + 0.052206385880708694, + 0.0456010177731514, + 0.08834584802389145, + 0.009068594314157963, + 0.02721724845468998, + 0.003163012443110347, + 0.024817200377583504, + 0.004634208977222443, + 0.025762483477592468, + 0.010066481307148933, + 0.02899768576025963, + 0.014108272269368172, + 0.04359622299671173, + 0.02107245847582817, + 0.046488646417856216, + 0.02715279720723629, + 0.04875380918383598, + 0.032857682555913925, + 0.03842322900891304, + 0.03946002945303917, + 0.05467399209737778, + 0.0617416612803936, + 0.025746947154402733, + 0.06822259724140167, + 0.032913338392972946, + 0.0761992558836937, + 0.024072155356407166, + 0.08795499801635742, + 0.022052984684705734, + 0.12569737434387207, + 0.025944042950868607, + 0.30153149366378784 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00042380566817482924, + 0.00033628063183600165, + 0.000202638705536708, + 0.00018804477448005897, + 0.00023208100123381636, + 0.0003223172253228011, + 0.0004741333361060697, + 0.0004905991387745841, + 0.0005898184863404294, + 0.0005622263144383919, + 0.000639932357888532, + 0.0005200270037733745, + 0.0004834555123403282, + 0.0004217229809514017, + 0.0003667204712485774, + 0.0003029597160326848 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002876924450068995, + "first_to_last_ratio": 0.7307130437944471, + "imbalance_abs_log_ratio": 0.3137344492502099, + "last_quartile_mean": 0.000393714670143248, + "mean": 0.0004097977077799118, + "normalized": [ + 1.0341826226183788, + 0.8206015442541381, + 0.4944847218265511, + 0.4588721969646822, + 0.5663306476044494, + 0.7865276432827353, + 1.156993626622993, + 1.1971739457314583, + 1.4392918143339164, + 1.371960613162688, + 1.561581106335074, + 1.2689846573096573, + 1.1797418657109118, + 1.029100390131744, + 0.894881704525127, + 0.739290899585495 + ], + "population_cv": 0.3269281757976474 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05859006941318512, + 0.08463093638420105, + 0.10210617631673813, + 0.16779249906539917, + 0.1711120903491974, + 0.1829725056886673, + 0.18304166197776794, + 0.18442119657993317, + 0.1841333508491516, + 0.1860278993844986, + 0.18528582155704498, + 0.1865086406469345, + 0.18522554636001587, + 0.18776042759418488, + 0.18429037928581238, + 0.17156346142292023, + 0.1677425056695938, + 0.15597018599510193, + 0.15884697437286377, + 0.1531117558479309, + 0.1616142988204956, + 0.15448127686977386, + 0.1705332249403, + 0.1696566343307495, + 0.18593955039978027, + 0.1874322146177292, + 0.20567089319229126, + 0.20625530183315277, + 0.23390550911426544, + 0.234551340341568, + 0.2939614951610565, + 0.2957085371017456 + ], + "loss_nats": 1.3572957515716553, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08463093638420105, + 0.10210617631673813, + 0.16779249906539917, + 0.1711120903491974, + 0.1829725056886673, + 0.18304166197776794, + 0.18442119657993317, + 0.1841333508491516, + 0.1860278993844986, + 0.18528582155704498, + 0.1865086406469345, + 0.18522554636001587, + 0.18776042759418488, + 0.18429037928581238, + 0.17156346142292023, + 0.1677425056695938, + 0.15597018599510193, + 0.15884697437286377, + 0.1531117558479309, + 0.1616142988204956, + 0.15448127686977386, + 0.1705332249403, + 0.1696566343307495, + 0.18593955039978027, + 0.1874322146177292, + 0.20567089319229126, + 0.20625530183315277, + 0.23390550911426544, + 0.234551340341568, + 0.2939614951610565, + 0.2957085371017456, + 0.5084910988807678 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.000266833056230098, + 0.0002936421660706401, + 0.00031041845795698464, + 0.0002957055112347007, + 0.0002881943073589355, + 0.0002774772292468697, + 0.00026020428049378097, + 0.00024136141291819513, + 0.00021430903871078044, + 0.00019666840671561658, + 0.00016092424630187452, + 0.0001381110487272963, + 0.00011530883784871548, + 9.161087655229494e-05, + 6.125834624981508e-05, + 3.126815863652155e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029164979787310585, + "first_to_last_ratio": 3.8958554703706607, + "imbalance_abs_log_ratio": 1.3599132881647322, + "last_quartile_mean": 7.486155482183676e-05, + "mean": 0.00020270596132831997, + "normalized": [ + 1.3163552491577308, + 1.4486113982362465, + 1.531373107741041, + 1.4587904040757373, + 1.4217357273087359, + 1.3688656585557626, + 1.2836538145631138, + 1.1906971622174716, + 1.05724092822148, + 0.9702152093942396, + 0.7938801860949112, + 0.6813368872936094, + 0.5688477886545793, + 0.4519397256596421, + 0.3022029833182646, + 0.15425376950743425 + ], + "population_cv": 0.4374351028998013 + }, + "activation_output_rms_by_block": [ + 0.10593125969171524, + 0.16277404129505157, + 0.16965323686599731, + 0.16991853713989258, + 0.16967523097991943, + 0.1679684966802597, + 0.16474221646785736, + 0.14505790174007416, + 0.1350426822900772, + 0.13761113584041595, + 0.1448632776737213, + 0.15946830809116364, + 0.17768383026123047, + 0.2030501663684845, + 0.25671765208244324, + 0.47381165623664856 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15206926874816418, + "first_to_last_ratio": 0.5473743911850654, + "imbalance_abs_log_ratio": 0.6026222661058785, + "last_quartile_mean": 0.2778158262372017, + "mean": 0.18399810185655951, + "normalized": [ + 0.5757193070083771, + 0.8846506548309193, + 0.9220379729691717, + 0.9234798371580865, + 0.9221575074301264, + 0.9128816818512828, + 0.8953473693782256, + 0.7883662944151337, + 0.7339351924149371, + 0.7478943231039105, + 0.7873085440123357, + 0.8666845281669334, + 0.9656829525325674, + 1.1035448970380004, + 1.3952190239580522, + 2.57508991373194 + ], + "population_cv": 0.4416592993774559 + }, + "bits_per_byte": 1.8703918160136477, + "branch_output_rms_by_sublayer": [ + 0.06665868312120438, + 0.03488925099372864, + 0.07055649906396866, + 0.008275732398033142, + 0.021501976996660233, + 0.0027294722385704517, + 0.020079251378774643, + 0.004083180334419012, + 0.019392874091863632, + 0.008691227994859219, + 0.02471928857266903, + 0.012035587802529335, + 0.03809819743037224, + 0.018656741827726364, + 0.04367014393210411, + 0.024902483448386192, + 0.04372077062726021, + 0.03141119331121445, + 0.03309999778866768, + 0.037526171654462814, + 0.0474277026951313, + 0.05794820562005043, + 0.02467101812362671, + 0.0651254877448082, + 0.02942940965294838, + 0.07373781502246857, + 0.023407138884067535, + 0.08589385449886322, + 0.019243014976382256, + 0.1193988099694252, + 0.02799036167562008, + 0.300537645816803 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.000552485054575078, + 0.0004878954173704819, + 0.00026717869748665266, + 0.00024508760509843496, + 0.0003479630309338483, + 0.0004644527429795662, + 0.0007202858637767138, + 0.0008028534351930377, + 0.0009144494502519988, + 0.0008971273720051169, + 0.001000538678877296, + 0.0008212401598758333, + 0.0007501935670996717, + 0.0006492653356624181, + 0.0005392275192896546, + 0.00040293680800761964 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00038816169363266187, + "first_to_last_ratio": 0.6630643028303428, + "imbalance_abs_log_ratio": 0.4108833058215335, + "last_quartile_mean": 0.000585405807514841, + "mean": 0.0006164487961552139, + "normalized": [ + 0.8962383542979121, + 0.7914613840005554, + 0.4334158800423393, + 0.3975798259758869, + 0.5644638015421408, + 0.7534328007068132, + 1.1684439458218279, + 1.3023846265909318, + 1.4834150962016837, + 1.4553153118320499, + 1.6230685908021032, + 1.3322114748181866, + 1.2169600650996852, + 1.0532348180609332, + 0.8747320501764496, + 0.6536419740305005 + ], + "population_cv": 0.37363268494405877 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060181986540555954, + 0.0962739959359169, + 0.10593125969171524, + 0.15956835448741913, + 0.16277404129505157, + 0.16968216001987457, + 0.16965323686599731, + 0.1704123467206955, + 0.16991853713989258, + 0.17062334716320038, + 0.16967523097991943, + 0.16979995369911194, + 0.1679684966802597, + 0.16850441694259644, + 0.16474221646785736, + 0.14905613660812378, + 0.14505790174007416, + 0.13280805945396423, + 0.1350426822900772, + 0.13030488789081573, + 0.13761113584041595, + 0.13074533641338348, + 0.1448632776737213, + 0.1443873643875122, + 0.15946830809116364, + 0.16084782779216766, + 0.17768383026123047, + 0.17834986746311188, + 0.2030501663684845, + 0.20354533195495605, + 0.25671765208244324, + 0.2584022283554077 + ], + "loss_nats": 1.2964568138122559, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0962739959359169, + 0.10593125969171524, + 0.15956835448741913, + 0.16277404129505157, + 0.16968216001987457, + 0.16965323686599731, + 0.1704123467206955, + 0.16991853713989258, + 0.17062334716320038, + 0.16967523097991943, + 0.16979995369911194, + 0.1679684966802597, + 0.16850441694259644, + 0.16474221646785736, + 0.14905613660812378, + 0.14505790174007416, + 0.13280805945396423, + 0.1350426822900772, + 0.13030488789081573, + 0.13761113584041595, + 0.13074533641338348, + 0.1448632776737213, + 0.1443873643875122, + 0.15946830809116364, + 0.16084782779216766, + 0.17768383026123047, + 0.17834986746311188, + 0.2030501663684845, + 0.20354533195495605, + 0.25671765208244324, + 0.2584022283554077, + 0.47381165623664856 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.032388224749795, + "cross_entropy_nats": 5.567627251148224, + "step": 0 + }, + { + "bits_per_byte": 5.232053968773903, + "cross_entropy_nats": 3.626583456993103, + "step": 100 + }, + { + "bits_per_byte": 3.1492899782815944, + "cross_entropy_nats": 2.1829214692115784, + "step": 500 + }, + { + "bits_per_byte": 1.9917407321684666, + "cross_entropy_nats": 1.3805694729089737, + "step": 2000 + }, + { + "bits_per_byte": 1.8310010171491868, + "cross_entropy_nats": 1.269153192639351, + "step": 4000 + }, + { + "bits_per_byte": 1.7368251448523782, + "cross_entropy_nats": 1.2038754522800446, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "final_optimizer_state": "34afebd4020a97001cb564ade8591d46cc164cbd65eb2e1cad4d8d953e6b7cce", + "final_public_parameters": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.682085667085165, + "measured_steps": 7980, + "median_ms": 21.630175004247576, + "p95_ms": 22.00388514320366, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.03605033732594, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.570165634155273, + "step": 1, + "unclipped_grad_norm": 14.13492488861084 + }, + { + "bits_per_byte": 7.579231395965356, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.253522872924805, + "step": 10, + "unclipped_grad_norm": 10.806709289550781 + }, + { + "bits_per_byte": 6.980917512225713, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.838803291320801, + "step": 20, + "unclipped_grad_norm": 4.377180576324463 + }, + { + "bits_per_byte": 6.73711765808501, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.669814109802246, + "step": 30, + "unclipped_grad_norm": 3.1878201961517334 + }, + { + "bits_per_byte": 6.622164454248309, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.590134620666504, + "step": 40, + "unclipped_grad_norm": 3.092231035232544 + }, + { + "bits_per_byte": 6.373620630666205, + "learning_rate": 3.75e-05, + "loss_nats": 4.4178571701049805, + "step": 50, + "unclipped_grad_norm": 3.1614222526550293 + }, + { + "bits_per_byte": 6.096217430338719, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.225575923919678, + "step": 60, + "unclipped_grad_norm": 2.885824203491211 + }, + { + "bits_per_byte": 5.853263042909657, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 4.057172775268555, + "step": 70, + "unclipped_grad_norm": 2.6939339637756348 + }, + { + "bits_per_byte": 5.640556620149043, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.909735918045044, + "step": 80, + "unclipped_grad_norm": 2.5527353286743164 + }, + { + "bits_per_byte": 5.452656287018279, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7794933319091797, + "step": 90, + "unclipped_grad_norm": 2.4638988971710205 + }, + { + "bits_per_byte": 5.271040199950041, + "learning_rate": 7.5e-05, + "loss_nats": 3.653606653213501, + "step": 100, + "unclipped_grad_norm": 2.4236807823181152 + }, + { + "bits_per_byte": 5.106934010531447, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.5398569107055664, + "step": 110, + "unclipped_grad_norm": 2.3251681327819824 + }, + { + "bits_per_byte": 4.82528283872765, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3446311950683594, + "step": 120, + "unclipped_grad_norm": 2.082082509994507 + }, + { + "bits_per_byte": 4.666372933122225, + "learning_rate": 9.75e-05, + "loss_nats": 3.234483242034912, + "step": 130, + "unclipped_grad_norm": 1.8279871940612793 + }, + { + "bits_per_byte": 4.4669253408472285, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0962367057800293, + "step": 140, + "unclipped_grad_norm": 1.706978440284729 + }, + { + "bits_per_byte": 4.329626087218621, + "learning_rate": 0.0001125, + "loss_nats": 3.001068115234375, + "step": 150, + "unclipped_grad_norm": 1.465320348739624 + }, + { + "bits_per_byte": 4.211352242359061, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.9190869331359863, + "step": 160, + "unclipped_grad_norm": 1.149477243423462 + }, + { + "bits_per_byte": 4.052300967014626, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.8088409900665283, + "step": 170, + "unclipped_grad_norm": 1.175323247909546 + }, + { + "bits_per_byte": 3.9086503940896553, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7092700004577637, + "step": 180, + "unclipped_grad_norm": 1.1839734315872192 + }, + { + "bits_per_byte": 3.9167359864373195, + "learning_rate": 0.0001425, + "loss_nats": 2.714874505996704, + "step": 190, + "unclipped_grad_norm": 0.9742223024368286 + }, + { + "bits_per_byte": 3.7936297730538158, + "learning_rate": 0.00015, + "loss_nats": 2.6295437812805176, + "step": 200, + "unclipped_grad_norm": 0.6956769227981567 + }, + { + "bits_per_byte": 3.735681250741498, + "learning_rate": 0.0001575, + "loss_nats": 2.589376926422119, + "step": 210, + "unclipped_grad_norm": 0.6548250317573547 + }, + { + "bits_per_byte": 3.7032831589719617, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.566920280456543, + "step": 220, + "unclipped_grad_norm": 0.6789838075637817 + }, + { + "bits_per_byte": 3.722889181170322, + "learning_rate": 0.0001725, + "loss_nats": 2.580510139465332, + "step": 230, + "unclipped_grad_norm": 0.730959415435791 + }, + { + "bits_per_byte": 3.721254658056522, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5793771743774414, + "step": 240, + "unclipped_grad_norm": 0.7681548595428467 + }, + { + "bits_per_byte": 3.588607879099055, + "learning_rate": 0.0001875, + "loss_nats": 2.487433433532715, + "step": 250, + "unclipped_grad_norm": 1.0810788869857788 + }, + { + "bits_per_byte": 3.588186865569743, + "learning_rate": 0.000195, + "loss_nats": 2.4871416091918945, + "step": 260, + "unclipped_grad_norm": 0.9089803695678711 + }, + { + "bits_per_byte": 3.5492888614934976, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.460179567337036, + "step": 270, + "unclipped_grad_norm": 0.6836683750152588 + }, + { + "bits_per_byte": 3.551394961035964, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.461639404296875, + "step": 280, + "unclipped_grad_norm": 0.6695814728736877 + }, + { + "bits_per_byte": 3.464650695559082, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.401512861251831, + "step": 290, + "unclipped_grad_norm": 0.8494355082511902 + }, + { + "bits_per_byte": 3.543094390374721, + "learning_rate": 0.000225, + "loss_nats": 2.455885887145996, + "step": 300, + "unclipped_grad_norm": 0.7996388077735901 + }, + { + "bits_per_byte": 3.442018122670838, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.3858251571655273, + "step": 310, + "unclipped_grad_norm": 0.7709511518478394 + }, + { + "bits_per_byte": 3.43619410218202, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3817882537841797, + "step": 320, + "unclipped_grad_norm": 0.8889051079750061 + }, + { + "bits_per_byte": 3.4344873463548575, + "learning_rate": 0.0002475, + "loss_nats": 2.3806052207946777, + "step": 330, + "unclipped_grad_norm": 0.977308988571167 + }, + { + "bits_per_byte": 3.4251934039022314, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3741631507873535, + "step": 340, + "unclipped_grad_norm": 0.7606520652770996 + }, + { + "bits_per_byte": 3.3985436602551338, + "learning_rate": 0.0002625, + "loss_nats": 2.3556909561157227, + "step": 350, + "unclipped_grad_norm": 0.6722884178161621 + }, + { + "bits_per_byte": 3.3928290207322647, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3517298698425293, + "step": 360, + "unclipped_grad_norm": 0.8897987008094788 + }, + { + "bits_per_byte": 3.4104885432881593, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3639705181121826, + "step": 370, + "unclipped_grad_norm": 1.0889073610305786 + }, + { + "bits_per_byte": 3.4536713231280003, + "learning_rate": 0.000285, + "loss_nats": 2.393902540206909, + "step": 380, + "unclipped_grad_norm": 1.0755773782730103 + }, + { + "bits_per_byte": 3.3990293392611703, + "learning_rate": 0.0002925, + "loss_nats": 2.356027603149414, + "step": 390, + "unclipped_grad_norm": 1.091948390007019 + }, + { + "bits_per_byte": 3.337470901107242, + "learning_rate": 0.0003, + "loss_nats": 2.3133585453033447, + "step": 400, + "unclipped_grad_norm": 0.9980842471122742 + }, + { + "bits_per_byte": 3.3053706832890004, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2911083698272705, + "step": 410, + "unclipped_grad_norm": 0.7333840727806091 + }, + { + "bits_per_byte": 3.321116382906094, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3020224571228027, + "step": 420, + "unclipped_grad_norm": 0.8894042372703552 + }, + { + "bits_per_byte": 3.3009490093353175, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.28804349899292, + "step": 430, + "unclipped_grad_norm": 0.9840139746665955 + }, + { + "bits_per_byte": 3.3110151538903176, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.295020818710327, + "step": 440, + "unclipped_grad_norm": 1.0280733108520508 + }, + { + "bits_per_byte": 3.2949894665176194, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.2839126586914062, + "step": 450, + "unclipped_grad_norm": 0.928855836391449 + }, + { + "bits_per_byte": 3.2291744577548367, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.238293170928955, + "step": 460, + "unclipped_grad_norm": 0.8088337779045105 + }, + { + "bits_per_byte": 3.242055958304461, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2472219467163086, + "step": 470, + "unclipped_grad_norm": 1.018490195274353 + }, + { + "bits_per_byte": 3.1972775234299147, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.21618390083313, + "step": 480, + "unclipped_grad_norm": 1.066487193107605 + }, + { + "bits_per_byte": 3.156052938052798, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1876091957092285, + "step": 490, + "unclipped_grad_norm": 1.2544180154800415 + }, + { + "bits_per_byte": 3.153048401142225, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1855266094207764, + "step": 500, + "unclipped_grad_norm": 0.7976559400558472 + }, + { + "bits_per_byte": 3.126950721804397, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1674370765686035, + "step": 510, + "unclipped_grad_norm": 0.9979055523872375 + }, + { + "bits_per_byte": 3.0098652770622984, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0862796306610107, + "step": 520, + "unclipped_grad_norm": 1.0761884450912476 + }, + { + "bits_per_byte": 3.0772147150360794, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.132962703704834, + "step": 530, + "unclipped_grad_norm": 1.010690450668335 + }, + { + "bits_per_byte": 3.0006050432092413, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0798609256744385, + "step": 540, + "unclipped_grad_norm": 1.0196499824523926 + }, + { + "bits_per_byte": 2.9739387892276614, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0613772869110107, + "step": 550, + "unclipped_grad_norm": 0.949600875377655 + }, + { + "bits_per_byte": 2.9792871057041928, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.065084457397461, + "step": 560, + "unclipped_grad_norm": 1.027385950088501 + }, + { + "bits_per_byte": 2.952135172716576, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.046264171600342, + "step": 570, + "unclipped_grad_norm": 1.2502241134643555 + }, + { + "bits_per_byte": 2.8483484584350953, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.9743247032165527, + "step": 580, + "unclipped_grad_norm": 0.9710299968719482 + }, + { + "bits_per_byte": 2.9027066709280014, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.012002944946289, + "step": 590, + "unclipped_grad_norm": 1.0665416717529297 + }, + { + "bits_per_byte": 2.824687945242644, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.957924485206604, + "step": 600, + "unclipped_grad_norm": 1.3016425371170044 + }, + { + "bits_per_byte": 2.889905658258981, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.0031299591064453, + "step": 610, + "unclipped_grad_norm": 1.458404779434204 + }, + { + "bits_per_byte": 2.7619725797963617, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9144535064697266, + "step": 620, + "unclipped_grad_norm": 1.0435335636138916 + }, + { + "bits_per_byte": 2.780679820661313, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9274203777313232, + "step": 630, + "unclipped_grad_norm": 1.2443503141403198 + }, + { + "bits_per_byte": 2.796278303129268, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.938232421875, + "step": 640, + "unclipped_grad_norm": 1.0311143398284912 + }, + { + "bits_per_byte": 2.704606046774563, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.874690055847168, + "step": 650, + "unclipped_grad_norm": 0.9809540510177612 + }, + { + "bits_per_byte": 2.691190368111359, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8653910160064697, + "step": 660, + "unclipped_grad_norm": 1.5349526405334473 + }, + { + "bits_per_byte": 2.7107573502479734, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8789538145065308, + "step": 670, + "unclipped_grad_norm": 1.2438547611236572 + }, + { + "bits_per_byte": 2.7307020062858824, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8927783966064453, + "step": 680, + "unclipped_grad_norm": 1.507369041442871 + }, + { + "bits_per_byte": 2.6187595307351854, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.815185785293579, + "step": 690, + "unclipped_grad_norm": 1.4796253442764282 + }, + { + "bits_per_byte": 2.562429709132788, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7761409282684326, + "step": 700, + "unclipped_grad_norm": 1.6683489084243774 + }, + { + "bits_per_byte": 2.6317965036011897, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8242223262786865, + "step": 710, + "unclipped_grad_norm": 1.1483558416366577 + }, + { + "bits_per_byte": 2.6874191325432393, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8627769947052002, + "step": 720, + "unclipped_grad_norm": 1.464730143547058 + }, + { + "bits_per_byte": 2.5059506065894417, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.736992597579956, + "step": 730, + "unclipped_grad_norm": 1.4666827917099 + }, + { + "bits_per_byte": 2.538973683312213, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7598824501037598, + "step": 740, + "unclipped_grad_norm": 1.090103030204773 + }, + { + "bits_per_byte": 2.545309180204697, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7642738819122314, + "step": 750, + "unclipped_grad_norm": 1.1345367431640625 + }, + { + "bits_per_byte": 2.5190732268155758, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7460885047912598, + "step": 760, + "unclipped_grad_norm": 1.0966333150863647 + }, + { + "bits_per_byte": 2.5090782830780327, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7391605377197266, + "step": 770, + "unclipped_grad_norm": 1.297254204750061 + }, + { + "bits_per_byte": 2.4228073138563255, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6793620586395264, + "step": 780, + "unclipped_grad_norm": 1.1724724769592285 + }, + { + "bits_per_byte": 2.510659835535367, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7402567863464355, + "step": 790, + "unclipped_grad_norm": 1.0964044332504272 + }, + { + "bits_per_byte": 2.4699436308742007, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.7120344638824463, + "step": 800, + "unclipped_grad_norm": 1.4117348194122314 + }, + { + "bits_per_byte": 2.5086447148152073, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.738860011100769, + "step": 810, + "unclipped_grad_norm": 1.889366626739502 + }, + { + "bits_per_byte": 2.4598160885128424, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7050145864486694, + "step": 820, + "unclipped_grad_norm": 1.144262671470642 + }, + { + "bits_per_byte": 2.470564488243815, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7124648094177246, + "step": 830, + "unclipped_grad_norm": 1.5465298891067505 + }, + { + "bits_per_byte": 2.4487553682879026, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.69734787940979, + "step": 840, + "unclipped_grad_norm": 1.3992143869400024 + }, + { + "bits_per_byte": 2.383892111514994, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.652388095855713, + "step": 850, + "unclipped_grad_norm": 1.1111818552017212 + }, + { + "bits_per_byte": 2.4110481720862316, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6712112426757812, + "step": 860, + "unclipped_grad_norm": 1.363444209098816 + }, + { + "bits_per_byte": 2.3893498089574745, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6561710834503174, + "step": 870, + "unclipped_grad_norm": 1.2841624021530151 + }, + { + "bits_per_byte": 2.4391853936806416, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6907144784927368, + "step": 880, + "unclipped_grad_norm": 1.406774640083313 + }, + { + "bits_per_byte": 2.4004903291324147, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6638931035995483, + "step": 890, + "unclipped_grad_norm": 1.1680277585983276 + }, + { + "bits_per_byte": 2.3361910034716855, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6193242073059082, + "step": 900, + "unclipped_grad_norm": 1.2712005376815796 + }, + { + "bits_per_byte": 2.3780436694897533, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.648334264755249, + "step": 910, + "unclipped_grad_norm": 1.335773229598999 + }, + { + "bits_per_byte": 2.3308695162886885, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.615635633468628, + "step": 920, + "unclipped_grad_norm": 1.1817855834960938 + }, + { + "bits_per_byte": 2.330381085560238, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6152970790863037, + "step": 930, + "unclipped_grad_norm": 1.0815848112106323 + }, + { + "bits_per_byte": 2.2884202424547775, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5862120389938354, + "step": 940, + "unclipped_grad_norm": 1.0937949419021606 + }, + { + "bits_per_byte": 2.296288964679704, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5916662216186523, + "step": 950, + "unclipped_grad_norm": 1.6217018365859985 + }, + { + "bits_per_byte": 2.2717374093731326, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.574648380279541, + "step": 960, + "unclipped_grad_norm": 1.6217092275619507 + }, + { + "bits_per_byte": 2.290087270289591, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5873675346374512, + "step": 970, + "unclipped_grad_norm": 1.0035263299942017 + }, + { + "bits_per_byte": 2.280217357939225, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5805262327194214, + "step": 980, + "unclipped_grad_norm": 1.3000710010528564 + }, + { + "bits_per_byte": 2.201290563857312, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5258183479309082, + "step": 990, + "unclipped_grad_norm": 1.0893296003341675 + }, + { + "bits_per_byte": 2.251006276672828, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5602786540985107, + "step": 1000, + "unclipped_grad_norm": 1.206483006477356 + }, + { + "bits_per_byte": 2.2491729416146304, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5590078830718994, + "step": 1010, + "unclipped_grad_norm": 1.1218043565750122 + }, + { + "bits_per_byte": 2.2577407733153136, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5649466514587402, + "step": 1020, + "unclipped_grad_norm": 1.0566829442977905 + }, + { + "bits_per_byte": 2.1604180269665116, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4974876642227173, + "step": 1030, + "unclipped_grad_norm": 1.102095127105713 + }, + { + "bits_per_byte": 2.2614810520062782, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5675392150878906, + "step": 1040, + "unclipped_grad_norm": 1.2485690116882324 + }, + { + "bits_per_byte": 2.2470793968056775, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5575567483901978, + "step": 1050, + "unclipped_grad_norm": 1.242668628692627 + }, + { + "bits_per_byte": 2.1863758844091876, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5154802799224854, + "step": 1060, + "unclipped_grad_norm": 1.1564915180206299 + }, + { + "bits_per_byte": 2.2939011575551245, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5900111198425293, + "step": 1070, + "unclipped_grad_norm": 1.2426202297210693 + }, + { + "bits_per_byte": 2.2229925386217375, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5408610105514526, + "step": 1080, + "unclipped_grad_norm": 1.0772974491119385 + }, + { + "bits_per_byte": 2.1433408376664382, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.485650658607483, + "step": 1090, + "unclipped_grad_norm": 1.0020748376846313 + }, + { + "bits_per_byte": 2.2307484402276976, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5462369918823242, + "step": 1100, + "unclipped_grad_norm": 1.1032538414001465 + }, + { + "bits_per_byte": 2.263051941539259, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5686280727386475, + "step": 1110, + "unclipped_grad_norm": 1.062880039215088 + }, + { + "bits_per_byte": 2.1486932817265902, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4893606901168823, + "step": 1120, + "unclipped_grad_norm": 1.1697474718093872 + }, + { + "bits_per_byte": 2.172017052888652, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5055274963378906, + "step": 1130, + "unclipped_grad_norm": 1.4915344715118408 + }, + { + "bits_per_byte": 2.1695033544636413, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5037851333618164, + "step": 1140, + "unclipped_grad_norm": 1.5511094331741333 + }, + { + "bits_per_byte": 2.1545989939745684, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4934542179107666, + "step": 1150, + "unclipped_grad_norm": 1.240431547164917 + }, + { + "bits_per_byte": 2.097475816403741, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4538594484329224, + "step": 1160, + "unclipped_grad_norm": 1.178969144821167 + }, + { + "bits_per_byte": 2.2105972330261023, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5322692394256592, + "step": 1170, + "unclipped_grad_norm": 1.1780297756195068 + }, + { + "bits_per_byte": 2.102323491383611, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4572196006774902, + "step": 1180, + "unclipped_grad_norm": 0.919143557548523 + }, + { + "bits_per_byte": 2.136596881978155, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4809761047363281, + "step": 1190, + "unclipped_grad_norm": 1.1034574508666992 + }, + { + "bits_per_byte": 2.1069721824364325, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4604418277740479, + "step": 1200, + "unclipped_grad_norm": 1.1095129251480103 + }, + { + "bits_per_byte": 2.254480670185558, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5626869201660156, + "step": 1210, + "unclipped_grad_norm": 1.2801767587661743 + }, + { + "bits_per_byte": 2.1061446019204806, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4598681926727295, + "step": 1220, + "unclipped_grad_norm": 1.0897294282913208 + }, + { + "bits_per_byte": 2.047843343191243, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4194568395614624, + "step": 1230, + "unclipped_grad_norm": 1.1728222370147705 + }, + { + "bits_per_byte": 2.0460125878728084, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4181878566741943, + "step": 1240, + "unclipped_grad_norm": 1.1321473121643066 + }, + { + "bits_per_byte": 2.217450741662985, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5370197296142578, + "step": 1250, + "unclipped_grad_norm": 1.0654293298721313 + }, + { + "bits_per_byte": 2.1496304151911416, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4900102615356445, + "step": 1260, + "unclipped_grad_norm": 1.0989246368408203 + }, + { + "bits_per_byte": 2.0288079594284456, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4062625169754028, + "step": 1270, + "unclipped_grad_norm": 1.0421466827392578 + }, + { + "bits_per_byte": 2.1597347398946334, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.497014045715332, + "step": 1280, + "unclipped_grad_norm": 1.076604962348938 + }, + { + "bits_per_byte": 2.060473061140004, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4282110929489136, + "step": 1290, + "unclipped_grad_norm": 1.1023766994476318 + }, + { + "bits_per_byte": 2.1029859685547345, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4576787948608398, + "step": 1300, + "unclipped_grad_norm": 1.0737801790237427 + }, + { + "bits_per_byte": 2.17370695441603, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.5066988468170166, + "step": 1310, + "unclipped_grad_norm": 1.1388311386108398 + }, + { + "bits_per_byte": 2.1091708086450627, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.461965799331665, + "step": 1320, + "unclipped_grad_norm": 1.289936900138855 + }, + { + "bits_per_byte": 1.9980870424701236, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3849684000015259, + "step": 1330, + "unclipped_grad_norm": 1.0864324569702148 + }, + { + "bits_per_byte": 2.0165677821838885, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3977782726287842, + "step": 1340, + "unclipped_grad_norm": 0.974994957447052 + }, + { + "bits_per_byte": 2.204823603453987, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.52826726436615, + "step": 1350, + "unclipped_grad_norm": 1.1115007400512695 + }, + { + "bits_per_byte": 2.0280969831497786, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4057697057724, + "step": 1360, + "unclipped_grad_norm": 0.916259229183197 + }, + { + "bits_per_byte": 2.124154625118834, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4723517894744873, + "step": 1370, + "unclipped_grad_norm": 1.1937594413757324 + }, + { + "bits_per_byte": 2.0873195529396886, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4468196630477905, + "step": 1380, + "unclipped_grad_norm": 0.9523230791091919 + }, + { + "bits_per_byte": 2.1050263707245707, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4590930938720703, + "step": 1390, + "unclipped_grad_norm": 0.8934341669082642 + }, + { + "bits_per_byte": 2.0970224701360665, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4535452127456665, + "step": 1400, + "unclipped_grad_norm": 0.9132420420646667 + }, + { + "bits_per_byte": 2.040656188211686, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4144750833511353, + "step": 1410, + "unclipped_grad_norm": 1.0989789962768555 + }, + { + "bits_per_byte": 2.069019738974631, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4341351985931396, + "step": 1420, + "unclipped_grad_norm": 1.0159250497817993 + }, + { + "bits_per_byte": 2.0709008852097686, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.435439109802246, + "step": 1430, + "unclipped_grad_norm": 0.9772002696990967 + }, + { + "bits_per_byte": 2.030792811202053, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4076383113861084, + "step": 1440, + "unclipped_grad_norm": 1.0684847831726074 + }, + { + "bits_per_byte": 2.0937577234747375, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.451282262802124, + "step": 1450, + "unclipped_grad_norm": 1.1867483854293823 + }, + { + "bits_per_byte": 2.0324108239813703, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4087598323822021, + "step": 1460, + "unclipped_grad_norm": 1.0155152082443237 + }, + { + "bits_per_byte": 2.0236820165194795, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4027094841003418, + "step": 1470, + "unclipped_grad_norm": 0.9900296330451965 + }, + { + "bits_per_byte": 1.9307554906920323, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.338297724723816, + "step": 1480, + "unclipped_grad_norm": 0.8600749969482422 + }, + { + "bits_per_byte": 2.042708801149734, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4158978462219238, + "step": 1490, + "unclipped_grad_norm": 1.3268300294876099 + }, + { + "bits_per_byte": 1.9310841495378313, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3385255336761475, + "step": 1500, + "unclipped_grad_norm": 0.9175688028335571 + }, + { + "bits_per_byte": 2.024790788669592, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4034780263900757, + "step": 1510, + "unclipped_grad_norm": 1.1049447059631348 + }, + { + "bits_per_byte": 2.021844897842963, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.4014360904693604, + "step": 1520, + "unclipped_grad_norm": 0.9541988968849182 + }, + { + "bits_per_byte": 1.9848854822072437, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3758177757263184, + "step": 1530, + "unclipped_grad_norm": 1.0897772312164307 + }, + { + "bits_per_byte": 2.0563004180647706, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4253188371658325, + "step": 1540, + "unclipped_grad_norm": 0.960680365562439 + }, + { + "bits_per_byte": 2.0173688773716076, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3983335494995117, + "step": 1550, + "unclipped_grad_norm": 1.0041202306747437 + }, + { + "bits_per_byte": 2.0953523466135375, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4523875713348389, + "step": 1560, + "unclipped_grad_norm": 0.9772875905036926 + }, + { + "bits_per_byte": 2.057308064416176, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4260172843933105, + "step": 1570, + "unclipped_grad_norm": 1.0609372854232788 + }, + { + "bits_per_byte": 2.019397928686487, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3997399806976318, + "step": 1580, + "unclipped_grad_norm": 1.008042573928833 + }, + { + "bits_per_byte": 2.0214770269527675, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.4011811017990112, + "step": 1590, + "unclipped_grad_norm": 0.9353208541870117 + }, + { + "bits_per_byte": 2.0397778728137297, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4138662815093994, + "step": 1600, + "unclipped_grad_norm": 0.9491218328475952 + }, + { + "bits_per_byte": 2.06155259623947, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4289593696594238, + "step": 1610, + "unclipped_grad_norm": 0.9174792170524597 + }, + { + "bits_per_byte": 2.0684931280976895, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4337701797485352, + "step": 1620, + "unclipped_grad_norm": 0.9907702803611755 + }, + { + "bits_per_byte": 1.9372454280049913, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3427962064743042, + "step": 1630, + "unclipped_grad_norm": 0.9023784399032593 + }, + { + "bits_per_byte": 2.056802951370592, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4256671667099, + "step": 1640, + "unclipped_grad_norm": 0.9690282344818115 + }, + { + "bits_per_byte": 2.0155706267741857, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3970870971679688, + "step": 1650, + "unclipped_grad_norm": 1.0204124450683594 + }, + { + "bits_per_byte": 1.9586099728084236, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.35760498046875, + "step": 1660, + "unclipped_grad_norm": 0.942615270614624 + }, + { + "bits_per_byte": 2.120022913914506, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4694879055023193, + "step": 1670, + "unclipped_grad_norm": 0.9669644236564636 + }, + { + "bits_per_byte": 2.02882687752004, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4062756299972534, + "step": 1680, + "unclipped_grad_norm": 0.8547722101211548 + }, + { + "bits_per_byte": 1.9310628236891243, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3385107517242432, + "step": 1690, + "unclipped_grad_norm": 0.9439292550086975 + }, + { + "bits_per_byte": 1.9561478691786718, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.355898380279541, + "step": 1700, + "unclipped_grad_norm": 0.8740168213844299 + }, + { + "bits_per_byte": 2.0381777462301023, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.412757158279419, + "step": 1710, + "unclipped_grad_norm": 1.1610065698623657 + }, + { + "bits_per_byte": 2.022566537045983, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.4019362926483154, + "step": 1720, + "unclipped_grad_norm": 0.8489123582839966 + }, + { + "bits_per_byte": 1.9268317065125973, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3355779647827148, + "step": 1730, + "unclipped_grad_norm": 0.9296807646751404 + }, + { + "bits_per_byte": 1.9993203300594609, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3858232498168945, + "step": 1740, + "unclipped_grad_norm": 0.9287921786308289 + }, + { + "bits_per_byte": 2.0917408829280695, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.449884295463562, + "step": 1750, + "unclipped_grad_norm": 0.9826183915138245 + }, + { + "bits_per_byte": 1.9484261921202333, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.35054612159729, + "step": 1760, + "unclipped_grad_norm": 0.8883485198020935 + }, + { + "bits_per_byte": 2.0047687404387946, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3895998001098633, + "step": 1770, + "unclipped_grad_norm": 1.015872836112976 + }, + { + "bits_per_byte": 1.9445588182502813, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3478654623031616, + "step": 1780, + "unclipped_grad_norm": 0.9133850932121277 + }, + { + "bits_per_byte": 1.9244168981118326, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3339041471481323, + "step": 1790, + "unclipped_grad_norm": 1.018885850906372 + }, + { + "bits_per_byte": 2.0032738672374952, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.388563632965088, + "step": 1800, + "unclipped_grad_norm": 1.0286787748336792 + }, + { + "bits_per_byte": 1.9831790703453829, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3746349811553955, + "step": 1810, + "unclipped_grad_norm": 0.8631220459938049 + }, + { + "bits_per_byte": 2.0513977086366246, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4219205379486084, + "step": 1820, + "unclipped_grad_norm": 0.8914260268211365 + }, + { + "bits_per_byte": 1.9040037613329734, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3197548389434814, + "step": 1830, + "unclipped_grad_norm": 1.014155626296997 + }, + { + "bits_per_byte": 1.9870995868544514, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3773524761199951, + "step": 1840, + "unclipped_grad_norm": 0.853826105594635 + }, + { + "bits_per_byte": 1.8864264464841909, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3075711727142334, + "step": 1850, + "unclipped_grad_norm": 0.9869223833084106 + }, + { + "bits_per_byte": 2.0198366564288377, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.4000440835952759, + "step": 1860, + "unclipped_grad_norm": 0.9129742980003357 + }, + { + "bits_per_byte": 2.033820393787842, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4097368717193604, + "step": 1870, + "unclipped_grad_norm": 0.9571877717971802 + }, + { + "bits_per_byte": 1.9046001971661655, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3201682567596436, + "step": 1880, + "unclipped_grad_norm": 0.9010704159736633 + }, + { + "bits_per_byte": 1.9853064957365558, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3761096000671387, + "step": 1890, + "unclipped_grad_norm": 0.9147026538848877 + }, + { + "bits_per_byte": 1.8987844318445888, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3161370754241943, + "step": 1900, + "unclipped_grad_norm": 1.0985209941864014 + }, + { + "bits_per_byte": 1.886329620251755, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3075040578842163, + "step": 1910, + "unclipped_grad_norm": 0.9578983783721924 + }, + { + "bits_per_byte": 1.954425118964979, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3547042608261108, + "step": 1920, + "unclipped_grad_norm": 0.9071367979049683 + }, + { + "bits_per_byte": 1.9264953084475098, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3353447914123535, + "step": 1930, + "unclipped_grad_norm": 0.8819456100463867 + }, + { + "bits_per_byte": 1.874642539212373, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.299403190612793, + "step": 1940, + "unclipped_grad_norm": 0.8353087902069092 + }, + { + "bits_per_byte": 1.9478835868567612, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3501700162887573, + "step": 1950, + "unclipped_grad_norm": 1.0307022333145142 + }, + { + "bits_per_byte": 1.9152383560179354, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3275420665740967, + "step": 1960, + "unclipped_grad_norm": 0.9104531407356262 + }, + { + "bits_per_byte": 1.8790318804276935, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.302445650100708, + "step": 1970, + "unclipped_grad_norm": 0.900970458984375 + }, + { + "bits_per_byte": 1.862672890677623, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2911064624786377, + "step": 1980, + "unclipped_grad_norm": 1.0306406021118164 + }, + { + "bits_per_byte": 1.9492345105792885, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3511064052581787, + "step": 1990, + "unclipped_grad_norm": 0.9717852473258972 + }, + { + "bits_per_byte": 1.8977989712551449, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3154540061950684, + "step": 2000, + "unclipped_grad_norm": 0.805345892906189 + }, + { + "bits_per_byte": 1.9588875528069163, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.357797384262085, + "step": 2010, + "unclipped_grad_norm": 0.8472830653190613 + }, + { + "bits_per_byte": 1.946676612613006, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.34933340549469, + "step": 2020, + "unclipped_grad_norm": 1.1016662120819092 + }, + { + "bits_per_byte": 1.9391907237689006, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.344144582748413, + "step": 2030, + "unclipped_grad_norm": 0.868341863155365 + }, + { + "bits_per_byte": 1.9511490214486933, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.352433443069458, + "step": 2040, + "unclipped_grad_norm": 0.8853706121444702 + }, + { + "bits_per_byte": 1.8935334575484455, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3124973773956299, + "step": 2050, + "unclipped_grad_norm": 0.8925303220748901 + }, + { + "bits_per_byte": 1.8037261810596972, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2502477169036865, + "step": 2060, + "unclipped_grad_norm": 0.8529354333877563 + }, + { + "bits_per_byte": 1.9080168045082062, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3225364685058594, + "step": 2070, + "unclipped_grad_norm": 0.9431800842285156 + }, + { + "bits_per_byte": 2.015688606872678, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3971688747406006, + "step": 2080, + "unclipped_grad_norm": 0.9268007874488831 + }, + { + "bits_per_byte": 1.8786344285215495, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3021701574325562, + "step": 2090, + "unclipped_grad_norm": 0.8720243573188782 + }, + { + "bits_per_byte": 1.9356492570223336, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3416898250579834, + "step": 2100, + "unclipped_grad_norm": 0.8494227528572083 + }, + { + "bits_per_byte": 1.8547725236449384, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2856303453445435, + "step": 2110, + "unclipped_grad_norm": 0.9150698781013489 + }, + { + "bits_per_byte": 1.8966766124756143, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.31467604637146, + "step": 2120, + "unclipped_grad_norm": 0.8588051199913025 + }, + { + "bits_per_byte": 1.924086863404827, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3336753845214844, + "step": 2130, + "unclipped_grad_norm": 0.870884358882904 + }, + { + "bits_per_byte": 1.8065243387892351, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2521872520446777, + "step": 2140, + "unclipped_grad_norm": 0.7779732346534729 + }, + { + "bits_per_byte": 1.8523434406841521, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2839466333389282, + "step": 2150, + "unclipped_grad_norm": 0.9460588693618774 + }, + { + "bits_per_byte": 1.9103203401338626, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3241331577301025, + "step": 2160, + "unclipped_grad_norm": 0.8497743606567383 + }, + { + "bits_per_byte": 1.8798684040414901, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.303025484085083, + "step": 2170, + "unclipped_grad_norm": 0.9395503401756287 + }, + { + "bits_per_byte": 1.8289068489031246, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2677016258239746, + "step": 2180, + "unclipped_grad_norm": 1.0030585527420044 + }, + { + "bits_per_byte": 1.947407538879173, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3498400449752808, + "step": 2190, + "unclipped_grad_norm": 0.8407843112945557 + }, + { + "bits_per_byte": 1.9944045499498493, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3824158906936646, + "step": 2200, + "unclipped_grad_norm": 0.9960005879402161 + }, + { + "bits_per_byte": 2.0001205653339253, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3863779306411743, + "step": 2210, + "unclipped_grad_norm": 0.9367930889129639 + }, + { + "bits_per_byte": 1.8803774726880442, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3033783435821533, + "step": 2220, + "unclipped_grad_norm": 0.9032418131828308 + }, + { + "bits_per_byte": 1.8995089947526738, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3166393041610718, + "step": 2230, + "unclipped_grad_norm": 0.9414434432983398 + }, + { + "bits_per_byte": 1.859138991167694, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2886569499969482, + "step": 2240, + "unclipped_grad_norm": 0.9088358283042908 + }, + { + "bits_per_byte": 1.8658401231759127, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2933018207550049, + "step": 2250, + "unclipped_grad_norm": 0.9021689295768738 + }, + { + "bits_per_byte": 1.8814774737229627, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3041408061981201, + "step": 2260, + "unclipped_grad_norm": 0.8373098969459534 + }, + { + "bits_per_byte": 1.9445300971475874, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.3478455543518066, + "step": 2270, + "unclipped_grad_norm": 0.9310195446014404 + }, + { + "bits_per_byte": 1.8355594818037997, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.272312879562378, + "step": 2280, + "unclipped_grad_norm": 0.8906505107879639 + }, + { + "bits_per_byte": 1.8521159076370606, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2837889194488525, + "step": 2290, + "unclipped_grad_norm": 0.8472009301185608 + }, + { + "bits_per_byte": 1.9052471958987114, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3206167221069336, + "step": 2300, + "unclipped_grad_norm": 0.8389962911605835 + }, + { + "bits_per_byte": 1.8722884406873628, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2977714538574219, + "step": 2310, + "unclipped_grad_norm": 0.9144077897071838 + }, + { + "bits_per_byte": 1.8873241959216946, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.3081934452056885, + "step": 2320, + "unclipped_grad_norm": 0.8859429955482483 + }, + { + "bits_per_byte": 1.8308435455344907, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.269044041633606, + "step": 2330, + "unclipped_grad_norm": 0.8459444046020508 + }, + { + "bits_per_byte": 1.8670001461559822, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.294105887413025, + "step": 2340, + "unclipped_grad_norm": 0.9549022912979126 + }, + { + "bits_per_byte": 1.7832834632648895, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.236077904701233, + "step": 2350, + "unclipped_grad_norm": 0.8211755752563477 + }, + { + "bits_per_byte": 1.8681859665336813, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2949278354644775, + "step": 2360, + "unclipped_grad_norm": 0.8336057662963867 + }, + { + "bits_per_byte": 1.9109924483334344, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.324599027633667, + "step": 2370, + "unclipped_grad_norm": 1.0507880449295044 + }, + { + "bits_per_byte": 1.8140403245972416, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.257396936416626, + "step": 2380, + "unclipped_grad_norm": 0.8251904249191284 + }, + { + "bits_per_byte": 1.8016899064734815, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2488362789154053, + "step": 2390, + "unclipped_grad_norm": 0.7763776183128357 + }, + { + "bits_per_byte": 1.9132727663012243, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3261796236038208, + "step": 2400, + "unclipped_grad_norm": 0.8198488354682922 + }, + { + "bits_per_byte": 1.8847781647583217, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.3064286708831787, + "step": 2410, + "unclipped_grad_norm": 0.8494725823402405 + }, + { + "bits_per_byte": 1.9646795845226739, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.3618121147155762, + "step": 2420, + "unclipped_grad_norm": 0.9295504689216614 + }, + { + "bits_per_byte": 1.8742646933284273, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.29914128780365, + "step": 2430, + "unclipped_grad_norm": 0.8960878849029541 + }, + { + "bits_per_byte": 1.802365110360769, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2493042945861816, + "step": 2440, + "unclipped_grad_norm": 0.8283300399780273 + }, + { + "bits_per_byte": 1.9114973893963676, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.324949026107788, + "step": 2450, + "unclipped_grad_norm": 0.8929033875465393 + }, + { + "bits_per_byte": 1.9097386948086446, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3237299919128418, + "step": 2460, + "unclipped_grad_norm": 0.8874521255493164 + }, + { + "bits_per_byte": 1.848397298760105, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2812113761901855, + "step": 2470, + "unclipped_grad_norm": 0.8395382761955261 + }, + { + "bits_per_byte": 1.8498309461376972, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2822051048278809, + "step": 2480, + "unclipped_grad_norm": 0.8023180961608887 + }, + { + "bits_per_byte": 1.7784197938984894, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2327066659927368, + "step": 2490, + "unclipped_grad_norm": 0.8236642479896545 + }, + { + "bits_per_byte": 1.8541779796209057, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2852182388305664, + "step": 2500, + "unclipped_grad_norm": 0.8692671656608582 + }, + { + "bits_per_byte": 1.8219728523856378, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2628953456878662, + "step": 2510, + "unclipped_grad_norm": 0.809110701084137 + }, + { + "bits_per_byte": 1.8139226884640514, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2573153972625732, + "step": 2520, + "unclipped_grad_norm": 0.8132631778717041 + }, + { + "bits_per_byte": 1.8159424527157844, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2587153911590576, + "step": 2530, + "unclipped_grad_norm": 0.8300133347511292 + }, + { + "bits_per_byte": 1.8460886036549224, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2796111106872559, + "step": 2540, + "unclipped_grad_norm": 0.8400688171386719 + }, + { + "bits_per_byte": 1.8887621428655585, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.3091901540756226, + "step": 2550, + "unclipped_grad_norm": 0.872266411781311 + }, + { + "bits_per_byte": 1.8326880594649941, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.270322561264038, + "step": 2560, + "unclipped_grad_norm": 0.8239099383354187 + }, + { + "bits_per_byte": 1.8127545822993913, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2565057277679443, + "step": 2570, + "unclipped_grad_norm": 0.8143022060394287 + }, + { + "bits_per_byte": 1.852552915552903, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.284091830253601, + "step": 2580, + "unclipped_grad_norm": 0.9730299115180969 + }, + { + "bits_per_byte": 1.8163588227135232, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.25900399684906, + "step": 2590, + "unclipped_grad_norm": 0.8007379770278931 + }, + { + "bits_per_byte": 1.8542342179477378, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2852572202682495, + "step": 2600, + "unclipped_grad_norm": 0.8254386782646179 + }, + { + "bits_per_byte": 1.7603803616668383, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2202026844024658, + "step": 2610, + "unclipped_grad_norm": 0.7949259877204895 + }, + { + "bits_per_byte": 1.8049837182028061, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2511193752288818, + "step": 2620, + "unclipped_grad_norm": 0.865028977394104 + }, + { + "bits_per_byte": 1.8277927452908351, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2669293880462646, + "step": 2630, + "unclipped_grad_norm": 0.826142430305481 + }, + { + "bits_per_byte": 1.8414027643495146, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2763631343841553, + "step": 2640, + "unclipped_grad_norm": 0.841355562210083 + }, + { + "bits_per_byte": 1.7851867952619882, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2373971939086914, + "step": 2650, + "unclipped_grad_norm": 0.788905143737793 + }, + { + "bits_per_byte": 1.7802099613113203, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.233947515487671, + "step": 2660, + "unclipped_grad_norm": 0.7999458909034729 + }, + { + "bits_per_byte": 1.815510432296817, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.258415937423706, + "step": 2670, + "unclipped_grad_norm": 0.8260414004325867 + }, + { + "bits_per_byte": 1.8079201499836377, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2531547546386719, + "step": 2680, + "unclipped_grad_norm": 0.8360624313354492 + }, + { + "bits_per_byte": 1.8778747811526886, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3016436100006104, + "step": 2690, + "unclipped_grad_norm": 0.8545008301734924 + }, + { + "bits_per_byte": 1.8673174541468243, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.294325828552246, + "step": 2700, + "unclipped_grad_norm": 0.8579524755477905 + }, + { + "bits_per_byte": 1.7489412796100003, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2122737169265747, + "step": 2710, + "unclipped_grad_norm": 0.8124077916145325 + }, + { + "bits_per_byte": 1.8645750187961658, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2924249172210693, + "step": 2720, + "unclipped_grad_norm": 0.9173712730407715 + }, + { + "bits_per_byte": 1.7690415799469403, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2262061834335327, + "step": 2730, + "unclipped_grad_norm": 0.7701802849769592 + }, + { + "bits_per_byte": 1.7960051919318623, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2448959350585938, + "step": 2740, + "unclipped_grad_norm": 0.841517448425293 + }, + { + "bits_per_byte": 1.8115809726899035, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2556922435760498, + "step": 2750, + "unclipped_grad_norm": 0.8873427510261536 + }, + { + "bits_per_byte": 1.9016966140716487, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3181556463241577, + "step": 2760, + "unclipped_grad_norm": 0.8982214331626892 + }, + { + "bits_per_byte": 1.8168608400713917, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2593519687652588, + "step": 2770, + "unclipped_grad_norm": 0.8194716572761536 + }, + { + "bits_per_byte": 1.8166995203448824, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2592401504516602, + "step": 2780, + "unclipped_grad_norm": 0.8212807774543762 + }, + { + "bits_per_byte": 1.8228974311166763, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2635362148284912, + "step": 2790, + "unclipped_grad_norm": 0.9329227209091187 + }, + { + "bits_per_byte": 1.757803201643659, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2184163331985474, + "step": 2800, + "unclipped_grad_norm": 0.8613669276237488 + }, + { + "bits_per_byte": 1.8264712306016053, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2660133838653564, + "step": 2810, + "unclipped_grad_norm": 0.8505002856254578 + }, + { + "bits_per_byte": 1.8298420905585164, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2683498859405518, + "step": 2820, + "unclipped_grad_norm": 0.8405188322067261 + }, + { + "bits_per_byte": 1.7699738978972677, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2268524169921875, + "step": 2830, + "unclipped_grad_norm": 0.8156670928001404 + }, + { + "bits_per_byte": 1.8402193517289274, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2755428552627563, + "step": 2840, + "unclipped_grad_norm": 0.9026433825492859 + }, + { + "bits_per_byte": 1.7471456087523418, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.211029052734375, + "step": 2850, + "unclipped_grad_norm": 0.7569094300270081 + }, + { + "bits_per_byte": 1.7464477031551406, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.210545301437378, + "step": 2860, + "unclipped_grad_norm": 0.8251471519470215 + }, + { + "bits_per_byte": 1.8558431156465593, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.286372423171997, + "step": 2870, + "unclipped_grad_norm": 0.8680534958839417 + }, + { + "bits_per_byte": 1.7726970711910293, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2287399768829346, + "step": 2880, + "unclipped_grad_norm": 0.8200762271881104 + }, + { + "bits_per_byte": 1.8185927053655802, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2605524063110352, + "step": 2890, + "unclipped_grad_norm": 0.856894314289093 + }, + { + "bits_per_byte": 1.732919547838272, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2011682987213135, + "step": 2900, + "unclipped_grad_norm": 0.8436375856399536 + }, + { + "bits_per_byte": 1.7766230911449254, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2314612865447998, + "step": 2910, + "unclipped_grad_norm": 0.8930034041404724 + }, + { + "bits_per_byte": 1.8530752268635733, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2844538688659668, + "step": 2920, + "unclipped_grad_norm": 0.8661487698554993 + }, + { + "bits_per_byte": 1.7993667648256269, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2472259998321533, + "step": 2930, + "unclipped_grad_norm": 0.9391220808029175 + }, + { + "bits_per_byte": 1.8410732455904613, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.276134729385376, + "step": 2940, + "unclipped_grad_norm": 0.8410946726799011 + }, + { + "bits_per_byte": 1.7968802396594523, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2455024719238281, + "step": 2950, + "unclipped_grad_norm": 0.8708962798118591 + }, + { + "bits_per_byte": 1.7886298879322615, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.239783763885498, + "step": 2960, + "unclipped_grad_norm": 0.8532378673553467 + }, + { + "bits_per_byte": 1.841481188438308, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2764174938201904, + "step": 2970, + "unclipped_grad_norm": 0.9022923111915588 + }, + { + "bits_per_byte": 1.775313615241256, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2305536270141602, + "step": 2980, + "unclipped_grad_norm": 0.7898789644241333 + }, + { + "bits_per_byte": 1.9084053133165053, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.3228057622909546, + "step": 2990, + "unclipped_grad_norm": 0.835300624370575 + }, + { + "bits_per_byte": 1.8358795415170555, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.272534728050232, + "step": 3000, + "unclipped_grad_norm": 0.8411970138549805 + }, + { + "bits_per_byte": 1.7909437425169699, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2413876056671143, + "step": 3010, + "unclipped_grad_norm": 0.8151383399963379 + }, + { + "bits_per_byte": 1.7661882157864754, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2242283821105957, + "step": 3020, + "unclipped_grad_norm": 0.8463819622993469 + }, + { + "bits_per_byte": 1.805475244618972, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.251460075378418, + "step": 3030, + "unclipped_grad_norm": 0.9359927177429199 + }, + { + "bits_per_byte": 1.7607282825695338, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.220443844795227, + "step": 3040, + "unclipped_grad_norm": 0.7992194890975952 + }, + { + "bits_per_byte": 1.7895066554863601, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.240391492843628, + "step": 3050, + "unclipped_grad_norm": 0.8191037178039551 + }, + { + "bits_per_byte": 1.7907009030139516, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2412192821502686, + "step": 3060, + "unclipped_grad_norm": 0.8547786474227905 + }, + { + "bits_per_byte": 1.763532287709201, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.222387433052063, + "step": 3070, + "unclipped_grad_norm": 0.8228699564933777 + }, + { + "bits_per_byte": 1.772629653991891, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2286932468414307, + "step": 3080, + "unclipped_grad_norm": 0.857837438583374 + }, + { + "bits_per_byte": 1.7447516102523315, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2093696594238281, + "step": 3090, + "unclipped_grad_norm": 0.8343800902366638 + }, + { + "bits_per_byte": 1.8342075261853672, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2713757753372192, + "step": 3100, + "unclipped_grad_norm": 0.847975492477417 + }, + { + "bits_per_byte": 1.8219030274293875, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2628469467163086, + "step": 3110, + "unclipped_grad_norm": 0.872373104095459 + }, + { + "bits_per_byte": 1.7073493392906405, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1834443807601929, + "step": 3120, + "unclipped_grad_norm": 0.8234026432037354 + }, + { + "bits_per_byte": 1.8002469720327425, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2478361129760742, + "step": 3130, + "unclipped_grad_norm": 0.8703857660293579 + }, + { + "bits_per_byte": 1.7476684360109647, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2113914489746094, + "step": 3140, + "unclipped_grad_norm": 0.7984506487846375 + }, + { + "bits_per_byte": 1.731277285505183, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.200029969215393, + "step": 3150, + "unclipped_grad_norm": 0.7827578783035278 + }, + { + "bits_per_byte": 1.7956350852672056, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2446393966674805, + "step": 3160, + "unclipped_grad_norm": 0.8662806153297424 + }, + { + "bits_per_byte": 1.8529331691939606, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2843554019927979, + "step": 3170, + "unclipped_grad_norm": 0.8483284711837769 + }, + { + "bits_per_byte": 1.7207951149177458, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1927642822265625, + "step": 3180, + "unclipped_grad_norm": 0.8697596788406372 + }, + { + "bits_per_byte": 1.8014959100433083, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.248701810836792, + "step": 3190, + "unclipped_grad_norm": 0.9436899423599243 + }, + { + "bits_per_byte": 1.7115792525886113, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1863763332366943, + "step": 3200, + "unclipped_grad_norm": 0.7923902273178101 + }, + { + "bits_per_byte": 1.7455042063325072, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2098913192749023, + "step": 3210, + "unclipped_grad_norm": 0.8492945432662964 + }, + { + "bits_per_byte": 1.7423042971305538, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2076733112335205, + "step": 3220, + "unclipped_grad_norm": 0.8532925844192505 + }, + { + "bits_per_byte": 1.7197866086530862, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1920652389526367, + "step": 3230, + "unclipped_grad_norm": 0.8590078949928284 + }, + { + "bits_per_byte": 1.772480717016244, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2285900115966797, + "step": 3240, + "unclipped_grad_norm": 0.8679581880569458 + }, + { + "bits_per_byte": 1.7710855937524448, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2276229858398438, + "step": 3250, + "unclipped_grad_norm": 0.8697354793548584 + }, + { + "bits_per_byte": 1.8824907955018464, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.3048431873321533, + "step": 3260, + "unclipped_grad_norm": 0.882398247718811 + }, + { + "bits_per_byte": 1.726842196904733, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1969558000564575, + "step": 3270, + "unclipped_grad_norm": 0.8395750522613525 + }, + { + "bits_per_byte": 1.6700450984399413, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1575870513916016, + "step": 3280, + "unclipped_grad_norm": 0.7638692259788513 + }, + { + "bits_per_byte": 1.7148034113443364, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1886111497879028, + "step": 3290, + "unclipped_grad_norm": 0.8540530800819397 + }, + { + "bits_per_byte": 1.79605059535169, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2449274063110352, + "step": 3300, + "unclipped_grad_norm": 0.8206289410591125 + }, + { + "bits_per_byte": 1.7794766272880411, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2334392070770264, + "step": 3310, + "unclipped_grad_norm": 0.8657307028770447 + }, + { + "bits_per_byte": 1.796555880379925, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2452776432037354, + "step": 3320, + "unclipped_grad_norm": 0.8429055213928223 + }, + { + "bits_per_byte": 1.7430792509553417, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2082104682922363, + "step": 3330, + "unclipped_grad_norm": 0.8332963585853577 + }, + { + "bits_per_byte": 1.747674627386396, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2113957405090332, + "step": 3340, + "unclipped_grad_norm": 0.8822024464607239 + }, + { + "bits_per_byte": 1.7362042444871009, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2034450769424438, + "step": 3350, + "unclipped_grad_norm": 0.7802554368972778 + }, + { + "bits_per_byte": 1.7416999500954218, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.207254409790039, + "step": 3360, + "unclipped_grad_norm": 0.8200514316558838 + }, + { + "bits_per_byte": 1.676567712456564, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1621081829071045, + "step": 3370, + "unclipped_grad_norm": 0.8079729676246643 + }, + { + "bits_per_byte": 1.7870502272840867, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2386888265609741, + "step": 3380, + "unclipped_grad_norm": 0.9383370876312256 + }, + { + "bits_per_byte": 1.7544094679941835, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2160639762878418, + "step": 3390, + "unclipped_grad_norm": 0.848736047744751 + }, + { + "bits_per_byte": 1.758961848762522, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.219219446182251, + "step": 3400, + "unclipped_grad_norm": 0.8597305417060852 + }, + { + "bits_per_byte": 1.7916406162182659, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.241870641708374, + "step": 3410, + "unclipped_grad_norm": 0.9396098256111145 + }, + { + "bits_per_byte": 1.7669203459311984, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2247358560562134, + "step": 3420, + "unclipped_grad_norm": 0.8953562378883362 + }, + { + "bits_per_byte": 1.7096045477914046, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1850075721740723, + "step": 3430, + "unclipped_grad_norm": 0.8465718626976013 + }, + { + "bits_per_byte": 1.713755865017931, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.187885046005249, + "step": 3440, + "unclipped_grad_norm": 0.8348708152770996 + }, + { + "bits_per_byte": 1.7182920794170884, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1910293102264404, + "step": 3450, + "unclipped_grad_norm": 0.9096519351005554 + }, + { + "bits_per_byte": 1.707162050183851, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.183314561843872, + "step": 3460, + "unclipped_grad_norm": 0.8655852675437927 + }, + { + "bits_per_byte": 1.7530255236026904, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2151046991348267, + "step": 3470, + "unclipped_grad_norm": 0.8270760774612427 + }, + { + "bits_per_byte": 1.8250635526042933, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2650376558303833, + "step": 3480, + "unclipped_grad_norm": 0.9349867701530457 + }, + { + "bits_per_byte": 1.751566594775421, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2140934467315674, + "step": 3490, + "unclipped_grad_norm": 0.7542157769203186 + }, + { + "bits_per_byte": 1.6978185767277763, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1768381595611572, + "step": 3500, + "unclipped_grad_norm": 0.8014698028564453 + }, + { + "bits_per_byte": 1.7200215369541647, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.192228078842163, + "step": 3510, + "unclipped_grad_norm": 0.8212472796440125 + }, + { + "bits_per_byte": 1.7550949908405227, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2165391445159912, + "step": 3520, + "unclipped_grad_norm": 0.8387999534606934 + }, + { + "bits_per_byte": 1.7221352037332687, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1936931610107422, + "step": 3530, + "unclipped_grad_norm": 0.8480421900749207 + }, + { + "bits_per_byte": 1.728760147427155, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1982852220535278, + "step": 3540, + "unclipped_grad_norm": 0.8651905059814453 + }, + { + "bits_per_byte": 1.782876724295599, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2357959747314453, + "step": 3550, + "unclipped_grad_norm": 0.9422439932823181 + }, + { + "bits_per_byte": 1.717021471592514, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1901485919952393, + "step": 3560, + "unclipped_grad_norm": 0.8451233506202698 + }, + { + "bits_per_byte": 1.7490260670568758, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2123324871063232, + "step": 3570, + "unclipped_grad_norm": 0.869125247001648 + }, + { + "bits_per_byte": 1.759480720420175, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2195791006088257, + "step": 3580, + "unclipped_grad_norm": 0.8907663226127625 + }, + { + "bits_per_byte": 1.7336618249593958, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2016828060150146, + "step": 3590, + "unclipped_grad_norm": 0.8389977216720581 + }, + { + "bits_per_byte": 1.7850502410372031, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.237302541732788, + "step": 3600, + "unclipped_grad_norm": 0.8462889790534973 + }, + { + "bits_per_byte": 1.760744448938715, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.2204550504684448, + "step": 3610, + "unclipped_grad_norm": 0.8433146476745605 + }, + { + "bits_per_byte": 1.7017459725428794, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1795604228973389, + "step": 3620, + "unclipped_grad_norm": 0.7866216897964478 + }, + { + "bits_per_byte": 1.7213356563894076, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1931389570236206, + "step": 3630, + "unclipped_grad_norm": 0.881050169467926 + }, + { + "bits_per_byte": 1.6603886165592963, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1508936882019043, + "step": 3640, + "unclipped_grad_norm": 0.8100939989089966 + }, + { + "bits_per_byte": 1.7286516263744605, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1982100009918213, + "step": 3650, + "unclipped_grad_norm": 0.8479801416397095 + }, + { + "bits_per_byte": 1.7281064413712255, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1978321075439453, + "step": 3660, + "unclipped_grad_norm": 0.8173186182975769 + }, + { + "bits_per_byte": 1.7549983365907376, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2164721488952637, + "step": 3670, + "unclipped_grad_norm": 0.8759186267852783 + }, + { + "bits_per_byte": 1.6966509865110686, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.176028847694397, + "step": 3680, + "unclipped_grad_norm": 0.8159555792808533 + }, + { + "bits_per_byte": 1.7680449404851903, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.225515365600586, + "step": 3690, + "unclipped_grad_norm": 0.8133695721626282 + }, + { + "bits_per_byte": 1.6566674279425777, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.148314356803894, + "step": 3700, + "unclipped_grad_norm": 0.8174366354942322 + }, + { + "bits_per_byte": 1.7250475579429796, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1957118511199951, + "step": 3710, + "unclipped_grad_norm": 0.8113536238670349 + }, + { + "bits_per_byte": 1.7713626578029849, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.22781503200531, + "step": 3720, + "unclipped_grad_norm": 0.8423733115196228 + }, + { + "bits_per_byte": 1.7265372716647531, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.196744441986084, + "step": 3730, + "unclipped_grad_norm": 0.8644699454307556 + }, + { + "bits_per_byte": 1.7046167069510814, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1815502643585205, + "step": 3740, + "unclipped_grad_norm": 0.8081701993942261 + }, + { + "bits_per_byte": 1.72418454900095, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1951136589050293, + "step": 3750, + "unclipped_grad_norm": 0.7984616756439209 + }, + { + "bits_per_byte": 1.7538620472164872, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2156845331192017, + "step": 3760, + "unclipped_grad_norm": 0.8375996351242065 + }, + { + "bits_per_byte": 1.7548705534811466, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2163835763931274, + "step": 3770, + "unclipped_grad_norm": 0.817222535610199 + }, + { + "bits_per_byte": 1.8002908276087126, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2478665113449097, + "step": 3780, + "unclipped_grad_norm": 0.8592872023582458 + }, + { + "bits_per_byte": 1.6997967211780003, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1782093048095703, + "step": 3790, + "unclipped_grad_norm": 0.8284241557121277 + }, + { + "bits_per_byte": 1.7288383995332977, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1983394622802734, + "step": 3800, + "unclipped_grad_norm": 0.8618725538253784 + }, + { + "bits_per_byte": 1.6712219757197955, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1584028005599976, + "step": 3810, + "unclipped_grad_norm": 0.9065259099006653 + }, + { + "bits_per_byte": 1.6792523616365327, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1639690399169922, + "step": 3820, + "unclipped_grad_norm": 0.8587555885314941 + }, + { + "bits_per_byte": 1.7510613097471863, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.2137432098388672, + "step": 3830, + "unclipped_grad_norm": 0.8374478220939636 + }, + { + "bits_per_byte": 1.6819891215597125, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1658660173416138, + "step": 3840, + "unclipped_grad_norm": 0.8430510759353638 + }, + { + "bits_per_byte": 1.6493425148596799, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1432371139526367, + "step": 3850, + "unclipped_grad_norm": 0.7972609400749207 + }, + { + "bits_per_byte": 1.629205754200757, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.129279375076294, + "step": 3860, + "unclipped_grad_norm": 0.801466703414917 + }, + { + "bits_per_byte": 1.6121130864621058, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.117431640625, + "step": 3870, + "unclipped_grad_norm": 0.7957305312156677 + }, + { + "bits_per_byte": 1.7045224604584086, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1814849376678467, + "step": 3880, + "unclipped_grad_norm": 0.8071211576461792 + }, + { + "bits_per_byte": 1.6850354502544453, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1679775714874268, + "step": 3890, + "unclipped_grad_norm": 0.8125332593917847 + }, + { + "bits_per_byte": 1.6717119542921035, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1587424278259277, + "step": 3900, + "unclipped_grad_norm": 0.8193724751472473 + }, + { + "bits_per_byte": 1.7034513525088348, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1807425022125244, + "step": 3910, + "unclipped_grad_norm": 0.8426622748374939 + }, + { + "bits_per_byte": 1.7239871129177595, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.194976806640625, + "step": 3920, + "unclipped_grad_norm": 0.8514105081558228 + }, + { + "bits_per_byte": 1.7258452134776812, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1962647438049316, + "step": 3930, + "unclipped_grad_norm": 0.8463394045829773 + }, + { + "bits_per_byte": 1.6514083704618439, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1446690559387207, + "step": 3940, + "unclipped_grad_norm": 0.7947182059288025 + }, + { + "bits_per_byte": 1.7619474675815003, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2212889194488525, + "step": 3950, + "unclipped_grad_norm": 0.8610087633132935 + }, + { + "bits_per_byte": 1.7314143556779207, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.200124979019165, + "step": 3960, + "unclipped_grad_norm": 0.8606525659561157 + }, + { + "bits_per_byte": 1.6963023776777697, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1757872104644775, + "step": 3970, + "unclipped_grad_norm": 0.8503133654594421 + }, + { + "bits_per_byte": 1.6634555831721327, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1530195474624634, + "step": 3980, + "unclipped_grad_norm": 0.8032745718955994 + }, + { + "bits_per_byte": 1.7136887917840944, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1878385543823242, + "step": 3990, + "unclipped_grad_norm": 0.8245929479598999 + }, + { + "bits_per_byte": 1.6397087346889487, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1365594863891602, + "step": 4000, + "unclipped_grad_norm": 0.8357985615730286 + }, + { + "bits_per_byte": 1.7015423450842577, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1794192790985107, + "step": 4010, + "unclipped_grad_norm": 0.8669785261154175 + }, + { + "bits_per_byte": 1.7224059044257267, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1938807964324951, + "step": 4020, + "unclipped_grad_norm": 0.8723159432411194 + }, + { + "bits_per_byte": 1.6827126525718923, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.166367530822754, + "step": 4030, + "unclipped_grad_norm": 0.8873636722564697 + }, + { + "bits_per_byte": 1.663583022316422, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1531078815460205, + "step": 4040, + "unclipped_grad_norm": 0.867052435874939 + }, + { + "bits_per_byte": 1.6966081628310037, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1759991645812988, + "step": 4050, + "unclipped_grad_norm": 0.8396835923194885 + }, + { + "bits_per_byte": 1.6707194424139744, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1580544710159302, + "step": 4060, + "unclipped_grad_norm": 0.7937865853309631 + }, + { + "bits_per_byte": 1.8029612022286594, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2497174739837646, + "step": 4070, + "unclipped_grad_norm": 0.8503637909889221 + }, + { + "bits_per_byte": 1.6233989319770252, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1252543926239014, + "step": 4080, + "unclipped_grad_norm": 0.8034883141517639 + }, + { + "bits_per_byte": 1.6627509702515477, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1525311470031738, + "step": 4090, + "unclipped_grad_norm": 0.8565743565559387 + }, + { + "bits_per_byte": 1.7695604516045933, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.2265658378601074, + "step": 4100, + "unclipped_grad_norm": 0.9050058722496033 + }, + { + "bits_per_byte": 1.6734001359929729, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1599125862121582, + "step": 4110, + "unclipped_grad_norm": 0.8419733643531799 + }, + { + "bits_per_byte": 1.686295739119968, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1688511371612549, + "step": 4120, + "unclipped_grad_norm": 0.8891176581382751 + }, + { + "bits_per_byte": 1.7589159293947416, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.219187617301941, + "step": 4130, + "unclipped_grad_norm": 0.8800218105316162 + }, + { + "bits_per_byte": 1.728608458729094, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.198180079460144, + "step": 4140, + "unclipped_grad_norm": 0.9665619730949402 + }, + { + "bits_per_byte": 1.6689839654841176, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1568515300750732, + "step": 4150, + "unclipped_grad_norm": 0.8343537449836731 + }, + { + "bits_per_byte": 1.6883550593813994, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.170278549194336, + "step": 4160, + "unclipped_grad_norm": 0.8785508275032043 + }, + { + "bits_per_byte": 1.7045477419080854, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1815024614334106, + "step": 4170, + "unclipped_grad_norm": 0.8620172739028931 + }, + { + "bits_per_byte": 1.6884666761218092, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1703559160232544, + "step": 4180, + "unclipped_grad_norm": 0.917255699634552 + }, + { + "bits_per_byte": 1.707792538581914, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1837515830993652, + "step": 4190, + "unclipped_grad_norm": 0.8766590356826782 + }, + { + "bits_per_byte": 1.696188009214946, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1757079362869263, + "step": 4200, + "unclipped_grad_norm": 0.8971944451332092 + }, + { + "bits_per_byte": 1.720210545887463, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1923590898513794, + "step": 4210, + "unclipped_grad_norm": 0.9026775360107422 + }, + { + "bits_per_byte": 1.6943541582087958, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1744368076324463, + "step": 4220, + "unclipped_grad_norm": 0.8253030180931091 + }, + { + "bits_per_byte": 1.693875014543492, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1741046905517578, + "step": 4230, + "unclipped_grad_norm": 0.9072017669677734 + }, + { + "bits_per_byte": 1.7273019065304893, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1972744464874268, + "step": 4240, + "unclipped_grad_norm": 0.8421373963356018 + }, + { + "bits_per_byte": 1.655899869371777, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.147782325744629, + "step": 4250, + "unclipped_grad_norm": 0.8799253106117249 + }, + { + "bits_per_byte": 1.6871874691646918, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1694692373275757, + "step": 4260, + "unclipped_grad_norm": 0.8348552584648132 + }, + { + "bits_per_byte": 1.6070542167869764, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1139250993728638, + "step": 4270, + "unclipped_grad_norm": 0.9256986975669861 + }, + { + "bits_per_byte": 1.6846880452997026, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1677367687225342, + "step": 4280, + "unclipped_grad_norm": 0.8782404661178589 + }, + { + "bits_per_byte": 1.655903652990096, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.147784948348999, + "step": 4290, + "unclipped_grad_norm": 0.8491708636283875 + }, + { + "bits_per_byte": 1.6260650070307003, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1271023750305176, + "step": 4300, + "unclipped_grad_norm": 0.8707545399665833 + }, + { + "bits_per_byte": 1.6222229146104252, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1244392395019531, + "step": 4310, + "unclipped_grad_norm": 0.8290966153144836 + }, + { + "bits_per_byte": 1.7449810351085826, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.2095286846160889, + "step": 4320, + "unclipped_grad_norm": 0.8566304445266724 + }, + { + "bits_per_byte": 1.683775849319526, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1671044826507568, + "step": 4330, + "unclipped_grad_norm": 0.8478822708129883 + }, + { + "bits_per_byte": 1.6416012317790432, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.137871265411377, + "step": 4340, + "unclipped_grad_norm": 0.8954984545707703 + }, + { + "bits_per_byte": 1.6099557360896828, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.115936279296875, + "step": 4350, + "unclipped_grad_norm": 0.8437728881835938 + }, + { + "bits_per_byte": 1.6930343633460747, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1735219955444336, + "step": 4360, + "unclipped_grad_norm": 0.9155061841011047 + }, + { + "bits_per_byte": 1.666669594951457, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1552473306655884, + "step": 4370, + "unclipped_grad_norm": 0.8761768341064453 + }, + { + "bits_per_byte": 1.5925817047342201, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.103893518447876, + "step": 4380, + "unclipped_grad_norm": 0.8261715769767761 + }, + { + "bits_per_byte": 1.7381402531878634, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.204787015914917, + "step": 4390, + "unclipped_grad_norm": 0.8470990061759949 + }, + { + "bits_per_byte": 1.684359214471253, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.167508840560913, + "step": 4400, + "unclipped_grad_norm": 0.8836275339126587 + }, + { + "bits_per_byte": 1.6966413554826203, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1760221719741821, + "step": 4410, + "unclipped_grad_norm": 0.868717610836029 + }, + { + "bits_per_byte": 1.7161705014360449, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.189558744430542, + "step": 4420, + "unclipped_grad_norm": 0.9417465329170227 + }, + { + "bits_per_byte": 1.6679764911153634, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1561532020568848, + "step": 4430, + "unclipped_grad_norm": 0.9034039974212646 + }, + { + "bits_per_byte": 1.6769285320580742, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.162358283996582, + "step": 4440, + "unclipped_grad_norm": 0.8574883341789246 + }, + { + "bits_per_byte": 1.6926945256279697, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1732864379882812, + "step": 4450, + "unclipped_grad_norm": 0.8669698238372803 + }, + { + "bits_per_byte": 1.689570116809745, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1711207628250122, + "step": 4460, + "unclipped_grad_norm": 0.9505860805511475 + }, + { + "bits_per_byte": 1.6610163532349456, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1513288021087646, + "step": 4470, + "unclipped_grad_norm": 0.8719004988670349 + }, + { + "bits_per_byte": 1.7079321884944145, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1838483810424805, + "step": 4480, + "unclipped_grad_norm": 0.862112283706665 + }, + { + "bits_per_byte": 1.747874987174651, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.2115346193313599, + "step": 4490, + "unclipped_grad_norm": 0.8834897875785828 + }, + { + "bits_per_byte": 1.6443725602150463, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1397922039031982, + "step": 4500, + "unclipped_grad_norm": 0.8092275261878967 + }, + { + "bits_per_byte": 1.6392098130187958, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1362136602401733, + "step": 4510, + "unclipped_grad_norm": 0.8292563557624817 + }, + { + "bits_per_byte": 1.7170035853968242, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.190136194229126, + "step": 4520, + "unclipped_grad_norm": 0.929835319519043 + }, + { + "bits_per_byte": 1.6814485800880505, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1654913425445557, + "step": 4530, + "unclipped_grad_norm": 0.9049495458602905 + }, + { + "bits_per_byte": 1.6774269377802746, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1627037525177002, + "step": 4540, + "unclipped_grad_norm": 0.9091315865516663 + }, + { + "bits_per_byte": 1.6939209339112724, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1741365194320679, + "step": 4550, + "unclipped_grad_norm": 0.8590887188911438 + }, + { + "bits_per_byte": 1.692378077550382, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1730670928955078, + "step": 4560, + "unclipped_grad_norm": 0.8829112648963928 + }, + { + "bits_per_byte": 1.680521765582551, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1648489236831665, + "step": 4570, + "unclipped_grad_norm": 0.8655811548233032 + }, + { + "bits_per_byte": 1.6629215770412038, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1526494026184082, + "step": 4580, + "unclipped_grad_norm": 0.8624271154403687 + }, + { + "bits_per_byte": 1.6390448816566183, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1360993385314941, + "step": 4590, + "unclipped_grad_norm": 0.8392674326896667 + }, + { + "bits_per_byte": 1.6424092062727966, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1384313106536865, + "step": 4600, + "unclipped_grad_norm": 0.863978385925293 + }, + { + "bits_per_byte": 1.6535189415532323, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.146131992340088, + "step": 4610, + "unclipped_grad_norm": 0.847902238368988 + }, + { + "bits_per_byte": 1.7059622992114318, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1824829578399658, + "step": 4620, + "unclipped_grad_norm": 0.884921133518219 + }, + { + "bits_per_byte": 1.687283435483873, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.169535756111145, + "step": 4630, + "unclipped_grad_norm": 0.8606892824172974 + }, + { + "bits_per_byte": 1.71471243252203, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1885480880737305, + "step": 4640, + "unclipped_grad_norm": 0.916991114616394 + }, + { + "bits_per_byte": 1.7302961244820108, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1993498802185059, + "step": 4650, + "unclipped_grad_norm": 0.8595409989356995 + }, + { + "bits_per_byte": 1.6534050890383611, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1460530757904053, + "step": 4660, + "unclipped_grad_norm": 0.8465806245803833 + }, + { + "bits_per_byte": 1.6640274254862515, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1534159183502197, + "step": 4670, + "unclipped_grad_norm": 0.8708443641662598 + }, + { + "bits_per_byte": 1.6364145789943225, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1342761516571045, + "step": 4680, + "unclipped_grad_norm": 0.8731908202171326 + }, + { + "bits_per_byte": 1.653688516446983, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.146249532699585, + "step": 4690, + "unclipped_grad_norm": 0.8683258891105652 + }, + { + "bits_per_byte": 1.6763891944649685, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1619844436645508, + "step": 4700, + "unclipped_grad_norm": 0.907905101776123 + }, + { + "bits_per_byte": 1.661433239180637, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1516177654266357, + "step": 4710, + "unclipped_grad_norm": 0.862278401851654 + }, + { + "bits_per_byte": 1.68449817645315, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1676051616668701, + "step": 4720, + "unclipped_grad_norm": 0.8695031404495239 + }, + { + "bits_per_byte": 1.7284700126951493, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1980841159820557, + "step": 4730, + "unclipped_grad_norm": 0.8831793069839478 + }, + { + "bits_per_byte": 1.66034751270574, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1508651971817017, + "step": 4740, + "unclipped_grad_norm": 0.9113041758537292 + }, + { + "bits_per_byte": 1.6300096011108898, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1298365592956543, + "step": 4750, + "unclipped_grad_norm": 0.9407509565353394 + }, + { + "bits_per_byte": 1.6017535395047338, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1102509498596191, + "step": 4760, + "unclipped_grad_norm": 0.8554308414459229 + }, + { + "bits_per_byte": 1.6115298932930302, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1170274019241333, + "step": 4770, + "unclipped_grad_norm": 0.846508264541626 + }, + { + "bits_per_byte": 1.6498288817963198, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1435742378234863, + "step": 4780, + "unclipped_grad_norm": 0.8984644412994385 + }, + { + "bits_per_byte": 1.6369418778018678, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1346416473388672, + "step": 4790, + "unclipped_grad_norm": 0.9358733296394348 + }, + { + "bits_per_byte": 1.6726428963812239, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1593877077102661, + "step": 4800, + "unclipped_grad_norm": 0.9628081321716309 + }, + { + "bits_per_byte": 1.6377882044267638, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1352282762527466, + "step": 4810, + "unclipped_grad_norm": 0.874860942363739 + }, + { + "bits_per_byte": 1.6475561310651679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1419988870620728, + "step": 4820, + "unclipped_grad_norm": 0.8858349919319153 + }, + { + "bits_per_byte": 1.624869727607204, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1262738704681396, + "step": 4830, + "unclipped_grad_norm": 0.8662204742431641 + }, + { + "bits_per_byte": 1.5894763859902412, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.101741075515747, + "step": 4840, + "unclipped_grad_norm": 0.836323082447052 + }, + { + "bits_per_byte": 1.6522122173719767, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.145226240158081, + "step": 4850, + "unclipped_grad_norm": 0.9080801010131836 + }, + { + "bits_per_byte": 1.6432429781641795, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1390092372894287, + "step": 4860, + "unclipped_grad_norm": 0.8762933611869812 + }, + { + "bits_per_byte": 1.6506518187806984, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1441446542739868, + "step": 4870, + "unclipped_grad_norm": 0.8573883771896362 + }, + { + "bits_per_byte": 1.6124140561011158, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1176402568817139, + "step": 4880, + "unclipped_grad_norm": 0.8266409635543823 + }, + { + "bits_per_byte": 1.606751183356156, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1137150526046753, + "step": 4890, + "unclipped_grad_norm": 0.9300702810287476 + }, + { + "bits_per_byte": 1.5799279092143381, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0951225757598877, + "step": 4900, + "unclipped_grad_norm": 0.8544939160346985 + }, + { + "bits_per_byte": 1.6316014725272758, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1309399604797363, + "step": 4910, + "unclipped_grad_norm": 0.8843865990638733 + }, + { + "bits_per_byte": 1.6408737451658935, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1373670101165771, + "step": 4920, + "unclipped_grad_norm": 0.8914017677307129 + }, + { + "bits_per_byte": 1.6335975031731897, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1323235034942627, + "step": 4930, + "unclipped_grad_norm": 0.9028998017311096 + }, + { + "bits_per_byte": 1.697724846183056, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.176773190498352, + "step": 4940, + "unclipped_grad_norm": 0.9006686806678772 + }, + { + "bits_per_byte": 1.5654046622795774, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0850558280944824, + "step": 4950, + "unclipped_grad_norm": 0.8810124397277832 + }, + { + "bits_per_byte": 1.6525025240866331, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1454274654388428, + "step": 4960, + "unclipped_grad_norm": 0.9115336537361145 + }, + { + "bits_per_byte": 1.721798977650832, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1934601068496704, + "step": 4970, + "unclipped_grad_norm": 0.9325219988822937 + }, + { + "bits_per_byte": 1.6130810048211617, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1181025505065918, + "step": 4980, + "unclipped_grad_norm": 0.9085396528244019 + }, + { + "bits_per_byte": 1.6527175024002116, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1455764770507812, + "step": 4990, + "unclipped_grad_norm": 0.903756856918335 + }, + { + "bits_per_byte": 1.663184366531722, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1528315544128418, + "step": 5000, + "unclipped_grad_norm": 0.9185003638267517 + }, + { + "bits_per_byte": 1.6328528182949538, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1318073272705078, + "step": 5010, + "unclipped_grad_norm": 0.9199827313423157 + }, + { + "bits_per_byte": 1.6437107709745262, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1393334865570068, + "step": 5020, + "unclipped_grad_norm": 0.8566684722900391 + }, + { + "bits_per_byte": 1.6317774107791085, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1310619115829468, + "step": 5030, + "unclipped_grad_norm": 0.9012371897697449 + }, + { + "bits_per_byte": 1.6211884389654854, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1237221956253052, + "step": 5040, + "unclipped_grad_norm": 0.9131078124046326 + }, + { + "bits_per_byte": 1.6615727171104868, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1517144441604614, + "step": 5050, + "unclipped_grad_norm": 0.9046566486358643 + }, + { + "bits_per_byte": 1.602131729353981, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1105130910873413, + "step": 5060, + "unclipped_grad_norm": 0.895842969417572 + }, + { + "bits_per_byte": 1.6828842912574533, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1664865016937256, + "step": 5070, + "unclipped_grad_norm": 0.9286125898361206 + }, + { + "bits_per_byte": 1.6860735375350533, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1686971187591553, + "step": 5080, + "unclipped_grad_norm": 0.9972010254859924 + }, + { + "bits_per_byte": 1.6658614484750525, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1546871662139893, + "step": 5090, + "unclipped_grad_norm": 0.9360038042068481 + }, + { + "bits_per_byte": 1.5959157604038465, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1062045097351074, + "step": 5100, + "unclipped_grad_norm": 0.8976820707321167 + }, + { + "bits_per_byte": 1.6220873922815453, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.124345302581787, + "step": 5110, + "unclipped_grad_norm": 0.8928531408309937 + }, + { + "bits_per_byte": 1.5977783325126909, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1074955463409424, + "step": 5120, + "unclipped_grad_norm": 0.8989635705947876 + }, + { + "bits_per_byte": 1.6638141669991817, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1532680988311768, + "step": 5130, + "unclipped_grad_norm": 0.9084221720695496 + }, + { + "bits_per_byte": 1.5980395741593514, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.10767662525177, + "step": 5140, + "unclipped_grad_norm": 0.896472156047821 + }, + { + "bits_per_byte": 1.590172399778283, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.102223515510559, + "step": 5150, + "unclipped_grad_norm": 0.8930872678756714 + }, + { + "bits_per_byte": 1.6326397317905348, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1316596269607544, + "step": 5160, + "unclipped_grad_norm": 0.9028102159500122 + }, + { + "bits_per_byte": 1.6614396025387188, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1516221761703491, + "step": 5170, + "unclipped_grad_norm": 0.9357115030288696 + }, + { + "bits_per_byte": 1.6782567540706879, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1632789373397827, + "step": 5180, + "unclipped_grad_norm": 0.8988713622093201 + }, + { + "bits_per_byte": 1.6215858908716294, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.123997688293457, + "step": 5190, + "unclipped_grad_norm": 0.8884453177452087 + }, + { + "bits_per_byte": 1.5925564232845433, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.103875994682312, + "step": 5200, + "unclipped_grad_norm": 0.9432880282402039 + }, + { + "bits_per_byte": 1.6471872282790672, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1417431831359863, + "step": 5210, + "unclipped_grad_norm": 0.932002067565918 + }, + { + "bits_per_byte": 1.634469455213064, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1329278945922852, + "step": 5220, + "unclipped_grad_norm": 0.8859370946884155 + }, + { + "bits_per_byte": 1.684877914146255, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1678683757781982, + "step": 5230, + "unclipped_grad_norm": 0.9864789843559265 + }, + { + "bits_per_byte": 1.612974375577627, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1180286407470703, + "step": 5240, + "unclipped_grad_norm": 0.9477531313896179 + }, + { + "bits_per_byte": 1.5751990742462143, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0918447971343994, + "step": 5250, + "unclipped_grad_norm": 0.8995106220245361 + }, + { + "bits_per_byte": 1.5678148271487689, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.086726427078247, + "step": 5260, + "unclipped_grad_norm": 0.9177948236465454 + }, + { + "bits_per_byte": 1.6615842399480945, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1517224311828613, + "step": 5270, + "unclipped_grad_norm": 0.9501162171363831 + }, + { + "bits_per_byte": 1.6530886409607735, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1458337306976318, + "step": 5280, + "unclipped_grad_norm": 0.9527469277381897 + }, + { + "bits_per_byte": 1.6249691335794025, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1263427734375, + "step": 5290, + "unclipped_grad_norm": 0.9307261109352112 + }, + { + "bits_per_byte": 1.5681051338634253, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0869276523590088, + "step": 5300, + "unclipped_grad_norm": 0.9039522409439087 + }, + { + "bits_per_byte": 1.6362377808292354, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1341536045074463, + "step": 5310, + "unclipped_grad_norm": 0.9477429986000061 + }, + { + "bits_per_byte": 1.60507762018061, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1125550270080566, + "step": 5320, + "unclipped_grad_norm": 0.926771342754364 + }, + { + "bits_per_byte": 1.6965259551238914, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1759421825408936, + "step": 5330, + "unclipped_grad_norm": 0.9526844620704651 + }, + { + "bits_per_byte": 1.5996515675458884, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1087939739227295, + "step": 5340, + "unclipped_grad_norm": 0.9416574835777283 + }, + { + "bits_per_byte": 1.628412054267025, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.128729224205017, + "step": 5350, + "unclipped_grad_norm": 0.9132721424102783 + }, + { + "bits_per_byte": 1.5519294776191745, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0757155418395996, + "step": 5360, + "unclipped_grad_norm": 0.8944959044456482 + }, + { + "bits_per_byte": 1.5965125402023406, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1066181659698486, + "step": 5370, + "unclipped_grad_norm": 0.9166159629821777 + }, + { + "bits_per_byte": 1.6489008634122642, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1429309844970703, + "step": 5380, + "unclipped_grad_norm": 0.9673669934272766 + }, + { + "bits_per_byte": 1.5567627060563722, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0790656805038452, + "step": 5390, + "unclipped_grad_norm": 0.8981189131736755 + }, + { + "bits_per_byte": 1.6274061277421286, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1280319690704346, + "step": 5400, + "unclipped_grad_norm": 0.95365971326828 + }, + { + "bits_per_byte": 1.5903326876088872, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1023346185684204, + "step": 5410, + "unclipped_grad_norm": 0.9384826421737671 + }, + { + "bits_per_byte": 1.5563373929607884, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0787708759307861, + "step": 5420, + "unclipped_grad_norm": 0.8839442133903503 + }, + { + "bits_per_byte": 1.6100503265476573, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.116001844406128, + "step": 5430, + "unclipped_grad_norm": 0.9247893691062927 + }, + { + "bits_per_byte": 1.5906383007794704, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1025464534759521, + "step": 5440, + "unclipped_grad_norm": 0.9033803343772888 + }, + { + "bits_per_byte": 1.6134788006926075, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1183782815933228, + "step": 5450, + "unclipped_grad_norm": 0.9274419546127319 + }, + { + "bits_per_byte": 1.588039986890235, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.100745439529419, + "step": 5460, + "unclipped_grad_norm": 0.93131422996521 + }, + { + "bits_per_byte": 1.6180344491313123, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1215360164642334, + "step": 5470, + "unclipped_grad_norm": 0.9031676054000854 + }, + { + "bits_per_byte": 1.5764667583657241, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0927234888076782, + "step": 5480, + "unclipped_grad_norm": 0.9524510502815247 + }, + { + "bits_per_byte": 1.667538279320965, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1558494567871094, + "step": 5490, + "unclipped_grad_norm": 1.0016353130340576 + }, + { + "bits_per_byte": 1.6287254066568972, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1289464235305786, + "step": 5500, + "unclipped_grad_norm": 0.9340007305145264 + }, + { + "bits_per_byte": 1.5745544832707805, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.091398000717163, + "step": 5510, + "unclipped_grad_norm": 0.919238805770874 + }, + { + "bits_per_byte": 1.54915986900968, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0737957954406738, + "step": 5520, + "unclipped_grad_norm": 0.9040963649749756 + }, + { + "bits_per_byte": 1.5578374256416139, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.079810619354248, + "step": 5530, + "unclipped_grad_norm": 0.8987631797790527 + }, + { + "bits_per_byte": 1.6786796594091595, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.163572072982788, + "step": 5540, + "unclipped_grad_norm": 0.9474910497665405 + }, + { + "bits_per_byte": 1.6142570221877617, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.11891770362854, + "step": 5550, + "unclipped_grad_norm": 0.9275277256965637 + }, + { + "bits_per_byte": 1.6535560898058186, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1461577415466309, + "step": 5560, + "unclipped_grad_norm": 0.9428616166114807 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.109092354774475, + "step": 5570, + "unclipped_grad_norm": 0.8811440467834473 + }, + { + "bits_per_byte": 1.625784675309794, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1269080638885498, + "step": 5580, + "unclipped_grad_norm": 0.9328234791755676 + }, + { + "bits_per_byte": 1.6288614449337298, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1290407180786133, + "step": 5590, + "unclipped_grad_norm": 0.9723306894302368 + }, + { + "bits_per_byte": 1.6197528997787336, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1227271556854248, + "step": 5600, + "unclipped_grad_norm": 1.0117963552474976 + }, + { + "bits_per_byte": 1.5616874292638288, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0824792385101318, + "step": 5610, + "unclipped_grad_norm": 0.901840329170227 + }, + { + "bits_per_byte": 1.6073563903045422, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1141345500946045, + "step": 5620, + "unclipped_grad_norm": 0.9225190281867981 + }, + { + "bits_per_byte": 1.5324476268947373, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0622117519378662, + "step": 5630, + "unclipped_grad_norm": 0.8962357640266418 + }, + { + "bits_per_byte": 1.5325635432014189, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0622920989990234, + "step": 5640, + "unclipped_grad_norm": 0.8800385594367981 + }, + { + "bits_per_byte": 1.6207170345194706, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1233954429626465, + "step": 5650, + "unclipped_grad_norm": 0.9619011282920837 + }, + { + "bits_per_byte": 1.5448582389462995, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0708141326904297, + "step": 5660, + "unclipped_grad_norm": 0.9848581552505493 + }, + { + "bits_per_byte": 1.6303492668463437, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.130071997642517, + "step": 5670, + "unclipped_grad_norm": 0.942057728767395 + }, + { + "bits_per_byte": 1.6277949805157295, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1283015012741089, + "step": 5680, + "unclipped_grad_norm": 1.0017359256744385 + }, + { + "bits_per_byte": 1.6320572265520623, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.131255865097046, + "step": 5690, + "unclipped_grad_norm": 0.9255995154380798 + }, + { + "bits_per_byte": 1.6423192593463953, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1383689641952515, + "step": 5700, + "unclipped_grad_norm": 0.9918621182441711 + }, + { + "bits_per_byte": 1.6211487109731362, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.123694658279419, + "step": 5710, + "unclipped_grad_norm": 1.0109361410140991 + }, + { + "bits_per_byte": 1.6355581053930257, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1336824893951416, + "step": 5720, + "unclipped_grad_norm": 0.9666855335235596 + }, + { + "bits_per_byte": 1.5611546270114558, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0821099281311035, + "step": 5730, + "unclipped_grad_norm": 0.9373083114624023 + }, + { + "bits_per_byte": 1.6439162902423072, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.13947594165802, + "step": 5740, + "unclipped_grad_norm": 0.923280656337738 + }, + { + "bits_per_byte": 1.5432997321641806, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0697338581085205, + "step": 5750, + "unclipped_grad_norm": 0.9531146287918091 + }, + { + "bits_per_byte": 1.5628747974853856, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0833022594451904, + "step": 5760, + "unclipped_grad_norm": 0.9156620502471924 + }, + { + "bits_per_byte": 1.5744538734200257, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0913282632827759, + "step": 5770, + "unclipped_grad_norm": 0.9679534435272217 + }, + { + "bits_per_byte": 1.6685440338632107, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1565465927124023, + "step": 5780, + "unclipped_grad_norm": 0.9836651682853699 + }, + { + "bits_per_byte": 1.5013796489469744, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0406770706176758, + "step": 5790, + "unclipped_grad_norm": 0.9446811676025391 + }, + { + "bits_per_byte": 1.6245429605705646, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.1260473728179932, + "step": 5800, + "unclipped_grad_norm": 0.9269219040870667 + }, + { + "bits_per_byte": 1.5340637478648953, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.063331961631775, + "step": 5810, + "unclipped_grad_norm": 0.87967449426651 + }, + { + "bits_per_byte": 1.5414010436986552, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0684177875518799, + "step": 5820, + "unclipped_grad_norm": 0.9367535710334778 + }, + { + "bits_per_byte": 1.6061841565562613, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.1133220195770264, + "step": 5830, + "unclipped_grad_norm": 0.9435480833053589 + }, + { + "bits_per_byte": 1.5806048329281341, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0955917835235596, + "step": 5840, + "unclipped_grad_norm": 0.9508634805679321 + }, + { + "bits_per_byte": 1.5486303344276733, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.073428750038147, + "step": 5850, + "unclipped_grad_norm": 0.9543959498405457 + }, + { + "bits_per_byte": 1.563623781929893, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.083821415901184, + "step": 5860, + "unclipped_grad_norm": 0.900787353515625 + }, + { + "bits_per_byte": 1.6201161271373559, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.122978925704956, + "step": 5870, + "unclipped_grad_norm": 0.9706017374992371 + }, + { + "bits_per_byte": 1.5594879431419442, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.080954670906067, + "step": 5880, + "unclipped_grad_norm": 0.9371631741523743 + }, + { + "bits_per_byte": 1.6214689426690427, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1239166259765625, + "step": 5890, + "unclipped_grad_norm": 1.0151838064193726 + }, + { + "bits_per_byte": 1.6434954486956461, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1391842365264893, + "step": 5900, + "unclipped_grad_norm": 1.0114811658859253 + }, + { + "bits_per_byte": 1.5747577667641004, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.091538906097412, + "step": 5910, + "unclipped_grad_norm": 0.9295753836631775 + }, + { + "bits_per_byte": 1.5403519495283922, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0676906108856201, + "step": 5920, + "unclipped_grad_norm": 0.9814701080322266 + }, + { + "bits_per_byte": 1.6536778535226295, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1462421417236328, + "step": 5930, + "unclipped_grad_norm": 0.9983100891113281 + }, + { + "bits_per_byte": 1.5856325737434576, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.099076747894287, + "step": 5940, + "unclipped_grad_norm": 0.964913010597229 + }, + { + "bits_per_byte": 1.5932624120663352, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.104365348815918, + "step": 5950, + "unclipped_grad_norm": 0.9470323920249939 + }, + { + "bits_per_byte": 1.5376077943512252, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0657885074615479, + "step": 5960, + "unclipped_grad_norm": 0.97988361120224 + }, + { + "bits_per_byte": 1.5721553252912444, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0897350311279297, + "step": 5970, + "unclipped_grad_norm": 0.9889833927154541 + }, + { + "bits_per_byte": 1.552403117839651, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0760438442230225, + "step": 5980, + "unclipped_grad_norm": 0.9984597563743591 + }, + { + "bits_per_byte": 1.542794103170644, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0693833827972412, + "step": 5990, + "unclipped_grad_norm": 0.9274381399154663 + }, + { + "bits_per_byte": 1.648261775881658, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1424880027770996, + "step": 6000, + "unclipped_grad_norm": 0.9919959902763367 + }, + { + "bits_per_byte": 1.595717292424751, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1060669422149658, + "step": 6010, + "unclipped_grad_norm": 1.009833574295044 + }, + { + "bits_per_byte": 1.5605256864572503, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0816739797592163, + "step": 6020, + "unclipped_grad_norm": 0.9331597685813904 + }, + { + "bits_per_byte": 1.5469388850564376, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.072256326675415, + "step": 6030, + "unclipped_grad_norm": 0.9432128667831421 + }, + { + "bits_per_byte": 1.5694490062972672, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0878591537475586, + "step": 6040, + "unclipped_grad_norm": 0.9656267166137695 + }, + { + "bits_per_byte": 1.5872409554943265, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.100191593170166, + "step": 6050, + "unclipped_grad_norm": 0.9372153878211975 + }, + { + "bits_per_byte": 1.5733865490887713, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0905884504318237, + "step": 6060, + "unclipped_grad_norm": 0.9818388819694519 + }, + { + "bits_per_byte": 1.4709371719003996, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0195759534835815, + "step": 6070, + "unclipped_grad_norm": 0.9397172331809998 + }, + { + "bits_per_byte": 1.5520641400348003, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0758088827133179, + "step": 6080, + "unclipped_grad_norm": 1.003225564956665 + }, + { + "bits_per_byte": 1.548305115234892, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0732033252716064, + "step": 6090, + "unclipped_grad_norm": 0.9615646600723267 + }, + { + "bits_per_byte": 1.6580690865471095, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.149285912513733, + "step": 6100, + "unclipped_grad_norm": 1.030853271484375 + }, + { + "bits_per_byte": 1.5849845431150065, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0986275672912598, + "step": 6110, + "unclipped_grad_norm": 1.0033804178237915 + }, + { + "bits_per_byte": 1.5391577020008007, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0668628215789795, + "step": 6120, + "unclipped_grad_norm": 0.9861558079719543 + }, + { + "bits_per_byte": 1.6089128332948508, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.115213394165039, + "step": 6130, + "unclipped_grad_norm": 0.9338588714599609 + }, + { + "bits_per_byte": 1.5704974125369269, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0885858535766602, + "step": 6140, + "unclipped_grad_norm": 0.9351722598075867 + }, + { + "bits_per_byte": 1.5301674809095982, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.060631275177002, + "step": 6150, + "unclipped_grad_norm": 0.9382454752922058 + }, + { + "bits_per_byte": 1.594941134721407, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1055289506912231, + "step": 6160, + "unclipped_grad_norm": 0.9520907998085022 + }, + { + "bits_per_byte": 1.53183020917814, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.061783790588379, + "step": 6170, + "unclipped_grad_norm": 0.9805156588554382 + }, + { + "bits_per_byte": 1.5374516341042417, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0656802654266357, + "step": 6180, + "unclipped_grad_norm": 0.9368215799331665 + }, + { + "bits_per_byte": 1.5636041759076946, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0838078260421753, + "step": 6190, + "unclipped_grad_norm": 1.028733491897583 + }, + { + "bits_per_byte": 1.544540243024854, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0705937147140503, + "step": 6200, + "unclipped_grad_norm": 0.9240782856941223 + }, + { + "bits_per_byte": 1.5709191139968426, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0888781547546387, + "step": 6210, + "unclipped_grad_norm": 0.9899703860282898 + }, + { + "bits_per_byte": 1.604423398176728, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.1121015548706055, + "step": 6220, + "unclipped_grad_norm": 0.9920126795768738 + }, + { + "bits_per_byte": 1.531685743751415, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0616836547851562, + "step": 6230, + "unclipped_grad_norm": 0.9534332156181335 + }, + { + "bits_per_byte": 1.5783843649228444, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0940526723861694, + "step": 6240, + "unclipped_grad_norm": 0.9837333559989929 + }, + { + "bits_per_byte": 1.5384501653751512, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0663723945617676, + "step": 6250, + "unclipped_grad_norm": 0.9426394104957581 + }, + { + "bits_per_byte": 1.5118525324712653, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0479363203048706, + "step": 6260, + "unclipped_grad_norm": 0.9544605612754822 + }, + { + "bits_per_byte": 1.5631276119821538, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.08347749710083, + "step": 6270, + "unclipped_grad_norm": 0.9525503516197205 + }, + { + "bits_per_byte": 1.563399516553168, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0836659669876099, + "step": 6280, + "unclipped_grad_norm": 0.9565526247024536 + }, + { + "bits_per_byte": 1.5618869291388295, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0826175212860107, + "step": 6290, + "unclipped_grad_norm": 0.9895926713943481 + }, + { + "bits_per_byte": 1.5244160370994442, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0566446781158447, + "step": 6300, + "unclipped_grad_norm": 0.9698922038078308 + }, + { + "bits_per_byte": 1.5852545558768611, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0988147258758545, + "step": 6310, + "unclipped_grad_norm": 1.004137396812439 + }, + { + "bits_per_byte": 1.55847651317222, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0802536010742188, + "step": 6320, + "unclipped_grad_norm": 0.9893865585327148 + }, + { + "bits_per_byte": 1.5695085122944659, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0879004001617432, + "step": 6330, + "unclipped_grad_norm": 0.9685529470443726 + }, + { + "bits_per_byte": 1.4838095573695282, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0284984111785889, + "step": 6340, + "unclipped_grad_norm": 0.916477382183075 + }, + { + "bits_per_byte": 1.5924496220583575, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.103801965713501, + "step": 6350, + "unclipped_grad_norm": 0.9532552361488342 + }, + { + "bits_per_byte": 1.6071991981616538, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.114025592803955, + "step": 6360, + "unclipped_grad_norm": 0.9765400290489197 + }, + { + "bits_per_byte": 1.53600216432277, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0646755695343018, + "step": 6370, + "unclipped_grad_norm": 1.0022367238998413 + }, + { + "bits_per_byte": 1.5289498437414897, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0597872734069824, + "step": 6380, + "unclipped_grad_norm": 0.9823477864265442 + }, + { + "bits_per_byte": 1.6382666601614642, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1355599164962769, + "step": 6390, + "unclipped_grad_norm": 0.9935681223869324 + }, + { + "bits_per_byte": 1.5996092598137763, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1087646484375, + "step": 6400, + "unclipped_grad_norm": 1.055086374282837 + }, + { + "bits_per_byte": 1.61815156931655, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.1216171979904175, + "step": 6410, + "unclipped_grad_norm": 1.0302053689956665 + }, + { + "bits_per_byte": 1.538316190890129, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0662795305252075, + "step": 6420, + "unclipped_grad_norm": 1.0392645597457886 + }, + { + "bits_per_byte": 1.5869190039719114, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.099968433380127, + "step": 6430, + "unclipped_grad_norm": 0.9931192994117737 + }, + { + "bits_per_byte": 1.5112646957706162, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.047528862953186, + "step": 6440, + "unclipped_grad_norm": 0.9700445532798767 + }, + { + "bits_per_byte": 1.5261810950452492, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0578681230545044, + "step": 6450, + "unclipped_grad_norm": 0.9057909846305847 + }, + { + "bits_per_byte": 1.548995453595455, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0736818313598633, + "step": 6460, + "unclipped_grad_norm": 0.9796255826950073 + }, + { + "bits_per_byte": 1.5656127612871213, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0852000713348389, + "step": 6470, + "unclipped_grad_norm": 1.0005474090576172 + }, + { + "bits_per_byte": 1.5309916217725328, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0612025260925293, + "step": 6480, + "unclipped_grad_norm": 0.9810837507247925 + }, + { + "bits_per_byte": 1.6052196778502228, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1126534938812256, + "step": 6490, + "unclipped_grad_norm": 1.013913869857788 + }, + { + "bits_per_byte": 1.648418280093943, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.1425964832305908, + "step": 6500, + "unclipped_grad_norm": 1.0267438888549805 + }, + { + "bits_per_byte": 1.6007087449007422, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.1095267534255981, + "step": 6510, + "unclipped_grad_norm": 1.015332818031311 + }, + { + "bits_per_byte": 1.5803760960024866, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.095433235168457, + "step": 6520, + "unclipped_grad_norm": 1.0154883861541748 + }, + { + "bits_per_byte": 1.5618320666732044, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.082579493522644, + "step": 6530, + "unclipped_grad_norm": 0.9579733610153198 + }, + { + "bits_per_byte": 1.5301038473287791, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0605871677398682, + "step": 6540, + "unclipped_grad_norm": 0.9741249084472656 + }, + { + "bits_per_byte": 1.5804765338705906, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0955028533935547, + "step": 6550, + "unclipped_grad_norm": 0.9633719325065613 + }, + { + "bits_per_byte": 1.594095152061813, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.1049425601959229, + "step": 6560, + "unclipped_grad_norm": 1.0173745155334473 + }, + { + "bits_per_byte": 1.5579932419232956, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.079918622970581, + "step": 6570, + "unclipped_grad_norm": 0.988563060760498 + }, + { + "bits_per_byte": 1.5634815522776295, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0837228298187256, + "step": 6580, + "unclipped_grad_norm": 0.980972409248352 + }, + { + "bits_per_byte": 1.5783241709950424, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0940109491348267, + "step": 6590, + "unclipped_grad_norm": 1.0017447471618652 + }, + { + "bits_per_byte": 1.4742578129232586, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.021877646446228, + "step": 6600, + "unclipped_grad_norm": 0.9747391939163208 + }, + { + "bits_per_byte": 1.5148066784651357, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0499839782714844, + "step": 6610, + "unclipped_grad_norm": 0.9631054401397705 + }, + { + "bits_per_byte": 1.6442318784066405, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1396946907043457, + "step": 6620, + "unclipped_grad_norm": 1.0293588638305664 + }, + { + "bits_per_byte": 1.593316414618706, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.104402780532837, + "step": 6630, + "unclipped_grad_norm": 0.9871839880943298 + }, + { + "bits_per_byte": 1.541715771949734, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0686359405517578, + "step": 6640, + "unclipped_grad_norm": 0.9633845090866089 + }, + { + "bits_per_byte": 1.535648567992596, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0644304752349854, + "step": 6650, + "unclipped_grad_norm": 0.9820731282234192 + }, + { + "bits_per_byte": 1.568001256342304, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0868556499481201, + "step": 6660, + "unclipped_grad_norm": 1.0003972053527832 + }, + { + "bits_per_byte": 1.5410072034281794, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0681447982788086, + "step": 6670, + "unclipped_grad_norm": 1.0044373273849487 + }, + { + "bits_per_byte": 1.5651386051186926, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0848714113235474, + "step": 6680, + "unclipped_grad_norm": 1.0404566526412964 + }, + { + "bits_per_byte": 1.5524736307265046, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0760927200317383, + "step": 6690, + "unclipped_grad_norm": 1.033932089805603 + }, + { + "bits_per_byte": 1.565877786552101, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0853837728500366, + "step": 6700, + "unclipped_grad_norm": 0.9689918756484985 + }, + { + "bits_per_byte": 1.5892436934636238, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.1015797853469849, + "step": 6710, + "unclipped_grad_norm": 1.0171167850494385 + }, + { + "bits_per_byte": 1.6393943504031716, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1363415718078613, + "step": 6720, + "unclipped_grad_norm": 1.0878738164901733 + }, + { + "bits_per_byte": 1.4992198908174392, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.039180040359497, + "step": 6730, + "unclipped_grad_norm": 0.971998393535614 + }, + { + "bits_per_byte": 1.5310298019210244, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0612289905548096, + "step": 6740, + "unclipped_grad_norm": 0.973441481590271 + }, + { + "bits_per_byte": 1.5191564636707577, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0529990196228027, + "step": 6750, + "unclipped_grad_norm": 0.9789940714836121 + }, + { + "bits_per_byte": 1.5175835103459665, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0519087314605713, + "step": 6760, + "unclipped_grad_norm": 0.9706876277923584 + }, + { + "bits_per_byte": 1.5403151452411075, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0676651000976562, + "step": 6770, + "unclipped_grad_norm": 1.0128401517868042 + }, + { + "bits_per_byte": 1.5160401380371238, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0508389472961426, + "step": 6780, + "unclipped_grad_norm": 0.994110107421875 + }, + { + "bits_per_byte": 1.5638086632795705, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0839495658874512, + "step": 6790, + "unclipped_grad_norm": 0.9864916801452637 + }, + { + "bits_per_byte": 1.5414822195098625, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0684740543365479, + "step": 6800, + "unclipped_grad_norm": 1.023742914199829 + }, + { + "bits_per_byte": 1.5324796156677978, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0622339248657227, + "step": 6810, + "unclipped_grad_norm": 1.0296359062194824 + }, + { + "bits_per_byte": 1.5483856031154957, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0732591152191162, + "step": 6820, + "unclipped_grad_norm": 0.9808952212333679 + }, + { + "bits_per_byte": 1.549455679169164, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0740008354187012, + "step": 6830, + "unclipped_grad_norm": 0.996810793876648 + }, + { + "bits_per_byte": 1.5259360197677698, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0576982498168945, + "step": 6840, + "unclipped_grad_norm": 0.9733325242996216 + }, + { + "bits_per_byte": 1.6203460679415593, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.1231383085250854, + "step": 6850, + "unclipped_grad_norm": 1.0648897886276245 + }, + { + "bits_per_byte": 1.568673364541876, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0873215198516846, + "step": 6860, + "unclipped_grad_norm": 1.0356100797653198 + }, + { + "bits_per_byte": 1.4980275350990075, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0383535623550415, + "step": 6870, + "unclipped_grad_norm": 0.9937211275100708 + }, + { + "bits_per_byte": 1.521422507078526, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0545697212219238, + "step": 6880, + "unclipped_grad_norm": 1.069592833518982 + }, + { + "bits_per_byte": 1.5148947335823775, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0500450134277344, + "step": 6890, + "unclipped_grad_norm": 1.0049850940704346 + }, + { + "bits_per_byte": 1.5822056474423651, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0967013835906982, + "step": 6900, + "unclipped_grad_norm": 1.0225300788879395 + }, + { + "bits_per_byte": 1.5606532975841907, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.081762433052063, + "step": 6910, + "unclipped_grad_norm": 1.021737813949585 + }, + { + "bits_per_byte": 1.5130916674707318, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.048795223236084, + "step": 6920, + "unclipped_grad_norm": 1.0000710487365723 + }, + { + "bits_per_byte": 1.5005744261756349, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.040118932723999, + "step": 6930, + "unclipped_grad_norm": 1.0489689111709595 + }, + { + "bits_per_byte": 1.568557792200496, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0872414112091064, + "step": 6940, + "unclipped_grad_norm": 1.0565794706344604 + }, + { + "bits_per_byte": 1.484857619643886, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0292248725891113, + "step": 6950, + "unclipped_grad_norm": 1.0331164598464966 + }, + { + "bits_per_byte": 1.498764136792653, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0388641357421875, + "step": 6960, + "unclipped_grad_norm": 1.0262477397918701 + }, + { + "bits_per_byte": 1.4731473209466375, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0211079120635986, + "step": 6970, + "unclipped_grad_norm": 0.9956484436988831 + }, + { + "bits_per_byte": 1.5915678670073838, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1031907796859741, + "step": 6980, + "unclipped_grad_norm": 1.0641905069351196 + }, + { + "bits_per_byte": 1.5488172795691613, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0735583305358887, + "step": 6990, + "unclipped_grad_norm": 1.0450271368026733 + }, + { + "bits_per_byte": 1.5813185609292149, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0960865020751953, + "step": 7000, + "unclipped_grad_norm": 1.0753706693649292 + }, + { + "bits_per_byte": 1.4760245906955722, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0231022834777832, + "step": 7010, + "unclipped_grad_norm": 0.977759838104248 + }, + { + "bits_per_byte": 1.5697365612895098, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0880584716796875, + "step": 7020, + "unclipped_grad_norm": 1.0638940334320068 + }, + { + "bits_per_byte": 1.5177874817698898, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0520501136779785, + "step": 7030, + "unclipped_grad_norm": 1.0119246244430542 + }, + { + "bits_per_byte": 1.5230066392756236, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.055667757987976, + "step": 7040, + "unclipped_grad_norm": 1.0224649906158447 + }, + { + "bits_per_byte": 1.5100723400521843, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0467023849487305, + "step": 7050, + "unclipped_grad_norm": 1.0288535356521606 + }, + { + "bits_per_byte": 1.4969820525644124, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0376288890838623, + "step": 7060, + "unclipped_grad_norm": 0.9655771851539612 + }, + { + "bits_per_byte": 1.5399431467672913, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.067407250404358, + "step": 7070, + "unclipped_grad_norm": 0.986509382724762 + }, + { + "bits_per_byte": 1.5402030125527448, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0675873756408691, + "step": 7080, + "unclipped_grad_norm": 1.0742125511169434 + }, + { + "bits_per_byte": 1.5012656244494524, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0405980348587036, + "step": 7090, + "unclipped_grad_norm": 1.0513901710510254 + }, + { + "bits_per_byte": 1.5043721470719873, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0427513122558594, + "step": 7100, + "unclipped_grad_norm": 1.0241730213165283 + }, + { + "bits_per_byte": 1.4832691878805173, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0281238555908203, + "step": 7110, + "unclipped_grad_norm": 0.991193950176239 + }, + { + "bits_per_byte": 1.5470104298391967, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0723059177398682, + "step": 7120, + "unclipped_grad_norm": 1.0775213241577148 + }, + { + "bits_per_byte": 1.564362103450047, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0843331813812256, + "step": 7130, + "unclipped_grad_norm": 1.0102075338363647 + }, + { + "bits_per_byte": 1.530404644985138, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0607956647872925, + "step": 7140, + "unclipped_grad_norm": 1.0219485759735107 + }, + { + "bits_per_byte": 1.5796152447550695, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0949058532714844, + "step": 7150, + "unclipped_grad_norm": 1.0659542083740234 + }, + { + "bits_per_byte": 1.568367407405991, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0871094465255737, + "step": 7160, + "unclipped_grad_norm": 1.0356900691986084 + }, + { + "bits_per_byte": 1.5848232233884971, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0985157489776611, + "step": 7170, + "unclipped_grad_norm": 1.0675727128982544 + }, + { + "bits_per_byte": 1.5187308066098724, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0527039766311646, + "step": 7180, + "unclipped_grad_norm": 0.9808175563812256 + }, + { + "bits_per_byte": 1.5693248348233442, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.087773084640503, + "step": 7190, + "unclipped_grad_norm": 1.0627371072769165 + }, + { + "bits_per_byte": 1.5123168856285947, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0482581853866577, + "step": 7200, + "unclipped_grad_norm": 1.0047924518585205 + }, + { + "bits_per_byte": 1.5017243021793036, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0409159660339355, + "step": 7210, + "unclipped_grad_norm": 1.0128343105316162 + }, + { + "bits_per_byte": 1.5058675362212395, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0437878370285034, + "step": 7220, + "unclipped_grad_norm": 1.020466685295105 + }, + { + "bits_per_byte": 1.5492668422185167, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0738699436187744, + "step": 7230, + "unclipped_grad_norm": 1.0896223783493042 + }, + { + "bits_per_byte": 1.5598331123222258, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0811939239501953, + "step": 7240, + "unclipped_grad_norm": 1.0040799379348755 + }, + { + "bits_per_byte": 1.5635771746315092, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0837891101837158, + "step": 7250, + "unclipped_grad_norm": 1.0716665983200073 + }, + { + "bits_per_byte": 1.530704410745592, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0610034465789795, + "step": 7260, + "unclipped_grad_norm": 1.0284863710403442 + }, + { + "bits_per_byte": 1.5407076096503765, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0679371356964111, + "step": 7270, + "unclipped_grad_norm": 1.0729715824127197 + }, + { + "bits_per_byte": 1.5375348737072594, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0657379627227783, + "step": 7280, + "unclipped_grad_norm": 1.0802364349365234 + }, + { + "bits_per_byte": 1.5291964668628268, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0599582195281982, + "step": 7290, + "unclipped_grad_norm": 1.0581231117248535 + }, + { + "bits_per_byte": 1.5223175047936162, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.055190086364746, + "step": 7300, + "unclipped_grad_norm": 1.0030993223190308 + }, + { + "bits_per_byte": 1.518025505758684, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0522150993347168, + "step": 7310, + "unclipped_grad_norm": 1.0415269136428833 + }, + { + "bits_per_byte": 1.5002450793992326, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0398906469345093, + "step": 7320, + "unclipped_grad_norm": 1.0742172002792358 + }, + { + "bits_per_byte": 1.5556656287265183, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0783052444458008, + "step": 7330, + "unclipped_grad_norm": 1.098557472229004 + }, + { + "bits_per_byte": 1.5089695872948519, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0459380149841309, + "step": 7340, + "unclipped_grad_norm": 1.0178015232086182 + }, + { + "bits_per_byte": 1.5128952632834465, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.048659086227417, + "step": 7350, + "unclipped_grad_norm": 1.0542628765106201 + }, + { + "bits_per_byte": 1.5165268489390655, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0511763095855713, + "step": 7360, + "unclipped_grad_norm": 1.0132694244384766 + }, + { + "bits_per_byte": 1.4757258568310236, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0228952169418335, + "step": 7370, + "unclipped_grad_norm": 1.0333343744277954 + }, + { + "bits_per_byte": 1.4706363742440405, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0193674564361572, + "step": 7380, + "unclipped_grad_norm": 1.0362695455551147 + }, + { + "bits_per_byte": 1.4849742238811712, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0293056964874268, + "step": 7390, + "unclipped_grad_norm": 1.0089579820632935 + }, + { + "bits_per_byte": 1.4991997688472884, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0391660928726196, + "step": 7400, + "unclipped_grad_norm": 1.0394701957702637 + }, + { + "bits_per_byte": 1.5501245196983695, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0744644403457642, + "step": 7410, + "unclipped_grad_norm": 1.044879674911499 + }, + { + "bits_per_byte": 1.5165729402894967, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.051208257675171, + "step": 7420, + "unclipped_grad_norm": 1.0228228569030762 + }, + { + "bits_per_byte": 1.4806290822071222, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0262938737869263, + "step": 7430, + "unclipped_grad_norm": 1.033545970916748 + }, + { + "bits_per_byte": 1.4643294264715996, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.014995813369751, + "step": 7440, + "unclipped_grad_norm": 1.0252758264541626 + }, + { + "bits_per_byte": 1.5693753977226979, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0878081321716309, + "step": 7450, + "unclipped_grad_norm": 1.039492130279541 + }, + { + "bits_per_byte": 1.4916665847741957, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0339444875717163, + "step": 7460, + "unclipped_grad_norm": 1.036054253578186 + }, + { + "bits_per_byte": 1.466042717639495, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0161833763122559, + "step": 7470, + "unclipped_grad_norm": 1.0448886156082153 + }, + { + "bits_per_byte": 1.5608368030726611, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0818896293640137, + "step": 7480, + "unclipped_grad_norm": 1.0651273727416992 + }, + { + "bits_per_byte": 1.5231578120257319, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0557725429534912, + "step": 7490, + "unclipped_grad_norm": 1.077128291130066 + }, + { + "bits_per_byte": 1.5418131141301226, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0687034130096436, + "step": 7500, + "unclipped_grad_norm": 1.0931757688522339 + }, + { + "bits_per_byte": 1.528281175194935, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.059323787689209, + "step": 7510, + "unclipped_grad_norm": 1.0170495510101318 + }, + { + "bits_per_byte": 1.541195696413525, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0682754516601562, + "step": 7520, + "unclipped_grad_norm": 1.072600245475769 + }, + { + "bits_per_byte": 1.5347975978361268, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.063840627670288, + "step": 7530, + "unclipped_grad_norm": 1.0730668306350708 + }, + { + "bits_per_byte": 1.5420986053305548, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0689013004302979, + "step": 7540, + "unclipped_grad_norm": 1.0506013631820679 + }, + { + "bits_per_byte": 1.5157248938380923, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.050620436668396, + "step": 7550, + "unclipped_grad_norm": 1.036077857017517 + }, + { + "bits_per_byte": 1.443441617576359, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.00051748752594, + "step": 7560, + "unclipped_grad_norm": 1.0049386024475098 + }, + { + "bits_per_byte": 1.4652058500603964, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0156033039093018, + "step": 7570, + "unclipped_grad_norm": 1.0102496147155762 + }, + { + "bits_per_byte": 1.4617899306489595, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0132355690002441, + "step": 7580, + "unclipped_grad_norm": 1.0245665311813354 + }, + { + "bits_per_byte": 1.5365071053857031, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0650255680084229, + "step": 7590, + "unclipped_grad_norm": 1.048311471939087 + }, + { + "bits_per_byte": 1.567407228266224, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0864439010620117, + "step": 7600, + "unclipped_grad_norm": 1.0463460683822632 + }, + { + "bits_per_byte": 1.5308137917115408, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0610792636871338, + "step": 7610, + "unclipped_grad_norm": 1.0807271003723145 + }, + { + "bits_per_byte": 1.5336785067269625, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0630649328231812, + "step": 7620, + "unclipped_grad_norm": 1.048875331878662 + }, + { + "bits_per_byte": 1.510936380890119, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0473012924194336, + "step": 7630, + "unclipped_grad_norm": 1.0328357219696045 + }, + { + "bits_per_byte": 1.4914947741059836, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.033825397491455, + "step": 7640, + "unclipped_grad_norm": 1.0975395441055298 + }, + { + "bits_per_byte": 1.462546310347454, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0137598514556885, + "step": 7650, + "unclipped_grad_norm": 1.0314407348632812 + }, + { + "bits_per_byte": 1.4939462148113822, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.035524606704712, + "step": 7660, + "unclipped_grad_norm": 1.1010568141937256 + }, + { + "bits_per_byte": 1.5304318182439745, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0608144998550415, + "step": 7670, + "unclipped_grad_norm": 1.0511856079101562 + }, + { + "bits_per_byte": 1.5306942637691912, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.060996413230896, + "step": 7680, + "unclipped_grad_norm": 1.062692403793335 + }, + { + "bits_per_byte": 1.5040376408160592, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.042519450187683, + "step": 7690, + "unclipped_grad_norm": 1.0179529190063477 + }, + { + "bits_per_byte": 1.5428006385113768, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0693879127502441, + "step": 7700, + "unclipped_grad_norm": 1.0779086351394653 + }, + { + "bits_per_byte": 1.4839846357081066, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0286197662353516, + "step": 7710, + "unclipped_grad_norm": 1.0490714311599731 + }, + { + "bits_per_byte": 1.496933553456869, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.037595272064209, + "step": 7720, + "unclipped_grad_norm": 0.9670684337615967 + }, + { + "bits_per_byte": 1.4854798528747077, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.029656171798706, + "step": 7730, + "unclipped_grad_norm": 0.9940106868743896 + }, + { + "bits_per_byte": 1.5407578285844283, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.06797194480896, + "step": 7740, + "unclipped_grad_norm": 1.0928970575332642 + }, + { + "bits_per_byte": 1.4753865350608713, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0226600170135498, + "step": 7750, + "unclipped_grad_norm": 1.013372778892517 + }, + { + "bits_per_byte": 1.4801974057534566, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0259946584701538, + "step": 7760, + "unclipped_grad_norm": 1.0438604354858398 + }, + { + "bits_per_byte": 1.538979355991856, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0667392015457153, + "step": 7770, + "unclipped_grad_norm": 1.1242064237594604 + }, + { + "bits_per_byte": 1.51728718423853, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0517033338546753, + "step": 7780, + "unclipped_grad_norm": 1.1380839347839355 + }, + { + "bits_per_byte": 1.5213089985289567, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0544910430908203, + "step": 7790, + "unclipped_grad_norm": 1.054468035697937 + }, + { + "bits_per_byte": 1.5100510142034773, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0466876029968262, + "step": 7800, + "unclipped_grad_norm": 1.0544580221176147 + }, + { + "bits_per_byte": 1.498489308516574, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0386736392974854, + "step": 7810, + "unclipped_grad_norm": 1.0938669443130493 + }, + { + "bits_per_byte": 1.5401904578192318, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.067578673362732, + "step": 7820, + "unclipped_grad_norm": 1.1261787414550781 + }, + { + "bits_per_byte": 1.5279041892242438, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0590624809265137, + "step": 7830, + "unclipped_grad_norm": 1.0831990242004395 + }, + { + "bits_per_byte": 1.5007298984920148, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.040226697921753, + "step": 7840, + "unclipped_grad_norm": 1.0495885610580444 + }, + { + "bits_per_byte": 1.485421722738716, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.029615879058838, + "step": 7850, + "unclipped_grad_norm": 1.0680148601531982 + }, + { + "bits_per_byte": 1.5337288976436654, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0630998611450195, + "step": 7860, + "unclipped_grad_norm": 1.059203028678894 + }, + { + "bits_per_byte": 1.528788867980282, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.059675693511963, + "step": 7870, + "unclipped_grad_norm": 1.0786525011062622 + }, + { + "bits_per_byte": 1.4810511276323397, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0265864133834839, + "step": 7880, + "unclipped_grad_norm": 1.0475893020629883 + }, + { + "bits_per_byte": 1.50947349646188, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0462872982025146, + "step": 7890, + "unclipped_grad_norm": 1.0943796634674072 + }, + { + "bits_per_byte": 1.4635482812713807, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0144543647766113, + "step": 7900, + "unclipped_grad_norm": 1.0396785736083984 + }, + { + "bits_per_byte": 1.530891011921778, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.061132788658142, + "step": 7910, + "unclipped_grad_norm": 1.1283469200134277 + }, + { + "bits_per_byte": 1.5486817572402813, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0734643936157227, + "step": 7920, + "unclipped_grad_norm": 1.0839927196502686 + }, + { + "bits_per_byte": 1.5089941808139253, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0459550619125366, + "step": 7930, + "unclipped_grad_norm": 1.0844900608062744 + }, + { + "bits_per_byte": 1.5398654106091012, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.067353367805481, + "step": 7940, + "unclipped_grad_norm": 1.0818886756896973 + }, + { + "bits_per_byte": 1.473885126518839, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0216193199157715, + "step": 7950, + "unclipped_grad_norm": 1.0441930294036865 + }, + { + "bits_per_byte": 1.5289575829607784, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0597926378250122, + "step": 7960, + "unclipped_grad_norm": 1.1338133811950684 + }, + { + "bits_per_byte": 1.498011712695128, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0383425951004028, + "step": 7970, + "unclipped_grad_norm": 1.0597225427627563 + }, + { + "bits_per_byte": 1.5385745088317249, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0664585828781128, + "step": 7980, + "unclipped_grad_norm": 1.1261745691299438 + }, + { + "bits_per_byte": 1.500032680825417, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.039743423461914, + "step": 7990, + "unclipped_grad_norm": 1.0900131464004517 + }, + { + "bits_per_byte": 1.5133369147308622, + "learning_rate": 3e-05, + "loss_nats": 1.0489652156829834, + "step": 8000, + "unclipped_grad_norm": 1.0884610414505005 + } + ] + }, + "depth-16-baseline-seed-2026073003": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "14cb0082a87883b57c59c7a7d75d194347e04cee762e0db4354d50b7946c7443", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00031658337684348226, + 0.0002727926184888929, + 0.00024126221251208335, + 0.00021358189405873418, + 0.0001949657453224063, + 0.00017900438979268074, + 0.00016492439317516983, + 0.00015548785449936986, + 0.00014800082135479897, + 0.0001436655584257096, + 0.00013764512550551444, + 0.0001334012922598049, + 0.00013005948858335614, + 0.00012698007049039006, + 0.00012402374704834074, + 0.000122041383292526 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002610550254757982, + "first_to_last_ratio": 2.075552313213765, + "imbalance_abs_log_ratio": 0.7302272933071482, + "last_quartile_mean": 0.00012577617235365324, + "mean": 0.00017527624822832877, + "normalized": [ + 1.8061966754963605, + 1.5563581560322512, + 1.3764683746413606, + 1.2185444189820027, + 1.1123340857252286, + 1.021270089941082, + 0.9409397727427746, + 0.887101681323182, + 0.8443860640033859, + 0.8196521769370567, + 0.7853039239304516, + 0.7610916687697798, + 0.7420257444918054, + 0.7244568033255466, + 0.7075901515576575, + 0.6962802121000742 + ], + "population_cv": 0.32194911387722014 + }, + "activation_output_rms_by_block": [ + 0.0284949392080307, + 0.028932223096489906, + 0.029410898685455322, + 0.029886392876505852, + 0.030463071539998055, + 0.031026815995573997, + 0.031805604696273804, + 0.03248070180416107, + 0.03315235301852226, + 0.034095361828804016, + 0.034913208335638046, + 0.03547592833638191, + 0.03669917955994606, + 0.03757266327738762, + 0.038674499839544296, + 0.039735738188028336 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029181113466620445, + "first_to_last_ratio": 0.7644934703880544, + "imbalance_abs_log_ratio": 0.2685417946723145, + "last_quartile_mean": 0.03817052021622658, + "mean": 0.03330122376792133, + "normalized": [ + 0.8556724343409726, + 0.8688036000755015, + 0.8831777141411389, + 0.8974562942427076, + 0.9147733354274736, + 0.9317019762337311, + 0.9550881648653333, + 0.9753606062804618, + 0.9955295712122606, + 1.0238471134399472, + 1.0484061660601651, + 1.065304043587595, + 1.102036964638457, + 1.1282667429651916, + 1.1613537120758601, + 1.193221560413203 + ], + "population_cv": 0.1043395636179365 + }, + "bits_per_byte": 8.022455452740537, + "branch_output_rms_by_sublayer": [ + 0.003436586121097207, + 0.0038391812704503536, + 0.003560001263394952, + 0.0038144157733768225, + 0.004080869723111391, + 0.0038664413150399923, + 0.004083882085978985, + 0.003824100596830249, + 0.004302335437387228, + 0.0039005596190690994, + 0.005134448409080505, + 0.003882436081767082, + 0.005223615560680628, + 0.0038928077556192875, + 0.00572318397462368, + 0.0038727927021682262, + 0.005704221315681934, + 0.003765099449083209, + 0.0069826096296310425, + 0.003909832797944546, + 0.006672552786767483, + 0.0038559692911803722, + 0.0066586704924702644, + 0.00382705288939178, + 0.007329727057367563, + 0.0038791403640061617, + 0.007807865273207426, + 0.00388751900754869, + 0.008622666820883751, + 0.0039123171009123325, + 0.008360983803868294, + 0.0037942787166684866 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007247792155837073, + 0.0070599825250150985, + 0.007115739546591776, + 0.006247289691276946, + 0.006127370040903895, + 0.0065579111453454445, + 0.0059278656112939756, + 0.005972691938247438, + 0.0055663256066952145, + 0.006567633669083912, + 0.005568350992310226, + 0.005243662280059585, + 0.00519845242839581, + 0.005229357864659918, + 0.005505343111599652, + 0.004934663781812996 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006917700979680223, + "first_to_last_ratio": 1.3260037536012093, + "imbalance_abs_log_ratio": 0.2821697225298069, + "last_quartile_mean": 0.005216954296617094, + "mean": 0.00600440202432056, + "normalized": [ + 1.2070797602292813, + 1.1758011033270186, + 1.1850871273725165, + 1.0404515996718042, + 1.0204796441153106, + 1.092183887551653, + 0.9872532830552356, + 0.9947188602720667, + 0.927040791763953, + 1.0938031201911544, + 0.9273781085536696, + 0.8733029965049588, + 0.8657735453655023, + 0.8709206751111334, + 0.9168844939596827, + 0.821841002955059 + ], + "population_cv": 0.119259291012628 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028142500668764114, + 0.02826075814664364, + 0.0284949392080307, + 0.028706196695566177, + 0.028932223096489906, + 0.029190780594944954, + 0.029410898685455322, + 0.02965042181313038, + 0.029886392876505852, + 0.03018481843173504, + 0.030463071539998055, + 0.030850933864712715, + 0.031026815995573997, + 0.031584836542606354, + 0.031805604696273804, + 0.03225463256239891, + 0.03248070180416107, + 0.03296073526144028, + 0.03315235301852226, + 0.03385457023978233, + 0.034095361828804016, + 0.03471403941512108, + 0.034913208335638046, + 0.03529877960681915, + 0.03547592833638191, + 0.03643115609884262, + 0.03669917955994606, + 0.03748895600438118, + 0.03757266327738762, + 0.03844280168414116, + 0.038674499839544296, + 0.03956866264343262 + ], + "loss_nats": 5.560742378234863, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.02826075814664364, + 0.0284949392080307, + 0.028706196695566177, + 0.028932223096489906, + 0.029190780594944954, + 0.029410898685455322, + 0.02965042181313038, + 0.029886392876505852, + 0.03018481843173504, + 0.030463071539998055, + 0.030850933864712715, + 0.031026815995573997, + 0.031584836542606354, + 0.031805604696273804, + 0.03225463256239891, + 0.03248070180416107, + 0.03296073526144028, + 0.03315235301852226, + 0.03385457023978233, + 0.034095361828804016, + 0.03471403941512108, + 0.034913208335638046, + 0.03529877960681915, + 0.03547592833638191, + 0.03643115609884262, + 0.03669917955994606, + 0.03748895600438118, + 0.03757266327738762, + 0.03844280168414116, + 0.038674499839544296, + 0.03956866264343262, + 0.039735738188028336 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.706054748268798e-05, + 6.966097134863958e-05, + 6.121626211097464e-05, + 5.34596256329678e-05, + 4.684355008066632e-05, + 4.049177005072124e-05, + 3.4754324587993324e-05, + 3.020181247848086e-05, + 2.6929572413791902e-05, + 2.4378721718676388e-05, + 2.2718366381013766e-05, + 2.1439738702611066e-05, + 2.0652638340834528e-05, + 2.0034611225128174e-05, + 1.9586463167797774e-05, + 1.9270755728939548e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.53493516438175e-05, + "first_to_last_ratio": 3.2861795625404726, + "imbalance_abs_log_ratio": 1.1897256628392332, + "last_quartile_mean": 1.9886117115675006e-05, + "mean": 3.6793733215745306e-05, + "normalized": [ + 2.0943932769972666, + 1.893283590990143, + 1.663768711699472, + 1.45295464636599, + 1.2731393633256098, + 1.1005072470709065, + 0.9445718482603106, + 0.8208412095991516, + 0.7319064976605258, + 0.6625780965396546, + 0.6174520603223757, + 0.5827008250806217, + 0.5613085853434561, + 0.5445115098175108, + 0.5323314993058669, + 0.5237510316211383 + ], + "population_cv": 0.5070097402008534 + }, + "activation_output_rms_by_block": [ + 0.03020417131483555, + 0.03345569595694542, + 0.03829808160662651, + 0.04440607130527496, + 0.05133094638586044, + 0.059150952845811844, + 0.06815594434738159, + 0.07766561955213547, + 0.08808834105730057, + 0.10039259493350983, + 0.11500746011734009, + 0.132473424077034, + 0.1519528478384018, + 0.17349907755851746, + 0.19798777997493744, + 0.22741958498954773 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03659100504592061, + "first_to_last_ratio": 0.19492869311536967, + "imbalance_abs_log_ratio": 1.6351214636257212, + "last_quartile_mean": 0.1877148225903511, + "mean": 0.09934303711634129, + "normalized": [ + 0.30403913743308697, + 0.33676940959400337, + 0.3855134966507554, + 0.44699731953303096, + 0.5167040174717673, + 0.5954212249071846, + 0.6860666466997954, + 0.7817922806324186, + 0.8867087579992116, + 1.010564986209747, + 1.1576801299385893, + 1.3334948067059142, + 1.5295772330822623, + 1.746464398459366, + 1.9929708787045903, + 2.2892352759782764 + ], + "population_cv": 0.6013462998472742 + }, + "bits_per_byte": 5.091903414776641, + "branch_output_rms_by_sublayer": [ + 0.0044672503136098385, + 0.008439179509878159, + 0.004589807707816362, + 0.008912671357393265, + 0.005376614164561033, + 0.009684479795396328, + 0.004654362332075834, + 0.010457071475684643, + 0.00613226555287838, + 0.009992391802370548, + 0.005980229936540127, + 0.011163929477334023, + 0.008545844815671444, + 0.010891783982515335, + 0.00866044219583273, + 0.011625969782471657, + 0.010422462597489357, + 0.011488622985780239, + 0.010880537331104279, + 0.012680526822805405, + 0.014226034283638, + 0.012272600084543228, + 0.01669861003756523, + 0.013891858980059624, + 0.01948261633515358, + 0.012507432140409946, + 0.01911107264459133, + 0.013616994023323059, + 0.02317427285015583, + 0.013388233259320259, + 0.027934540063142776, + 0.013881584629416466 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00027208547857035954, + 0.0002379277587721014, + 0.0002494129668009364, + 0.00019949675850635897, + 0.0002037090250478814, + 0.0001757519905869641, + 0.00017904243730119992, + 0.00016456038574532694, + 0.0001493779474782176, + 0.00013508785060359837, + 0.0001355896704444294, + 0.00013648423482290936, + 0.00013660889397982336, + 0.00013630230473358484, + 0.0001407085634636149, + 0.00014945333748469326 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023973074066243908, + "first_to_last_ratio": 1.703016825392403, + "imbalance_abs_log_ratio": 0.532401281486357, + "last_quartile_mean": 0.0001407682749154291, + "mean": 0.00017509997527137498, + "normalized": [ + 1.5538864477203589, + 1.358810921608379, + 1.4244032097342618, + 1.1393305921212904, + 1.163386943557058, + 1.0037236745155367, + 1.0225154916425019, + 0.9398081609679643, + 0.8531009056209594, + 0.7714898325612858, + 0.774355737254038, + 0.7794646150656627, + 0.7801765463878734, + 0.7784256081266694, + 0.8035898534282527, + 0.8535314596879079 + ], + "population_cv": 0.2495872477003493 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028559360653162003, + 0.028688903898000717, + 0.03020417131483555, + 0.03060002811253071, + 0.03345569595694542, + 0.034075960516929626, + 0.03829808160662651, + 0.03887331485748291, + 0.04440607130527496, + 0.04547364264726639, + 0.05133094638586044, + 0.05236440151929855, + 0.059150952845811844, + 0.061456043273210526, + 0.06815594434738159, + 0.07020121067762375, + 0.07766561955213547, + 0.08064334839582443, + 0.08808834105730057, + 0.09190437197685242, + 0.10039259493350983, + 0.10653280466794968, + 0.11500746011734009, + 0.1223740428686142, + 0.132473424077034, + 0.14304450154304504, + 0.1519528478384018, + 0.1632441133260727, + 0.17349907755851746, + 0.1877153217792511, + 0.19798777997493744, + 0.21634350717067719 + ], + "loss_nats": 3.5294384956359863, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028688903898000717, + 0.03020417131483555, + 0.03060002811253071, + 0.03345569595694542, + 0.034075960516929626, + 0.03829808160662651, + 0.03887331485748291, + 0.04440607130527496, + 0.04547364264726639, + 0.05133094638586044, + 0.05236440151929855, + 0.059150952845811844, + 0.061456043273210526, + 0.06815594434738159, + 0.07020121067762375, + 0.07766561955213547, + 0.08064334839582443, + 0.08808834105730057, + 0.09190437197685242, + 0.10039259493350983, + 0.10653280466794968, + 0.11500746011734009, + 0.1223740428686142, + 0.132473424077034, + 0.14304450154304504, + 0.1519528478384018, + 0.1632441133260727, + 0.17349907755851746, + 0.1877153217792511, + 0.19798777997493744, + 0.21634350717067719, + 0.22741958498954773 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015226140385493636, + 0.00015351864567492157, + 0.00014040769019629806, + 0.00012548675294965506, + 0.00010927267430815846, + 9.748537559062243e-05, + 8.759690535953268e-05, + 7.801534957252443e-05, + 7.03658297425136e-05, + 6.262787064770237e-05, + 5.5923766922205687e-05, + 4.834980791201815e-05, + 4.202092532068491e-05, + 3.7704347050748765e-05, + 3.55200027115643e-05, + 3.515387652441859e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014291862316895276, + "first_to_last_ratio": 3.801048653306501, + "imbalance_abs_log_ratio": 1.3352769900584454, + "last_quartile_mean": 3.759978790185414e-05, + "mean": 8.323195152115659e-05, + "normalized": [ + 1.8293624151805847, + 1.8444676938266782, + 1.686944588348479, + 1.5076752455787101, + 1.3128693045288338, + 1.1712494277614383, + 1.0524432475581997, + 0.9373245267797649, + 0.845418477597842, + 0.752449864542502, + 0.671902627538302, + 0.5809044126488876, + 0.504865313773205, + 0.4530032801305249, + 0.4267592200158317, + 0.4223603541902161 + ], + "population_cv": 0.48987170747402153 + }, + "activation_output_rms_by_block": [ + 0.05162646621465683, + 0.06355457007884979, + 0.07474122941493988, + 0.08428152650594711, + 0.08889172971248627, + 0.09293187409639359, + 0.0973934531211853, + 0.09854210913181305, + 0.10090499371290207, + 0.10472240298986435, + 0.11040307581424713, + 0.1186341941356659, + 0.130356103181839, + 0.149788036942482, + 0.17963695526123047, + 0.2188950628042221 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0685509480535984, + "first_to_last_ratio": 0.40402744210990815, + "imbalance_abs_log_ratio": 0.9062724773132301, + "last_quartile_mean": 0.1696690395474434, + "mean": 0.1103314864449203, + "normalized": [ + 0.4679214236856113, + 0.576032936079199, + 0.6774242949427878, + 0.7638936918340364, + 0.8056787103730612, + 0.8422969461468014, + 0.8827348951725225, + 0.8931458489957648, + 0.9145620798218456, + 0.9491615346100125, + 1.0006488571090046, + 1.0752523867689434, + 1.1814950326706182, + 1.3576182263914227, + 1.6281567578708265, + 1.9839763775275423 + ], + "population_cv": 0.3749688391491054 + }, + "bits_per_byte": 3.129900568231996, + "branch_output_rms_by_sublayer": [ + 0.014640412293374538, + 0.020567819476127625, + 0.01939546875655651, + 0.012996131554245949, + 0.0218212753534317, + 0.010227516293525696, + 0.02053862437605858, + 0.010292341001331806, + 0.02414301224052906, + 0.009784159250557423, + 0.019980086013674736, + 0.010961603373289108, + 0.01658599078655243, + 0.011390469036996365, + 0.021502232179045677, + 0.012452049180865288, + 0.020629581063985825, + 0.014398678205907345, + 0.02130005694925785, + 0.015666954219341278, + 0.016051532700657845, + 0.0179822389036417, + 0.018369123339653015, + 0.02199208177626133, + 0.01977314054965973, + 0.026316894218325615, + 0.016924018040299416, + 0.03431078419089317, + 0.013718036003410816, + 0.04166935011744499, + 0.022049644961953163, + 0.04074697569012642 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0009002454290164998, + 0.0005806619682142476, + 0.0004904299887450306, + 0.00047545530221616405, + 0.00040490888378542537, + 0.0003357588676961225, + 0.0003168596431849841, + 0.00029935138955839515, + 0.00029222591787988014, + 0.00027100955094896284, + 0.0002328601817845373, + 0.0002299739939536432, + 0.0002128774552133389, + 0.00018778710115789697, + 0.00017922956471394958, + 0.0001849955243427181 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006116981720479855, + "first_to_last_ratio": 3.1988832674328185, + "imbalance_abs_log_ratio": 1.1628017699711202, + "last_quartile_mean": 0.0001912224113569759, + "mean": 0.0003496644226507373, + "normalized": [ + 2.5745983025436674, + 1.6606263909046375, + 1.4025733159444054, + 1.359747436161308, + 1.157992799827591, + 0.9602317134548619, + 0.9061821067837941, + 0.8561105167322174, + 0.8357324879224854, + 0.7750561206498867, + 0.6659533160945287, + 0.6576991511182508, + 0.6088050182502319, + 0.537049493724068, + 0.512575924525708, + 0.5290659053623568 + ], + "population_cv": 0.5241885524055848 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04364648833870888, + 0.045909684151411057, + 0.05162646621465683, + 0.0588536262512207, + 0.06355457007884979, + 0.07136592268943787, + 0.07474122941493988, + 0.08130704611539841, + 0.08428152650594711, + 0.08667806535959244, + 0.08889172971248627, + 0.09049459546804428, + 0.09293187409639359, + 0.09443560987710953, + 0.0973934531211853, + 0.09589134901762009, + 0.09854210913181305, + 0.09718576818704605, + 0.10090499371290207, + 0.09916814416646957, + 0.10472240298986435, + 0.10267213732004166, + 0.11040307581424713, + 0.10699481517076492, + 0.1186341941356659, + 0.11436477303504944, + 0.130356103181839, + 0.12692348659038544, + 0.149788036942482, + 0.15056778490543365, + 0.17963695526123047, + 0.18913158774375916 + ], + "loss_nats": 2.1694817543029785, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.045909684151411057, + 0.05162646621465683, + 0.0588536262512207, + 0.06355457007884979, + 0.07136592268943787, + 0.07474122941493988, + 0.08130704611539841, + 0.08428152650594711, + 0.08667806535959244, + 0.08889172971248627, + 0.09049459546804428, + 0.09293187409639359, + 0.09443560987710953, + 0.0973934531211853, + 0.09589134901762009, + 0.09854210913181305, + 0.09718576818704605, + 0.10090499371290207, + 0.09916814416646957, + 0.10472240298986435, + 0.10267213732004166, + 0.11040307581424713, + 0.10699481517076492, + 0.1186341941356659, + 0.11436477303504944, + 0.130356103181839, + 0.12692348659038544, + 0.149788036942482, + 0.15056778490543365, + 0.17963695526123047, + 0.18913158774375916, + 0.2188950628042221 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001594285131432116, + 0.00017026996647473425, + 0.00018808410095516592, + 0.00017561657296027988, + 0.0001699679996818304, + 0.00016689945186953992, + 0.00015736353816464543, + 0.00014705183275509626, + 0.0001390052493661642, + 0.00012381834676489234, + 0.000104674712929409, + 9.053898247657344e-05, + 7.686029857723042e-05, + 6.236408080440015e-05, + 4.532423918135464e-05, + 3.028271021321416e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00017334978838334791, + "first_to_last_ratio": 3.227644485017083, + "imbalance_abs_log_ratio": 1.171752609589353, + "last_quartile_mean": 5.370783219404984e-05, + "mean": 0.00012547191226985888, + "normalized": [ + 1.2706310939162266, + 1.3570365143437513, + 1.499013584415959, + 1.3996484933024078, + 1.3546298658162754, + 1.3301738122120965, + 1.2541734266884814, + 1.1719900501621776, + 1.1078594950174865, + 0.9868212297473388, + 0.834248167863097, + 0.7215876512812414, + 0.6125697551490491, + 0.4970361866349061, + 0.3612301618857412, + 0.24135051156376403 + ], + "population_cv": 0.3906610370884368 + }, + "activation_output_rms_by_block": [ + 0.08273108303546906, + 0.13354359567165375, + 0.1704990118741989, + 0.17198410630226135, + 0.17284893989562988, + 0.17146942019462585, + 0.16773979365825653, + 0.15899653732776642, + 0.14656992256641388, + 0.13979844748973846, + 0.1417049616575241, + 0.149912029504776, + 0.16512760519981384, + 0.19090111553668976, + 0.2512649893760681, + 0.390801340341568 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.13968944922089577, + "first_to_last_ratio": 0.5598242338035286, + "imbalance_abs_log_ratio": 0.5801324127278619, + "last_quartile_mean": 0.24952376261353493, + "mean": 0.17536830622702837, + "normalized": [ + 0.4717561845432151, + 0.7615035951751215, + 0.9722338975748231, + 0.9807023287298795, + 0.9856338560507224, + 0.9777674420407808, + 0.9565000499069877, + 0.906643513577263, + 0.8357834190213785, + 0.7971705406606262, + 0.8080420271270434, + 0.8548410641014165, + 0.9416046077678534, + 1.0885725000363118, + 1.4327844910059484, + 2.228460482680629 + ], + "population_cv": 0.3692959780115816 + }, + "bits_per_byte": 2.102960171157105, + "branch_output_rms_by_sublayer": [ + 0.027171069756150246, + 0.04243958741426468, + 0.07314105331897736, + 0.007419656030833721, + 0.04879630357027054, + 0.0038894445169717073, + 0.024768881499767303, + 0.005487008485943079, + 0.02289147861301899, + 0.00865041371434927, + 0.025303618982434273, + 0.011297631077468395, + 0.03125236928462982, + 0.014728116802871227, + 0.035565298050642014, + 0.019090818241238594, + 0.043166082352399826, + 0.02257433719933033, + 0.04396531730890274, + 0.02886364422738552, + 0.04178466275334358, + 0.03710934892296791, + 0.035476140677928925, + 0.04854284226894379, + 0.026270829141139984, + 0.058104008436203, + 0.018585238605737686, + 0.06995242089033127, + 0.01513067539781332, + 0.10811682045459747, + 0.016200197860598564, + 0.20113423466682434 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004582000765955337, + 0.0003576182203712263, + 0.0003389913908322493, + 0.00022415790052185902, + 0.00024709072048410314, + 0.00027475536769928975, + 0.00033936291802973487, + 0.00039410580789665003, + 0.0004814106753566502, + 0.0005191302840338224, + 0.0005153457080868455, + 0.0004830357094583275, + 0.00041821493179685683, + 0.0003600919520715316, + 0.0003136989944772832, + 0.0002645229655850915 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034474189708021705, + "first_to_last_ratio": 1.0165412954471984, + "imbalance_abs_log_ratio": 0.016405978393597878, + "last_quartile_mean": 0.00033913221098269076, + "mean": 0.00037435835145606595, + "normalized": [ + 1.2239611452859689, + 0.9552831370804766, + 0.9055264548359695, + 0.5987789497683099, + 0.6600379543371862, + 0.7339367924627015, + 0.9065188921518208, + 1.052750142647483, + 1.2859621629494962, + 1.3867201893978494, + 1.3766106888824827, + 1.2903030146904997, + 1.1171513341967951, + 0.9618910615215468, + 0.8379644617440793, + 0.7066036180473336 + ], + "population_cv": 0.2506116126131003 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.055472806096076965, + 0.06610024720430374, + 0.08273108303546906, + 0.13110069930553436, + 0.13354359567165375, + 0.1698971837759018, + 0.1704990118741989, + 0.17238332331180573, + 0.17198410630226135, + 0.17322638630867004, + 0.17284893989562988, + 0.1726299524307251, + 0.17146942019462585, + 0.170275017619133, + 0.16773979365825653, + 0.16162297129631042, + 0.15899653732776642, + 0.14726711809635162, + 0.14656992256641388, + 0.13564367592334747, + 0.13979844748973846, + 0.13359025120735168, + 0.1417049616575241, + 0.13732580840587616, + 0.149912029504776, + 0.14843478798866272, + 0.16512760519981384, + 0.1652906984090805, + 0.19090111553668976, + 0.191492959856987, + 0.2512649893760681, + 0.25225311517715454 + ], + "loss_nats": 1.4576609134674072, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06610024720430374, + 0.08273108303546906, + 0.13110069930553436, + 0.13354359567165375, + 0.1698971837759018, + 0.1704990118741989, + 0.17238332331180573, + 0.17198410630226135, + 0.17322638630867004, + 0.17284893989562988, + 0.1726299524307251, + 0.17146942019462585, + 0.170275017619133, + 0.16773979365825653, + 0.16162297129631042, + 0.15899653732776642, + 0.14726711809635162, + 0.14656992256641388, + 0.13564367592334747, + 0.13979844748973846, + 0.13359025120735168, + 0.1417049616575241, + 0.13732580840587616, + 0.149912029504776, + 0.14843478798866272, + 0.16512760519981384, + 0.1652906984090805, + 0.19090111553668976, + 0.191492959856987, + 0.2512649893760681, + 0.25225311517715454, + 0.390801340341568 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00016188873269129544, + 0.00017867473070509732, + 0.00019946519751101732, + 0.00018539158918429166, + 0.00018016427929978818, + 0.0001743910543154925, + 0.00016342199523933232, + 0.00015197545872069895, + 0.0001410306867910549, + 0.00012255505134817213, + 0.00010388438386144117, + 8.946077286964282e-05, + 7.552531315013766e-05, + 6.115865835454315e-05, + 4.3820087739732116e-05, + 2.6790259653353132e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00018135506252292544, + "first_to_last_ratio": 3.499469999703496, + "imbalance_abs_log_ratio": 1.2526115283727475, + "last_quartile_mean": 5.1823579724441515e-05, + "mean": 0.00012872489071469317, + "normalized": [ + 1.257633483256217, + 1.3880355983452597, + 1.5495464506014887, + 1.4402155492615254, + 1.3996071645467982, + 1.3547578354679994, + 1.269544641537448, + 1.1806221615486823, + 1.0955976424453635, + 0.9520695699777604, + 0.8070263900374272, + 0.6949764911273015, + 0.58671879797927, + 0.475111363583098, + 0.3404165852962757, + 0.208120274988085 + ], + "population_cv": 0.4148571258695616 + }, + "activation_output_rms_by_block": [ + 0.09483804553747177, + 0.15792693197727203, + 0.18785393238067627, + 0.19029200077056885, + 0.19116424024105072, + 0.18977893888950348, + 0.18693965673446655, + 0.177143394947052, + 0.16181440651416779, + 0.1581212431192398, + 0.165044903755188, + 0.18009524047374725, + 0.1996685266494751, + 0.2291477918624878, + 0.29672855138778687, + 0.4953230619430542 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15772772766649723, + "first_to_last_ratio": 0.516772448690399, + "imbalance_abs_log_ratio": 0.6601526392924331, + "last_quartile_mean": 0.305216982960701, + "mean": 0.20386755419895053, + "normalized": [ + 0.46519440481893015, + 0.774654567264559, + 0.9214508562620668, + 0.9334099362611985, + 0.9376883977059789, + 0.9308932931245242, + 0.9169662012623924, + 0.8689141126114707, + 0.7937231951890498, + 0.7756076916728606, + 0.8095692539388771, + 0.8833933441806815, + 0.9794031592424085, + 1.1240032420208796, + 1.4554966951642292, + 2.429631649279893 + ], + "population_cv": 0.4175842970930077 + }, + "bits_per_byte": 1.9507894057257391, + "branch_output_rms_by_sublayer": [ + 0.03931046277284622, + 0.047202516347169876, + 0.08608254790306091, + 0.0076385182328522205, + 0.041328247636556625, + 0.004047342110425234, + 0.034119248390197754, + 0.0045016431249678135, + 0.029622135683894157, + 0.008596057072281837, + 0.02270515263080597, + 0.012007404118776321, + 0.02946399338543415, + 0.01667790301144123, + 0.038835883140563965, + 0.02453153021633625, + 0.05270649492740631, + 0.02996903844177723, + 0.05063421651721001, + 0.03910558298230171, + 0.04952584207057953, + 0.05079483985900879, + 0.03944792598485947, + 0.06468769162893295, + 0.0361105240881443, + 0.07707343250513077, + 0.028930658474564552, + 0.0911106988787651, + 0.02081119269132614, + 0.1352817416191101, + 0.02383771352469921, + 0.28301650285720825 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00041699002778082224, + 0.00029759414151732746, + 0.000265991514011251, + 0.0001953867836934275, + 0.00023560860135608575, + 0.00025046058190266666, + 0.00033611798182792255, + 0.0004636386291792892, + 0.0005491632897177901, + 0.0005951115480573408, + 0.0005882338016339408, + 0.0005513321353847482, + 0.0004945066783331371, + 0.00043544338635105514, + 0.0003756714670850321, + 0.0003149065782712711 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002939906167507071, + "first_to_last_ratio": 0.7256661946909654, + "imbalance_abs_log_ratio": 0.3206651568188065, + "last_quartile_mean": 0.00040513202751012384, + "mean": 0.00039788482163144424, + "normalized": [ + 1.0480169262829406, + 0.7479404222988562, + 0.6685138501152368, + 0.4910636774036378, + 0.5921527752428994, + 0.6294801115451073, + 0.8447620103972311, + 1.165258396332474, + 1.380206682592364, + 1.4956879873356546, + 1.4784022150480887, + 1.3856576210274245, + 1.2428387599846482, + 1.094395570470896, + 0.9441713949898092, + 0.7914515989327312 + ], + "population_cv": 0.32327688011764555 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.059103816747665405, + 0.07607264071702957, + 0.09483804553747177, + 0.15534481406211853, + 0.15792693197727203, + 0.18721190094947815, + 0.18785393238067627, + 0.19058401882648468, + 0.19029200077056885, + 0.1914086490869522, + 0.19116424024105072, + 0.19123691320419312, + 0.18977893888950348, + 0.19008098542690277, + 0.18693965673446655, + 0.18065746128559113, + 0.177143394947052, + 0.16187894344329834, + 0.16181440651416779, + 0.15150673687458038, + 0.1581212431192398, + 0.15270332992076874, + 0.165044903755188, + 0.16355812549591064, + 0.18009524047374725, + 0.17952413856983185, + 0.1996685266494751, + 0.2009342908859253, + 0.2291477918624878, + 0.23023094236850739, + 0.29672855138778687, + 0.29810962080955505 + ], + "loss_nats": 1.3521841764450073, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.07607264071702957, + 0.09483804553747177, + 0.15534481406211853, + 0.15792693197727203, + 0.18721190094947815, + 0.18785393238067627, + 0.19058401882648468, + 0.19029200077056885, + 0.1914086490869522, + 0.19116424024105072, + 0.19123691320419312, + 0.18977893888950348, + 0.19008098542690277, + 0.18693965673446655, + 0.18065746128559113, + 0.177143394947052, + 0.16187894344329834, + 0.16181440651416779, + 0.15150673687458038, + 0.1581212431192398, + 0.15270332992076874, + 0.165044903755188, + 0.16355812549591064, + 0.18009524047374725, + 0.17952413856983185, + 0.1996685266494751, + 0.2009342908859253, + 0.2291477918624878, + 0.23023094236850739, + 0.29672855138778687, + 0.29810962080955505, + 0.4953230619430542 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00025817437563091516, + 0.00027957011479884386, + 0.0003037556598428637, + 0.0002888414019253105, + 0.00028409145306795835, + 0.00028240776737220585, + 0.00026678573340177536, + 0.0002519928675610572, + 0.00023668054200243205, + 0.00020713888807222247, + 0.00017311860574409366, + 0.0001455697201890871, + 0.00011939310206798837, + 9.376799425808713e-05, + 6.36211916571483e-05, + 3.1806663173483685e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002825853880494833, + "first_to_last_ratio": 3.662935915109688, + "imbalance_abs_log_ratio": 1.2982649884510844, + "last_quartile_mean": 7.714723778917687e-05, + "mean": 0.00020541975504784205, + "normalized": [ + 1.2568137644346165, + 1.3609699550743417, + 1.478707146603886, + 1.4061033314835745, + 1.3829801958521164, + 1.3747838775605266, + 1.2987345513075956, + 1.226721682646191, + 1.1521800420183999, + 1.0083688787574498, + 0.8427553898298366, + 0.7086451843698482, + 0.5812152878878755, + 0.4564701882554999, + 0.30971311226776116, + 0.15483741165048096 + ], + "population_cv": 0.41926708997335715 + }, + "activation_output_rms_by_block": [ + 0.0951872169971466, + 0.14984266459941864, + 0.17071209847927094, + 0.17147330939769745, + 0.17152385413646698, + 0.17004960775375366, + 0.1669105589389801, + 0.15593038499355316, + 0.1393769234418869, + 0.1344291716814041, + 0.13867342472076416, + 0.1521374136209488, + 0.16997921466827393, + 0.1961352825164795, + 0.25573307275772095, + 0.4589640200138092 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1468038223683834, + "first_to_last_ratio": 0.5433095785892573, + "imbalance_abs_log_ratio": 0.6100759951930727, + "last_quartile_mean": 0.2702028974890709, + "mean": 0.18106613866984844, + "normalized": [ + 0.5257041305260761, + 0.8275576300472066, + 0.9428162534052995, + 0.9470203024009789, + 0.94729945309774, + 0.9391574206142317, + 0.9218209443529402, + 0.861179158836942, + 0.7697569764605372, + 0.7424313163629063, + 0.7658716629154935, + 0.8402311704363035, + 0.9387686505990491, + 1.0832245275529278, + 1.4123738134385158, + 2.534786588952852 + ], + "population_cv": 0.4350736575110673 + }, + "bits_per_byte": 1.8670605120664352, + "branch_output_rms_by_sublayer": [ + 0.051385972648859024, + 0.036380402743816376, + 0.07075809687376022, + 0.0071685765869915485, + 0.030611267313361168, + 0.003892687615007162, + 0.023587750270962715, + 0.004152141511440277, + 0.018766717985272408, + 0.007657408714294434, + 0.01761440560221672, + 0.010538195259869099, + 0.02535838447511196, + 0.014725946821272373, + 0.03455089405179024, + 0.021837059408426285, + 0.046537015587091446, + 0.02783774584531784, + 0.04248657822608948, + 0.03665589541196823, + 0.044122546911239624, + 0.04793383553624153, + 0.034650206565856934, + 0.06113889440894127, + 0.031748466193675995, + 0.07430189102888107, + 0.0256944727152586, + 0.0880647599697113, + 0.019152166321873665, + 0.12767614424228668, + 0.024809958413243294, + 0.2864501476287842 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005002075550754018, + 0.0003412904728571966, + 0.0002592425753327288, + 0.0002585649119364213, + 0.00030071075691208866, + 0.0003425364164715924, + 0.0005240790570344413, + 0.0006975962934822548, + 0.0008810870404936847, + 0.000990036099264455, + 0.0009754970927650687, + 0.0008971247584696028, + 0.0007874986100445877, + 0.000666994667011454, + 0.0005527789362036235, + 0.0004234912235697502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00033982637880043714, + "first_to_last_ratio": 0.5592092980363276, + "imbalance_abs_log_ratio": 0.5812314608110569, + "last_quartile_mean": 0.0006076908592073538, + "mean": 0.0005874210291827721, + "normalized": [ + 0.8515315765445054, + 0.580998050635001, + 0.44132327998776366, + 0.4401696553086297, + 0.5119169079296352, + 0.5831190908301898, + 0.8921693827739654, + 1.1875575759566523, + 1.4999242395517653, + 1.6853943765714456, + 1.6606438045335101, + 1.5272261527948612, + 1.3406033678095697, + 1.135462698602033, + 0.9410268082718403, + 0.7209330318986312 + ], + "population_cv": 0.42806100897327076 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060872070491313934, + 0.0852167010307312, + 0.0951872169971466, + 0.14736860990524292, + 0.14984266459941864, + 0.1702466458082199, + 0.17071209847927094, + 0.17185615003108978, + 0.17147330939769745, + 0.1719079613685608, + 0.17152385413646698, + 0.17163211107254028, + 0.17004960775375366, + 0.16988061368465424, + 0.1669105589389801, + 0.1597038209438324, + 0.15593038499355316, + 0.14019693434238434, + 0.1393769234418869, + 0.12975449860095978, + 0.1344291716814041, + 0.1281583607196808, + 0.13867342472076416, + 0.13766533136367798, + 0.1521374136209488, + 0.15170666575431824, + 0.16997921466827393, + 0.1708848923444748, + 0.1961352825164795, + 0.19691267609596252, + 0.25573307275772095, + 0.25681471824645996 + ], + "loss_nats": 1.2941477298736572, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0852167010307312, + 0.0951872169971466, + 0.14736860990524292, + 0.14984266459941864, + 0.1702466458082199, + 0.17071209847927094, + 0.17185615003108978, + 0.17147330939769745, + 0.1719079613685608, + 0.17152385413646698, + 0.17163211107254028, + 0.17004960775375366, + 0.16988061368465424, + 0.1669105589389801, + 0.1597038209438324, + 0.15593038499355316, + 0.14019693434238434, + 0.1393769234418869, + 0.12975449860095978, + 0.1344291716814041, + 0.1281583607196808, + 0.13867342472076416, + 0.13766533136367798, + 0.1521374136209488, + 0.15170666575431824, + 0.16997921466827393, + 0.1708848923444748, + 0.1961352825164795, + 0.19691267609596252, + 0.25573307275772095, + 0.25681471824645996, + 0.4589640200138092 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.018429854831792, + "cross_entropy_nats": 5.557952046394348, + "step": 0 + }, + { + "bits_per_byte": 5.108882101013433, + "cross_entropy_nats": 3.5412072241306305, + "step": 100 + }, + { + "bits_per_byte": 3.154257052216826, + "cross_entropy_nats": 2.186364382505417, + "step": 500 + }, + { + "bits_per_byte": 1.977161870343983, + "cross_entropy_nats": 1.37046417593956, + "step": 2000 + }, + { + "bits_per_byte": 1.8329381437371801, + "cross_entropy_nats": 1.2704959064722061, + "step": 4000 + }, + { + "bits_per_byte": 1.7363699927668699, + "cross_entropy_nats": 1.2035599648952484, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "final_optimizer_state": "de12f865acf312961efff64f13ff04f15f5b52fc60a03097833a902e65a3e5f3", + "final_public_parameters": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 21.482708441361407, + "measured_steps": 7980, + "median_ms": 21.420384509838186, + "p95_ms": 21.788083588762674, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3282042880, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.034161279888863, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.568856239318848, + "step": 1, + "unclipped_grad_norm": 18.562814712524414 + }, + { + "bits_per_byte": 7.448981367230313, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.163240432739258, + "step": 10, + "unclipped_grad_norm": 12.974101066589355 + }, + { + "bits_per_byte": 6.770661154309292, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6930646896362305, + "step": 20, + "unclipped_grad_norm": 4.495212554931641 + }, + { + "bits_per_byte": 6.551721736316109, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.54130744934082, + "step": 30, + "unclipped_grad_norm": 3.012619972229004 + }, + { + "bits_per_byte": 6.419676896636662, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.4497809410095215, + "step": 40, + "unclipped_grad_norm": 2.8561618328094482 + }, + { + "bits_per_byte": 6.250720452498435, + "learning_rate": 3.75e-05, + "loss_nats": 4.332669258117676, + "step": 50, + "unclipped_grad_norm": 2.8167924880981445 + }, + { + "bits_per_byte": 5.9407609363626195, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.11782169342041, + "step": 60, + "unclipped_grad_norm": 2.718289375305176 + }, + { + "bits_per_byte": 5.7053297580129, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9546332359313965, + "step": 70, + "unclipped_grad_norm": 2.7082417011260986 + }, + { + "bits_per_byte": 5.509554339299127, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.818932056427002, + "step": 80, + "unclipped_grad_norm": 2.5339207649230957 + }, + { + "bits_per_byte": 5.312337705971124, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.682231903076172, + "step": 90, + "unclipped_grad_norm": 2.2857937812805176 + }, + { + "bits_per_byte": 5.088000784463262, + "learning_rate": 7.5e-05, + "loss_nats": 3.5267333984375, + "step": 100, + "unclipped_grad_norm": 2.166426658630371 + }, + { + "bits_per_byte": 4.935865619440625, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.421281337738037, + "step": 110, + "unclipped_grad_norm": 2.0095791816711426 + }, + { + "bits_per_byte": 4.7340020148863085, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.281360149383545, + "step": 120, + "unclipped_grad_norm": 1.7170522212982178 + }, + { + "bits_per_byte": 4.584287677657216, + "learning_rate": 9.75e-05, + "loss_nats": 3.177586078643799, + "step": 130, + "unclipped_grad_norm": 1.5826245546340942 + }, + { + "bits_per_byte": 4.439357553809826, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0771281719207764, + "step": 140, + "unclipped_grad_norm": 1.4182634353637695 + }, + { + "bits_per_byte": 4.232185532753978, + "learning_rate": 0.0001125, + "loss_nats": 2.9335274696350098, + "step": 150, + "unclipped_grad_norm": 1.1410893201828003 + }, + { + "bits_per_byte": 4.110705931282335, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8493242263793945, + "step": 160, + "unclipped_grad_norm": 0.887942910194397 + }, + { + "bits_per_byte": 4.028603133586938, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.792414903640747, + "step": 170, + "unclipped_grad_norm": 0.9387072920799255 + }, + { + "bits_per_byte": 3.9322216482863084, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7256083488464355, + "step": 180, + "unclipped_grad_norm": 0.7101461887359619 + }, + { + "bits_per_byte": 3.8536510302743325, + "learning_rate": 0.0001425, + "loss_nats": 2.671147346496582, + "step": 190, + "unclipped_grad_norm": 0.90241938829422 + }, + { + "bits_per_byte": 3.794223801129896, + "learning_rate": 0.00015, + "loss_nats": 2.629955530166626, + "step": 200, + "unclipped_grad_norm": 0.8553160429000854 + }, + { + "bits_per_byte": 3.7222494057091122, + "learning_rate": 0.0001575, + "loss_nats": 2.580066680908203, + "step": 210, + "unclipped_grad_norm": 0.7590162754058838 + }, + { + "bits_per_byte": 3.70871849866983, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.570687770843506, + "step": 220, + "unclipped_grad_norm": 0.8053720593452454 + }, + { + "bits_per_byte": 3.6584135732924583, + "learning_rate": 0.0001725, + "loss_nats": 2.5358190536499023, + "step": 230, + "unclipped_grad_norm": 0.757310152053833 + }, + { + "bits_per_byte": 3.6516113154855327, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.53110408782959, + "step": 240, + "unclipped_grad_norm": 0.6297867894172668 + }, + { + "bits_per_byte": 3.5768769424837026, + "learning_rate": 0.0001875, + "loss_nats": 2.479302167892456, + "step": 250, + "unclipped_grad_norm": 0.5848793387413025 + }, + { + "bits_per_byte": 3.595435590338308, + "learning_rate": 0.000195, + "loss_nats": 2.492166042327881, + "step": 260, + "unclipped_grad_norm": 0.7728506326675415 + }, + { + "bits_per_byte": 3.581017252820574, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4821720123291016, + "step": 270, + "unclipped_grad_norm": 0.6333379149436951 + }, + { + "bits_per_byte": 3.5262535052369333, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4442126750946045, + "step": 280, + "unclipped_grad_norm": 0.6880936026573181 + }, + { + "bits_per_byte": 3.5151664716664115, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.436527729034424, + "step": 290, + "unclipped_grad_norm": 0.6548541188240051 + }, + { + "bits_per_byte": 3.500701698832944, + "learning_rate": 0.000225, + "loss_nats": 2.426501512527466, + "step": 300, + "unclipped_grad_norm": 0.6513921618461609 + }, + { + "bits_per_byte": 3.4243479371905896, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.373577117919922, + "step": 310, + "unclipped_grad_norm": 0.6903060674667358 + }, + { + "bits_per_byte": 3.495896847533139, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.423171043395996, + "step": 320, + "unclipped_grad_norm": 1.2031209468841553 + }, + { + "bits_per_byte": 3.486328076804434, + "learning_rate": 0.0002475, + "loss_nats": 2.4165384769439697, + "step": 330, + "unclipped_grad_norm": 0.8043278455734253 + }, + { + "bits_per_byte": 3.4281229563770284, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3761937618255615, + "step": 340, + "unclipped_grad_norm": 0.7758461236953735 + }, + { + "bits_per_byte": 3.3969462853939203, + "learning_rate": 0.0002625, + "loss_nats": 2.354583740234375, + "step": 350, + "unclipped_grad_norm": 0.7247553467750549 + }, + { + "bits_per_byte": 3.3922525348865724, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.351330280303955, + "step": 360, + "unclipped_grad_norm": 0.7819244265556335 + }, + { + "bits_per_byte": 3.4131862631495933, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.365840435028076, + "step": 370, + "unclipped_grad_norm": 0.8280189037322998 + }, + { + "bits_per_byte": 3.3082022056528055, + "learning_rate": 0.000285, + "loss_nats": 2.2930710315704346, + "step": 380, + "unclipped_grad_norm": 0.7057737112045288 + }, + { + "bits_per_byte": 3.3912378372464818, + "learning_rate": 0.0002925, + "loss_nats": 2.3506269454956055, + "step": 390, + "unclipped_grad_norm": 0.7360323071479797 + }, + { + "bits_per_byte": 3.361113700086654, + "learning_rate": 0.0003, + "loss_nats": 2.3297464847564697, + "step": 400, + "unclipped_grad_norm": 0.6539704203605652 + }, + { + "bits_per_byte": 3.4030207126223946, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.3587942123413086, + "step": 410, + "unclipped_grad_norm": 0.791742205619812 + }, + { + "bits_per_byte": 3.334176745412616, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.311075210571289, + "step": 420, + "unclipped_grad_norm": 0.8703852891921997 + }, + { + "bits_per_byte": 3.3591761435420335, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.3284034729003906, + "step": 430, + "unclipped_grad_norm": 0.8373144865036011 + }, + { + "bits_per_byte": 3.313656119476967, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.296851396560669, + "step": 440, + "unclipped_grad_norm": 1.3769525289535522 + }, + { + "bits_per_byte": 3.3562916505217624, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.326404094696045, + "step": 450, + "unclipped_grad_norm": 0.7585985660552979 + }, + { + "bits_per_byte": 3.2828990861619642, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2755322456359863, + "step": 460, + "unclipped_grad_norm": 0.9068207144737244 + }, + { + "bits_per_byte": 3.2566393991670237, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2573304176330566, + "step": 470, + "unclipped_grad_norm": 0.9806933999061584 + }, + { + "bits_per_byte": 3.2517141600116144, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2539165019989014, + "step": 480, + "unclipped_grad_norm": 1.2366756200790405 + }, + { + "bits_per_byte": 3.1895775161854854, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.2108466625213623, + "step": 490, + "unclipped_grad_norm": 0.7806023955345154 + }, + { + "bits_per_byte": 3.123298498230674, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.164905548095703, + "step": 500, + "unclipped_grad_norm": 1.1635692119598389 + }, + { + "bits_per_byte": 3.1673697404448724, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.195453405380249, + "step": 510, + "unclipped_grad_norm": 1.5141944885253906 + }, + { + "bits_per_byte": 3.1538006532570986, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1860480308532715, + "step": 520, + "unclipped_grad_norm": 1.1977208852767944 + }, + { + "bits_per_byte": 2.9931695452818388, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.07470703125, + "step": 530, + "unclipped_grad_norm": 1.6449142694473267 + }, + { + "bits_per_byte": 2.988713818763285, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0716185569763184, + "step": 540, + "unclipped_grad_norm": 1.1201231479644775 + }, + { + "bits_per_byte": 2.9807878263156216, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.066124677658081, + "step": 550, + "unclipped_grad_norm": 1.0647740364074707 + }, + { + "bits_per_byte": 2.990274733302516, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0727005004882812, + "step": 560, + "unclipped_grad_norm": 1.5665196180343628 + }, + { + "bits_per_byte": 2.920768976852217, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0245227813720703, + "step": 570, + "unclipped_grad_norm": 1.1736958026885986 + }, + { + "bits_per_byte": 2.9347649249794325, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.034224033355713, + "step": 580, + "unclipped_grad_norm": 1.3881648778915405 + }, + { + "bits_per_byte": 2.871172963961702, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9901454448699951, + "step": 590, + "unclipped_grad_norm": 0.9850356578826904 + }, + { + "bits_per_byte": 2.835541942321247, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9654479026794434, + "step": 600, + "unclipped_grad_norm": 1.141566514968872 + }, + { + "bits_per_byte": 2.8507572474430796, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.975994348526001, + "step": 610, + "unclipped_grad_norm": 1.0074148178100586 + }, + { + "bits_per_byte": 2.8309881856917016, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9622914791107178, + "step": 620, + "unclipped_grad_norm": 1.614481806755066 + }, + { + "bits_per_byte": 2.744016559150379, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9020073413848877, + "step": 630, + "unclipped_grad_norm": 1.23131263256073 + }, + { + "bits_per_byte": 2.7596193411846057, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9128223657608032, + "step": 640, + "unclipped_grad_norm": 1.2868338823318481 + }, + { + "bits_per_byte": 2.7369749014934524, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8971264362335205, + "step": 650, + "unclipped_grad_norm": 1.4851713180541992 + }, + { + "bits_per_byte": 2.7511890956046128, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.906978964805603, + "step": 660, + "unclipped_grad_norm": 0.9212717413902283 + }, + { + "bits_per_byte": 2.7285928110557007, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8913164138793945, + "step": 670, + "unclipped_grad_norm": 1.196967601776123 + }, + { + "bits_per_byte": 2.7059170705220903, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8755987882614136, + "step": 680, + "unclipped_grad_norm": 1.0040963888168335 + }, + { + "bits_per_byte": 2.678678114313137, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8567181825637817, + "step": 690, + "unclipped_grad_norm": 1.19021475315094 + }, + { + "bits_per_byte": 2.6497494285594567, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8366663455963135, + "step": 700, + "unclipped_grad_norm": 1.2448054552078247 + }, + { + "bits_per_byte": 2.6450725323518935, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8334245681762695, + "step": 710, + "unclipped_grad_norm": 1.2336615324020386 + }, + { + "bits_per_byte": 2.640725326886034, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8304113149642944, + "step": 720, + "unclipped_grad_norm": 1.0797665119171143 + }, + { + "bits_per_byte": 2.6141132474394757, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8119652271270752, + "step": 730, + "unclipped_grad_norm": 1.052964448928833 + }, + { + "bits_per_byte": 2.5497685183589187, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7673648595809937, + "step": 740, + "unclipped_grad_norm": 1.2775253057479858 + }, + { + "bits_per_byte": 2.5706501358787284, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7818388938903809, + "step": 750, + "unclipped_grad_norm": 1.240071415901184 + }, + { + "bits_per_byte": 2.470865801848127, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7126736640930176, + "step": 760, + "unclipped_grad_norm": 1.5425997972488403 + }, + { + "bits_per_byte": 2.521409439144896, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7477078437805176, + "step": 770, + "unclipped_grad_norm": 1.3344701528549194 + }, + { + "bits_per_byte": 2.472837410957618, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7140402793884277, + "step": 780, + "unclipped_grad_norm": 1.1049633026123047 + }, + { + "bits_per_byte": 2.473506767434776, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7145042419433594, + "step": 790, + "unclipped_grad_norm": 1.1233389377593994 + }, + { + "bits_per_byte": 2.401472866016794, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.664574146270752, + "step": 800, + "unclipped_grad_norm": 1.229269027709961 + }, + { + "bits_per_byte": 2.4415747486490784, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6923706531524658, + "step": 810, + "unclipped_grad_norm": 1.1753153800964355 + }, + { + "bits_per_byte": 2.4350285730092875, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6878331899642944, + "step": 820, + "unclipped_grad_norm": 1.128970742225647 + }, + { + "bits_per_byte": 2.4442745323023227, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.694242000579834, + "step": 830, + "unclipped_grad_norm": 1.1320509910583496 + }, + { + "bits_per_byte": 2.411402456347009, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6714568138122559, + "step": 840, + "unclipped_grad_norm": 1.2860697507858276 + }, + { + "bits_per_byte": 2.3597125546822393, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6356281042099, + "step": 850, + "unclipped_grad_norm": 1.1755540370941162 + }, + { + "bits_per_byte": 2.4397532803937905, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6911081075668335, + "step": 860, + "unclipped_grad_norm": 1.4042041301727295 + }, + { + "bits_per_byte": 2.4029735866282227, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.665614366531372, + "step": 870, + "unclipped_grad_norm": 1.2927662134170532 + }, + { + "bits_per_byte": 2.3685887512759196, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6417806148529053, + "step": 880, + "unclipped_grad_norm": 1.1862192153930664 + }, + { + "bits_per_byte": 2.3211230874816424, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6088799238204956, + "step": 890, + "unclipped_grad_norm": 1.135436773300171 + }, + { + "bits_per_byte": 2.364436574136139, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6389025449752808, + "step": 900, + "unclipped_grad_norm": 1.3485081195831299 + }, + { + "bits_per_byte": 2.347153177637681, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.626922607421875, + "step": 910, + "unclipped_grad_norm": 1.1310322284698486 + }, + { + "bits_per_byte": 2.338854326802947, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6211702823638916, + "step": 920, + "unclipped_grad_norm": 1.1499736309051514 + }, + { + "bits_per_byte": 2.28766523861749, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5856887102127075, + "step": 930, + "unclipped_grad_norm": 1.0895946025848389 + }, + { + "bits_per_byte": 2.3220005429663444, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6094881296157837, + "step": 940, + "unclipped_grad_norm": 1.0576175451278687 + }, + { + "bits_per_byte": 2.3596927766773903, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6356143951416016, + "step": 950, + "unclipped_grad_norm": 1.078567385673523 + }, + { + "bits_per_byte": 2.2833796029406397, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5827181339263916, + "step": 960, + "unclipped_grad_norm": 1.1438634395599365 + }, + { + "bits_per_byte": 2.2449070839426293, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5560510158538818, + "step": 970, + "unclipped_grad_norm": 1.2200326919555664 + }, + { + "bits_per_byte": 2.3829726922634817, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6517508029937744, + "step": 980, + "unclipped_grad_norm": 1.0979547500610352 + }, + { + "bits_per_byte": 2.2178129371257023, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5372707843780518, + "step": 990, + "unclipped_grad_norm": 0.9582167863845825 + }, + { + "bits_per_byte": 2.2343387500471095, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5487256050109863, + "step": 1000, + "unclipped_grad_norm": 1.1553164720535278 + }, + { + "bits_per_byte": 2.2444941535979077, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5557647943496704, + "step": 1010, + "unclipped_grad_norm": 1.5419076681137085 + }, + { + "bits_per_byte": 2.204213924956678, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278446674346924, + "step": 1020, + "unclipped_grad_norm": 1.2040239572525024 + }, + { + "bits_per_byte": 2.2117641353122064, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5330780744552612, + "step": 1030, + "unclipped_grad_norm": 1.308585286140442 + }, + { + "bits_per_byte": 2.2977618241016926, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5926871299743652, + "step": 1040, + "unclipped_grad_norm": 1.4551756381988525 + }, + { + "bits_per_byte": 2.2190181915429488, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5381062030792236, + "step": 1050, + "unclipped_grad_norm": 1.5392372608184814 + }, + { + "bits_per_byte": 2.226578548874878, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.543346643447876, + "step": 1060, + "unclipped_grad_norm": 1.1143345832824707 + }, + { + "bits_per_byte": 2.276697561006667, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.578086495399475, + "step": 1070, + "unclipped_grad_norm": 1.1605373620986938 + }, + { + "bits_per_byte": 2.2501855754629103, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5597097873687744, + "step": 1080, + "unclipped_grad_norm": 1.0904817581176758 + }, + { + "bits_per_byte": 2.178866605924565, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5102752447128296, + "step": 1090, + "unclipped_grad_norm": 1.1060513257980347 + }, + { + "bits_per_byte": 2.2050501046051734, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5284242630004883, + "step": 1100, + "unclipped_grad_norm": 1.2513083219528198 + }, + { + "bits_per_byte": 2.140159674573429, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.483445644378662, + "step": 1110, + "unclipped_grad_norm": 1.0150246620178223 + }, + { + "bits_per_byte": 2.197250347423243, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5230178833007812, + "step": 1120, + "unclipped_grad_norm": 1.0046241283416748 + }, + { + "bits_per_byte": 2.212073188135807, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.533292293548584, + "step": 1130, + "unclipped_grad_norm": 0.9860795736312866 + }, + { + "bits_per_byte": 2.1513696757393173, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4912158250808716, + "step": 1140, + "unclipped_grad_norm": 1.0693174600601196 + }, + { + "bits_per_byte": 2.217324850362554, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5369324684143066, + "step": 1150, + "unclipped_grad_norm": 1.2721859216690063 + }, + { + "bits_per_byte": 2.1164305403032837, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4669978618621826, + "step": 1160, + "unclipped_grad_norm": 1.2611263990402222 + }, + { + "bits_per_byte": 2.1545339845325424, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4934091567993164, + "step": 1170, + "unclipped_grad_norm": 1.396773099899292 + }, + { + "bits_per_byte": 2.157210722510571, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4952645301818848, + "step": 1180, + "unclipped_grad_norm": 1.0693202018737793 + }, + { + "bits_per_byte": 2.146851003570548, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4880837202072144, + "step": 1190, + "unclipped_grad_norm": 1.1468050479888916 + }, + { + "bits_per_byte": 2.1677991783762414, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.5026038885116577, + "step": 1200, + "unclipped_grad_norm": 1.1243788003921509 + }, + { + "bits_per_byte": 2.171953075342531, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5054831504821777, + "step": 1210, + "unclipped_grad_norm": 1.0842303037643433 + }, + { + "bits_per_byte": 2.1393570315418518, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4828892946243286, + "step": 1220, + "unclipped_grad_norm": 1.0247055292129517 + }, + { + "bits_per_byte": 2.133679196306267, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4789537191390991, + "step": 1230, + "unclipped_grad_norm": 1.0191700458526611 + }, + { + "bits_per_byte": 2.0961321159525497, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.452928066253662, + "step": 1240, + "unclipped_grad_norm": 1.0066498517990112 + }, + { + "bits_per_byte": 2.0423632880041502, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4156583547592163, + "step": 1250, + "unclipped_grad_norm": 1.1115199327468872 + }, + { + "bits_per_byte": 2.102061045858394, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4570376873016357, + "step": 1260, + "unclipped_grad_norm": 1.1423614025115967 + }, + { + "bits_per_byte": 2.1196135952054522, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4692041873931885, + "step": 1270, + "unclipped_grad_norm": 0.9267144203186035 + }, + { + "bits_per_byte": 2.1628902775726644, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4992012977600098, + "step": 1280, + "unclipped_grad_norm": 1.1472139358520508 + }, + { + "bits_per_byte": 2.104273774644395, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.458571434020996, + "step": 1290, + "unclipped_grad_norm": 1.0092620849609375 + }, + { + "bits_per_byte": 2.0607607881148975, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.428410530090332, + "step": 1300, + "unclipped_grad_norm": 1.0175256729125977 + }, + { + "bits_per_byte": 2.090551794880004, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.449060082435608, + "step": 1310, + "unclipped_grad_norm": 1.0536231994628906 + }, + { + "bits_per_byte": 2.0678022737891735, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4332913160324097, + "step": 1320, + "unclipped_grad_norm": 1.0326532125473022 + }, + { + "bits_per_byte": 2.079288479092348, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4412529468536377, + "step": 1330, + "unclipped_grad_norm": 1.0093759298324585 + }, + { + "bits_per_byte": 2.077430378532426, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.439965009689331, + "step": 1340, + "unclipped_grad_norm": 0.9901224970817566 + }, + { + "bits_per_byte": 2.085392143371469, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.445483684539795, + "step": 1350, + "unclipped_grad_norm": 0.9441472291946411 + }, + { + "bits_per_byte": 2.0594795173659697, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4275224208831787, + "step": 1360, + "unclipped_grad_norm": 1.1019998788833618 + }, + { + "bits_per_byte": 2.0796826633281253, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.441526174545288, + "step": 1370, + "unclipped_grad_norm": 1.3928191661834717 + }, + { + "bits_per_byte": 2.0298482824835147, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4069836139678955, + "step": 1380, + "unclipped_grad_norm": 1.0244042873382568 + }, + { + "bits_per_byte": 2.0804338835470944, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.442046880722046, + "step": 1390, + "unclipped_grad_norm": 1.095209002494812 + }, + { + "bits_per_byte": 2.195902519388431, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.5220836400985718, + "step": 1400, + "unclipped_grad_norm": 1.1566489934921265 + }, + { + "bits_per_byte": 2.0701701309262526, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4349325895309448, + "step": 1410, + "unclipped_grad_norm": 0.889271080493927 + }, + { + "bits_per_byte": 2.060555784795069, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4282684326171875, + "step": 1420, + "unclipped_grad_norm": 0.933581531047821 + }, + { + "bits_per_byte": 2.0393702739311848, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.413583755493164, + "step": 1430, + "unclipped_grad_norm": 1.1070539951324463 + }, + { + "bits_per_byte": 2.055492615553668, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4247589111328125, + "step": 1440, + "unclipped_grad_norm": 1.024774193763733 + }, + { + "bits_per_byte": 2.0178352943207476, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3986568450927734, + "step": 1450, + "unclipped_grad_norm": 0.9543858766555786 + }, + { + "bits_per_byte": 2.144940448302113, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4867594242095947, + "step": 1460, + "unclipped_grad_norm": 0.9325331449508667 + }, + { + "bits_per_byte": 2.1008117638825268, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4561717510223389, + "step": 1470, + "unclipped_grad_norm": 0.8672807216644287 + }, + { + "bits_per_byte": 2.0709982273901573, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4355065822601318, + "step": 1480, + "unclipped_grad_norm": 0.9597178101539612 + }, + { + "bits_per_byte": 2.0747749664031043, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.438124418258667, + "step": 1490, + "unclipped_grad_norm": 1.0223972797393799 + }, + { + "bits_per_byte": 1.9616643846877473, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3597221374511719, + "step": 1500, + "unclipped_grad_norm": 0.9415771961212158 + }, + { + "bits_per_byte": 1.978057942950641, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.371085286140442, + "step": 1510, + "unclipped_grad_norm": 1.0318301916122437 + }, + { + "bits_per_byte": 2.0205063568712975, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.4005082845687866, + "step": 1520, + "unclipped_grad_norm": 0.9729053378105164 + }, + { + "bits_per_byte": 2.0668845743641695, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4326552152633667, + "step": 1530, + "unclipped_grad_norm": 0.9350422620773315 + }, + { + "bits_per_byte": 2.045209256910628, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4176310300827026, + "step": 1540, + "unclipped_grad_norm": 0.9831364154815674 + }, + { + "bits_per_byte": 2.101370363532529, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4565589427947998, + "step": 1550, + "unclipped_grad_norm": 1.1195578575134277 + }, + { + "bits_per_byte": 1.9633480948396942, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.360889196395874, + "step": 1560, + "unclipped_grad_norm": 1.038403034210205 + }, + { + "bits_per_byte": 1.9669683296403562, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.363398551940918, + "step": 1570, + "unclipped_grad_norm": 0.8551042675971985 + }, + { + "bits_per_byte": 1.997066325437253, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.384260892868042, + "step": 1580, + "unclipped_grad_norm": 0.9270023107528687 + }, + { + "bits_per_byte": 2.0065728384463455, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.390850305557251, + "step": 1590, + "unclipped_grad_norm": 0.9501463174819946 + }, + { + "bits_per_byte": 1.9863786355820345, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3768527507781982, + "step": 1600, + "unclipped_grad_norm": 0.8800983428955078 + }, + { + "bits_per_byte": 1.9619350853802053, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3599097728729248, + "step": 1610, + "unclipped_grad_norm": 0.911471962928772 + }, + { + "bits_per_byte": 1.9591475905750209, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3579776287078857, + "step": 1620, + "unclipped_grad_norm": 0.9253746271133423 + }, + { + "bits_per_byte": 1.9975408255709834, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3845897912979126, + "step": 1630, + "unclipped_grad_norm": 0.932426393032074 + }, + { + "bits_per_byte": 1.9590067367839643, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3578799962997437, + "step": 1640, + "unclipped_grad_norm": 0.9996587634086609 + }, + { + "bits_per_byte": 1.9568615971797525, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3563930988311768, + "step": 1650, + "unclipped_grad_norm": 0.9020648002624512 + }, + { + "bits_per_byte": 1.9902011219801112, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.379502296447754, + "step": 1660, + "unclipped_grad_norm": 0.9605674743652344 + }, + { + "bits_per_byte": 1.9362635790512155, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3421156406402588, + "step": 1670, + "unclipped_grad_norm": 0.9840894937515259 + }, + { + "bits_per_byte": 1.9741790462430813, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3683966398239136, + "step": 1680, + "unclipped_grad_norm": 0.9542845487594604 + }, + { + "bits_per_byte": 1.975246198591685, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3691363334655762, + "step": 1690, + "unclipped_grad_norm": 0.8774324655532837 + }, + { + "bits_per_byte": 1.936552509904665, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.342315912246704, + "step": 1700, + "unclipped_grad_norm": 0.9619165658950806 + }, + { + "bits_per_byte": 1.9815170300074447, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3734829425811768, + "step": 1710, + "unclipped_grad_norm": 1.017353892326355 + }, + { + "bits_per_byte": 1.9011681113855474, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3177893161773682, + "step": 1720, + "unclipped_grad_norm": 0.890689492225647 + }, + { + "bits_per_byte": 1.9742225578537496, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.36842679977417, + "step": 1730, + "unclipped_grad_norm": 0.9096551537513733 + }, + { + "bits_per_byte": 1.9832260216090685, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3746675252914429, + "step": 1740, + "unclipped_grad_norm": 0.9014292359352112 + }, + { + "bits_per_byte": 1.9548246346629332, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3549811840057373, + "step": 1750, + "unclipped_grad_norm": 0.8635335564613342 + }, + { + "bits_per_byte": 1.9935654465962895, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3818342685699463, + "step": 1760, + "unclipped_grad_norm": 0.9282572865486145 + }, + { + "bits_per_byte": 1.9777825987266096, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.370894432067871, + "step": 1770, + "unclipped_grad_norm": 0.9273641705513 + }, + { + "bits_per_byte": 2.0529696300655105, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.4230101108551025, + "step": 1780, + "unclipped_grad_norm": 0.9726579189300537 + }, + { + "bits_per_byte": 1.9025646105105534, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3187572956085205, + "step": 1790, + "unclipped_grad_norm": 0.9649468660354614 + }, + { + "bits_per_byte": 2.051186341958714, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.4217740297317505, + "step": 1800, + "unclipped_grad_norm": 1.1441059112548828 + }, + { + "bits_per_byte": 1.9169516471858308, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3287296295166016, + "step": 1810, + "unclipped_grad_norm": 1.0408838987350464 + }, + { + "bits_per_byte": 1.9713771048952242, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3664544820785522, + "step": 1820, + "unclipped_grad_norm": 0.9737800359725952 + }, + { + "bits_per_byte": 1.9989835880290714, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.385589838027954, + "step": 1830, + "unclipped_grad_norm": 0.8691428899765015 + }, + { + "bits_per_byte": 1.9804913254776988, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.372771978378296, + "step": 1840, + "unclipped_grad_norm": 0.915309727191925 + }, + { + "bits_per_byte": 1.918886795973339, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.330070972442627, + "step": 1850, + "unclipped_grad_norm": 0.9071885347366333 + }, + { + "bits_per_byte": 1.9458363053808903, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3487509489059448, + "step": 1860, + "unclipped_grad_norm": 0.8704239726066589 + }, + { + "bits_per_byte": 2.024965007094916, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4035987854003906, + "step": 1870, + "unclipped_grad_norm": 0.9987708330154419 + }, + { + "bits_per_byte": 1.9165194547842126, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3284300565719604, + "step": 1880, + "unclipped_grad_norm": 0.868863582611084 + }, + { + "bits_per_byte": 1.9462939512148363, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3490681648254395, + "step": 1890, + "unclipped_grad_norm": 0.8501539826393127 + }, + { + "bits_per_byte": 1.9456569274760405, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3486266136169434, + "step": 1900, + "unclipped_grad_norm": 0.8660019040107727 + }, + { + "bits_per_byte": 1.9970934986960893, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.384279727935791, + "step": 1910, + "unclipped_grad_norm": 0.8706849217414856 + }, + { + "bits_per_byte": 1.9090278905126286, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3232372999191284, + "step": 1920, + "unclipped_grad_norm": 0.9953081607818604 + }, + { + "bits_per_byte": 1.9556608143114282, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3555607795715332, + "step": 1930, + "unclipped_grad_norm": 0.8388954401016235 + }, + { + "bits_per_byte": 1.9678466450383125, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3640073537826538, + "step": 1940, + "unclipped_grad_norm": 0.9170766472816467 + }, + { + "bits_per_byte": 1.9103982482747035, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.324187159538269, + "step": 1950, + "unclipped_grad_norm": 0.942866861820221 + }, + { + "bits_per_byte": 1.9025671902503163, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3187590837478638, + "step": 1960, + "unclipped_grad_norm": 0.8135740756988525 + }, + { + "bits_per_byte": 1.9075963069268465, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3222450017929077, + "step": 1970, + "unclipped_grad_norm": 0.9107059836387634 + }, + { + "bits_per_byte": 1.9379792779762228, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3433048725128174, + "step": 1980, + "unclipped_grad_norm": 0.9617857336997986 + }, + { + "bits_per_byte": 1.9109520324104816, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3245710134506226, + "step": 1990, + "unclipped_grad_norm": 0.9190393090248108 + }, + { + "bits_per_byte": 1.9811283492164946, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.373213529586792, + "step": 2000, + "unclipped_grad_norm": 0.878118097782135 + }, + { + "bits_per_byte": 1.9843879363982975, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.375472903251648, + "step": 2010, + "unclipped_grad_norm": 1.0113674402236938 + }, + { + "bits_per_byte": 1.8836764438968945, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3056650161743164, + "step": 2020, + "unclipped_grad_norm": 0.8797336220741272 + }, + { + "bits_per_byte": 1.9153360421636256, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3276097774505615, + "step": 2030, + "unclipped_grad_norm": 0.8666315674781799 + }, + { + "bits_per_byte": 1.906092662610353, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3212027549743652, + "step": 2040, + "unclipped_grad_norm": 0.8591557145118713 + }, + { + "bits_per_byte": 1.8798058023565762, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3029820919036865, + "step": 2050, + "unclipped_grad_norm": 0.9546817541122437 + }, + { + "bits_per_byte": 1.8574308594793247, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2874729633331299, + "step": 2060, + "unclipped_grad_norm": 0.8160566687583923 + }, + { + "bits_per_byte": 1.9859665651505671, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3765671253204346, + "step": 2070, + "unclipped_grad_norm": 0.8717494606971741 + }, + { + "bits_per_byte": 1.952941940583938, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.353676199913025, + "step": 2080, + "unclipped_grad_norm": 0.9155706763267517 + }, + { + "bits_per_byte": 1.8808098370723132, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.303678035736084, + "step": 2090, + "unclipped_grad_norm": 0.8717590570449829 + }, + { + "bits_per_byte": 1.8310299747280259, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.269173264503479, + "step": 2100, + "unclipped_grad_norm": 0.9355071783065796 + }, + { + "bits_per_byte": 1.868735623085839, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2953088283538818, + "step": 2110, + "unclipped_grad_norm": 0.839803159236908 + }, + { + "bits_per_byte": 1.8925072370707472, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3117860555648804, + "step": 2120, + "unclipped_grad_norm": 1.064001202583313 + }, + { + "bits_per_byte": 1.9234560310414621, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.333238124847412, + "step": 2130, + "unclipped_grad_norm": 0.9675858020782471 + }, + { + "bits_per_byte": 1.9467920129717349, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3494133949279785, + "step": 2140, + "unclipped_grad_norm": 0.9227033257484436 + }, + { + "bits_per_byte": 1.8425244351984418, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2771406173706055, + "step": 2150, + "unclipped_grad_norm": 0.9172095060348511 + }, + { + "bits_per_byte": 1.9385437250363546, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.343696117401123, + "step": 2160, + "unclipped_grad_norm": 0.9335131049156189 + }, + { + "bits_per_byte": 1.8590810330143535, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2886167764663696, + "step": 2170, + "unclipped_grad_norm": 0.8850465416908264 + }, + { + "bits_per_byte": 1.9348510855396792, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3411365747451782, + "step": 2180, + "unclipped_grad_norm": 0.8827643990516663 + }, + { + "bits_per_byte": 1.9013160164652894, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3178918361663818, + "step": 2190, + "unclipped_grad_norm": 0.9066281318664551 + }, + { + "bits_per_byte": 1.8761559865399657, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3004522323608398, + "step": 2200, + "unclipped_grad_norm": 0.844115674495697 + }, + { + "bits_per_byte": 1.8719651133037407, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2975473403930664, + "step": 2210, + "unclipped_grad_norm": 0.9099118709564209 + }, + { + "bits_per_byte": 1.8935372411667646, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3125, + "step": 2220, + "unclipped_grad_norm": 1.0520122051239014 + }, + { + "bits_per_byte": 1.8237762624625853, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2641453742980957, + "step": 2230, + "unclipped_grad_norm": 0.8068414330482483 + }, + { + "bits_per_byte": 1.9063850331168197, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3214054107666016, + "step": 2240, + "unclipped_grad_norm": 0.8128858804702759 + }, + { + "bits_per_byte": 1.9127232817317177, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.325798749923706, + "step": 2250, + "unclipped_grad_norm": 0.8365156650543213 + }, + { + "bits_per_byte": 1.965086839439917, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3620944023132324, + "step": 2260, + "unclipped_grad_norm": 0.9489718675613403 + }, + { + "bits_per_byte": 1.8335648270190927, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.270930290222168, + "step": 2270, + "unclipped_grad_norm": 0.8451220989227295 + }, + { + "bits_per_byte": 1.8897092513238603, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.3098466396331787, + "step": 2280, + "unclipped_grad_norm": 0.8222805261611938 + }, + { + "bits_per_byte": 1.8508822760824215, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2829338312149048, + "step": 2290, + "unclipped_grad_norm": 0.8723782896995544 + }, + { + "bits_per_byte": 1.839125714052091, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.274784803390503, + "step": 2300, + "unclipped_grad_norm": 0.8368175625801086 + }, + { + "bits_per_byte": 1.846592168856649, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2799601554870605, + "step": 2310, + "unclipped_grad_norm": 0.8264262080192566 + }, + { + "bits_per_byte": 1.893433707610945, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.3124282360076904, + "step": 2320, + "unclipped_grad_norm": 0.8465461134910583 + }, + { + "bits_per_byte": 1.8498577754312318, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2822237014770508, + "step": 2330, + "unclipped_grad_norm": 0.8474167585372925 + }, + { + "bits_per_byte": 1.8556085313107824, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2862098217010498, + "step": 2340, + "unclipped_grad_norm": 0.8308321237564087 + }, + { + "bits_per_byte": 1.9187394068415498, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.329968810081482, + "step": 2350, + "unclipped_grad_norm": 0.8866831660270691 + }, + { + "bits_per_byte": 1.8564204614055058, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.286772608757019, + "step": 2360, + "unclipped_grad_norm": 0.8822014927864075 + }, + { + "bits_per_byte": 1.767254508221825, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2249674797058105, + "step": 2370, + "unclipped_grad_norm": 0.8126892447471619 + }, + { + "bits_per_byte": 1.8880877988915257, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.308722734451294, + "step": 2380, + "unclipped_grad_norm": 0.8927111625671387 + }, + { + "bits_per_byte": 1.8776909316989163, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3015161752700806, + "step": 2390, + "unclipped_grad_norm": 0.864246666431427 + }, + { + "bits_per_byte": 1.9130975159799952, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3260581493377686, + "step": 2400, + "unclipped_grad_norm": 0.9290322065353394 + }, + { + "bits_per_byte": 1.848231851449975, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2810966968536377, + "step": 2410, + "unclipped_grad_norm": 0.9126319885253906 + }, + { + "bits_per_byte": 1.8675582298580322, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2944927215576172, + "step": 2420, + "unclipped_grad_norm": 0.839264452457428 + }, + { + "bits_per_byte": 1.8763154144573155, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.3005627393722534, + "step": 2430, + "unclipped_grad_norm": 0.8645790219306946 + }, + { + "bits_per_byte": 1.8291109923096986, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2678431272506714, + "step": 2440, + "unclipped_grad_norm": 0.8184406161308289 + }, + { + "bits_per_byte": 1.9499289765234726, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3515877723693848, + "step": 2450, + "unclipped_grad_norm": 0.9486745595932007 + }, + { + "bits_per_byte": 1.82168426549749, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2626953125, + "step": 2460, + "unclipped_grad_norm": 0.8475955128669739 + }, + { + "bits_per_byte": 1.8165447359591058, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2591328620910645, + "step": 2470, + "unclipped_grad_norm": 0.8975434899330139 + }, + { + "bits_per_byte": 1.9011218480524652, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.317757248878479, + "step": 2480, + "unclipped_grad_norm": 0.9190422296524048 + }, + { + "bits_per_byte": 1.8940924011637497, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.31288480758667, + "step": 2490, + "unclipped_grad_norm": 0.8232344388961792 + }, + { + "bits_per_byte": 1.8466803959565414, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2800213098526, + "step": 2500, + "unclipped_grad_norm": 0.7958891987800598 + }, + { + "bits_per_byte": 1.815657993411257, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2585182189941406, + "step": 2510, + "unclipped_grad_norm": 0.9414010643959045 + }, + { + "bits_per_byte": 1.8382814512190053, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2741996049880981, + "step": 2520, + "unclipped_grad_norm": 0.8523510694503784 + }, + { + "bits_per_byte": 1.8459320994426374, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2795026302337646, + "step": 2530, + "unclipped_grad_norm": 0.7949919700622559 + }, + { + "bits_per_byte": 1.8449038151731287, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2787898778915405, + "step": 2540, + "unclipped_grad_norm": 0.8316088318824768 + }, + { + "bits_per_byte": 1.8031964744950397, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2498805522918701, + "step": 2550, + "unclipped_grad_norm": 1.0299166440963745 + }, + { + "bits_per_byte": 1.7815225329027051, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2348573207855225, + "step": 2560, + "unclipped_grad_norm": 0.8839435577392578 + }, + { + "bits_per_byte": 1.8260208080389957, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.265701174736023, + "step": 2570, + "unclipped_grad_norm": 0.9318905472755432 + }, + { + "bits_per_byte": 1.7994856048373729, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.247308373451233, + "step": 2580, + "unclipped_grad_norm": 0.8335656523704529 + }, + { + "bits_per_byte": 1.796869232769797, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2454948425292969, + "step": 2590, + "unclipped_grad_norm": 0.8446885347366333 + }, + { + "bits_per_byte": 1.824228920799656, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2644591331481934, + "step": 2600, + "unclipped_grad_norm": 0.8396548628807068 + }, + { + "bits_per_byte": 1.8444822856958638, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2784976959228516, + "step": 2610, + "unclipped_grad_norm": 0.8530864119529724 + }, + { + "bits_per_byte": 1.7628988756060733, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2219483852386475, + "step": 2620, + "unclipped_grad_norm": 0.8868519067764282 + }, + { + "bits_per_byte": 1.7732109553518074, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2290961742401123, + "step": 2630, + "unclipped_grad_norm": 0.7921182513237 + }, + { + "bits_per_byte": 1.7536496486426716, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2155373096466064, + "step": 2640, + "unclipped_grad_norm": 0.8794499039649963 + }, + { + "bits_per_byte": 1.832483744075769, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2701809406280518, + "step": 2650, + "unclipped_grad_norm": 0.8381353616714478 + }, + { + "bits_per_byte": 1.733659417202284, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.201681137084961, + "step": 2660, + "unclipped_grad_norm": 0.7765254974365234 + }, + { + "bits_per_byte": 1.8051608603331948, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2512421607971191, + "step": 2670, + "unclipped_grad_norm": 0.8680312633514404 + }, + { + "bits_per_byte": 1.923795868759567, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.3334736824035645, + "step": 2680, + "unclipped_grad_norm": 0.9085569977760315 + }, + { + "bits_per_byte": 1.8284822237381442, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2674072980880737, + "step": 2690, + "unclipped_grad_norm": 0.8529220819473267 + }, + { + "bits_per_byte": 1.7587946816258833, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2191035747528076, + "step": 2700, + "unclipped_grad_norm": 0.8387420773506165 + }, + { + "bits_per_byte": 1.8291632750355609, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2678793668746948, + "step": 2710, + "unclipped_grad_norm": 0.7956082820892334 + }, + { + "bits_per_byte": 1.872990129902883, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.298257827758789, + "step": 2720, + "unclipped_grad_norm": 0.7902501821517944 + }, + { + "bits_per_byte": 1.7687435340129951, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2259995937347412, + "step": 2730, + "unclipped_grad_norm": 0.8540245294570923 + }, + { + "bits_per_byte": 1.8104493268472261, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2549078464508057, + "step": 2740, + "unclipped_grad_norm": 0.8416579961776733 + }, + { + "bits_per_byte": 1.7908353934469263, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2413125038146973, + "step": 2750, + "unclipped_grad_norm": 0.7708072662353516 + }, + { + "bits_per_byte": 1.9183405790741987, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3296923637390137, + "step": 2760, + "unclipped_grad_norm": 0.9251072406768799 + }, + { + "bits_per_byte": 1.8352038216818154, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.272066354751587, + "step": 2770, + "unclipped_grad_norm": 0.8847647905349731 + }, + { + "bits_per_byte": 1.8372603902208329, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2734918594360352, + "step": 2780, + "unclipped_grad_norm": 0.8566639423370361 + }, + { + "bits_per_byte": 1.8040237110456898, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2504539489746094, + "step": 2790, + "unclipped_grad_norm": 0.8849772214889526 + }, + { + "bits_per_byte": 1.8313404034128333, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2693884372711182, + "step": 2800, + "unclipped_grad_norm": 0.8829222321510315 + }, + { + "bits_per_byte": 1.7673478948012433, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2250322103500366, + "step": 2810, + "unclipped_grad_norm": 0.7698772549629211 + }, + { + "bits_per_byte": 1.8410065163219265, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2760884761810303, + "step": 2820, + "unclipped_grad_norm": 0.875986635684967 + }, + { + "bits_per_byte": 1.7626952481474518, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2218072414398193, + "step": 2830, + "unclipped_grad_norm": 0.815294623374939 + }, + { + "bits_per_byte": 1.808952733819418, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2538704872131348, + "step": 2840, + "unclipped_grad_norm": 0.7992775440216064 + }, + { + "bits_per_byte": 1.7569166310784614, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.217801809310913, + "step": 2850, + "unclipped_grad_norm": 0.8472921848297119 + }, + { + "bits_per_byte": 1.792434144169347, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2424206733703613, + "step": 2860, + "unclipped_grad_norm": 0.8268847465515137 + }, + { + "bits_per_byte": 1.793870887234655, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2434165477752686, + "step": 2870, + "unclipped_grad_norm": 0.8292641639709473 + }, + { + "bits_per_byte": 1.79901058875569, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2469791173934937, + "step": 2880, + "unclipped_grad_norm": 0.8117106556892395 + }, + { + "bits_per_byte": 1.7566032786885895, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2175846099853516, + "step": 2890, + "unclipped_grad_norm": 0.8036413788795471 + }, + { + "bits_per_byte": 1.7856263829175936, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2377018928527832, + "step": 2900, + "unclipped_grad_norm": 0.7987298369407654 + }, + { + "bits_per_byte": 1.7581136303284666, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2186315059661865, + "step": 2910, + "unclipped_grad_norm": 0.8029706478118896 + }, + { + "bits_per_byte": 1.8051701473963413, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2512485980987549, + "step": 2920, + "unclipped_grad_norm": 0.7970700263977051 + }, + { + "bits_per_byte": 1.756548588205615, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2175467014312744, + "step": 2930, + "unclipped_grad_norm": 0.8035452961921692 + }, + { + "bits_per_byte": 1.7718779178249697, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2281721830368042, + "step": 2940, + "unclipped_grad_norm": 0.8500884771347046 + }, + { + "bits_per_byte": 1.8224836408587004, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.263249397277832, + "step": 2950, + "unclipped_grad_norm": 0.8720842599868774 + }, + { + "bits_per_byte": 1.8354507887684544, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2722375392913818, + "step": 2960, + "unclipped_grad_norm": 0.8293792009353638 + }, + { + "bits_per_byte": 1.8121727649915225, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.256102442741394, + "step": 2970, + "unclipped_grad_norm": 0.8630385994911194 + }, + { + "bits_per_byte": 1.8230131754407068, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2636164426803589, + "step": 2980, + "unclipped_grad_norm": 0.7837274670600891 + }, + { + "bits_per_byte": 1.755593396562723, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2168846130371094, + "step": 2990, + "unclipped_grad_norm": 0.8064510226249695 + }, + { + "bits_per_byte": 1.8344553531852605, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.271547555923462, + "step": 3000, + "unclipped_grad_norm": 0.793539822101593 + }, + { + "bits_per_byte": 1.7345470196633868, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2022963762283325, + "step": 3010, + "unclipped_grad_norm": 0.8690526485443115 + }, + { + "bits_per_byte": 1.8061236192127248, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2519094944000244, + "step": 3020, + "unclipped_grad_norm": 0.9625511169433594 + }, + { + "bits_per_byte": 1.7339495519342893, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.201882243156433, + "step": 3030, + "unclipped_grad_norm": 0.7808479070663452 + }, + { + "bits_per_byte": 1.725635050678327, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1961190700531006, + "step": 3040, + "unclipped_grad_norm": 0.8198943138122559 + }, + { + "bits_per_byte": 1.8739061095013785, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2988927364349365, + "step": 3050, + "unclipped_grad_norm": 0.8317142724990845 + }, + { + "bits_per_byte": 1.8166181725510242, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2591837644577026, + "step": 3060, + "unclipped_grad_norm": 0.8308979868888855 + }, + { + "bits_per_byte": 1.7248370511783235, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.195565938949585, + "step": 3070, + "unclipped_grad_norm": 0.8185245394706726 + }, + { + "bits_per_byte": 1.8189339189448919, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.260788917541504, + "step": 3080, + "unclipped_grad_norm": 0.8484242558479309 + }, + { + "bits_per_byte": 1.7918220579149262, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.24199640750885, + "step": 3090, + "unclipped_grad_norm": 0.8042747974395752 + }, + { + "bits_per_byte": 1.7567866121944091, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2177116870880127, + "step": 3100, + "unclipped_grad_norm": 0.8309313058853149 + }, + { + "bits_per_byte": 1.9063415215061514, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.3213752508163452, + "step": 3110, + "unclipped_grad_norm": 0.8743271231651306 + }, + { + "bits_per_byte": 1.781927724028138, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.235138177871704, + "step": 3120, + "unclipped_grad_norm": 0.8567306995391846 + }, + { + "bits_per_byte": 1.731061619261001, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1998804807662964, + "step": 3130, + "unclipped_grad_norm": 0.8699595332145691 + }, + { + "bits_per_byte": 1.7586823769548698, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.219025731086731, + "step": 3140, + "unclipped_grad_norm": 0.7885372042655945 + }, + { + "bits_per_byte": 1.744890228268927, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.209465742111206, + "step": 3150, + "unclipped_grad_norm": 0.7910884022712708 + }, + { + "bits_per_byte": 1.7638991267034914, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2226417064666748, + "step": 3160, + "unclipped_grad_norm": 0.8217395544052124 + }, + { + "bits_per_byte": 1.816583260072899, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2591595649719238, + "step": 3170, + "unclipped_grad_norm": 0.8992500305175781 + }, + { + "bits_per_byte": 1.8054272614593814, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2514268159866333, + "step": 3180, + "unclipped_grad_norm": 0.8302581906318665 + }, + { + "bits_per_byte": 1.7809063190646637, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.234430193901062, + "step": 3190, + "unclipped_grad_norm": 0.8399643898010254 + }, + { + "bits_per_byte": 1.7853408917171614, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.237504005432129, + "step": 3200, + "unclipped_grad_norm": 0.8274973630905151 + }, + { + "bits_per_byte": 1.7397025435883011, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2058699131011963, + "step": 3210, + "unclipped_grad_norm": 0.8199328184127808 + }, + { + "bits_per_byte": 1.7292408389363165, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1986184120178223, + "step": 3220, + "unclipped_grad_norm": 0.81409752368927 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2331104278564453, + "step": 3230, + "unclipped_grad_norm": 0.816062867641449 + }, + { + "bits_per_byte": 1.7934072220079287, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2430951595306396, + "step": 3240, + "unclipped_grad_norm": 0.9001877307891846 + }, + { + "bits_per_byte": 1.765244718963842, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2235743999481201, + "step": 3250, + "unclipped_grad_norm": 0.8105912804603577 + }, + { + "bits_per_byte": 1.7706184888727015, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2272992134094238, + "step": 3260, + "unclipped_grad_norm": 0.8049122095108032 + }, + { + "bits_per_byte": 1.8044983831620713, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2507829666137695, + "step": 3270, + "unclipped_grad_norm": 0.8440065979957581 + }, + { + "bits_per_byte": 1.793506112032175, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2431637048721313, + "step": 3280, + "unclipped_grad_norm": 0.8675442337989807 + }, + { + "bits_per_byte": 1.7443828794488816, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2091140747070312, + "step": 3290, + "unclipped_grad_norm": 0.8013374209403992 + }, + { + "bits_per_byte": 1.7796785349201543, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.233579158782959, + "step": 3300, + "unclipped_grad_norm": 0.8424150943756104 + }, + { + "bits_per_byte": 1.7358951916635004, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.203230857849121, + "step": 3310, + "unclipped_grad_norm": 0.7571165561676025 + }, + { + "bits_per_byte": 1.7447595214542713, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2093751430511475, + "step": 3320, + "unclipped_grad_norm": 0.7720819115638733 + }, + { + "bits_per_byte": 1.7607893364105902, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2204861640930176, + "step": 3330, + "unclipped_grad_norm": 0.8520592451095581 + }, + { + "bits_per_byte": 1.6976324914995427, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1767091751098633, + "step": 3340, + "unclipped_grad_norm": 0.80418860912323 + }, + { + "bits_per_byte": 1.7457931371859567, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2100915908813477, + "step": 3350, + "unclipped_grad_norm": 0.8466405868530273 + }, + { + "bits_per_byte": 1.7522414546974068, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2145612239837646, + "step": 3360, + "unclipped_grad_norm": 0.7725750803947449 + }, + { + "bits_per_byte": 1.7121540186077948, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.186774730682373, + "step": 3370, + "unclipped_grad_norm": 0.779502809047699 + }, + { + "bits_per_byte": 1.7592602386617688, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2194262742996216, + "step": 3380, + "unclipped_grad_norm": 0.8456810712814331 + }, + { + "bits_per_byte": 1.7687302913488787, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2259904146194458, + "step": 3390, + "unclipped_grad_norm": 0.8990693688392639 + }, + { + "bits_per_byte": 1.7458971866897286, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2101637125015259, + "step": 3400, + "unclipped_grad_norm": 0.8017266392707825 + }, + { + "bits_per_byte": 1.738469084016313, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.205014944076538, + "step": 3410, + "unclipped_grad_norm": 0.8378265500068665 + }, + { + "bits_per_byte": 1.7225225086630118, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1939616203308105, + "step": 3420, + "unclipped_grad_norm": 0.8268731832504272 + }, + { + "bits_per_byte": 1.7602646173428078, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.2201224565505981, + "step": 3430, + "unclipped_grad_norm": 0.8129444122314453 + }, + { + "bits_per_byte": 1.768388733804265, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.225753664970398, + "step": 3440, + "unclipped_grad_norm": 0.8220692873001099 + }, + { + "bits_per_byte": 1.8502017407329574, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2824621200561523, + "step": 3450, + "unclipped_grad_norm": 0.8645455837249756 + }, + { + "bits_per_byte": 1.8368216624784819, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2731877565383911, + "step": 3460, + "unclipped_grad_norm": 0.8615949153900146 + }, + { + "bits_per_byte": 1.785747458703801, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.237785816192627, + "step": 3470, + "unclipped_grad_norm": 0.8510363101959229 + }, + { + "bits_per_byte": 1.7659735814381987, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2240796089172363, + "step": 3480, + "unclipped_grad_norm": 0.8455166816711426 + }, + { + "bits_per_byte": 1.6893135186946577, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1709429025650024, + "step": 3490, + "unclipped_grad_norm": 0.8278471231460571 + }, + { + "bits_per_byte": 1.7031111708254283, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.180506706237793, + "step": 3500, + "unclipped_grad_norm": 0.814669132232666 + }, + { + "bits_per_byte": 1.7385170671759038, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2050482034683228, + "step": 3510, + "unclipped_grad_norm": 0.9123345017433167 + }, + { + "bits_per_byte": 1.700717172325418, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.178847312927246, + "step": 3520, + "unclipped_grad_norm": 0.7641705870628357 + }, + { + "bits_per_byte": 1.742950951897798, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2081215381622314, + "step": 3530, + "unclipped_grad_norm": 0.855086624622345 + }, + { + "bits_per_byte": 1.733061433525234, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2012666463851929, + "step": 3540, + "unclipped_grad_norm": 0.8321052193641663 + }, + { + "bits_per_byte": 1.7782098030817857, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2325611114501953, + "step": 3550, + "unclipped_grad_norm": 0.8241682052612305 + }, + { + "bits_per_byte": 1.7088610667917246, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1844922304153442, + "step": 3560, + "unclipped_grad_norm": 0.7756084203720093 + }, + { + "bits_per_byte": 1.7644671853992913, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.223035454750061, + "step": 3570, + "unclipped_grad_norm": 0.8851686716079712 + }, + { + "bits_per_byte": 1.8350844657221166, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2719836235046387, + "step": 3580, + "unclipped_grad_norm": 0.8648787140846252 + }, + { + "bits_per_byte": 1.758608252432348, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2189743518829346, + "step": 3590, + "unclipped_grad_norm": 0.8309826254844666 + }, + { + "bits_per_byte": 1.7417491371335687, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2072885036468506, + "step": 3600, + "unclipped_grad_norm": 0.9055862426757812 + }, + { + "bits_per_byte": 1.7217079988285255, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.193397045135498, + "step": 3610, + "unclipped_grad_norm": 0.8098222613334656 + }, + { + "bits_per_byte": 1.7173626851718258, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.190385103225708, + "step": 3620, + "unclipped_grad_norm": 0.8190554976463318 + }, + { + "bits_per_byte": 1.7436209963055593, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2085859775543213, + "step": 3630, + "unclipped_grad_norm": 0.8753142356872559 + }, + { + "bits_per_byte": 1.6961333187319716, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1756700277328491, + "step": 3640, + "unclipped_grad_norm": 0.841607391834259 + }, + { + "bits_per_byte": 1.7716670670950119, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.228026032447815, + "step": 3650, + "unclipped_grad_norm": 0.8459575772285461 + }, + { + "bits_per_byte": 1.6801827877777005, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.164613962173462, + "step": 3660, + "unclipped_grad_norm": 0.8714450001716614 + }, + { + "bits_per_byte": 1.6935544388822839, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1738824844360352, + "step": 3670, + "unclipped_grad_norm": 0.7989607453346252 + }, + { + "bits_per_byte": 1.7486903569223915, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.2120997905731201, + "step": 3680, + "unclipped_grad_norm": 0.8391386270523071 + }, + { + "bits_per_byte": 1.7119070515211559, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1866035461425781, + "step": 3690, + "unclipped_grad_norm": 0.8321404457092285 + }, + { + "bits_per_byte": 1.6629633688253633, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.152678370475769, + "step": 3700, + "unclipped_grad_norm": 0.8227249383926392 + }, + { + "bits_per_byte": 1.726790602109474, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1969200372695923, + "step": 3710, + "unclipped_grad_norm": 0.8174790143966675 + }, + { + "bits_per_byte": 1.7194202856067484, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1918113231658936, + "step": 3720, + "unclipped_grad_norm": 0.822218120098114 + }, + { + "bits_per_byte": 1.767769424278508, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2253243923187256, + "step": 3730, + "unclipped_grad_norm": 0.8156639933586121 + }, + { + "bits_per_byte": 1.7041993050574373, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1812609434127808, + "step": 3740, + "unclipped_grad_norm": 0.7992775440216064 + }, + { + "bits_per_byte": 1.6957810982630046, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1754258871078491, + "step": 3750, + "unclipped_grad_norm": 0.8203845620155334 + }, + { + "bits_per_byte": 1.6945959658159089, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1746044158935547, + "step": 3760, + "unclipped_grad_norm": 0.8056831359863281 + }, + { + "bits_per_byte": 1.7464552703917786, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2105505466461182, + "step": 3770, + "unclipped_grad_norm": 0.9159992337226868 + }, + { + "bits_per_byte": 1.6766162115641075, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.1621417999267578, + "step": 3780, + "unclipped_grad_norm": 0.7913095951080322 + }, + { + "bits_per_byte": 1.7293030966559289, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1986615657806396, + "step": 3790, + "unclipped_grad_norm": 0.8222495913505554 + }, + { + "bits_per_byte": 1.6807076788281337, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.164977788925171, + "step": 3800, + "unclipped_grad_norm": 0.829034686088562 + }, + { + "bits_per_byte": 1.6730726810257301, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1596856117248535, + "step": 3810, + "unclipped_grad_norm": 0.8109799027442932 + }, + { + "bits_per_byte": 1.7832320404522815, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.2360422611236572, + "step": 3820, + "unclipped_grad_norm": 0.8464360237121582 + }, + { + "bits_per_byte": 1.7013447370184165, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.179282307624817, + "step": 3830, + "unclipped_grad_norm": 0.849337637424469 + }, + { + "bits_per_byte": 1.7095980124506718, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1850030422210693, + "step": 3840, + "unclipped_grad_norm": 0.8549852967262268 + }, + { + "bits_per_byte": 1.7414235739754853, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.207062840461731, + "step": 3850, + "unclipped_grad_norm": 0.8230528235435486 + }, + { + "bits_per_byte": 1.6685270075807752, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1565347909927368, + "step": 3860, + "unclipped_grad_norm": 0.8480414748191833 + }, + { + "bits_per_byte": 1.7316664822440857, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2002997398376465, + "step": 3870, + "unclipped_grad_norm": 0.8396956324577332 + }, + { + "bits_per_byte": 1.7011863409969716, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1791725158691406, + "step": 3880, + "unclipped_grad_norm": 0.8941359519958496 + }, + { + "bits_per_byte": 1.7358762735719053, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.2032177448272705, + "step": 3890, + "unclipped_grad_norm": 0.894798755645752 + }, + { + "bits_per_byte": 1.723932594417436, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1949390172958374, + "step": 3900, + "unclipped_grad_norm": 0.856995165348053 + }, + { + "bits_per_byte": 1.7425093004503822, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.207815408706665, + "step": 3910, + "unclipped_grad_norm": 0.815839409828186 + }, + { + "bits_per_byte": 1.6972770033602091, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1764627695083618, + "step": 3920, + "unclipped_grad_norm": 0.7732813358306885 + }, + { + "bits_per_byte": 1.6785904004133616, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1635102033615112, + "step": 3930, + "unclipped_grad_norm": 0.8116664886474609 + }, + { + "bits_per_byte": 1.6934890854749558, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1738371849060059, + "step": 3940, + "unclipped_grad_norm": 0.8463517427444458 + }, + { + "bits_per_byte": 1.7793810049341614, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2333729267120361, + "step": 3950, + "unclipped_grad_norm": 0.8989379405975342 + }, + { + "bits_per_byte": 1.7841349493693113, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2366681098937988, + "step": 3960, + "unclipped_grad_norm": 0.8764044642448425 + }, + { + "bits_per_byte": 1.7434844420807742, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.208491325378418, + "step": 3970, + "unclipped_grad_norm": 0.842854380607605 + }, + { + "bits_per_byte": 1.7365705675334386, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.203698992729187, + "step": 3980, + "unclipped_grad_norm": 0.8656630516052246 + }, + { + "bits_per_byte": 1.7578855813334224, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2184734344482422, + "step": 3990, + "unclipped_grad_norm": 0.8737640976905823 + }, + { + "bits_per_byte": 1.7628734221737457, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.221930742263794, + "step": 4000, + "unclipped_grad_norm": 0.8737133741378784 + }, + { + "bits_per_byte": 1.7856131402534772, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2376927137374878, + "step": 4010, + "unclipped_grad_norm": 0.8777877688407898 + }, + { + "bits_per_byte": 1.6452555191445761, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.140404224395752, + "step": 4020, + "unclipped_grad_norm": 0.7845610976219177 + }, + { + "bits_per_byte": 1.6881839366437907, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.170159935951233, + "step": 4030, + "unclipped_grad_norm": 0.8164089322090149 + }, + { + "bits_per_byte": 1.7054205538612142, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1821074485778809, + "step": 4040, + "unclipped_grad_norm": 0.8447781205177307 + }, + { + "bits_per_byte": 1.6980753468255143, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1770161390304565, + "step": 4050, + "unclipped_grad_norm": 0.8900085091590881 + }, + { + "bits_per_byte": 1.715704084486905, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1892354488372803, + "step": 4060, + "unclipped_grad_norm": 0.864895761013031 + }, + { + "bits_per_byte": 1.6694995694714043, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1572089195251465, + "step": 4070, + "unclipped_grad_norm": 0.8258009552955627 + }, + { + "bits_per_byte": 1.7115804564671673, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1863771677017212, + "step": 4080, + "unclipped_grad_norm": 0.8420462012290955 + }, + { + "bits_per_byte": 1.656621852540099, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.148282766342163, + "step": 4090, + "unclipped_grad_norm": 0.8618301153182983 + }, + { + "bits_per_byte": 1.6356337777594054, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.133734941482544, + "step": 4100, + "unclipped_grad_norm": 0.7950634360313416 + }, + { + "bits_per_byte": 1.7423063609223641, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.2076747417449951, + "step": 4110, + "unclipped_grad_norm": 0.9093177914619446 + }, + { + "bits_per_byte": 1.6838860901987291, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1671808958053589, + "step": 4120, + "unclipped_grad_norm": 0.8371159434318542 + }, + { + "bits_per_byte": 1.6537941137946128, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1463227272033691, + "step": 4130, + "unclipped_grad_norm": 0.8010553121566772 + }, + { + "bits_per_byte": 1.6821887934173643, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1660044193267822, + "step": 4140, + "unclipped_grad_norm": 0.8702026009559631 + }, + { + "bits_per_byte": 1.7209915191050311, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1929004192352295, + "step": 4150, + "unclipped_grad_norm": 0.8466006517410278 + }, + { + "bits_per_byte": 1.6966901985554654, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1760560274124146, + "step": 4160, + "unclipped_grad_norm": 0.869064450263977 + }, + { + "bits_per_byte": 1.6446666505480219, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.13999605178833, + "step": 4170, + "unclipped_grad_norm": 0.8484132289886475 + }, + { + "bits_per_byte": 1.6167019275524273, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.120612382888794, + "step": 4180, + "unclipped_grad_norm": 0.8314529657363892 + }, + { + "bits_per_byte": 1.7785682149261839, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.2328095436096191, + "step": 4190, + "unclipped_grad_norm": 0.8785865902900696 + }, + { + "bits_per_byte": 1.6748427264684103, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1609125137329102, + "step": 4200, + "unclipped_grad_norm": 0.8246812224388123 + }, + { + "bits_per_byte": 1.679696764806362, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1642770767211914, + "step": 4210, + "unclipped_grad_norm": 0.846137285232544 + }, + { + "bits_per_byte": 1.73478487166953, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.2024612426757812, + "step": 4220, + "unclipped_grad_norm": 0.9032177925109863 + }, + { + "bits_per_byte": 1.6973831166557916, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1765363216400146, + "step": 4230, + "unclipped_grad_norm": 0.8501660227775574 + }, + { + "bits_per_byte": 1.7410484798139536, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.2068028450012207, + "step": 4240, + "unclipped_grad_norm": 0.8855612874031067 + }, + { + "bits_per_byte": 1.756008390699255, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.2171722650527954, + "step": 4250, + "unclipped_grad_norm": 0.849215567111969 + }, + { + "bits_per_byte": 1.7259122867115178, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1963112354278564, + "step": 4260, + "unclipped_grad_norm": 0.8852341175079346 + }, + { + "bits_per_byte": 1.7235939605778872, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.194704294204712, + "step": 4270, + "unclipped_grad_norm": 0.8789628148078918 + }, + { + "bits_per_byte": 1.751234840241907, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.213863492012024, + "step": 4280, + "unclipped_grad_norm": 0.923163115978241 + }, + { + "bits_per_byte": 1.688365378340451, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.170285701751709, + "step": 4290, + "unclipped_grad_norm": 0.8880194425582886 + }, + { + "bits_per_byte": 1.629311007583085, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.129352331161499, + "step": 4300, + "unclipped_grad_norm": 0.8913658261299133 + }, + { + "bits_per_byte": 1.771773524355896, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2280998229980469, + "step": 4310, + "unclipped_grad_norm": 0.9543031454086304 + }, + { + "bits_per_byte": 1.6490332900534286, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1430227756500244, + "step": 4320, + "unclipped_grad_norm": 0.8419790267944336 + }, + { + "bits_per_byte": 1.6371308867351662, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1347726583480835, + "step": 4330, + "unclipped_grad_norm": 0.8086377382278442 + }, + { + "bits_per_byte": 1.6438610838113803, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1394376754760742, + "step": 4340, + "unclipped_grad_norm": 0.8752440810203552 + }, + { + "bits_per_byte": 1.5801602577756537, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0952836275100708, + "step": 4350, + "unclipped_grad_norm": 0.8255703449249268 + }, + { + "bits_per_byte": 1.7363834504092999, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.2035692930221558, + "step": 4360, + "unclipped_grad_norm": 0.9419499039649963 + }, + { + "bits_per_byte": 1.6755530148164737, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1614048480987549, + "step": 4370, + "unclipped_grad_norm": 0.8658137917518616 + }, + { + "bits_per_byte": 1.644153798283149, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1396405696868896, + "step": 4380, + "unclipped_grad_norm": 0.8454349040985107 + }, + { + "bits_per_byte": 1.6710379542833722, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1582752466201782, + "step": 4390, + "unclipped_grad_norm": 0.8288133144378662 + }, + { + "bits_per_byte": 1.6521485837911574, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1451821327209473, + "step": 4400, + "unclipped_grad_norm": 0.8934062719345093 + }, + { + "bits_per_byte": 1.6486657631285346, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1427680253982544, + "step": 4410, + "unclipped_grad_norm": 0.945034921169281 + }, + { + "bits_per_byte": 1.6360498037918425, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1340233087539673, + "step": 4420, + "unclipped_grad_norm": 0.8596903681755066 + }, + { + "bits_per_byte": 1.6605176035474434, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1509830951690674, + "step": 4430, + "unclipped_grad_norm": 0.8684608340263367 + }, + { + "bits_per_byte": 1.6658086498012377, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1546505689620972, + "step": 4440, + "unclipped_grad_norm": 0.8835514783859253 + }, + { + "bits_per_byte": 1.6458364765391906, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1408069133758545, + "step": 4450, + "unclipped_grad_norm": 0.8609577417373657 + }, + { + "bits_per_byte": 1.6156653881156773, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1198939085006714, + "step": 4460, + "unclipped_grad_norm": 0.8464177250862122 + }, + { + "bits_per_byte": 1.7173920942051233, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1904054880142212, + "step": 4470, + "unclipped_grad_norm": 0.940384030342102 + }, + { + "bits_per_byte": 1.6389153787205186, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1360095739364624, + "step": 4480, + "unclipped_grad_norm": 0.8790894746780396 + }, + { + "bits_per_byte": 1.590423322465892, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1023974418640137, + "step": 4490, + "unclipped_grad_norm": 0.8159270286560059 + }, + { + "bits_per_byte": 1.6820450159212428, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1659047603607178, + "step": 4500, + "unclipped_grad_norm": 0.8648387789726257 + }, + { + "bits_per_byte": 1.6443777196945721, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1397957801818848, + "step": 4510, + "unclipped_grad_norm": 0.874046266078949 + }, + { + "bits_per_byte": 1.6401994011918604, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1368995904922485, + "step": 4520, + "unclipped_grad_norm": 0.8307194709777832 + }, + { + "bits_per_byte": 1.6802708428949422, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.164674997329712, + "step": 4530, + "unclipped_grad_norm": 0.8542748689651489 + }, + { + "bits_per_byte": 1.6174216749462882, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.121111273765564, + "step": 4540, + "unclipped_grad_norm": 0.7951624989509583 + }, + { + "bits_per_byte": 1.6783786897701496, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1633634567260742, + "step": 4550, + "unclipped_grad_norm": 0.9085310697555542 + }, + { + "bits_per_byte": 1.678087351159588, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1631615161895752, + "step": 4560, + "unclipped_grad_norm": 0.8672650456428528 + }, + { + "bits_per_byte": 1.6438235915934922, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1394116878509521, + "step": 4570, + "unclipped_grad_norm": 0.8864105343818665 + }, + { + "bits_per_byte": 1.5942666187647232, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.105061411857605, + "step": 4580, + "unclipped_grad_norm": 0.8459784984588623 + }, + { + "bits_per_byte": 1.7013707063986967, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.179300308227539, + "step": 4590, + "unclipped_grad_norm": 0.864471971988678 + }, + { + "bits_per_byte": 1.685760529110483, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1684801578521729, + "step": 4600, + "unclipped_grad_norm": 0.8711076378822327 + }, + { + "bits_per_byte": 1.6521852160957913, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1452075242996216, + "step": 4610, + "unclipped_grad_norm": 0.8594344258308411 + }, + { + "bits_per_byte": 1.5889760884588813, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1013942956924438, + "step": 4620, + "unclipped_grad_norm": 0.8217349052429199 + }, + { + "bits_per_byte": 1.6772566749559206, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.162585735321045, + "step": 4630, + "unclipped_grad_norm": 0.8906145095825195 + }, + { + "bits_per_byte": 1.6271617403952525, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.127862572669983, + "step": 4640, + "unclipped_grad_norm": 0.8258018493652344 + }, + { + "bits_per_byte": 1.61576359020932, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1199619770050049, + "step": 4650, + "unclipped_grad_norm": 0.9025488495826721 + }, + { + "bits_per_byte": 1.6536071686531248, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1461931467056274, + "step": 4660, + "unclipped_grad_norm": 0.905563473701477 + }, + { + "bits_per_byte": 1.6824091031931194, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1661571264266968, + "step": 4670, + "unclipped_grad_norm": 0.8757683038711548 + }, + { + "bits_per_byte": 1.6350793056930237, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1333506107330322, + "step": 4680, + "unclipped_grad_norm": 0.8878620862960815 + }, + { + "bits_per_byte": 1.6161242378281793, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1202119588851929, + "step": 4690, + "unclipped_grad_norm": 0.8490695953369141 + }, + { + "bits_per_byte": 1.7156000349831328, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.189163327217102, + "step": 4700, + "unclipped_grad_norm": 0.8805282115936279 + }, + { + "bits_per_byte": 1.6530434095235966, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.14580237865448, + "step": 4710, + "unclipped_grad_norm": 0.9434417486190796 + }, + { + "bits_per_byte": 1.6037863744379321, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1116600036621094, + "step": 4720, + "unclipped_grad_norm": 0.8885965347290039 + }, + { + "bits_per_byte": 1.7271340514632472, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1971580982208252, + "step": 4730, + "unclipped_grad_norm": 0.9324085116386414 + }, + { + "bits_per_byte": 1.6733065774309035, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1598477363586426, + "step": 4740, + "unclipped_grad_norm": 0.8685431480407715 + }, + { + "bits_per_byte": 1.7120205600707252, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1866822242736816, + "step": 4750, + "unclipped_grad_norm": 0.9381123185157776 + }, + { + "bits_per_byte": 1.5687653752600876, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0873852968215942, + "step": 4760, + "unclipped_grad_norm": 0.8193861246109009 + }, + { + "bits_per_byte": 1.681102207029213, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1652512550354004, + "step": 4770, + "unclipped_grad_norm": 0.8750576376914978 + }, + { + "bits_per_byte": 1.6366992102815003, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.134473443031311, + "step": 4780, + "unclipped_grad_norm": 0.8815762996673584 + }, + { + "bits_per_byte": 1.6734204299457747, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1599266529083252, + "step": 4790, + "unclipped_grad_norm": 0.9372621774673462 + }, + { + "bits_per_byte": 1.6264474964462192, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1273674964904785, + "step": 4800, + "unclipped_grad_norm": 0.8206531405448914 + }, + { + "bits_per_byte": 1.6050609378634764, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1125434637069702, + "step": 4810, + "unclipped_grad_norm": 0.867507815361023 + }, + { + "bits_per_byte": 1.6801829597603513, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1646140813827515, + "step": 4820, + "unclipped_grad_norm": 0.9113548398017883 + }, + { + "bits_per_byte": 1.6755392562044045, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1613953113555908, + "step": 4830, + "unclipped_grad_norm": 0.9283871650695801 + }, + { + "bits_per_byte": 1.6755334087942753, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.161391258239746, + "step": 4840, + "unclipped_grad_norm": 0.8912690281867981 + }, + { + "bits_per_byte": 1.6043515094286673, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1120517253875732, + "step": 4850, + "unclipped_grad_norm": 0.8777358531951904 + }, + { + "bits_per_byte": 1.6436579723007114, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1392968893051147, + "step": 4860, + "unclipped_grad_norm": 0.9008035063743591 + }, + { + "bits_per_byte": 1.639443709423969, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1363757848739624, + "step": 4870, + "unclipped_grad_norm": 0.8822212219238281 + }, + { + "bits_per_byte": 1.8005314313372698, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.2480332851409912, + "step": 4880, + "unclipped_grad_norm": 0.953372061252594 + }, + { + "bits_per_byte": 1.587463501044543, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1003458499908447, + "step": 4890, + "unclipped_grad_norm": 0.898897111415863 + }, + { + "bits_per_byte": 1.6710314189426394, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1582707166671753, + "step": 4900, + "unclipped_grad_norm": 0.8573687672615051 + }, + { + "bits_per_byte": 1.6542582949692914, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1466444730758667, + "step": 4910, + "unclipped_grad_norm": 0.8744381070137024 + }, + { + "bits_per_byte": 1.5936046575415521, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.104602575302124, + "step": 4920, + "unclipped_grad_norm": 0.8256805539131165 + }, + { + "bits_per_byte": 1.5994107918346805, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1086270809173584, + "step": 4930, + "unclipped_grad_norm": 0.8097874522209167 + }, + { + "bits_per_byte": 1.6321965324992613, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.131352424621582, + "step": 4940, + "unclipped_grad_norm": 0.9138422012329102 + }, + { + "bits_per_byte": 1.6452641182771193, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1404101848602295, + "step": 4950, + "unclipped_grad_norm": 0.890544593334198 + }, + { + "bits_per_byte": 1.6569370967391306, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1485012769699097, + "step": 4960, + "unclipped_grad_norm": 0.8706549406051636 + }, + { + "bits_per_byte": 1.6359316517106997, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.133941411972046, + "step": 4970, + "unclipped_grad_norm": 0.9410244226455688 + }, + { + "bits_per_byte": 1.6744512939550464, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1606411933898926, + "step": 4980, + "unclipped_grad_norm": 0.9115959405899048 + }, + { + "bits_per_byte": 1.637535905877948, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1350533962249756, + "step": 4990, + "unclipped_grad_norm": 0.8625267744064331 + }, + { + "bits_per_byte": 1.6562579372508734, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1480305194854736, + "step": 5000, + "unclipped_grad_norm": 0.9143829345703125 + }, + { + "bits_per_byte": 1.691347557506412, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1723527908325195, + "step": 5010, + "unclipped_grad_norm": 0.8908807635307312 + }, + { + "bits_per_byte": 1.640470617832271, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1370875835418701, + "step": 5020, + "unclipped_grad_norm": 0.9010583162307739 + }, + { + "bits_per_byte": 1.5978712031441566, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1075599193572998, + "step": 5030, + "unclipped_grad_norm": 0.8482128381729126 + }, + { + "bits_per_byte": 1.6379517599277342, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1353416442871094, + "step": 5040, + "unclipped_grad_norm": 0.8523857593536377 + }, + { + "bits_per_byte": 1.624761722502462, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1261990070343018, + "step": 5050, + "unclipped_grad_norm": 0.8824069499969482 + }, + { + "bits_per_byte": 1.6994286783051538, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1779541969299316, + "step": 5060, + "unclipped_grad_norm": 0.9452667832374573 + }, + { + "bits_per_byte": 1.6270504676201443, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1277854442596436, + "step": 5070, + "unclipped_grad_norm": 0.8711076378822327 + }, + { + "bits_per_byte": 1.7018825267676645, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1796550750732422, + "step": 5080, + "unclipped_grad_norm": 0.9225792288780212 + }, + { + "bits_per_byte": 1.6371355302667394, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1347758769989014, + "step": 5090, + "unclipped_grad_norm": 0.9880733489990234 + }, + { + "bits_per_byte": 1.6193408293472662, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1224415302276611, + "step": 5100, + "unclipped_grad_norm": 0.8858473896980286 + }, + { + "bits_per_byte": 1.648101144085752, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1423766613006592, + "step": 5110, + "unclipped_grad_norm": 0.8950205445289612 + }, + { + "bits_per_byte": 1.5621068949492831, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0827699899673462, + "step": 5120, + "unclipped_grad_norm": 0.8509450554847717 + }, + { + "bits_per_byte": 1.6969497203756174, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1762359142303467, + "step": 5130, + "unclipped_grad_norm": 0.9032918810844421 + }, + { + "bits_per_byte": 1.6594853636769649, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1502676010131836, + "step": 5140, + "unclipped_grad_norm": 0.9204651713371277 + }, + { + "bits_per_byte": 1.6102843949354817, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1161640882492065, + "step": 5150, + "unclipped_grad_norm": 0.8756265640258789 + }, + { + "bits_per_byte": 1.6252881613967531, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1265639066696167, + "step": 5160, + "unclipped_grad_norm": 0.9101666212081909 + }, + { + "bits_per_byte": 1.6855433150224433, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1683295965194702, + "step": 5170, + "unclipped_grad_norm": 0.9160894155502319 + }, + { + "bits_per_byte": 1.6357713638800955, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1338303089141846, + "step": 5180, + "unclipped_grad_norm": 0.9035648107528687 + }, + { + "bits_per_byte": 1.5386909410863592, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0665392875671387, + "step": 5190, + "unclipped_grad_norm": 0.8579265475273132 + }, + { + "bits_per_byte": 1.6228207263048244, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1248536109924316, + "step": 5200, + "unclipped_grad_norm": 0.8737999200820923 + }, + { + "bits_per_byte": 1.5871657990758994, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1001394987106323, + "step": 5210, + "unclipped_grad_norm": 0.8650596737861633 + }, + { + "bits_per_byte": 1.6000187505054806, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1090484857559204, + "step": 5220, + "unclipped_grad_norm": 0.857958197593689 + }, + { + "bits_per_byte": 1.6870952864638293, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1694053411483765, + "step": 5230, + "unclipped_grad_norm": 0.9485795497894287 + }, + { + "bits_per_byte": 1.65525063486477, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1473323106765747, + "step": 5240, + "unclipped_grad_norm": 0.9584298133850098 + }, + { + "bits_per_byte": 1.648363245645667, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1425583362579346, + "step": 5250, + "unclipped_grad_norm": 0.9243587851524353 + }, + { + "bits_per_byte": 1.5949392429122475, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.105527639389038, + "step": 5260, + "unclipped_grad_norm": 0.97037672996521 + }, + { + "bits_per_byte": 1.6406128474845345, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1371861696243286, + "step": 5270, + "unclipped_grad_norm": 0.9107886552810669 + }, + { + "bits_per_byte": 1.5702697075071845, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.088428020477295, + "step": 5280, + "unclipped_grad_norm": 0.9339945316314697 + }, + { + "bits_per_byte": 1.5822967982473224, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.0967645645141602, + "step": 5290, + "unclipped_grad_norm": 0.9108784198760986 + }, + { + "bits_per_byte": 1.5641275191142703, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0841705799102783, + "step": 5300, + "unclipped_grad_norm": 0.9299371838569641 + }, + { + "bits_per_byte": 1.568704665384333, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0873432159423828, + "step": 5310, + "unclipped_grad_norm": 0.841905951499939 + }, + { + "bits_per_byte": 1.5839983945949592, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0979440212249756, + "step": 5320, + "unclipped_grad_norm": 0.9190206527709961 + }, + { + "bits_per_byte": 1.6212247273048175, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1237473487854004, + "step": 5330, + "unclipped_grad_norm": 0.9011974930763245 + }, + { + "bits_per_byte": 1.6958863516453326, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1754988431930542, + "step": 5340, + "unclipped_grad_norm": 0.9635267853736877 + }, + { + "bits_per_byte": 1.5503265993131332, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0746045112609863, + "step": 5350, + "unclipped_grad_norm": 0.8525294661521912 + }, + { + "bits_per_byte": 1.6023642498979476, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.110674262046814, + "step": 5360, + "unclipped_grad_norm": 0.9040093421936035 + }, + { + "bits_per_byte": 1.6756675552619482, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1614842414855957, + "step": 5370, + "unclipped_grad_norm": 0.9546249508857727 + }, + { + "bits_per_byte": 1.5762295942901843, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0925590991973877, + "step": 5380, + "unclipped_grad_norm": 0.9253390431404114 + }, + { + "bits_per_byte": 1.5803738602280255, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0954316854476929, + "step": 5390, + "unclipped_grad_norm": 0.9185824394226074 + }, + { + "bits_per_byte": 1.6346944085203927, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1330838203430176, + "step": 5400, + "unclipped_grad_norm": 0.9316674470901489 + }, + { + "bits_per_byte": 1.5477485793766999, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0728175640106201, + "step": 5410, + "unclipped_grad_norm": 0.9008721709251404 + }, + { + "bits_per_byte": 1.6909000586488672, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1720426082611084, + "step": 5420, + "unclipped_grad_norm": 0.9377346038818359 + }, + { + "bits_per_byte": 1.6161049757712826, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1201986074447632, + "step": 5430, + "unclipped_grad_norm": 0.9362180233001709 + }, + { + "bits_per_byte": 1.6092089874196365, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1154186725616455, + "step": 5440, + "unclipped_grad_norm": 0.8880205154418945 + }, + { + "bits_per_byte": 1.6243469003485809, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1259114742279053, + "step": 5450, + "unclipped_grad_norm": 0.9006999731063843 + }, + { + "bits_per_byte": 1.6430875058477994, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.1389014720916748, + "step": 5460, + "unclipped_grad_norm": 0.9130494594573975 + }, + { + "bits_per_byte": 1.6231913489174337, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1251105070114136, + "step": 5470, + "unclipped_grad_norm": 0.9382801651954651 + }, + { + "bits_per_byte": 1.5843513029945295, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0981886386871338, + "step": 5480, + "unclipped_grad_norm": 0.9530964493751526 + }, + { + "bits_per_byte": 1.6238423032509495, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1255617141723633, + "step": 5490, + "unclipped_grad_norm": 1.0083582401275635 + }, + { + "bits_per_byte": 1.6558995254064752, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1477820873260498, + "step": 5500, + "unclipped_grad_norm": 0.9359451532363892 + }, + { + "bits_per_byte": 1.6193308543535163, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1224346160888672, + "step": 5510, + "unclipped_grad_norm": 0.9424907565116882 + }, + { + "bits_per_byte": 1.622570491547819, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1246801614761353, + "step": 5520, + "unclipped_grad_norm": 0.9104092121124268 + }, + { + "bits_per_byte": 1.6496316176957802, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.1434375047683716, + "step": 5530, + "unclipped_grad_norm": 0.9198633432388306 + }, + { + "bits_per_byte": 1.6739231352342467, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1602751016616821, + "step": 5540, + "unclipped_grad_norm": 1.0088523626327515 + }, + { + "bits_per_byte": 1.5961751102413477, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.10638427734375, + "step": 5550, + "unclipped_grad_norm": 0.8725464344024658 + }, + { + "bits_per_byte": 1.5912085952497315, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1029417514801025, + "step": 5560, + "unclipped_grad_norm": 0.9294795989990234 + }, + { + "bits_per_byte": 1.6449143055652642, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1401677131652832, + "step": 5570, + "unclipped_grad_norm": 0.9388754963874817 + }, + { + "bits_per_byte": 1.5525186901810306, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0761239528656006, + "step": 5580, + "unclipped_grad_norm": 0.9197003245353699 + }, + { + "bits_per_byte": 1.6195045568308877, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1225550174713135, + "step": 5590, + "unclipped_grad_norm": 0.9434337615966797 + }, + { + "bits_per_byte": 1.5951824263805676, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.105696201324463, + "step": 5600, + "unclipped_grad_norm": 0.9404297471046448 + }, + { + "bits_per_byte": 1.6108896018838679, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1165835857391357, + "step": 5610, + "unclipped_grad_norm": 0.9016186594963074 + }, + { + "bits_per_byte": 1.5417312503883118, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0686466693878174, + "step": 5620, + "unclipped_grad_norm": 0.9296513795852661 + }, + { + "bits_per_byte": 1.5687026015925227, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0873417854309082, + "step": 5630, + "unclipped_grad_norm": 0.9855212569236755 + }, + { + "bits_per_byte": 1.548401425519375, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0732700824737549, + "step": 5640, + "unclipped_grad_norm": 0.9301762580871582 + }, + { + "bits_per_byte": 1.6637563808284919, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1532280445098877, + "step": 5650, + "unclipped_grad_norm": 1.0132923126220703 + }, + { + "bits_per_byte": 1.614469592744228, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1190650463104248, + "step": 5660, + "unclipped_grad_norm": 0.9275335073471069 + }, + { + "bits_per_byte": 1.6100616774026144, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.1160097122192383, + "step": 5670, + "unclipped_grad_norm": 0.9310563802719116 + }, + { + "bits_per_byte": 1.6255531866617328, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1267476081848145, + "step": 5680, + "unclipped_grad_norm": 0.9579228758811951 + }, + { + "bits_per_byte": 1.6417694308115869, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1379878520965576, + "step": 5690, + "unclipped_grad_norm": 0.9637964367866516 + }, + { + "bits_per_byte": 1.6007787418396433, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1095752716064453, + "step": 5700, + "unclipped_grad_norm": 0.955059826374054 + }, + { + "bits_per_byte": 1.6229964925740061, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1249754428863525, + "step": 5710, + "unclipped_grad_norm": 0.9087291955947876 + }, + { + "bits_per_byte": 1.6524653758340466, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1454017162322998, + "step": 5720, + "unclipped_grad_norm": 0.9723272919654846 + }, + { + "bits_per_byte": 1.5500029279642094, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0743801593780518, + "step": 5730, + "unclipped_grad_norm": 0.9230121374130249 + }, + { + "bits_per_byte": 1.6318054439511993, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1310813426971436, + "step": 5740, + "unclipped_grad_norm": 0.9736878275871277 + }, + { + "bits_per_byte": 1.5815806624891298, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0962681770324707, + "step": 5750, + "unclipped_grad_norm": 0.9681398868560791 + }, + { + "bits_per_byte": 1.5890875332166405, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.1014715433120728, + "step": 5760, + "unclipped_grad_norm": 0.9480424523353577 + }, + { + "bits_per_byte": 1.528980628635994, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.059808611869812, + "step": 5770, + "unclipped_grad_norm": 0.9915046691894531 + }, + { + "bits_per_byte": 1.610684942529341, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1164417266845703, + "step": 5780, + "unclipped_grad_norm": 0.9259656667709351 + }, + { + "bits_per_byte": 1.5796823179889061, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0949523448944092, + "step": 5790, + "unclipped_grad_norm": 0.9487527012825012 + }, + { + "bits_per_byte": 1.5821124328455975, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0966367721557617, + "step": 5800, + "unclipped_grad_norm": 0.9434526562690735 + }, + { + "bits_per_byte": 1.6104013431380684, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.116245150566101, + "step": 5810, + "unclipped_grad_norm": 0.9532226920127869 + }, + { + "bits_per_byte": 1.6033849669308184, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.1113817691802979, + "step": 5820, + "unclipped_grad_norm": 0.914564311504364 + }, + { + "bits_per_byte": 1.5785037208825432, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0941354036331177, + "step": 5830, + "unclipped_grad_norm": 0.9744407534599304 + }, + { + "bits_per_byte": 1.5595204478629572, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.080977201461792, + "step": 5840, + "unclipped_grad_norm": 0.9682079553604126 + }, + { + "bits_per_byte": 1.5459948722858516, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0716019868850708, + "step": 5850, + "unclipped_grad_norm": 0.908660352230072 + }, + { + "bits_per_byte": 1.5771918372217617, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0932260751724243, + "step": 5860, + "unclipped_grad_norm": 0.9774399995803833 + }, + { + "bits_per_byte": 1.617923176356204, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.121458888053894, + "step": 5870, + "unclipped_grad_norm": 0.9498292803764343 + }, + { + "bits_per_byte": 1.6179303996275405, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1214638948440552, + "step": 5880, + "unclipped_grad_norm": 0.929435670375824 + }, + { + "bits_per_byte": 1.6226986186227117, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1247689723968506, + "step": 5890, + "unclipped_grad_norm": 0.9540143013000488 + }, + { + "bits_per_byte": 1.5934973403674138, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1045281887054443, + "step": 5900, + "unclipped_grad_norm": 1.0085638761520386 + }, + { + "bits_per_byte": 1.5806963276983932, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0956552028656006, + "step": 5910, + "unclipped_grad_norm": 1.0136828422546387 + }, + { + "bits_per_byte": 1.602695832448811, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.1109040975570679, + "step": 5920, + "unclipped_grad_norm": 1.0095367431640625 + }, + { + "bits_per_byte": 1.6274630539995643, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1280714273452759, + "step": 5930, + "unclipped_grad_norm": 0.947801411151886 + }, + { + "bits_per_byte": 1.6238797954688375, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.1255877017974854, + "step": 5940, + "unclipped_grad_norm": 0.9719098806381226 + }, + { + "bits_per_byte": 1.597647625698035, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.1074049472808838, + "step": 5950, + "unclipped_grad_norm": 0.9620362520217896 + }, + { + "bits_per_byte": 1.588192019553598, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1008508205413818, + "step": 5960, + "unclipped_grad_norm": 0.9313361644744873 + }, + { + "bits_per_byte": 1.619017845928946, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.1222176551818848, + "step": 5970, + "unclipped_grad_norm": 1.0069472789764404 + }, + { + "bits_per_byte": 1.5640157303912094, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0840930938720703, + "step": 5980, + "unclipped_grad_norm": 0.9776687026023865 + }, + { + "bits_per_byte": 1.5433833157324999, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0697917938232422, + "step": 5990, + "unclipped_grad_norm": 0.9335829615592957 + }, + { + "bits_per_byte": 1.6083124418606887, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1147972345352173, + "step": 6000, + "unclipped_grad_norm": 0.9547005295753479 + }, + { + "bits_per_byte": 1.591078920330981, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.1028518676757812, + "step": 6010, + "unclipped_grad_norm": 0.9792258739471436 + }, + { + "bits_per_byte": 1.553183919074568, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.076585054397583, + "step": 6020, + "unclipped_grad_norm": 0.9365350604057312 + }, + { + "bits_per_byte": 1.621642989111716, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.124037265777588, + "step": 6030, + "unclipped_grad_norm": 0.9456943869590759 + }, + { + "bits_per_byte": 1.55026812521184, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.074563980102539, + "step": 6040, + "unclipped_grad_norm": 0.986164391040802 + }, + { + "bits_per_byte": 1.573968538379291, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0909918546676636, + "step": 6050, + "unclipped_grad_norm": 0.9421037435531616 + }, + { + "bits_per_byte": 1.6258569080231564, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.1269581317901611, + "step": 6060, + "unclipped_grad_norm": 0.9926829934120178 + }, + { + "bits_per_byte": 1.5943486544891847, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.1051182746887207, + "step": 6070, + "unclipped_grad_norm": 0.9500434994697571 + }, + { + "bits_per_byte": 1.5776276412590482, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.093528151512146, + "step": 6080, + "unclipped_grad_norm": 0.9243369102478027 + }, + { + "bits_per_byte": 1.583595095278686, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.097664475440979, + "step": 6090, + "unclipped_grad_norm": 0.9602344036102295 + }, + { + "bits_per_byte": 1.5816112754009832, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0962893962860107, + "step": 6100, + "unclipped_grad_norm": 0.9188515543937683 + }, + { + "bits_per_byte": 1.5481370881849987, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0730868577957153, + "step": 6110, + "unclipped_grad_norm": 0.9290796518325806 + }, + { + "bits_per_byte": 1.557837081676312, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.079810380935669, + "step": 6120, + "unclipped_grad_norm": 0.9658544063568115 + }, + { + "bits_per_byte": 1.5877624068917426, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.100553035736084, + "step": 6130, + "unclipped_grad_norm": 0.9524391889572144 + }, + { + "bits_per_byte": 1.6144964220377627, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.1190836429595947, + "step": 6140, + "unclipped_grad_norm": 0.9724568724632263 + }, + { + "bits_per_byte": 1.5513643426284394, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0753238201141357, + "step": 6150, + "unclipped_grad_norm": 0.9667541980743408 + }, + { + "bits_per_byte": 1.6439771721007128, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.139518141746521, + "step": 6160, + "unclipped_grad_norm": 1.0203982591629028 + }, + { + "bits_per_byte": 1.5065515112237209, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0442619323730469, + "step": 6170, + "unclipped_grad_norm": 0.892124593257904 + }, + { + "bits_per_byte": 1.5721656442502963, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0897421836853027, + "step": 6180, + "unclipped_grad_norm": 0.9671148657798767 + }, + { + "bits_per_byte": 1.5890949284706275, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.1014766693115234, + "step": 6190, + "unclipped_grad_norm": 1.0115299224853516 + }, + { + "bits_per_byte": 1.5390813417038176, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.066809892654419, + "step": 6200, + "unclipped_grad_norm": 0.9446512460708618 + }, + { + "bits_per_byte": 1.5691769297436022, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0876705646514893, + "step": 6210, + "unclipped_grad_norm": 0.9595397710800171 + }, + { + "bits_per_byte": 1.5550289489530242, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0778639316558838, + "step": 6220, + "unclipped_grad_norm": 0.927939772605896 + }, + { + "bits_per_byte": 1.5332487220824564, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0627670288085938, + "step": 6230, + "unclipped_grad_norm": 0.9231003522872925 + }, + { + "bits_per_byte": 1.5454189023881122, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0712027549743652, + "step": 6240, + "unclipped_grad_norm": 0.9556372761726379 + }, + { + "bits_per_byte": 1.547023500520662, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.072314977645874, + "step": 6250, + "unclipped_grad_norm": 0.9621478915214539 + }, + { + "bits_per_byte": 1.6040983509665974, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.1118762493133545, + "step": 6260, + "unclipped_grad_norm": 0.9583669304847717 + }, + { + "bits_per_byte": 1.562768856172454, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0832288265228271, + "step": 6270, + "unclipped_grad_norm": 0.9386000633239746 + }, + { + "bits_per_byte": 1.6650833989625493, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1541478633880615, + "step": 6280, + "unclipped_grad_norm": 0.9854296445846558 + }, + { + "bits_per_byte": 1.5598107545776136, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0811784267425537, + "step": 6290, + "unclipped_grad_norm": 0.9466595649719238 + }, + { + "bits_per_byte": 1.5877661905100615, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.100555658340454, + "step": 6300, + "unclipped_grad_norm": 1.077492117881775 + }, + { + "bits_per_byte": 1.6323781461785725, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.1314783096313477, + "step": 6310, + "unclipped_grad_norm": 1.0482228994369507 + }, + { + "bits_per_byte": 1.5889704130314029, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.1013903617858887, + "step": 6320, + "unclipped_grad_norm": 0.9478169083595276 + }, + { + "bits_per_byte": 1.5225548408518068, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0553545951843262, + "step": 6330, + "unclipped_grad_norm": 0.9578908085823059 + }, + { + "bits_per_byte": 1.580892387920377, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0957911014556885, + "step": 6340, + "unclipped_grad_norm": 1.0281808376312256 + }, + { + "bits_per_byte": 1.5773070655978398, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0933059453964233, + "step": 6350, + "unclipped_grad_norm": 0.9984999299049377 + }, + { + "bits_per_byte": 1.5368639693862434, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0652729272842407, + "step": 6360, + "unclipped_grad_norm": 0.9870114922523499 + }, + { + "bits_per_byte": 1.5938194638724799, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.104751467704773, + "step": 6370, + "unclipped_grad_norm": 0.9943422675132751 + }, + { + "bits_per_byte": 1.54489022771936, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0708363056182861, + "step": 6380, + "unclipped_grad_norm": 0.9544562697410583 + }, + { + "bits_per_byte": 1.5900604390725714, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1021459102630615, + "step": 6390, + "unclipped_grad_norm": 0.9881879091262817 + }, + { + "bits_per_byte": 1.5576724942794364, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0796962976455688, + "step": 6400, + "unclipped_grad_norm": 0.9512554407119751 + }, + { + "bits_per_byte": 1.5486053969432982, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.073411464691162, + "step": 6410, + "unclipped_grad_norm": 0.9827495217323303 + }, + { + "bits_per_byte": 1.5454973264769056, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0712571144104004, + "step": 6420, + "unclipped_grad_norm": 1.0222079753875732 + }, + { + "bits_per_byte": 1.5609757650545584, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0819859504699707, + "step": 6430, + "unclipped_grad_norm": 1.0177998542785645 + }, + { + "bits_per_byte": 1.5622200595335507, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0828484296798706, + "step": 6440, + "unclipped_grad_norm": 0.9765349626541138 + }, + { + "bits_per_byte": 1.5515923916234835, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.07548189163208, + "step": 6450, + "unclipped_grad_norm": 1.028122901916504 + }, + { + "bits_per_byte": 1.5316869476299713, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.061684489250183, + "step": 6460, + "unclipped_grad_norm": 1.1802783012390137 + }, + { + "bits_per_byte": 1.566511026672578, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0858227014541626, + "step": 6470, + "unclipped_grad_norm": 0.9929527044296265 + }, + { + "bits_per_byte": 1.5861486936786968, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.099434494972229, + "step": 6480, + "unclipped_grad_norm": 0.996606171131134 + }, + { + "bits_per_byte": 1.5680002244463989, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0868549346923828, + "step": 6490, + "unclipped_grad_norm": 0.9814039468765259 + }, + { + "bits_per_byte": 1.568809918766661, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.087416172027588, + "step": 6500, + "unclipped_grad_norm": 1.001226782798767 + }, + { + "bits_per_byte": 1.513912368680649, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0493640899658203, + "step": 6510, + "unclipped_grad_norm": 0.9533669948577881 + }, + { + "bits_per_byte": 1.4992986588715345, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0392346382141113, + "step": 6520, + "unclipped_grad_norm": 1.0103793144226074 + }, + { + "bits_per_byte": 1.5872271968822576, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.100182056427002, + "step": 6530, + "unclipped_grad_norm": 1.0064735412597656 + }, + { + "bits_per_byte": 1.551986575859261, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0757551193237305, + "step": 6540, + "unclipped_grad_norm": 0.9814162254333496 + }, + { + "bits_per_byte": 1.6102771716641453, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.1161590814590454, + "step": 6550, + "unclipped_grad_norm": 1.0333638191223145 + }, + { + "bits_per_byte": 1.5466836628025573, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0720794200897217, + "step": 6560, + "unclipped_grad_norm": 0.9711053967475891 + }, + { + "bits_per_byte": 1.4930591282982317, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.034909725189209, + "step": 6570, + "unclipped_grad_norm": 0.9939249157905579 + }, + { + "bits_per_byte": 1.5761067986774682, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0924739837646484, + "step": 6580, + "unclipped_grad_norm": 0.9629629850387573 + }, + { + "bits_per_byte": 1.5551524324963437, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0779495239257812, + "step": 6590, + "unclipped_grad_norm": 1.007703185081482 + }, + { + "bits_per_byte": 1.4715650805586997, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0200111865997314, + "step": 6600, + "unclipped_grad_norm": 0.9724071621894836 + }, + { + "bits_per_byte": 1.5956089433547074, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.1059918403625488, + "step": 6610, + "unclipped_grad_norm": 0.9970710277557373 + }, + { + "bits_per_byte": 1.6125643689379698, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1177444458007812, + "step": 6620, + "unclipped_grad_norm": 1.0349808931350708 + }, + { + "bits_per_byte": 1.59399093057539, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.104870319366455, + "step": 6630, + "unclipped_grad_norm": 1.039212703704834 + }, + { + "bits_per_byte": 1.5557672704731782, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0783756971359253, + "step": 6640, + "unclipped_grad_norm": 1.023613452911377 + }, + { + "bits_per_byte": 1.5432283593640725, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.069684386253357, + "step": 6650, + "unclipped_grad_norm": 0.9931045174598694 + }, + { + "bits_per_byte": 1.5957387902561087, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1060818433761597, + "step": 6660, + "unclipped_grad_norm": 1.01187002658844 + }, + { + "bits_per_byte": 1.5766402888604447, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.092843770980835, + "step": 6670, + "unclipped_grad_norm": 1.0118197202682495 + }, + { + "bits_per_byte": 1.5937730285567469, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.1047192811965942, + "step": 6680, + "unclipped_grad_norm": 1.0237627029418945 + }, + { + "bits_per_byte": 1.5627511419594151, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0832165479660034, + "step": 6690, + "unclipped_grad_norm": 1.0074224472045898 + }, + { + "bits_per_byte": 1.5937723406261435, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.104718804359436, + "step": 6700, + "unclipped_grad_norm": 1.01242196559906 + }, + { + "bits_per_byte": 1.5383141270983187, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.066278100013733, + "step": 6710, + "unclipped_grad_norm": 0.9823194146156311 + }, + { + "bits_per_byte": 1.5376188012408802, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.065796136856079, + "step": 6720, + "unclipped_grad_norm": 1.027299404144287 + }, + { + "bits_per_byte": 1.5285384612406259, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.059502124786377, + "step": 6730, + "unclipped_grad_norm": 0.9929317235946655 + }, + { + "bits_per_byte": 1.5454023920536293, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0711913108825684, + "step": 6740, + "unclipped_grad_norm": 0.9901378154754639 + }, + { + "bits_per_byte": 1.5351463786520767, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.064082384109497, + "step": 6750, + "unclipped_grad_norm": 1.026999831199646 + }, + { + "bits_per_byte": 1.501298817101069, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.040621042251587, + "step": 6760, + "unclipped_grad_norm": 0.9864338040351868 + }, + { + "bits_per_byte": 1.5231093129181885, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.055738925933838, + "step": 6770, + "unclipped_grad_norm": 0.9597692489624023 + }, + { + "bits_per_byte": 1.5314098835794312, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0614924430847168, + "step": 6780, + "unclipped_grad_norm": 0.9848454594612122 + }, + { + "bits_per_byte": 1.6182862317321756, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.1217105388641357, + "step": 6790, + "unclipped_grad_norm": 0.994420051574707 + }, + { + "bits_per_byte": 1.6079612532876268, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.1145538091659546, + "step": 6800, + "unclipped_grad_norm": 1.0759928226470947 + }, + { + "bits_per_byte": 1.5649406530875496, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0847342014312744, + "step": 6810, + "unclipped_grad_norm": 1.0181862115859985 + }, + { + "bits_per_byte": 1.5209997737227055, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.054276704788208, + "step": 6820, + "unclipped_grad_norm": 0.9867210984230042 + }, + { + "bits_per_byte": 1.5408068436399243, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.068005919456482, + "step": 6830, + "unclipped_grad_norm": 1.0285433530807495 + }, + { + "bits_per_byte": 1.5661266454478995, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0855562686920166, + "step": 6840, + "unclipped_grad_norm": 0.9897852540016174 + }, + { + "bits_per_byte": 1.5403562490946636, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0676935911178589, + "step": 6850, + "unclipped_grad_norm": 0.9739700555801392 + }, + { + "bits_per_byte": 1.5589986525002395, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.080615520477295, + "step": 6860, + "unclipped_grad_norm": 1.0206658840179443 + }, + { + "bits_per_byte": 1.5592929148158658, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0808194875717163, + "step": 6870, + "unclipped_grad_norm": 1.015761137008667 + }, + { + "bits_per_byte": 1.5959116328202259, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.1062016487121582, + "step": 6880, + "unclipped_grad_norm": 1.0070849657058716 + }, + { + "bits_per_byte": 1.4912815156189139, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.033677577972412, + "step": 6890, + "unclipped_grad_norm": 0.9983416199684143 + }, + { + "bits_per_byte": 1.5029766798428865, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0417840480804443, + "step": 6900, + "unclipped_grad_norm": 0.9461802840232849 + }, + { + "bits_per_byte": 1.691495806551456, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1724555492401123, + "step": 6910, + "unclipped_grad_norm": 1.0603437423706055 + }, + { + "bits_per_byte": 1.4926446501096524, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0346224308013916, + "step": 6920, + "unclipped_grad_norm": 0.9669191241264343 + }, + { + "bits_per_byte": 1.5121873826924952, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.048168420791626, + "step": 6930, + "unclipped_grad_norm": 1.012385606765747 + }, + { + "bits_per_byte": 1.5487118542041824, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.073485255241394, + "step": 6940, + "unclipped_grad_norm": 1.0546537637710571 + }, + { + "bits_per_byte": 1.5353376233598361, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0642149448394775, + "step": 6950, + "unclipped_grad_norm": 0.9847853779792786 + }, + { + "bits_per_byte": 1.5735119244412503, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0906753540039062, + "step": 6960, + "unclipped_grad_norm": 1.0091943740844727 + }, + { + "bits_per_byte": 1.5592397721767493, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0807826519012451, + "step": 6970, + "unclipped_grad_norm": 1.0198149681091309 + }, + { + "bits_per_byte": 1.6070454456717824, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1139190196990967, + "step": 6980, + "unclipped_grad_norm": 1.0826783180236816 + }, + { + "bits_per_byte": 1.5476336949659235, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0727379322052002, + "step": 6990, + "unclipped_grad_norm": 1.0041176080703735 + }, + { + "bits_per_byte": 1.5120640711318265, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.048082947731018, + "step": 7000, + "unclipped_grad_norm": 1.0215020179748535 + }, + { + "bits_per_byte": 1.5377993830242862, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0659213066101074, + "step": 7010, + "unclipped_grad_norm": 1.0378373861312866 + }, + { + "bits_per_byte": 1.5074898485668284, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.044912338256836, + "step": 7020, + "unclipped_grad_norm": 0.982413113117218 + }, + { + "bits_per_byte": 1.5236237130269192, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0560954809188843, + "step": 7030, + "unclipped_grad_norm": 0.9606820344924927 + }, + { + "bits_per_byte": 1.6050487270952651, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.112534999847412, + "step": 7040, + "unclipped_grad_norm": 1.0613288879394531 + }, + { + "bits_per_byte": 1.5624771735965908, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.083026647567749, + "step": 7050, + "unclipped_grad_norm": 1.0451524257659912 + }, + { + "bits_per_byte": 1.5626594752065053, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0831530094146729, + "step": 7060, + "unclipped_grad_norm": 1.073291540145874 + }, + { + "bits_per_byte": 1.519674131449855, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0533578395843506, + "step": 7070, + "unclipped_grad_norm": 1.009161353111267 + }, + { + "bits_per_byte": 1.4484239549718545, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.003970980644226, + "step": 7080, + "unclipped_grad_norm": 0.9980688691139221 + }, + { + "bits_per_byte": 1.539162345532374, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0668660402297974, + "step": 7090, + "unclipped_grad_norm": 1.0500024557113647 + }, + { + "bits_per_byte": 1.5110017342974469, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.047346591949463, + "step": 7100, + "unclipped_grad_norm": 1.038030743598938 + }, + { + "bits_per_byte": 1.5757193217650745, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0922054052352905, + "step": 7110, + "unclipped_grad_norm": 1.0732139348983765 + }, + { + "bits_per_byte": 1.5620382738715888, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0827224254608154, + "step": 7120, + "unclipped_grad_norm": 1.0506101846694946 + }, + { + "bits_per_byte": 1.4981061311704518, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0384080410003662, + "step": 7130, + "unclipped_grad_norm": 1.0116451978683472 + }, + { + "bits_per_byte": 1.6131864301861407, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.1181756258010864, + "step": 7140, + "unclipped_grad_norm": 1.0758471488952637 + }, + { + "bits_per_byte": 1.623974385926812, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.1256532669067383, + "step": 7150, + "unclipped_grad_norm": 1.0434077978134155 + }, + { + "bits_per_byte": 1.5482879169698056, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0731914043426514, + "step": 7160, + "unclipped_grad_norm": 1.0636146068572998 + }, + { + "bits_per_byte": 1.5506334163622726, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.074817180633545, + "step": 7170, + "unclipped_grad_norm": 1.0567522048950195 + }, + { + "bits_per_byte": 1.468200068011918, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0176787376403809, + "step": 7180, + "unclipped_grad_norm": 0.9437705874443054 + }, + { + "bits_per_byte": 1.4717380951054677, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0201311111450195, + "step": 7190, + "unclipped_grad_norm": 1.010156512260437 + }, + { + "bits_per_byte": 1.5698447383769025, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.088133454322815, + "step": 7200, + "unclipped_grad_norm": 1.056505560874939 + }, + { + "bits_per_byte": 1.56738452655631, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.086428165435791, + "step": 7210, + "unclipped_grad_norm": 1.0318564176559448 + }, + { + "bits_per_byte": 1.523764394835325, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0561929941177368, + "step": 7220, + "unclipped_grad_norm": 1.0528209209442139 + }, + { + "bits_per_byte": 1.5338092135416184, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0631555318832397, + "step": 7230, + "unclipped_grad_norm": 1.025399923324585 + }, + { + "bits_per_byte": 1.490086580160719, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0328493118286133, + "step": 7240, + "unclipped_grad_norm": 0.9961045980453491 + }, + { + "bits_per_byte": 1.5541721313864256, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0772700309753418, + "step": 7250, + "unclipped_grad_norm": 1.0621839761734009 + }, + { + "bits_per_byte": 1.5116575041451867, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.04780113697052, + "step": 7260, + "unclipped_grad_norm": 1.0816527605056763 + }, + { + "bits_per_byte": 1.5896555919124404, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.101865291595459, + "step": 7270, + "unclipped_grad_norm": 1.059293270111084 + }, + { + "bits_per_byte": 1.5429449319554507, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0694879293441772, + "step": 7280, + "unclipped_grad_norm": 1.0573333501815796 + }, + { + "bits_per_byte": 1.5487371356538593, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.073502779006958, + "step": 7290, + "unclipped_grad_norm": 1.0477967262268066 + }, + { + "bits_per_byte": 1.5296157605656304, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.060248851776123, + "step": 7300, + "unclipped_grad_norm": 1.0505754947662354 + }, + { + "bits_per_byte": 1.57645196785775, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0927132368087769, + "step": 7310, + "unclipped_grad_norm": 1.0852391719818115 + }, + { + "bits_per_byte": 1.5598148821612343, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.081181287765503, + "step": 7320, + "unclipped_grad_norm": 1.079657793045044 + }, + { + "bits_per_byte": 1.5641130725715977, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.084160566329956, + "step": 7330, + "unclipped_grad_norm": 1.0734436511993408 + }, + { + "bits_per_byte": 1.5372211773520854, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0655205249786377, + "step": 7340, + "unclipped_grad_norm": 1.0373533964157104 + }, + { + "bits_per_byte": 1.585016531888067, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0986497402191162, + "step": 7350, + "unclipped_grad_norm": 1.034577488899231 + }, + { + "bits_per_byte": 1.5482229075277794, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0731463432312012, + "step": 7360, + "unclipped_grad_norm": 0.9948747754096985 + }, + { + "bits_per_byte": 1.5566985565276004, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0790212154388428, + "step": 7370, + "unclipped_grad_norm": 1.0546287298202515 + }, + { + "bits_per_byte": 1.614560055618582, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.1191277503967285, + "step": 7380, + "unclipped_grad_norm": 1.1286121606826782 + }, + { + "bits_per_byte": 1.5888789182611438, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.1013269424438477, + "step": 7390, + "unclipped_grad_norm": 1.063069224357605 + }, + { + "bits_per_byte": 1.5174531474966126, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0518183708190918, + "step": 7400, + "unclipped_grad_norm": 1.038301944732666 + }, + { + "bits_per_byte": 1.5128250943618944, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0486104488372803, + "step": 7410, + "unclipped_grad_norm": 0.9619468450546265 + }, + { + "bits_per_byte": 1.6064108296900985, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.1134791374206543, + "step": 7420, + "unclipped_grad_norm": 1.0911381244659424 + }, + { + "bits_per_byte": 1.5182607780250643, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0523781776428223, + "step": 7430, + "unclipped_grad_norm": 1.0977153778076172 + }, + { + "bits_per_byte": 1.527175670715189, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0585575103759766, + "step": 7440, + "unclipped_grad_norm": 1.035995602607727 + }, + { + "bits_per_byte": 1.4906346888690187, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0332292318344116, + "step": 7450, + "unclipped_grad_norm": 1.1210148334503174 + }, + { + "bits_per_byte": 1.5053576076614312, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0434343814849854, + "step": 7460, + "unclipped_grad_norm": 1.0129456520080566 + }, + { + "bits_per_byte": 1.555529418467035, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0782108306884766, + "step": 7470, + "unclipped_grad_norm": 1.0514966249465942 + }, + { + "bits_per_byte": 1.4886658314819414, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0318645238876343, + "step": 7480, + "unclipped_grad_norm": 1.032109260559082 + }, + { + "bits_per_byte": 1.51988842183283, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0535063743591309, + "step": 7490, + "unclipped_grad_norm": 1.0880528688430786 + }, + { + "bits_per_byte": 1.5815053340880518, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0962159633636475, + "step": 7500, + "unclipped_grad_norm": 1.0377991199493408 + }, + { + "bits_per_byte": 1.5370581377990675, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0654075145721436, + "step": 7510, + "unclipped_grad_norm": 1.0226150751113892 + }, + { + "bits_per_byte": 1.5380518535557528, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.066096305847168, + "step": 7520, + "unclipped_grad_norm": 1.0527992248535156 + }, + { + "bits_per_byte": 1.562738587225902, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0832078456878662, + "step": 7530, + "unclipped_grad_norm": 1.072679042816162 + }, + { + "bits_per_byte": 1.6011416252329638, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.1098268032073975, + "step": 7540, + "unclipped_grad_norm": 1.109021544456482 + }, + { + "bits_per_byte": 1.5189222233002826, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0528366565704346, + "step": 7550, + "unclipped_grad_norm": 1.0227179527282715 + }, + { + "bits_per_byte": 1.5563348132210255, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0787690877914429, + "step": 7560, + "unclipped_grad_norm": 1.0419527292251587 + }, + { + "bits_per_byte": 1.5726612982500827, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.090085744857788, + "step": 7570, + "unclipped_grad_norm": 1.03652822971344 + }, + { + "bits_per_byte": 1.5146027070412125, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0498425960540771, + "step": 7580, + "unclipped_grad_norm": 1.0102300643920898 + }, + { + "bits_per_byte": 1.5228090312097822, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0555307865142822, + "step": 7590, + "unclipped_grad_norm": 1.049957513809204 + }, + { + "bits_per_byte": 1.5153031923781766, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0503281354904175, + "step": 7600, + "unclipped_grad_norm": 1.020882248878479 + }, + { + "bits_per_byte": 1.4615728885435706, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.013085126876831, + "step": 7610, + "unclipped_grad_norm": 1.0289204120635986 + }, + { + "bits_per_byte": 1.5704728190178536, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0885688066482544, + "step": 7620, + "unclipped_grad_norm": 1.062748670578003 + }, + { + "bits_per_byte": 1.5187743182205407, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.052734136581421, + "step": 7630, + "unclipped_grad_norm": 1.036866307258606 + }, + { + "bits_per_byte": 1.5251964943690597, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0571856498718262, + "step": 7640, + "unclipped_grad_norm": 1.0375516414642334 + }, + { + "bits_per_byte": 1.625032595177571, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1263867616653442, + "step": 7650, + "unclipped_grad_norm": 1.15651273727417 + }, + { + "bits_per_byte": 1.5325501285546517, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0622828006744385, + "step": 7660, + "unclipped_grad_norm": 1.0661544799804688 + }, + { + "bits_per_byte": 1.5264065643005305, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0580244064331055, + "step": 7670, + "unclipped_grad_norm": 1.0767730474472046 + }, + { + "bits_per_byte": 1.490845023651024, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0333750247955322, + "step": 7680, + "unclipped_grad_norm": 1.040376901626587 + }, + { + "bits_per_byte": 1.53468460523451, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0637623071670532, + "step": 7690, + "unclipped_grad_norm": 1.0646589994430542 + }, + { + "bits_per_byte": 1.5464913861988927, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.071946144104004, + "step": 7700, + "unclipped_grad_norm": 1.1067028045654297 + }, + { + "bits_per_byte": 1.4665762078224713, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0165531635284424, + "step": 7710, + "unclipped_grad_norm": 1.0252745151519775 + }, + { + "bits_per_byte": 1.4804371495687594, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0261608362197876, + "step": 7720, + "unclipped_grad_norm": 1.0019656419754028 + }, + { + "bits_per_byte": 1.5129640563437916, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0487067699432373, + "step": 7730, + "unclipped_grad_norm": 1.0048695802688599 + }, + { + "bits_per_byte": 1.5117082390271912, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0478363037109375, + "step": 7740, + "unclipped_grad_norm": 1.034838080406189 + }, + { + "bits_per_byte": 1.5481713127325205, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.073110580444336, + "step": 7750, + "unclipped_grad_norm": 1.047455072402954 + }, + { + "bits_per_byte": 1.5013908278192805, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0406848192214966, + "step": 7760, + "unclipped_grad_norm": 1.096320629119873 + }, + { + "bits_per_byte": 1.5005262710333933, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0400855541229248, + "step": 7770, + "unclipped_grad_norm": 1.0564236640930176 + }, + { + "bits_per_byte": 1.6389760885962732, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.1360516548156738, + "step": 7780, + "unclipped_grad_norm": 1.134232997894287 + }, + { + "bits_per_byte": 1.5295576304296388, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0602085590362549, + "step": 7790, + "unclipped_grad_norm": 1.0709397792816162 + }, + { + "bits_per_byte": 1.5624484524938966, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.083006739616394, + "step": 7800, + "unclipped_grad_norm": 1.099833607673645 + }, + { + "bits_per_byte": 1.476959144420361, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0237500667572021, + "step": 7810, + "unclipped_grad_norm": 1.057004690170288 + }, + { + "bits_per_byte": 1.5143643390871164, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0496773719787598, + "step": 7820, + "unclipped_grad_norm": 1.0959510803222656 + }, + { + "bits_per_byte": 1.5593489811600472, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0808583498001099, + "step": 7830, + "unclipped_grad_norm": 1.063362956047058 + }, + { + "bits_per_byte": 1.4978546925348903, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.038233757019043, + "step": 7840, + "unclipped_grad_norm": 1.0525480508804321 + }, + { + "bits_per_byte": 1.4918236049344333, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0340533256530762, + "step": 7850, + "unclipped_grad_norm": 1.030691146850586 + }, + { + "bits_per_byte": 1.5064691315339576, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.044204831123352, + "step": 7860, + "unclipped_grad_norm": 1.0765224695205688 + }, + { + "bits_per_byte": 1.5562261201856802, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0786937475204468, + "step": 7870, + "unclipped_grad_norm": 1.0955042839050293 + }, + { + "bits_per_byte": 1.5252102529811287, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0571951866149902, + "step": 7880, + "unclipped_grad_norm": 1.043196201324463 + }, + { + "bits_per_byte": 1.5900611270031748, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.1021463871002197, + "step": 7890, + "unclipped_grad_norm": 1.104681372642517 + }, + { + "bits_per_byte": 1.5606699799013244, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0817739963531494, + "step": 7900, + "unclipped_grad_norm": 1.0641751289367676 + }, + { + "bits_per_byte": 1.5589052659208211, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0805507898330688, + "step": 7910, + "unclipped_grad_norm": 1.1519651412963867 + }, + { + "bits_per_byte": 1.481516856650876, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0269092321395874, + "step": 7920, + "unclipped_grad_norm": 1.0578134059906006 + }, + { + "bits_per_byte": 1.4791508913229565, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0252692699432373, + "step": 7930, + "unclipped_grad_norm": 1.0473592281341553 + }, + { + "bits_per_byte": 1.5474438261193708, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0726063251495361, + "step": 7940, + "unclipped_grad_norm": 1.1337641477584839 + }, + { + "bits_per_byte": 1.4870933941051028, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0307745933532715, + "step": 7950, + "unclipped_grad_norm": 1.0619428157806396 + }, + { + "bits_per_byte": 1.5210572159280935, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.054316520690918, + "step": 7960, + "unclipped_grad_norm": 1.0414892435073853 + }, + { + "bits_per_byte": 1.5042326691421377, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0426546335220337, + "step": 7970, + "unclipped_grad_norm": 1.0393376350402832 + }, + { + "bits_per_byte": 1.5355154534208282, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.064338207244873, + "step": 7980, + "unclipped_grad_norm": 1.0560662746429443 + }, + { + "bits_per_byte": 1.5624764856659872, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0830261707305908, + "step": 7990, + "unclipped_grad_norm": 1.1011888980865479 + }, + { + "bits_per_byte": 1.4795096471326563, + "learning_rate": 3e-05, + "loss_nats": 1.0255179405212402, + "step": 8000, + "unclipped_grad_norm": 1.0237547159194946 + } + ] + }, + "depth-16-block-seed-2026073001": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "6b2a0f3320fece089c65a158888f4aafe0879ff220d9835ed3614be007b68068", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027713447343558073, + 0.00024001447309274226, + 0.00020889485313091427, + 0.00019106797117274255, + 0.00017709653184283525, + 0.00016345064796041697, + 0.0001520135992905125, + 0.00014396758342627436, + 0.00013824697816744447, + 0.0001333381951553747, + 0.00012879272981081158, + 0.00012578140012919903, + 0.00012338913802523166, + 0.0001212825154652819, + 0.00011928759340662509, + 0.00011723516217898577 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022927794270799495, + "first_to_last_ratio": 1.9059069547229377, + "imbalance_abs_log_ratio": 0.6449579870010378, + "last_quartile_mean": 0.00012029860226903111, + "mean": 0.00016006211535568582, + "normalized": [ + 1.7314182860806244, + 1.4995083162520277, + 1.3050861702453052, + 1.1937113960299495, + 1.1064237870985023, + 1.021170110098828, + 0.9497162957812465, + 0.8994482117542517, + 0.8637083038683806, + 0.8330403159990363, + 0.8046421823465957, + 0.7858286756343993, + 0.7708828397715447, + 0.7577215582573746, + 0.7452581339534802, + 0.7324354168284537 + ], + "population_cv": 0.28697332073733334 + }, + "activation_output_rms_by_block": [ + 0.005004490725696087, + 0.007150472607463598, + 0.0056586493737995625, + 0.007886149920523167, + 0.005775847006589174, + 0.008285986259579659, + 0.006597128231078386, + 0.009296777658164501, + 0.007501539774239063, + 0.01081777922809124, + 0.007698386441916227, + 0.011321330443024635, + 0.009009703993797302, + 0.013792803511023521, + 0.00953624676913023, + 0.012954792007803917 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0064249406568706036, + "first_to_last_ratio": 0.567404514268178, + "imbalance_abs_log_ratio": 0.5666828005795043, + "last_quartile_mean": 0.011323386570438743, + "mean": 0.008643005246995017, + "normalized": [ + 0.5790220626599802, + 0.8273132322752738, + 0.6547085431617609, + 0.9124314628022479, + 0.6682683674867963, + 0.9586927258271091, + 0.7632910130850682, + 1.0756417926966766, + 0.8679318778438997, + 1.251622429808468, + 0.8907071350665657, + 1.3098835554867694, + 1.04242722714183, + 1.5958342170183197, + 1.103348487778111, + 1.4988758698611242 + ], + "population_cv": 0.28641618387108014 + }, + "bits_per_byte": 8.092078157393424, + "branch_output_rms_by_sublayer": [ + 0.0033052791841328144, + 0.003737553022801876, + 0.0033692142460495234, + 0.0038969451561570168, + 0.004222586750984192, + 0.003809630637988448, + 0.004224879201501608, + 0.003801452461630106, + 0.004351162351667881, + 0.0037875697016716003, + 0.005037518683820963, + 0.0037579576019197702, + 0.005422801710665226, + 0.003782435320317745, + 0.005750134587287903, + 0.003670723643153906, + 0.006515308283269405, + 0.003754077944904566, + 0.006654165219515562, + 0.0036852110642939806, + 0.006751408334821463, + 0.003720880253240466, + 0.007695567328482866, + 0.003703791182488203, + 0.008140962570905685, + 0.003749529365450144, + 0.009426429867744446, + 0.0036915375385433435, + 0.008798080496490002, + 0.003752112854272127, + 0.008380400948226452, + 0.0036098985001444817 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.007836087849617155, + 0.00707663314616961, + 0.006860545956961424, + 0.006477463491545069, + 0.00630808320659222, + 0.0065403273634858875, + 0.006076989561376891, + 0.0053044433268254945, + 0.006002429844215364, + 0.005811298045810947, + 0.00541486468124986, + 0.005552686577658202, + 0.005724443119797102, + 0.005942467552368739, + 0.006312250195815329, + 0.004848514440357715 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.007062682611073314, + "first_to_last_ratio": 1.237564932158175, + "imbalance_abs_log_ratio": 0.21314568451503188, + "last_quartile_mean": 0.005706918827084721, + "mean": 0.006130595522490438, + "normalized": [ + 1.2781935818257821, + 1.154314147819503, + 1.119066807097797, + 1.0565798163950184, + 1.0289511326347756, + 1.0668339379905794, + 0.9912559944760846, + 0.865241118479574, + 0.9790940899942118, + 0.947917380047643, + 0.883252640201938, + 0.9057336366895933, + 0.9337499267072272, + 0.9693132633801169, + 1.0296308364592772, + 0.7908716898008789 + ], + "population_cv": 0.1166547247084935 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02823692187666893, + 0.014202049933373928, + 0.014330258592963219, + 0.014431006275117397, + 0.0145475585013628, + 0.009805543348193169, + 0.009861846454441547, + 0.009975572116672993, + 0.010043247602880001, + 0.007605585735291243, + 0.007654537446796894, + 0.0077590313740074635, + 0.007819762453436852, + 0.006375181954354048, + 0.006421996746212244, + 0.0065260399132966995, + 0.006563509814441204, + 0.005602568853646517, + 0.0056375316344201565, + 0.0057973419316112995, + 0.005820456892251968, + 0.005056646186858416, + 0.00509054446592927, + 0.005272198468446732, + 0.005295110866427422, + 0.0046798414550721645, + 0.004703808110207319, + 0.004869809839874506, + 0.004889858886599541, + 0.004459173884242773, + 0.00446688337251544, + 0.00453129643574357 + ], + "loss_nats": 5.609001159667969, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0033052791841328144, + 0.005004490725696087, + 0.005999269895255566, + 0.007150472607463598, + 0.004222586750984192, + 0.0056586493737995625, + 0.006890070624649525, + 0.007886149920523167, + 0.004351162351667881, + 0.005775847006589174, + 0.007411441300064325, + 0.008285986259579659, + 0.005422801710665226, + 0.006597128231078386, + 0.008578632958233356, + 0.009296777658164501, + 0.006515308283269405, + 0.007501539774239063, + 0.010256913490593433, + 0.01081777922809124, + 0.006751408334821463, + 0.007698386441916227, + 0.01081965770572424, + 0.011321330443024635, + 0.008140962570905685, + 0.009009703993797302, + 0.013233783654868603, + 0.013792803511023521, + 0.008798080496490002, + 0.00953624676913023, + 0.01231932733207941, + 0.012954792007803917 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.295638690469787e-05, + 7.495813042623922e-05, + 6.424964522011578e-05, + 5.6337856221944094e-05, + 4.736588743980974e-05, + 4.000150511274114e-05, + 3.3406991860829294e-05, + 2.9257240385049954e-05, + 2.6081905161845498e-05, + 2.4221646526711993e-05, + 2.3017932107904926e-05, + 2.205361488449853e-05, + 2.1097952412674204e-05, + 2.0617559130187146e-05, + 2.0049723389092833e-05, + 1.9771381630562246e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.962550469324924e-05, + "first_to_last_ratio": 3.415668082811133, + "imbalance_abs_log_ratio": 1.2283731057520397, + "last_quartile_mean": 2.0384154140629107e-05, + "mean": 3.784033492593153e-05, + "normalized": [ + 2.1922741187961536, + 1.980905575306399, + 1.6979142850050797, + 1.4888308026929338, + 1.2517301322127163, + 1.0571128715176512, + 0.8828408079954885, + 0.7731760419752606, + 0.6892620060815551, + 0.640101272203941, + 0.6082909190143296, + 0.5828070741885916, + 0.557551946989138, + 0.5448566766267753, + 0.5298505795030106, + 0.5224948898909759 + ], + "population_cv": 0.5389270020989329 + }, + "activation_output_rms_by_block": [ + 0.00972363818436861, + 0.018413539975881577, + 0.011301152408123016, + 0.020593363791704178, + 0.012286243960261345, + 0.022828614339232445, + 0.014672646299004555, + 0.02904798462986946, + 0.01802690140902996, + 0.03463379293680191, + 0.0215163491666317, + 0.04299422353506088, + 0.024622667580842972, + 0.048123423010110855, + 0.032037004828453064, + 0.06016550958156586 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.015007923590019345, + "first_to_last_ratio": 0.36394181302547757, + "imbalance_abs_log_ratio": 1.010761278449072, + "last_quartile_mean": 0.04123715125024319, + "mean": 0.0263116909773089, + "normalized": [ + 0.3695558066851058, + 0.6998235116002747, + 0.4295106847320868, + 0.7826697193070493, + 0.4669499946186262, + 0.8676224708978132, + 0.5576474089657784, + 1.1039953553316026, + 0.6851289574879962, + 1.3162891342358027, + 0.8177486268437599, + 1.6340349836176982, + 0.9358071133503911, + 1.8289749241739084, + 1.2175958153385753, + 2.2866454928135314 + ], + "population_cv": 0.5242218345998277 + }, + "bits_per_byte": 5.094476963164152, + "branch_output_rms_by_sublayer": [ + 0.004319524392485619, + 0.008673875592648983, + 0.004246409051120281, + 0.009991174563765526, + 0.005452096462249756, + 0.010073051787912846, + 0.006387368775904179, + 0.010035603307187557, + 0.0060663470067083836, + 0.01051078736782074, + 0.007258581463247538, + 0.010655198246240616, + 0.007839187048375607, + 0.011261194944381714, + 0.010483811609447002, + 0.011669226922094822, + 0.01079322025179863, + 0.012236819602549076, + 0.012645936571061611, + 0.012139025144279003, + 0.013178177177906036, + 0.012943001464009285, + 0.017010189592838287, + 0.012494352646172047, + 0.016437800601124763, + 0.012577332556247711, + 0.018056757748126984, + 0.01252446137368679, + 0.022778941318392754, + 0.013186860829591751, + 0.022304536774754524, + 0.012990611605346203 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00047768131303286524, + 0.00037801938737084786, + 0.0003757481810571057, + 0.0003849183374966076, + 0.0003205908784645981, + 0.0003135875371373823, + 0.0002702256708410831, + 0.00025052984822069896, + 0.00022846424191684566, + 0.00020301436013832287, + 0.00018877625003321395, + 0.00018901092530252878, + 0.00017529332745812984, + 0.0001779656532508539, + 0.00018745944157007413, + 0.00017444725355961893 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004040918047393566, + "first_to_last_ratio": 2.260129748343848, + "imbalance_abs_log_ratio": 0.8154222224078733, + "last_quartile_mean": 0.0001787914189596692, + "mean": 0.00026848328792817357, + "normalized": [ + 1.779184532188304, + 1.407981071328276, + 1.39952167584311, + 1.4336770845848084, + 1.1940813185749002, + 1.167996487071014, + 1.0064897257715932, + 0.9331301406280542, + 0.8509439961043904, + 0.7561526890740202, + 0.703121045223927, + 0.7039951229779867, + 0.6529021929477618, + 0.6628556087202881, + 0.6982164253747687, + 0.649750883586796 + ], + "population_cv": 0.3443291255737099 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6925109028816223, + "mean_weights": [ + 0.516845703125, + 0.4831542670726776 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6916897296905518, + "mean_weights": [ + 0.5256155133247375, + 0.47438448667526245 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6921632289886475, + "mean_weights": [ + 0.5142087936401367, + 0.4857912063598633 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927341222763062, + "mean_weights": [ + 0.5126981735229492, + 0.4873018264770508 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0980398654937744, + "mean_weights": [ + 0.3447725176811218, + 0.324658066034317, + 0.33056944608688354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098333477973938, + "mean_weights": [ + 0.3432988226413727, + 0.3276415765285492, + 0.3290596008300781 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0971107482910156, + "mean_weights": [ + 0.354239284992218, + 0.3338475227355957, + 0.3119131922721863 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098445177078247, + "mean_weights": [ + 0.33804112672805786, + 0.3361448645591736, + 0.3258139491081238 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3858705759048462, + "mean_weights": [ + 0.25839337706565857, + 0.24941343069076538, + 0.24149224162101746, + 0.2507009506225586 + ], + "sources": 4 + }, + { + "entropy_mean": 1.38621187210083, + "mean_weights": [ + 0.2517760992050171, + 0.2535240054130554, + 0.24683426320552826, + 0.24786563217639923 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385884165763855, + "mean_weights": [ + 0.2573123574256897, + 0.2553749680519104, + 0.24531587958335876, + 0.24199676513671875 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861134052276611, + "mean_weights": [ + 0.2514210641384125, + 0.2551700472831726, + 0.24895039200782776, + 0.24445852637290955 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6088017225265503, + "mean_weights": [ + 0.20787756145000458, + 0.20764237642288208, + 0.1969079077243805, + 0.1933688223361969, + 0.19420333206653595 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093868017196655, + "mean_weights": [ + 0.1996297687292099, + 0.200723797082901, + 0.20193718373775482, + 0.19867070019245148, + 0.199038565158844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.607616901397705, + "mean_weights": [ + 0.20617185533046722, + 0.2148669958114624, + 0.20431196689605713, + 0.19360044598579407, + 0.18104872107505798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094070672988892, + "mean_weights": [ + 0.1992160826921463, + 0.20051369071006775, + 0.198721244931221, + 0.19902712106704712, + 0.20252184569835663 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7898081541061401, + "mean_weights": [ + 0.17206060886383057, + 0.18212765455245972, + 0.17207278311252594, + 0.1640574187040329, + 0.15447911620140076, + 0.15520241856575012 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917404174804688, + "mean_weights": [ + 0.16550754010677338, + 0.16615179181098938, + 0.16667258739471436, + 0.16704785823822021, + 0.16769331693649292, + 0.16692689061164856 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7875205278396606, + "mean_weights": [ + 0.1741587072610855, + 0.18743160367012024, + 0.17662332952022552, + 0.16453337669372559, + 0.15226861834526062, + 0.14498436450958252 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916977405548096, + "mean_weights": [ + 0.1655852198600769, + 0.16538481414318085, + 0.16465555131435394, + 0.16688036918640137, + 0.16788282990455627, + 0.16961121559143066 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941636562347412, + "mean_weights": [ + 0.15022630989551544, + 0.1620834320783615, + 0.15417584776878357, + 0.14502081274986267, + 0.1326843798160553, + 0.12827157974243164, + 0.12753766775131226 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458396434783936, + "mean_weights": [ + 0.14176656305789948, + 0.14105038344860077, + 0.14134296774864197, + 0.14280946552753448, + 0.145027756690979, + 0.14322641491889954, + 0.14477643370628357 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395363330841064, + "mean_weights": [ + 0.1491325944662094, + 0.16522708535194397, + 0.15644297003746033, + 0.1476437896490097, + 0.13585588335990906, + 0.12666445970535278, + 0.11903323233127594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456722736358643, + "mean_weights": [ + 0.14162756502628326, + 0.1389600932598114, + 0.139722540974617, + 0.1427886188030243, + 0.14271582663059235, + 0.14634007215499878, + 0.14784526824951172 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0722482204437256, + "mean_weights": [ + 0.13311916589736938, + 0.1474711298942566, + 0.1391790360212326, + 0.13093729317188263, + 0.12214824557304382, + 0.11368396133184433, + 0.1066313236951828, + 0.10682985186576843 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079228401184082, + "mean_weights": [ + 0.12360984086990356, + 0.12237860262393951, + 0.12229418754577637, + 0.12343960255384445, + 0.1245395839214325, + 0.12723875045776367, + 0.12765389680862427, + 0.12884552776813507 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0708298683166504, + "mean_weights": [ + 0.13347145915031433, + 0.14857617020606995, + 0.14165782928466797, + 0.1326693594455719, + 0.12101326882839203, + 0.11364143341779709, + 0.10599445551633835, + 0.10297602415084839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079227924346924, + "mean_weights": [ + 0.12400860339403152, + 0.12165012210607529, + 0.12233424186706543, + 0.12391965091228485, + 0.12502582371234894, + 0.12579283118247986, + 0.12804390490055084, + 0.12922482192516327 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1904914379119873, + "mean_weights": [ + 0.1202651709318161, + 0.13243305683135986, + 0.1248447597026825, + 0.11750722676515579, + 0.10865370184183121, + 0.1034422218799591, + 0.09818274527788162, + 0.09678354859352112, + 0.09788758307695389 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969027519226074, + "mean_weights": [ + 0.10986955463886261, + 0.108255535364151, + 0.10776711255311966, + 0.10903829336166382, + 0.11005579680204391, + 0.11137478053569794, + 0.11346087604761124, + 0.11412093043327332, + 0.1160571351647377 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1876320838928223, + "mean_weights": [ + 0.12014202773571014, + 0.1339646577835083, + 0.12840712070465088, + 0.12060266733169556, + 0.11135218292474747, + 0.10444460809230804, + 0.09650979936122894, + 0.09281164407730103, + 0.09176531434059143 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028673440217971802, + 0.01493624784052372, + 0.015959296375513077, + 0.015908053144812584, + 0.017468810081481934, + 0.011937789618968964, + 0.013394257053732872, + 0.013926870189607143, + 0.015381631441414356, + 0.011804917827248573, + 0.013283023610711098, + 0.013736946508288383, + 0.015293780714273453, + 0.01275910809636116, + 0.014138885773718357, + 0.014833617955446243, + 0.01643991842865944, + 0.014291302300989628, + 0.01566910557448864, + 0.016281748190522194, + 0.01796886697411537, + 0.016028419137001038, + 0.017591506242752075, + 0.01816963590681553, + 0.020232705399394035, + 0.018053309991955757, + 0.019960105419158936, + 0.01998855359852314, + 0.02254442311823368, + 0.020374728366732597, + 0.022746482864022255, + 0.022396961227059364 + ], + "loss_nats": 3.531222343444824, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1935343742370605, + "mean_weights": [ + 0.0999753400683403, + 0.09610259532928467, + 0.10262835025787354, + 0.1083034798502922, + 0.11600279062986374, + 0.11817552894353867, + 0.12095586210489273, + 0.11944535374641418, + 0.11841070652008057 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004319524392485619, + 0.00972363818436861, + 0.010906110517680645, + 0.018413539975881577, + 0.005452096462249756, + 0.011301152408123016, + 0.013437297195196152, + 0.020593363791704178, + 0.0060663470067083836, + 0.012286243960261345, + 0.014960306696593761, + 0.022828614339232445, + 0.007839187048375607, + 0.014672646299004555, + 0.020714404061436653, + 0.02904798462986946, + 0.01079322025179863, + 0.01802690140902996, + 0.026133937761187553, + 0.03463379293680191, + 0.013178177177906036, + 0.0215163491666317, + 0.03426522761583328, + 0.04299422353506088, + 0.016437800601124763, + 0.024622667580842972, + 0.03885738551616669, + 0.048123423010110855, + 0.022778941318392754, + 0.032037004828453064, + 0.050110094249248505, + 0.06016550958156586 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015777233056724072, + 0.00015884912863839418, + 0.00014271712279878557, + 0.0001244439627043903, + 0.00010494918387848884, + 9.392325591761619e-05, + 8.538185647921637e-05, + 7.606096187373623e-05, + 7.664377335458994e-05, + 6.531321560032666e-05, + 6.149107503006235e-05, + 5.037344453739934e-05, + 4.3381573050282896e-05, + 3.9420847315341234e-05, + 3.5541354009183124e-05, + 3.5066059353994206e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001459456361772027, + "first_to_last_ratio": 3.805378902507801, + "imbalance_abs_log_ratio": 1.3364155665156117, + "last_quartile_mean": 3.8352458432200365e-05, + "mean": 8.445807156931551e-05, + "normalized": [ + 1.8680550909542795, + 1.8808045896243943, + 1.6897985017531016, + 1.4734407309105797, + 1.2426187566022755, + 1.1120696242813515, + 1.0109377930698158, + 0.9005765874172524, + 0.907477188745515, + 0.7733211804004252, + 0.7280662923921495, + 0.5964313842526867, + 0.513646265468887, + 0.46675050214695213, + 0.42081654658683526, + 0.4151889653934991 + ], + "population_cv": 0.4877143313799778 + }, + "activation_output_rms_by_block": [ + 0.025262881070375443, + 0.04275903478264809, + 0.027615655213594437, + 0.03926205262541771, + 0.023017436265945435, + 0.03517371043562889, + 0.029550904408097267, + 0.04735429957509041, + 0.028369002044200897, + 0.04305245727300644, + 0.030713967978954315, + 0.05133935809135437, + 0.032636724412441254, + 0.06983363628387451, + 0.0380333848297596, + 0.08374716341495514 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03372490592300892, + "first_to_last_ratio": 0.601556641750376, + "imbalance_abs_log_ratio": 0.508234580502337, + "last_quartile_mean": 0.056062727235257626, + "mean": 0.04048260429408401, + "normalized": [ + 0.6240428823910242, + 1.056232313317272, + 0.6821610342304507, + 0.9698499716125064, + 0.5685759764548824, + 0.868859873246076, + 0.7299654981044719, + 1.1697443976451536, + 0.7007701836106094, + 1.0634804263148154, + 0.7586954573335775, + 1.2681831860026092, + 0.8061913254234652, + 1.7250282560027828, + 0.939499458914941, + 2.0687197593953623 + ], + "population_cv": 0.3940384688106746 + }, + "bits_per_byte": 3.0824777281177833, + "branch_output_rms_by_sublayer": [ + 0.014225655235350132, + 0.01885763369500637, + 0.016379276290535927, + 0.015279849991202354, + 0.023593612015247345, + 0.011461553163826466, + 0.02286384254693985, + 0.009474686346948147, + 0.01935836300253868, + 0.010116237215697765, + 0.017970046028494835, + 0.009506942704319954, + 0.022477619349956512, + 0.013764957897365093, + 0.025794917717576027, + 0.012601078487932682, + 0.02379908226430416, + 0.014191905967891216, + 0.0178237184882164, + 0.0165114663541317, + 0.023045850917696953, + 0.019447455182671547, + 0.020932989194989204, + 0.02472713589668274, + 0.017093855887651443, + 0.028764601796865463, + 0.021727578714489937, + 0.03578802943229675, + 0.008686712011694908, + 0.036414168775081635, + 0.008637740276753902, + 0.04455644637346268 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006504433508330581, + 0.00048523602227392907, + 0.0003980638932136608, + 0.0004387300445530515, + 0.000365155189968218, + 0.0002788779263831531, + 0.00029885208568229324, + 0.00026779671325771547, + 0.0002902285411340871, + 0.00023612702925164195, + 0.00026277969610552434, + 0.0002548116213942568, + 0.0001837802586554031, + 0.00016271499692659642, + 0.00013374164438063855, + 0.00014095740133328614 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004931183277184248, + "first_to_last_ratio": 3.1752920249892207, + "imbalance_abs_log_ratio": 1.1553996044896453, + "last_quartile_mean": 0.00015529857532398106, + "mean": 0.0003030185259591571, + "normalized": [ + 2.1465464818501867, + 1.6013411085609086, + 1.3136619022010376, + 1.4478654173513685, + 1.20505896071, + 0.9203329251913204, + 0.9862502127100129, + 0.8837635006310174, + 0.9577914096684838, + 0.7792494815431475, + 0.8672067005597657, + 0.8409110320489186, + 0.6064984247206531, + 0.536980359242221, + 0.4413645797968972, + 0.46517750321406204 + ], + "population_cv": 0.440031122144889 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6679806113243103, + "mean_weights": [ + 0.4191037714481354, + 0.580896258354187 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6774038076400757, + "mean_weights": [ + 0.4775225520133972, + 0.522477388381958 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686615526676178, + "mean_weights": [ + 0.49171629548072815, + 0.5082836747169495 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6786561012268066, + "mean_weights": [ + 0.48390668630599976, + 0.516093373298645 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0939385890960693, + "mean_weights": [ + 0.32296979427337646, + 0.3461979627609253, + 0.33083218336105347 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0876491069793701, + "mean_weights": [ + 0.3022151589393616, + 0.3410899341106415, + 0.35669487714767456 + ], + "sources": 3 + }, + { + "entropy_mean": 1.094268560409546, + "mean_weights": [ + 0.3278208076953888, + 0.3347434997558594, + 0.33743569254875183 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0785255432128906, + "mean_weights": [ + 0.30184268951416016, + 0.3319981098175049, + 0.3661591410636902 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3751009702682495, + "mean_weights": [ + 0.2335343211889267, + 0.2411448359489441, + 0.23957431316375732, + 0.2857465147972107 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3803684711456299, + "mean_weights": [ + 0.23392163217067719, + 0.24959179759025574, + 0.25211477279663086, + 0.264371782541275 + ], + "sources": 4 + }, + { + "entropy_mean": 1.377173662185669, + "mean_weights": [ + 0.2529916763305664, + 0.22492024302482605, + 0.24044713377952576, + 0.2816409468650818 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786447048187256, + "mean_weights": [ + 0.23216357827186584, + 0.24167919158935547, + 0.2545858919620514, + 0.2715713381767273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6062697172164917, + "mean_weights": [ + 0.20799711346626282, + 0.1897224485874176, + 0.18646025657653809, + 0.19469907879829407, + 0.22112110257148743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6025722026824951, + "mean_weights": [ + 0.20026245713233948, + 0.18544277548789978, + 0.1946466565132141, + 0.19751906394958496, + 0.22212904691696167 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6060729026794434, + "mean_weights": [ + 0.20172038674354553, + 0.20138651132583618, + 0.18545961380004883, + 0.19016142189502716, + 0.2212720513343811 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601726770401001, + "mean_weights": [ + 0.20073023438453674, + 0.1886606216430664, + 0.19257165491580963, + 0.19004762172698975, + 0.22798986732959747 + ], + "sources": 5 + }, + { + "entropy_mean": 1.788693904876709, + "mean_weights": [ + 0.16298983991146088, + 0.17133885622024536, + 0.15878048539161682, + 0.15357322990894318, + 0.17383432388305664, + 0.17948326468467712 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7880157232284546, + "mean_weights": [ + 0.16720382869243622, + 0.15951547026634216, + 0.1605137139558792, + 0.15542668104171753, + 0.17394030094146729, + 0.1834000051021576 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7797719240188599, + "mean_weights": [ + 0.15782463550567627, + 0.17797376215457916, + 0.15211433172225952, + 0.14025422930717468, + 0.16258476674556732, + 0.20924828946590424 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7884608507156372, + "mean_weights": [ + 0.16717270016670227, + 0.17295360565185547, + 0.1665436029434204, + 0.1547294557094574, + 0.1560850888490677, + 0.18251556158065796 + ], + "sources": 6 + }, + { + "entropy_mean": 1.927689790725708, + "mean_weights": [ + 0.1226624995470047, + 0.15493766963481903, + 0.12924720346927643, + 0.12375974655151367, + 0.1207568496465683, + 0.16289259493350983, + 0.18574345111846924 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9359079599380493, + "mean_weights": [ + 0.1374971866607666, + 0.15181128680706024, + 0.14001096785068512, + 0.13405972719192505, + 0.11918757855892181, + 0.14624495804309845, + 0.17118829488754272 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9246108531951904, + "mean_weights": [ + 0.12061166763305664, + 0.16360490024089813, + 0.1358117014169693, + 0.1262153834104538, + 0.1155187338590622, + 0.15234649181365967, + 0.18589113652706146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937231183052063, + "mean_weights": [ + 0.1359395533800125, + 0.16017206013202667, + 0.1499013900756836, + 0.1417488306760788, + 0.12114370614290237, + 0.13951142132282257, + 0.15158303081989288 + ], + "sources": 7 + }, + { + "entropy_mean": 2.052294969558716, + "mean_weights": [ + 0.0988478884100914, + 0.15235479176044464, + 0.12090037763118744, + 0.11009317636489868, + 0.09369668364524841, + 0.12056069076061249, + 0.13413047790527344, + 0.1694159209728241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.061406135559082, + "mean_weights": [ + 0.1149308905005455, + 0.14702443778514862, + 0.13390076160430908, + 0.12344712018966675, + 0.0966324657201767, + 0.11953848600387573, + 0.1299745738506317, + 0.1345512568950653 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0511577129364014, + "mean_weights": [ + 0.09969542920589447, + 0.16962948441505432, + 0.1319108009338379, + 0.11863738298416138, + 0.09143088757991791, + 0.12074249982833862, + 0.14481820166110992, + 0.12313534319400787 + ], + "sources": 8 + }, + { + "entropy_mean": 2.069282054901123, + "mean_weights": [ + 0.12384641170501709, + 0.13645249605178833, + 0.13362589478492737, + 0.12747827172279358, + 0.10400287806987762, + 0.11596200615167618, + 0.12676866352558136, + 0.13186340034008026 + ], + "sources": 8 + }, + { + "entropy_mean": 2.165989875793457, + "mean_weights": [ + 0.08691617846488953, + 0.1522500216960907, + 0.1198691874742508, + 0.10617777705192566, + 0.08211278915405273, + 0.11005747318267822, + 0.12842655181884766, + 0.08939606696367264, + 0.12479399144649506 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907525062561035, + "mean_weights": [ + 0.1032973974943161, + 0.12799270451068878, + 0.11672255396842957, + 0.11367214471101761, + 0.10446079820394516, + 0.11008220911026001, + 0.12035597860813141, + 0.1041758805513382, + 0.09924033284187317 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1538312435150146, + "mean_weights": [ + 0.09009353071451187, + 0.16798722743988037, + 0.13133394718170166, + 0.11392836272716522, + 0.08597822487354279, + 0.10755141079425812, + 0.13270318508148193, + 0.09302861988544464, + 0.07739552855491638 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04356693848967552, + 0.020052054896950722, + 0.025759555399417877, + 0.02808709815144539, + 0.03107464499771595, + 0.023514779284596443, + 0.025064069777727127, + 0.026588447391986847, + 0.02779453620314598, + 0.02102271094918251, + 0.022771678864955902, + 0.023088837042450905, + 0.024504151195287704, + 0.02003801427781582, + 0.021661879494786263, + 0.021999599412083626, + 0.02357223629951477, + 0.01831224001944065, + 0.01920042186975479, + 0.01775558479130268, + 0.01933184079825878, + 0.014507194049656391, + 0.0161737073212862, + 0.015147161670029163, + 0.017492808401584625, + 0.01415062416344881, + 0.01712540164589882, + 0.017588311806321144, + 0.021003147587180138, + 0.01741618849337101, + 0.020855708047747612, + 0.02042423188686371 + ], + "loss_nats": 2.136610746383667, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1821343898773193, + "mean_weights": [ + 0.12100096046924591, + 0.09246665984392166, + 0.10230658948421478, + 0.10203823447227478, + 0.09980043023824692, + 0.10532397031784058, + 0.11232466995716095, + 0.1358221024274826, + 0.12891638278961182 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.014225655235350132, + 0.025262881070375443, + 0.03417503461241722, + 0.04275903478264809, + 0.023593612015247345, + 0.027615655213594437, + 0.035288337618112564, + 0.03926205262541771, + 0.01935836300253868, + 0.023017436265945435, + 0.030542491003870964, + 0.03517371043562889, + 0.022477619349956512, + 0.029550904408097267, + 0.04085070639848709, + 0.04735429957509041, + 0.02379908226430416, + 0.028369002044200897, + 0.03469245135784149, + 0.04305245727300644, + 0.023045850917696953, + 0.030713967978954315, + 0.037714507430791855, + 0.05133935809135437, + 0.017093855887651443, + 0.032636724412441254, + 0.041826508939266205, + 0.06983363628387451, + 0.008686712011694908, + 0.0380333848297596, + 0.04225953668355942, + 0.08374716341495514 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.445879600709304e-05, + 7.514825847465545e-05, + 8.149984205374494e-05, + 7.616886432515457e-05, + 7.93620856711641e-05, + 7.107412966433913e-05, + 6.609321280848235e-05, + 6.663892418146133e-05, + 0.00013160602247808129, + 0.00010466711682965979, + 0.00017213824321515858, + 0.0001231937640113756, + 0.00011169029312441126, + 7.821891631465405e-05, + 5.0732542149489745e-05, + 4.2012750782305375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.6818940215162e-05, + "first_to_last_ratio": 1.087107257387618, + "imbalance_abs_log_ratio": 0.08352027612092149, + "last_quartile_mean": 7.066362559271511e-05, + "mean": 8.779398513070191e-05, + "normalized": [ + 0.8481081693266763, + 0.8559613550151489, + 0.9283078098392884, + 0.8675863638238926, + 0.9039581191469443, + 0.8095557976839601, + 0.752821650709751, + 0.7590374680253287, + 1.499032334415107, + 1.1921900648869996, + 1.9607065672994624, + 1.403214170401, + 1.2721861635297007, + 0.8909370749967309, + 0.5778589737550069, + 0.47853791714500205 + ], + "population_cv": 0.365612445298668 + }, + "activation_output_rms_by_block": [ + 0.10262583196163177, + 0.17122715711593628, + 0.1563396155834198, + 0.23706908524036407, + 0.04811444878578186, + 0.07098451256752014, + 0.05486894026398659, + 0.08282698690891266, + 0.05263432487845421, + 0.09547916799783707, + 0.05760422348976135, + 0.09091533720493317, + 0.05383981764316559, + 0.1016460508108139, + 0.07459317892789841, + 0.18281812965869904 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16681542247533798, + "first_to_last_ratio": 1.6160480792916094, + "imbalance_abs_log_ratio": 0.4799837116932076, + "last_quartile_mean": 0.10322429426014423, + "mean": 0.10209917556494474, + "normalized": [ + 1.005158282559804, + 1.6770669906831872, + 1.531252477978855, + 2.3219490650006835, + 0.4712520793586283, + 0.6952505950683928, + 0.5374082597668455, + 0.8112405066016116, + 0.5155215464494499, + 0.9351610085930936, + 0.5641987133688422, + 0.8904610316574242, + 0.5273286228341615, + 0.9955619156411052, + 0.7305953110311854, + 1.7905935934067303 + ], + "population_cv": 0.5286889875537891 + }, + "bits_per_byte": 2.0839361382492645, + "branch_output_rms_by_sublayer": [ + 0.05184043571352959, + 0.0848206952214241, + 0.0633782222867012, + 0.04741951450705528, + 0.15336942672729492, + 0.017866143956780434, + 0.08626767247915268, + 0.013284381479024887, + 0.043669264763593674, + 0.012495848350226879, + 0.03683627396821976, + 0.02719804272055626, + 0.04707624390721321, + 0.01943410001695156, + 0.031199999153614044, + 0.02732040360569954, + 0.04137895256280899, + 0.029010025784373283, + 0.04094765707850456, + 0.04834925755858421, + 0.03695840761065483, + 0.041233282536268234, + 0.026400581002235413, + 0.05263383314013481, + 0.021495802327990532, + 0.04698725789785385, + 0.018388623371720314, + 0.06700151413679123, + 0.015488670207560062, + 0.0718570277094841, + 0.015049737878143787, + 0.12199355661869049 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003583741677462985, + 0.0002652231972340971, + 0.00033417461006948805, + 0.0002275253539728081, + 0.0001893834795064905, + 0.00020636019253143534, + 0.0002096112726133392, + 0.0001923796657115476, + 0.00047635112247231166, + 0.000565001794959564, + 0.0009549288128880833, + 0.0007263576654831343, + 0.0005471283815528999, + 0.0004183078461688176, + 0.0002607600761312772, + 0.0002482750049477455 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00029632433225567294, + "first_to_last_ratio": 0.8038795478406102, + "imbalance_abs_log_ratio": 0.21830583714537038, + "last_quartile_mean": 0.00036861782720018504, + "mean": 0.0003862589152493336, + "normalized": [ + 0.9278081452565691, + 0.6866461504529756, + 0.8651570148323317, + 0.5890488089470723, + 0.49030189862217616, + 0.5342535392308757, + 0.5426703807678671, + 0.49805883596852335, + 1.2332430493282536, + 1.462754056032209, + 2.472250542804088, + 1.8804942405387945, + 1.4164809146210218, + 1.0829726632945058, + 0.6750914110628468, + 0.6427683482398892 + ], + "population_cv": 0.550745478202196 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4576239287853241, + "mean_weights": [ + 0.8160475492477417, + 0.18395249545574188 + ], + "sources": 2 + }, + { + "entropy_mean": 0.622429370880127, + "mean_weights": [ + 0.626032829284668, + 0.37396717071533203 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5034195184707642, + "mean_weights": [ + 0.7919335961341858, + 0.20806637406349182 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6750276684761047, + "mean_weights": [ + 0.4605996012687683, + 0.5394004583358765 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9881983399391174, + "mean_weights": [ + 0.5066143870353699, + 0.2128569334745407, + 0.2805286943912506 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0713173151016235, + "mean_weights": [ + 0.2439693659543991, + 0.33285707235336304, + 0.42317354679107666 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0050503015518188, + "mean_weights": [ + 0.46526122093200684, + 0.19465598464012146, + 0.3400827646255493 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091313123703003, + "mean_weights": [ + 0.29163461923599243, + 0.3559022545814514, + 0.35246312618255615 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3032760620117188, + "mean_weights": [ + 0.25388818979263306, + 0.13041767477989197, + 0.27188149094581604, + 0.34381261467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3785908222198486, + "mean_weights": [ + 0.2572174668312073, + 0.24623332917690277, + 0.2131483256816864, + 0.28340083360671997 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2377310991287231, + "mean_weights": [ + 0.36895960569381714, + 0.12289825826883316, + 0.12992563843727112, + 0.3782165050506592 + ], + "sources": 4 + }, + { + "entropy_mean": 1.370632529258728, + "mean_weights": [ + 0.2267947793006897, + 0.24114219844341278, + 0.21443253755569458, + 0.31763046979904175 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5590848922729492, + "mean_weights": [ + 0.22265344858169556, + 0.11684703826904297, + 0.1529940664768219, + 0.24088799953460693, + 0.26661744713783264 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5987765789031982, + "mean_weights": [ + 0.19780412316322327, + 0.19732677936553955, + 0.1696196049451828, + 0.187029629945755, + 0.24821987748146057 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5600793361663818, + "mean_weights": [ + 0.19789859652519226, + 0.13498559594154358, + 0.15099790692329407, + 0.2153477668762207, + 0.300770103931427 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6049537658691406, + "mean_weights": [ + 0.21525533497333527, + 0.2069544643163681, + 0.18156641721725464, + 0.19566501677036285, + 0.20055876672267914 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7536540031433105, + "mean_weights": [ + 0.1706119179725647, + 0.12355935573577881, + 0.11797749251127243, + 0.17735670506954193, + 0.15967294573783875, + 0.250821590423584 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7486846446990967, + "mean_weights": [ + 0.1656067669391632, + 0.16547001898288727, + 0.14594469964504242, + 0.13893218338489532, + 0.11710066348314285, + 0.26694566011428833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.502744197845459, + "mean_weights": [ + 0.14790554344654083, + 0.12157414853572845, + 0.05193060636520386, + 0.11843410134315491, + 0.09838350117206573, + 0.4617721140384674 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7703553438186646, + "mean_weights": [ + 0.17112275958061218, + 0.18443143367767334, + 0.1734921634197235, + 0.14492332935333252, + 0.11953528970479965, + 0.2064950168132782 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6120144128799438, + "mean_weights": [ + 0.08869323134422302, + 0.07363369315862656, + 0.0430244579911232, + 0.08643164485692978, + 0.0696420818567276, + 0.21115350723266602, + 0.4274213910102844 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8458226919174194, + "mean_weights": [ + 0.12519317865371704, + 0.13179585337638855, + 0.1208491250872612, + 0.10168728232383728, + 0.08638136088848114, + 0.13217970728874207, + 0.3019134998321533 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5814783573150635, + "mean_weights": [ + 0.09110043942928314, + 0.07703252136707306, + 0.04712430387735367, + 0.08346284925937653, + 0.06837025284767151, + 0.1662226915359497, + 0.4666869342327118 + ], + "sources": 7 + }, + { + "entropy_mean": 1.864192008972168, + "mean_weights": [ + 0.13281965255737305, + 0.1363598257303238, + 0.09730516374111176, + 0.10794492065906525, + 0.09621791541576385, + 0.14952389895915985, + 0.27982860803604126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7858508825302124, + "mean_weights": [ + 0.0760645866394043, + 0.06870019435882568, + 0.041292984038591385, + 0.06624549627304077, + 0.05510660633444786, + 0.12923449277877808, + 0.3349960446357727, + 0.22835959494113922 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9164016246795654, + "mean_weights": [ + 0.10010629147291183, + 0.09289439767599106, + 0.06040576845407486, + 0.08482401818037033, + 0.06471984833478928, + 0.12172460556030273, + 0.20661231875419617, + 0.26871275901794434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7764818668365479, + "mean_weights": [ + 0.07597766816616058, + 0.07146216928958893, + 0.04444725811481476, + 0.06381020694971085, + 0.052670422941446304, + 0.10254879295825958, + 0.2695360779762268, + 0.3195474147796631 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9026113748550415, + "mean_weights": [ + 0.09874680638313293, + 0.09077301621437073, + 0.052286967635154724, + 0.08586552739143372, + 0.061087287962436676, + 0.1283089518547058, + 0.24689394235610962, + 0.2360374927520752 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0007686614990234, + "mean_weights": [ + 0.0738106444478035, + 0.07652983069419861, + 0.04215084761381149, + 0.06585545837879181, + 0.053016483783721924, + 0.10676231980323792, + 0.23121783137321472, + 0.19090119004249573, + 0.1597554087638855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.05625581741333, + "mean_weights": [ + 0.08243454247713089, + 0.09636277705430984, + 0.04735179618000984, + 0.07971908152103424, + 0.057041361927986145, + 0.1233949214220047, + 0.21250130236148834, + 0.17240752279758453, + 0.12878668308258057 + ], + "sources": 9 + }, + { + "entropy_mean": 2.023190498352051, + "mean_weights": [ + 0.08014722913503647, + 0.08487560600042343, + 0.04445280879735947, + 0.06905653327703476, + 0.057849444448947906, + 0.1067991554737091, + 0.20276135206222534, + 0.2105080783367157, + 0.14354978501796722 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05538630113005638, + 0.04719197377562523, + 0.05779172107577324, + 0.05863569676876068, + 0.10090170800685883, + 0.07870100438594818, + 0.1107865646481514, + 0.10731016099452972, + 0.12961514294147491, + 0.08656422793865204, + 0.08832070231437683, + 0.06243300065398216, + 0.09176014363765717, + 0.06557498127222061, + 0.0774947926402092, + 0.06959337741136551, + 0.08217275887727737, + 0.054640792310237885, + 0.061008501797914505, + 0.04349073767662048, + 0.07030727714300156, + 0.028683170676231384, + 0.04603966698050499, + 0.03691008314490318, + 0.04661448672413826, + 0.035560719668865204, + 0.03472456708550453, + 0.035963866859674454, + 0.04182035103440285, + 0.0347055159509182, + 0.03997879475355148, + 0.04123349115252495 + ], + "loss_nats": 1.444474458694458, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.9153306484222412, + "mean_weights": [ + 0.0914863795042038, + 0.05659400671720505, + 0.05275696888566017, + 0.06721317768096924, + 0.05635443329811096, + 0.0678911805152893, + 0.08835235983133316, + 0.2647843658924103, + 0.25456711649894714 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.05184043571352959, + 0.10262583196163177, + 0.1408688724040985, + 0.17122715711593628, + 0.15336942672729492, + 0.1563396155834198, + 0.2344740480184555, + 0.23706908524036407, + 0.043669264763593674, + 0.04811444878578186, + 0.059994228184223175, + 0.07098451256752014, + 0.04707624390721321, + 0.05486894026398659, + 0.069931261241436, + 0.08282698690891266, + 0.04137895256280899, + 0.05263432487845421, + 0.07298346608877182, + 0.09547916799783707, + 0.03695840761065483, + 0.05760422348976135, + 0.0662730410695076, + 0.09091533720493317, + 0.021495802327990532, + 0.05383981764316559, + 0.058252494782209396, + 0.1016460508108139, + 0.015488670207560062, + 0.07459317892789841, + 0.07778036594390869, + 0.18281812965869904 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.599572978098877e-05, + 5.6567834690213203e-05, + 7.252454088302329e-05, + 6.816006498411298e-05, + 6.961339386180043e-05, + 6.354664219543338e-05, + 5.091843559057452e-05, + 5.004410559195094e-05, + 0.00011432960309321061, + 8.808868005871773e-05, + 0.0001649916812311858, + 0.00011419985094107687, + 0.0001274511159863323, + 8.248462108895183e-05, + 5.212613905314356e-05, + 4.046529647894204e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.331204258458456e-05, + "first_to_last_ratio": 0.8371088393670095, + "imbalance_abs_log_ratio": 0.1778011818604578, + "last_quartile_mean": 7.563179315184243e-05, + "mean": 7.946923346935364e-05, + "normalized": [ + 0.7046214910652542, + 0.711820565276094, + 0.9126115569114117, + 0.8576912348147676, + 0.8759791786420843, + 0.7996382929745935, + 0.6407314298584572, + 0.6297293104160852, + 1.438664978910366, + 1.1084626869175487, + 2.076170538310438, + 1.4370322444989567, + 1.6037793548805563, + 1.0379440883969395, + 0.6559285496725645, + 0.5091944984538828 + ], + "population_cv": 0.418216483754139 + }, + "activation_output_rms_by_block": [ + 0.16897788643836975, + 0.2546124756336212, + 0.24958282709121704, + 0.3488057255744934, + 0.05895345285534859, + 0.09218393266201019, + 0.05982525274157524, + 0.10435724258422852, + 0.07404384016990662, + 0.13799543678760529, + 0.072679303586483, + 0.12079861760139465, + 0.06610427051782608, + 0.11929895728826523, + 0.08813745528459549, + 0.2320432960987091 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.25549472868442535, + "first_to_last_ratio": 2.0213831070680737, + "imbalance_abs_log_ratio": 0.7037819835779293, + "last_quartile_mean": 0.12639599479734898, + "mean": 0.1405249983072281, + "normalized": [ + 1.2024756340429583, + 1.8118660644063134, + 1.776074222364032, + 2.4821613931772033, + 0.419522886073689, + 0.6559966822448885, + 0.425726763652258, + 0.7426240444143152, + 0.5269086714950565, + 0.9819992061904, + 0.5171983950327836, + 0.8596236901372816, + 0.47040933153618064, + 0.8489518500291555, + 0.6272012549105437, + 1.6512599102929408 + ], + "population_cv": 0.5965464726510664 + }, + "bits_per_byte": 1.9646943750306483, + "branch_output_rms_by_sublayer": [ + 0.07875782251358032, + 0.1447884738445282, + 0.07949890196323395, + 0.06182052940130234, + 0.24504326283931732, + 0.02123360149562359, + 0.10607437044382095, + 0.01836485043168068, + 0.053805794566869736, + 0.015964502468705177, + 0.03685670718550682, + 0.03905382379889488, + 0.05105716362595558, + 0.02389056608080864, + 0.045697249472141266, + 0.039935220032930374, + 0.059221696108579636, + 0.040779970586299896, + 0.054663438349962234, + 0.080495685338974, + 0.04156494140625, + 0.05664549022912979, + 0.0355471596121788, + 0.07668386399745941, + 0.026702450588345528, + 0.05748681351542473, + 0.024266516789793968, + 0.08028528094291687, + 0.019828684628009796, + 0.08536289632320404, + 0.0202633123844862, + 0.16591297090053558 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00033760869044818646, + 0.0001988249533593431, + 0.00022180290878374173, + 0.00015787488110689858, + 0.0001349156990162873, + 0.00018159413720443885, + 0.00013882426062051804, + 0.00016817749051027943, + 0.0004737077931871952, + 0.0005939057773758512, + 0.0010653831085878915, + 0.0008603315741687424, + 0.000762827758322017, + 0.0005453805470031765, + 0.00031779802289983946, + 0.00029451637801964313 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022902785842454247, + "first_to_last_ratio": 0.47701150875195986, + "imbalance_abs_log_ratio": 0.7402146610222566, + "last_quartile_mean": 0.000480130676561169, + "mean": 0.00040334212378837815, + "normalized": [ + 0.8370280973313858, + 0.49294368634717844, + 0.5499125821534951, + 0.3914167943185893, + 0.33449444295353004, + 0.45022358562210574, + 0.34418488036066147, + 0.41695989729681016, + 1.1744565351565805, + 1.4724615713271154, + 2.6413881559935133, + 2.1330070018179734, + 1.89126727245144, + 1.3521537048516206, + 0.7879118102392371, + 0.7301899817787623 + ], + "population_cv": 0.6909894412084847 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.27112114429473877, + "mean_weights": [ + 0.9155579805374146, + 0.08444204181432724 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5555346012115479, + "mean_weights": [ + 0.7120839953422546, + 0.28791600465774536 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4612463414669037, + "mean_weights": [ + 0.8216869831085205, + 0.1783129870891571 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6770275831222534, + "mean_weights": [ + 0.48782873153686523, + 0.5121712684631348 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9962381720542908, + "mean_weights": [ + 0.465767502784729, + 0.19465303421020508, + 0.3395794630050659 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0414454936981201, + "mean_weights": [ + 0.22931429743766785, + 0.28697463870048523, + 0.4837110638618469 + ], + "sources": 3 + }, + { + "entropy_mean": 0.989799976348877, + "mean_weights": [ + 0.40070968866348267, + 0.17750780284404755, + 0.4217824935913086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.085202932357788, + "mean_weights": [ + 0.2848654091358185, + 0.3230235278606415, + 0.39211106300354004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2675609588623047, + "mean_weights": [ + 0.23358023166656494, + 0.10898500680923462, + 0.35628458857536316, + 0.3011501431465149 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3760833740234375, + "mean_weights": [ + 0.2731572389602661, + 0.2245836853981018, + 0.22189702093601227, + 0.280362069606781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2111725807189941, + "mean_weights": [ + 0.41297993063926697, + 0.11292652785778046, + 0.1222275048494339, + 0.3518660068511963 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647139072418213, + "mean_weights": [ + 0.23418807983398438, + 0.22505050897598267, + 0.2112433910369873, + 0.32951802015304565 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5535553693771362, + "mean_weights": [ + 0.21889275312423706, + 0.10524161159992218, + 0.1673421710729599, + 0.2413085401058197, + 0.26721489429473877 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5950126647949219, + "mean_weights": [ + 0.2038668543100357, + 0.17965060472488403, + 0.17214898765087128, + 0.19203421473503113, + 0.25229936838150024 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5395398139953613, + "mean_weights": [ + 0.2043805718421936, + 0.12131387740373611, + 0.1313788890838623, + 0.23064203560352325, + 0.3122846484184265 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6000796556472778, + "mean_weights": [ + 0.22543779015541077, + 0.20209266245365143, + 0.18844717741012573, + 0.20001280307769775, + 0.18400955200195312 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7217694520950317, + "mean_weights": [ + 0.1735105812549591, + 0.1090376153588295, + 0.10670766234397888, + 0.18005813658237457, + 0.14383366703987122, + 0.28685232996940613 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7343032360076904, + "mean_weights": [ + 0.16843369603157043, + 0.15050029754638672, + 0.15599164366722107, + 0.14163106679916382, + 0.10262320935726166, + 0.2808200716972351 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4446122646331787, + "mean_weights": [ + 0.1531161069869995, + 0.1061544418334961, + 0.04045919328927994, + 0.12559477984905243, + 0.08803897351026535, + 0.4866364896297455 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7605862617492676, + "mean_weights": [ + 0.1779545247554779, + 0.16784369945526123, + 0.1857004165649414, + 0.14843404293060303, + 0.10648007690906525, + 0.21358723938465118 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5263525247573853, + "mean_weights": [ + 0.08794503659009933, + 0.0601879358291626, + 0.033082835376262665, + 0.08346094191074371, + 0.05888189375400543, + 0.20315150916576385, + 0.4732898473739624 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7944886684417725, + "mean_weights": [ + 0.12276870757341385, + 0.11536462604999542, + 0.1304849088191986, + 0.09876619279384613, + 0.07423829287290573, + 0.11369137465953827, + 0.3446859121322632 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4918596744537354, + "mean_weights": [ + 0.09284307807683945, + 0.06384406983852386, + 0.03825954347848892, + 0.08267752081155777, + 0.05671031028032303, + 0.1521964967250824, + 0.5134689807891846 + ], + "sources": 7 + }, + { + "entropy_mean": 1.830867052078247, + "mean_weights": [ + 0.13634362816810608, + 0.12310001999139786, + 0.09669966250658035, + 0.1085672378540039, + 0.08431269228458405, + 0.13993649184703827, + 0.3110402822494507 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6549372673034668, + "mean_weights": [ + 0.06966839730739594, + 0.049915459007024765, + 0.029968995600938797, + 0.05769501253962517, + 0.040842454880476, + 0.10442843288183212, + 0.30951738357543945, + 0.33796387910842896 + ], + "sources": 8 + }, + { + "entropy_mean": 1.859694480895996, + "mean_weights": [ + 0.09671731293201447, + 0.07844845205545425, + 0.05707312002778053, + 0.0829043835401535, + 0.06170058995485306, + 0.10576222091913223, + 0.19003936648368835, + 0.3273545503616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.677255392074585, + "mean_weights": [ + 0.07509785890579224, + 0.05865459889173508, + 0.03770594298839569, + 0.059113647788763046, + 0.042673639953136444, + 0.08797852694988251, + 0.24025270342826843, + 0.39852309226989746 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8695217370986938, + "mean_weights": [ + 0.0984041839838028, + 0.08118931949138641, + 0.05154065042734146, + 0.08640529215335846, + 0.05755901336669922, + 0.11663993448019028, + 0.23645231127738953, + 0.27180927991867065 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8854753971099854, + "mean_weights": [ + 0.06882007420063019, + 0.0593036487698555, + 0.03391250967979431, + 0.060251303017139435, + 0.038275185972452164, + 0.09127645194530487, + 0.20795494318008423, + 0.23203802108764648, + 0.20816785097122192 + ], + "sources": 9 + }, + { + "entropy_mean": 2.032858371734619, + "mean_weights": [ + 0.08241330087184906, + 0.08549442887306213, + 0.041513457894325256, + 0.08080939948558807, + 0.054553937166929245, + 0.12166906148195267, + 0.2086121290922165, + 0.17801183462142944, + 0.14692245423793793 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9540549516677856, + "mean_weights": [ + 0.07821957767009735, + 0.07357673346996307, + 0.037884071469306946, + 0.06657841801643372, + 0.044617801904678345, + 0.09479035437107086, + 0.1742672622203827, + 0.2624226212501526, + 0.16764315962791443 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.058635130524635315, + 0.05468355491757393, + 0.07153555005788803, + 0.0688173845410347, + 0.13985061645507812, + 0.1219802051782608, + 0.16997404396533966, + 0.17499428987503052, + 0.1912512183189392, + 0.14618010818958282, + 0.12251991033554077, + 0.0794699639081955, + 0.12536334991455078, + 0.09043660014867783, + 0.10440115630626678, + 0.08707059919834137, + 0.11446695029735565, + 0.06907524168491364, + 0.0854191705584526, + 0.06048329547047615, + 0.10043052583932877, + 0.03875526040792465, + 0.06678051501512527, + 0.05217510461807251, + 0.06635956466197968, + 0.04468905180692673, + 0.04571816325187683, + 0.04713807255029678, + 0.05554596707224846, + 0.0446290485560894, + 0.052154671400785446, + 0.054206155240535736 + ], + "loss_nats": 1.3618223667144775, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.836374282836914, + "mean_weights": [ + 0.08612492680549622, + 0.0513404943048954, + 0.04786331206560135, + 0.0643724650144577, + 0.0475543811917305, + 0.059286996722221375, + 0.07609974592924118, + 0.2768789529800415, + 0.2904787063598633 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.07875782251358032, + 0.16897788643836975, + 0.21303832530975342, + 0.2546124756336212, + 0.24504326283931732, + 0.24958282709121704, + 0.345289409160614, + 0.3488057255744934, + 0.053805794566869736, + 0.05895345285534859, + 0.07887569069862366, + 0.09218393266201019, + 0.05105716362595558, + 0.05982525274157524, + 0.08438026905059814, + 0.10435724258422852, + 0.059221696108579636, + 0.07404384016990662, + 0.09824100881814957, + 0.13799543678760529, + 0.04156494140625, + 0.072679303586483, + 0.08400582522153854, + 0.12079861760139465, + 0.026702450588345528, + 0.06610427051782608, + 0.07228080183267593, + 0.11929895728826523, + 0.019828684628009796, + 0.08813745528459549, + 0.09191317111253738, + 0.2320432960987091 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.052410364849493e-05, + 7.120969530660659e-05, + 9.769770258571953e-05, + 9.110000246437266e-05, + 9.818621765589342e-05, + 8.837205677991733e-05, + 7.198825915111229e-05, + 6.996823503868654e-05, + 0.00017213664250448346, + 0.00013030946138314903, + 0.0002552570658735931, + 0.00017121800919994712, + 0.00020642777963075787, + 0.00012888041965197772, + 9.31880422285758e-05, + 5.8820565755013376e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.263287600129843e-05, + "first_to_last_ratio": 0.6782682211585493, + "imbalance_abs_log_ratio": 0.3882124628447938, + "last_quartile_mean": 0.00012182920181658119, + "mean": 0.0001172052661786438, + "normalized": [ + 0.6017144617120052, + 0.6075639570500958, + 0.8335606903260565, + 0.7772688500661601, + 0.8377287203651615, + 0.7539939088165287, + 0.6142066947861206, + 0.5969717685896574, + 1.4686766910465736, + 1.1118055155007445, + 2.1778634543991506, + 1.4608388750977905, + 1.7612500390223207, + 1.0996128745233913, + 0.7950840885130448, + 0.5018594101851985 + ], + "population_cv": 0.46767509062432555 + }, + "activation_output_rms_by_block": [ + 0.17702145874500275, + 0.2651609778404236, + 0.2740711271762848, + 0.3786091208457947, + 0.05278070271015167, + 0.09138669073581696, + 0.05598878860473633, + 0.10357969254255295, + 0.06921031326055527, + 0.1325109452009201, + 0.06760682165622711, + 0.11279971897602081, + 0.060408592224121094, + 0.10416899621486664, + 0.073752261698246, + 0.19092032313346863 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.27371567115187645, + "first_to_last_ratio": 2.5506400527811586, + "imbalance_abs_log_ratio": 0.9363443287654816, + "last_quartile_mean": 0.10731254331767559, + "mean": 0.13812353322282434, + "normalized": [ + 1.2816169309789325, + 1.9197378727103604, + 1.984246426234597, + 2.7410906165788056, + 0.38212679243445447, + 0.661630126333285, + 0.40535300030599286, + 0.7499061899571856, + 0.5010754622740751, + 0.9593654470679528, + 0.4894663500039642, + 0.8166582214056853, + 0.43735191834883375, + 0.7541726871902314, + 0.5339586960845188, + 1.3822432620911254 + ], + "population_cv": 0.6672200194546598 + }, + "bits_per_byte": 1.8951255009474826, + "branch_output_rms_by_sublayer": [ + 0.08229005336761475, + 0.1513136327266693, + 0.08296028524637222, + 0.06130215525627136, + 0.2681557536125183, + 0.023455867543816566, + 0.11001982539892197, + 0.01849420741200447, + 0.047267504036426544, + 0.01671545021235943, + 0.035008542239665985, + 0.04324181750416756, + 0.04728955402970314, + 0.02490479126572609, + 0.04628673940896988, + 0.041959211230278015, + 0.055535849183797836, + 0.03813145309686661, + 0.04947454109787941, + 0.083168163895607, + 0.037920478731393814, + 0.05371610447764397, + 0.03159874305129051, + 0.07563052326440811, + 0.023936081677675247, + 0.05285703018307686, + 0.021295147016644478, + 0.07199715077877045, + 0.018087781965732574, + 0.0706687644124031, + 0.01923547312617302, + 0.13923443853855133 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004973569892103084, + 0.0002771506747073045, + 0.0003595853430528865, + 0.00025621221416812444, + 0.0002736875635381804, + 0.00034154014112545435, + 0.0002723491519532143, + 0.0002954443301970981, + 0.0007531989220686374, + 0.0008991377204894639, + 0.0016884293947290697, + 0.0013942943414501116, + 0.0012489939262520248, + 0.0008593069460315814, + 0.000558781441282585, + 0.00045362324645161216 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00034757630528465595, + "first_to_last_ratio": 0.44550989973264, + "imbalance_abs_log_ratio": 0.8085358106960822, + "last_quartile_mean": 0.0007801763900044509, + "mean": 0.0006518182716692285, + "normalized": [ + 0.7630301432585446, + 0.42519623452339683, + 0.5516650248727019, + 0.39307307773376104, + 0.41988323346214174, + 0.523980618479459, + 0.417829882638547, + 0.45326181090398177, + 1.1555351465367567, + 1.3794300644363342, + 2.5903376264755575, + 2.139084468865058, + 1.916168939317244, + 1.3183228874989945, + 0.8572656913277571, + 0.6959351496697651 + ], + "population_cv": 0.6710654904340576 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2534913420677185, + "mean_weights": [ + 0.9232981204986572, + 0.07670190930366516 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5290617346763611, + "mean_weights": [ + 0.7383309602737427, + 0.26166898012161255 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4635298252105713, + "mean_weights": [ + 0.8211046457290649, + 0.17889533936977386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.680541455745697, + "mean_weights": [ + 0.4997203052043915, + 0.5002796649932861 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9984116554260254, + "mean_weights": [ + 0.45704665780067444, + 0.20008087158203125, + 0.3428725004196167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0224952697753906, + "mean_weights": [ + 0.22097927331924438, + 0.2672557532787323, + 0.5117650032043457 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9886391162872314, + "mean_weights": [ + 0.4091557264328003, + 0.17683157324790955, + 0.41401270031929016 + ], + "sources": 3 + }, + { + "entropy_mean": 1.083122730255127, + "mean_weights": [ + 0.2858731150627136, + 0.31187689304351807, + 0.4022499620914459 + ], + "sources": 3 + }, + { + "entropy_mean": 1.262911081314087, + "mean_weights": [ + 0.24318832159042358, + 0.1005454957485199, + 0.3312486708164215, + 0.325017511844635 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3697116374969482, + "mean_weights": [ + 0.2760387659072876, + 0.21072158217430115, + 0.21145658195018768, + 0.30178308486938477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1750009059906006, + "mean_weights": [ + 0.40358343720436096, + 0.10160854458808899, + 0.10028939694166183, + 0.3945186138153076 + ], + "sources": 4 + }, + { + "entropy_mean": 1.355452299118042, + "mean_weights": [ + 0.23967716097831726, + 0.20835967361927032, + 0.20370042324066162, + 0.348262757062912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5273280143737793, + "mean_weights": [ + 0.2132018804550171, + 0.09696971625089645, + 0.14153459668159485, + 0.2435910701751709, + 0.3047027587890625 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5883060693740845, + "mean_weights": [ + 0.2065391093492508, + 0.16525766253471375, + 0.16744214296340942, + 0.19437497854232788, + 0.26638609170913696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4673986434936523, + "mean_weights": [ + 0.19910070300102234, + 0.10532978177070618, + 0.08866854012012482, + 0.23074698448181152, + 0.37615397572517395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6001981496810913, + "mean_weights": [ + 0.22463412582874298, + 0.18846800923347473, + 0.19619622826576233, + 0.2022438496351242, + 0.18845780193805695 + ], + "sources": 5 + }, + { + "entropy_mean": 1.658735752105713, + "mean_weights": [ + 0.16469183564186096, + 0.09157001227140427, + 0.085207998752594, + 0.16954544186592102, + 0.1403937190771103, + 0.34859102964401245 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7217588424682617, + "mean_weights": [ + 0.1685413271188736, + 0.1354280561208725, + 0.15638956427574158, + 0.13712063431739807, + 0.10325069725513458, + 0.29926973581314087 + ], + "sources": 6 + }, + { + "entropy_mean": 1.383854627609253, + "mean_weights": [ + 0.14582018554210663, + 0.09375129640102386, + 0.03237149864435196, + 0.11780930310487747, + 0.08738291263580322, + 0.522864818572998 + ], + "sources": 6 + }, + { + "entropy_mean": 1.758126974105835, + "mean_weights": [ + 0.18006329238414764, + 0.15370121598243713, + 0.18850785493850708, + 0.14673401415348053, + 0.10883604735136032, + 0.2221575826406479 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4768933057785034, + "mean_weights": [ + 0.08639779686927795, + 0.05247560143470764, + 0.02685363218188286, + 0.07686029374599457, + 0.05649450421333313, + 0.2057446390390396, + 0.49517351388931274 + ], + "sources": 7 + }, + { + "entropy_mean": 1.754528522491455, + "mean_weights": [ + 0.1207476407289505, + 0.10044647008180618, + 0.12684977054595947, + 0.0954352393746376, + 0.07363080978393555, + 0.10486453771591187, + 0.37802553176879883 + ], + "sources": 7 + }, + { + "entropy_mean": 1.434983491897583, + "mean_weights": [ + 0.09186187386512756, + 0.054585013538599014, + 0.030214153230190277, + 0.07775386422872543, + 0.05580390244722366, + 0.14904344081878662, + 0.5407377481460571 + ], + "sources": 7 + }, + { + "entropy_mean": 1.814563274383545, + "mean_weights": [ + 0.13872049748897552, + 0.10858511924743652, + 0.09594539552927017, + 0.10938869416713715, + 0.08644531667232513, + 0.13206249475479126, + 0.32885247468948364 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5878477096557617, + "mean_weights": [ + 0.06617997586727142, + 0.04098459705710411, + 0.023611580953001976, + 0.05256733298301697, + 0.03869973123073578, + 0.09328385442495346, + 0.30499327182769775, + 0.37967967987060547 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8095037937164307, + "mean_weights": [ + 0.0948307067155838, + 0.0694318562746048, + 0.0552619993686676, + 0.07500732690095901, + 0.061008021235466, + 0.09275255352258682, + 0.18348734080791473, + 0.3682202100753784 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5925970077514648, + "mean_weights": [ + 0.07132496684789658, + 0.04804965853691101, + 0.030455447733402252, + 0.054331667721271515, + 0.038998860865831375, + 0.07944463938474655, + 0.22967059910297394, + 0.4477241635322571 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8567713499069214, + "mean_weights": [ + 0.09893427789211273, + 0.07625022530555725, + 0.05004327371716499, + 0.08632857352495193, + 0.05865295231342316, + 0.10661771893501282, + 0.2290721982717514, + 0.2941007912158966 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7706098556518555, + "mean_weights": [ + 0.0600857213139534, + 0.04508119821548462, + 0.026483699679374695, + 0.05021750181913376, + 0.03222868591547012, + 0.07324853539466858, + 0.17921699583530426, + 0.23399141430854797, + 0.2994462847709656 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0186803340911865, + "mean_weights": [ + 0.08094437420368195, + 0.07421140372753143, + 0.03662773221731186, + 0.07851459085941315, + 0.05622554570436478, + 0.11645162105560303, + 0.2003173828125, + 0.18591775000095367, + 0.17078962922096252 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8739352226257324, + "mean_weights": [ + 0.07170331478118896, + 0.05929408222436905, + 0.03108443133533001, + 0.05872419476509094, + 0.03831540048122406, + 0.07797900587320328, + 0.1519450843334198, + 0.2740318179130554, + 0.23692265152931213 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.061437346041202545, + 0.05765428766608238, + 0.07246167957782745, + 0.07205748558044434, + 0.1429983228445053, + 0.13230633735656738, + 0.18651185929775238, + 0.18575440347194672, + 0.20683544874191284, + 0.14580051600933075, + 0.1242423728108406, + 0.07512427866458893, + 0.12742595374584198, + 0.08523519337177277, + 0.10536280274391174, + 0.07712213695049286, + 0.12031000852584839, + 0.06152798980474472, + 0.08708149939775467, + 0.05821346119046211, + 0.10479474812746048, + 0.03660533204674721, + 0.06689487397670746, + 0.049551647156476974, + 0.06684304773807526, + 0.04078952595591545, + 0.04357287660241127, + 0.04368526116013527, + 0.05231116712093353, + 0.03767167031764984, + 0.04748373478651047, + 0.0490146204829216 + ], + "loss_nats": 1.3136008977890015, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.685392141342163, + "mean_weights": [ + 0.08037909120321274, + 0.03995431959629059, + 0.0371873639523983, + 0.05237603187561035, + 0.03903363645076752, + 0.04661020264029503, + 0.0628754124045372, + 0.26517099142074585, + 0.37641292810440063 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.08229005336761475, + 0.17702145874500275, + 0.22416120767593384, + 0.2651609778404236, + 0.2681557536125183, + 0.2740711271762848, + 0.37491148710250854, + 0.3786091208457947, + 0.047267504036426544, + 0.05278070271015167, + 0.07517243921756744, + 0.09138669073581696, + 0.04728955402970314, + 0.05598878860473633, + 0.08201033622026443, + 0.10357969254255295, + 0.055535849183797836, + 0.06921031326055527, + 0.0903589129447937, + 0.1325109452009201, + 0.037920478731393814, + 0.06760682165622711, + 0.07624635845422745, + 0.11279971897602081, + 0.023936081677675247, + 0.060408592224121094, + 0.0652509555220604, + 0.10416899621486664, + 0.018087781965732574, + 0.073752261698246, + 0.0771600753068924, + 0.19092032313346863 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.073347956844582, + "cross_entropy_nats": 5.596018373966217, + "step": 0 + }, + { + "bits_per_byte": 5.112499455104693, + "cross_entropy_nats": 3.5437145829200745, + "step": 100 + }, + { + "bits_per_byte": 3.1188867992707414, + "cross_entropy_nats": 2.1618475914001465, + "step": 500 + }, + { + "bits_per_byte": 1.962477669145846, + "cross_entropy_nats": 1.3602858632802963, + "step": 2000 + }, + { + "bits_per_byte": 1.8161008917328916, + "cross_entropy_nats": 1.2588252127170563, + "step": 4000 + }, + { + "bits_per_byte": 1.737390666804078, + "cross_entropy_nats": 1.20426744222641, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "0bff629038a5dd94843a123dbdfc2bdaca027f8a9025a2a7688adc4ec0647b31", + "final_model_state": "efa098925d29c4e5158cd73c7764d7014b6c4ee53df36a146dbd0cff04321b62", + "final_optimizer_state": "b86655a1b88951524b7f3bfca00df30a0958104385aa444bf7b109fbe1d9e51b", + "final_public_parameters": "87107ee0633f1863f2fcaf1d5f91af597c67acce2c9777fec3486aa4fd1463ef", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.63475258274364, + "measured_steps": 7980, + "median_ms": 54.539098986424506, + "p95_ms": 55.062895112496335, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.084436624250287, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.603704452514648, + "step": 1, + "unclipped_grad_norm": 18.78145980834961 + }, + { + "bits_per_byte": 7.484355446790379, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.187759876251221, + "step": 10, + "unclipped_grad_norm": 12.867269515991211 + }, + { + "bits_per_byte": 6.795359238834398, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.710184097290039, + "step": 20, + "unclipped_grad_norm": 4.549929141998291 + }, + { + "bits_per_byte": 6.560912489178218, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.547677993774414, + "step": 30, + "unclipped_grad_norm": 2.9244894981384277 + }, + { + "bits_per_byte": 6.418922236764676, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.449257850646973, + "step": 40, + "unclipped_grad_norm": 2.8415465354919434 + }, + { + "bits_per_byte": 6.252045406840682, + "learning_rate": 3.75e-05, + "loss_nats": 4.333587646484375, + "step": 50, + "unclipped_grad_norm": 2.8266561031341553 + }, + { + "bits_per_byte": 5.950088587414815, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.124287128448486, + "step": 60, + "unclipped_grad_norm": 2.6689200401306152 + }, + { + "bits_per_byte": 5.70102021674758, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.951646089553833, + "step": 70, + "unclipped_grad_norm": 2.5050199031829834 + }, + { + "bits_per_byte": 5.519442653793135, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8257861137390137, + "step": 80, + "unclipped_grad_norm": 2.352375030517578 + }, + { + "bits_per_byte": 5.302846327435307, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.6756529808044434, + "step": 90, + "unclipped_grad_norm": 2.2169463634490967 + }, + { + "bits_per_byte": 5.139077911943008, + "learning_rate": 7.5e-05, + "loss_nats": 3.5621373653411865, + "step": 100, + "unclipped_grad_norm": 2.0478744506835938 + }, + { + "bits_per_byte": 4.944703463903163, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4274072647094727, + "step": 110, + "unclipped_grad_norm": 1.971948504447937 + }, + { + "bits_per_byte": 4.730622899762156, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.279017925262451, + "step": 120, + "unclipped_grad_norm": 1.6964443922042847 + }, + { + "bits_per_byte": 4.549973699157069, + "learning_rate": 9.75e-05, + "loss_nats": 3.153801441192627, + "step": 130, + "unclipped_grad_norm": 1.537685751914978 + }, + { + "bits_per_byte": 4.410896832849143, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.057400703430176, + "step": 140, + "unclipped_grad_norm": 1.355125904083252 + }, + { + "bits_per_byte": 4.261557417798932, + "learning_rate": 0.0001125, + "loss_nats": 2.9538865089416504, + "step": 150, + "unclipped_grad_norm": 1.2107940912246704 + }, + { + "bits_per_byte": 4.165243693662742, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.887126922607422, + "step": 160, + "unclipped_grad_norm": 0.8971752524375916 + }, + { + "bits_per_byte": 4.038897327136983, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7995502948760986, + "step": 170, + "unclipped_grad_norm": 0.8723403215408325 + }, + { + "bits_per_byte": 3.967892225936462, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.750333309173584, + "step": 180, + "unclipped_grad_norm": 0.7337902784347534 + }, + { + "bits_per_byte": 3.8789046187617258, + "learning_rate": 0.0001425, + "loss_nats": 2.6886518001556396, + "step": 190, + "unclipped_grad_norm": 0.8901073336601257 + }, + { + "bits_per_byte": 3.824914105176364, + "learning_rate": 0.00015, + "loss_nats": 2.651228427886963, + "step": 200, + "unclipped_grad_norm": 0.642342746257782 + }, + { + "bits_per_byte": 3.7744440764541642, + "learning_rate": 0.0001575, + "loss_nats": 2.6162452697753906, + "step": 210, + "unclipped_grad_norm": 0.6421008110046387 + }, + { + "bits_per_byte": 3.68308276473222, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5529184341430664, + "step": 220, + "unclipped_grad_norm": 0.7395935654640198 + }, + { + "bits_per_byte": 3.6743831943209755, + "learning_rate": 0.0001725, + "loss_nats": 2.5468883514404297, + "step": 230, + "unclipped_grad_norm": 0.6173842549324036 + }, + { + "bits_per_byte": 3.575872219837362, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4786057472229004, + "step": 240, + "unclipped_grad_norm": 0.8246023058891296 + }, + { + "bits_per_byte": 3.6039487315560175, + "learning_rate": 0.0001875, + "loss_nats": 2.4980669021606445, + "step": 250, + "unclipped_grad_norm": 0.7212417721748352 + }, + { + "bits_per_byte": 3.6017153648519127, + "learning_rate": 0.000195, + "loss_nats": 2.496518850326538, + "step": 260, + "unclipped_grad_norm": 0.6610386967658997 + }, + { + "bits_per_byte": 3.6077832567396544, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.5007247924804688, + "step": 270, + "unclipped_grad_norm": 0.8672931790351868 + }, + { + "bits_per_byte": 3.524929582790591, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4432950019836426, + "step": 280, + "unclipped_grad_norm": 0.7419575452804565 + }, + { + "bits_per_byte": 3.446865969633359, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3891854286193848, + "step": 290, + "unclipped_grad_norm": 0.8831741809844971 + }, + { + "bits_per_byte": 3.491855255237863, + "learning_rate": 0.000225, + "loss_nats": 2.4203696250915527, + "step": 300, + "unclipped_grad_norm": 0.9954257607460022 + }, + { + "bits_per_byte": 3.448469191904702, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.390296697616577, + "step": 310, + "unclipped_grad_norm": 1.0782777070999146 + }, + { + "bits_per_byte": 3.4360496367552953, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.381688117980957, + "step": 320, + "unclipped_grad_norm": 0.9428723454475403 + }, + { + "bits_per_byte": 3.415760155502406, + "learning_rate": 0.0002475, + "loss_nats": 2.367624521255493, + "step": 330, + "unclipped_grad_norm": 1.0516632795333862 + }, + { + "bits_per_byte": 3.5107169365232886, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.433443546295166, + "step": 340, + "unclipped_grad_norm": 1.1474740505218506 + }, + { + "bits_per_byte": 3.4490202243180663, + "learning_rate": 0.0002625, + "loss_nats": 2.390678644180298, + "step": 350, + "unclipped_grad_norm": 1.2037273645401 + }, + { + "bits_per_byte": 3.401095538828636, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.357459783554077, + "step": 360, + "unclipped_grad_norm": 1.0446844100952148 + }, + { + "bits_per_byte": 3.376710806693402, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.34055757522583, + "step": 370, + "unclipped_grad_norm": 0.8781102895736694 + }, + { + "bits_per_byte": 3.422498435763211, + "learning_rate": 0.000285, + "loss_nats": 2.3722951412200928, + "step": 380, + "unclipped_grad_norm": 0.9067438244819641 + }, + { + "bits_per_byte": 3.35829834409203, + "learning_rate": 0.0002925, + "loss_nats": 2.3277950286865234, + "step": 390, + "unclipped_grad_norm": 1.0105773210525513 + }, + { + "bits_per_byte": 3.298973960572809, + "learning_rate": 0.0003, + "loss_nats": 2.2866744995117188, + "step": 400, + "unclipped_grad_norm": 1.1294934749603271 + }, + { + "bits_per_byte": 3.2943352445137375, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.283459186553955, + "step": 410, + "unclipped_grad_norm": 1.2423999309539795 + }, + { + "bits_per_byte": 3.324363071389082, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.3042728900909424, + "step": 420, + "unclipped_grad_norm": 1.016541838645935 + }, + { + "bits_per_byte": 3.29597423917646, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.284595251083374, + "step": 430, + "unclipped_grad_norm": 1.1856372356414795 + }, + { + "bits_per_byte": 3.3349637380229638, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.3116207122802734, + "step": 440, + "unclipped_grad_norm": 0.8890857696533203 + }, + { + "bits_per_byte": 3.2500166912475987, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.252739906311035, + "step": 450, + "unclipped_grad_norm": 1.1027717590332031 + }, + { + "bits_per_byte": 3.273506081702441, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.269021511077881, + "step": 460, + "unclipped_grad_norm": 0.9078090190887451 + }, + { + "bits_per_byte": 3.199572459923028, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.2177746295928955, + "step": 470, + "unclipped_grad_norm": 1.019633173942566 + }, + { + "bits_per_byte": 3.1991851549932853, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.217506170272827, + "step": 480, + "unclipped_grad_norm": 0.8520210981369019 + }, + { + "bits_per_byte": 3.1319430341936423, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1708974838256836, + "step": 490, + "unclipped_grad_norm": 2.670640707015991 + }, + { + "bits_per_byte": 3.0465694704441373, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1117210388183594, + "step": 500, + "unclipped_grad_norm": 0.961890697479248 + }, + { + "bits_per_byte": 3.080116406321437, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1349740028381348, + "step": 510, + "unclipped_grad_norm": 1.4813798666000366 + }, + { + "bits_per_byte": 2.9958545384271087, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.076568126678467, + "step": 520, + "unclipped_grad_norm": 1.3385756015777588 + }, + { + "bits_per_byte": 3.0340161127923593, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1030197143554688, + "step": 530, + "unclipped_grad_norm": 1.0723164081573486 + }, + { + "bits_per_byte": 2.9825217554016206, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.067326545715332, + "step": 540, + "unclipped_grad_norm": 1.0559418201446533 + }, + { + "bits_per_byte": 2.964598755424604, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.054903268814087, + "step": 550, + "unclipped_grad_norm": 1.3542453050613403 + }, + { + "bits_per_byte": 2.9389846913010023, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.037148952484131, + "step": 560, + "unclipped_grad_norm": 1.3118830919265747 + }, + { + "bits_per_byte": 2.888880297694537, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0024192333221436, + "step": 570, + "unclipped_grad_norm": 1.1606483459472656 + }, + { + "bits_per_byte": 2.9506829512126904, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.045257568359375, + "step": 580, + "unclipped_grad_norm": 1.3442156314849854 + }, + { + "bits_per_byte": 2.8292920927888927, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.961115837097168, + "step": 590, + "unclipped_grad_norm": 0.9916635751724243 + }, + { + "bits_per_byte": 2.904369399196543, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.013155460357666, + "step": 600, + "unclipped_grad_norm": 1.5107635259628296 + }, + { + "bits_per_byte": 2.8103640262002334, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.947995901107788, + "step": 610, + "unclipped_grad_norm": 1.3907541036605835 + }, + { + "bits_per_byte": 2.8376170849865576, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.966886281967163, + "step": 620, + "unclipped_grad_norm": 1.3457273244857788 + }, + { + "bits_per_byte": 2.772177686333259, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9215271472930908, + "step": 630, + "unclipped_grad_norm": 1.2430039644241333 + }, + { + "bits_per_byte": 2.7475067750669893, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9044265747070312, + "step": 640, + "unclipped_grad_norm": 1.7454813718795776 + }, + { + "bits_per_byte": 2.664434339186028, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8468451499938965, + "step": 650, + "unclipped_grad_norm": 1.3669214248657227 + }, + { + "bits_per_byte": 2.607502578305611, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8073830604553223, + "step": 660, + "unclipped_grad_norm": 2.4166994094848633 + }, + { + "bits_per_byte": 2.640132846653812, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.830000638961792, + "step": 670, + "unclipped_grad_norm": 1.6203631162643433 + }, + { + "bits_per_byte": 2.577274563624662, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.7864305973052979, + "step": 680, + "unclipped_grad_norm": 1.443710207939148 + }, + { + "bits_per_byte": 2.648628101675831, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8358891010284424, + "step": 690, + "unclipped_grad_norm": 1.1127547025680542 + }, + { + "bits_per_byte": 2.5587637269469963, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7735998630523682, + "step": 700, + "unclipped_grad_norm": 1.3421220779418945 + }, + { + "bits_per_byte": 2.584466534118443, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.7914156913757324, + "step": 710, + "unclipped_grad_norm": 1.5519825220108032 + }, + { + "bits_per_byte": 2.61796909847182, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8146378993988037, + "step": 720, + "unclipped_grad_norm": 2.2321839332580566 + }, + { + "bits_per_byte": 2.574501859327452, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.7845087051391602, + "step": 730, + "unclipped_grad_norm": 1.835390567779541 + }, + { + "bits_per_byte": 2.509336257054327, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7393393516540527, + "step": 740, + "unclipped_grad_norm": 1.5465850830078125 + }, + { + "bits_per_byte": 2.4645351204698667, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7082855701446533, + "step": 750, + "unclipped_grad_norm": 1.1801629066467285 + }, + { + "bits_per_byte": 2.4095216540771736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.6701531410217285, + "step": 760, + "unclipped_grad_norm": 1.296120524406433 + }, + { + "bits_per_byte": 2.584656058999694, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7915470600128174, + "step": 770, + "unclipped_grad_norm": 2.048929452896118 + }, + { + "bits_per_byte": 2.4472219709728096, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6962850093841553, + "step": 780, + "unclipped_grad_norm": 1.45075261592865 + }, + { + "bits_per_byte": 2.4007569022412523, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.664077877998352, + "step": 790, + "unclipped_grad_norm": 1.5680437088012695 + }, + { + "bits_per_byte": 2.3632249563608103, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6380627155303955, + "step": 800, + "unclipped_grad_norm": 1.358572006225586 + }, + { + "bits_per_byte": 2.4340216145884854, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6871352195739746, + "step": 810, + "unclipped_grad_norm": 1.6542795896530151 + }, + { + "bits_per_byte": 2.3610528154804133, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6365571022033691, + "step": 820, + "unclipped_grad_norm": 1.4101909399032593 + }, + { + "bits_per_byte": 2.429125956449025, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.683741807937622, + "step": 830, + "unclipped_grad_norm": 1.4574730396270752 + }, + { + "bits_per_byte": 2.509321810511654, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7393293380737305, + "step": 840, + "unclipped_grad_norm": 1.71798837184906 + }, + { + "bits_per_byte": 2.3876714302677042, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6550077199935913, + "step": 850, + "unclipped_grad_norm": 1.7788947820663452 + }, + { + "bits_per_byte": 2.402646303643631, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.665387511253357, + "step": 860, + "unclipped_grad_norm": 1.3990782499313354 + }, + { + "bits_per_byte": 2.3188210996998437, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6072843074798584, + "step": 870, + "unclipped_grad_norm": 1.5185577869415283 + }, + { + "bits_per_byte": 2.342377219423221, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6236121654510498, + "step": 880, + "unclipped_grad_norm": 1.4461203813552856 + }, + { + "bits_per_byte": 2.2667829331670766, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.571214199066162, + "step": 890, + "unclipped_grad_norm": 1.2836962938308716 + }, + { + "bits_per_byte": 2.284366783356592, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.583402395248413, + "step": 900, + "unclipped_grad_norm": 1.337744116783142 + }, + { + "bits_per_byte": 2.3096057533186616, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6008967161178589, + "step": 910, + "unclipped_grad_norm": 1.3567826747894287 + }, + { + "bits_per_byte": 2.2867041995644684, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5850225687026978, + "step": 920, + "unclipped_grad_norm": 1.568986177444458 + }, + { + "bits_per_byte": 2.304558234498489, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.597398042678833, + "step": 930, + "unclipped_grad_norm": 1.5530638694763184 + }, + { + "bits_per_byte": 2.297734650842856, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5926682949066162, + "step": 940, + "unclipped_grad_norm": 1.5203993320465088 + }, + { + "bits_per_byte": 2.280084587332759, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5804342031478882, + "step": 950, + "unclipped_grad_norm": 1.4065279960632324 + }, + { + "bits_per_byte": 2.325858113825197, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6121619939804077, + "step": 960, + "unclipped_grad_norm": 1.3770049810409546 + }, + { + "bits_per_byte": 2.231583072032335, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5468155145645142, + "step": 970, + "unclipped_grad_norm": 1.3417105674743652 + }, + { + "bits_per_byte": 2.3223450242160224, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.609726905822754, + "step": 980, + "unclipped_grad_norm": 1.444604754447937 + }, + { + "bits_per_byte": 2.2731906627729233, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5756556987762451, + "step": 990, + "unclipped_grad_norm": 1.3634493350982666 + }, + { + "bits_per_byte": 2.2346108266007745, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5489141941070557, + "step": 1000, + "unclipped_grad_norm": 1.291415810585022 + }, + { + "bits_per_byte": 2.2324534762283514, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5474188327789307, + "step": 1010, + "unclipped_grad_norm": 1.811213493347168 + }, + { + "bits_per_byte": 2.198316295893291, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.523756742477417, + "step": 1020, + "unclipped_grad_norm": 1.5226773023605347 + }, + { + "bits_per_byte": 2.1998723949182977, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5248353481292725, + "step": 1030, + "unclipped_grad_norm": 1.3728047609329224 + }, + { + "bits_per_byte": 2.256233517363152, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5639019012451172, + "step": 1040, + "unclipped_grad_norm": 1.3665989637374878 + }, + { + "bits_per_byte": 2.2423672441546874, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.554290533065796, + "step": 1050, + "unclipped_grad_norm": 1.274046778678894 + }, + { + "bits_per_byte": 2.1489992388624755, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.4895727634429932, + "step": 1060, + "unclipped_grad_norm": 1.2373967170715332 + }, + { + "bits_per_byte": 2.2260787672914706, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5430002212524414, + "step": 1070, + "unclipped_grad_norm": 1.3982683420181274 + }, + { + "bits_per_byte": 2.206438348562938, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5293865203857422, + "step": 1080, + "unclipped_grad_norm": 1.334183692932129 + }, + { + "bits_per_byte": 2.1702291212502822, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5042881965637207, + "step": 1090, + "unclipped_grad_norm": 1.4794831275939941 + }, + { + "bits_per_byte": 2.2378767771406594, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.551177978515625, + "step": 1100, + "unclipped_grad_norm": 1.3411470651626587 + }, + { + "bits_per_byte": 2.087035609583114, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4466228485107422, + "step": 1110, + "unclipped_grad_norm": 1.452073574066162 + }, + { + "bits_per_byte": 2.1415479185311934, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.484407901763916, + "step": 1120, + "unclipped_grad_norm": 1.1817387342453003 + }, + { + "bits_per_byte": 2.079985696758946, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4417362213134766, + "step": 1130, + "unclipped_grad_norm": 1.5416163206100464 + }, + { + "bits_per_byte": 2.2251825656978244, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5423790216445923, + "step": 1140, + "unclipped_grad_norm": 1.5179954767227173 + }, + { + "bits_per_byte": 2.0668831985029628, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4326542615890503, + "step": 1150, + "unclipped_grad_norm": 1.186457633972168 + }, + { + "bits_per_byte": 2.104958265594829, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4590458869934082, + "step": 1160, + "unclipped_grad_norm": 1.5302215814590454 + }, + { + "bits_per_byte": 2.146566716248672, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.487886667251587, + "step": 1170, + "unclipped_grad_norm": 1.187764048576355 + }, + { + "bits_per_byte": 2.1225221657968443, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4712202548980713, + "step": 1180, + "unclipped_grad_norm": 1.1978200674057007 + }, + { + "bits_per_byte": 2.061979457178911, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4292552471160889, + "step": 1190, + "unclipped_grad_norm": 1.333662509918213 + }, + { + "bits_per_byte": 2.04061714814994, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4144480228424072, + "step": 1200, + "unclipped_grad_norm": 1.397094964981079 + }, + { + "bits_per_byte": 2.105093787923709, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4591398239135742, + "step": 1210, + "unclipped_grad_norm": 1.355148196220398 + }, + { + "bits_per_byte": 2.113284805636352, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4648174047470093, + "step": 1220, + "unclipped_grad_norm": 1.2704886198043823 + }, + { + "bits_per_byte": 2.0731149898569767, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4369738101959229, + "step": 1230, + "unclipped_grad_norm": 1.2257503271102905 + }, + { + "bits_per_byte": 2.02175856255223, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4013762474060059, + "step": 1240, + "unclipped_grad_norm": 1.311407208442688 + }, + { + "bits_per_byte": 2.10808387829161, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.461212396621704, + "step": 1250, + "unclipped_grad_norm": 1.183249831199646 + }, + { + "bits_per_byte": 2.0709103442555663, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4354456663131714, + "step": 1260, + "unclipped_grad_norm": 1.2188260555267334 + }, + { + "bits_per_byte": 2.073173119992968, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.437014102935791, + "step": 1270, + "unclipped_grad_norm": 1.2204123735427856 + }, + { + "bits_per_byte": 1.983587529141182, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.3749181032180786, + "step": 1280, + "unclipped_grad_norm": 1.4762879610061646 + }, + { + "bits_per_byte": 2.0899118474361438, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4486165046691895, + "step": 1290, + "unclipped_grad_norm": 1.5250427722930908 + }, + { + "bits_per_byte": 2.0451232655851967, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4175714254379272, + "step": 1300, + "unclipped_grad_norm": 1.2546813488006592 + }, + { + "bits_per_byte": 2.037817958524497, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4125077724456787, + "step": 1310, + "unclipped_grad_norm": 1.2255067825317383 + }, + { + "bits_per_byte": 2.0185500542177333, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.3991522789001465, + "step": 1320, + "unclipped_grad_norm": 1.1987844705581665 + }, + { + "bits_per_byte": 2.097532054730573, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4538984298706055, + "step": 1330, + "unclipped_grad_norm": 1.270599603652954 + }, + { + "bits_per_byte": 2.036339251692379, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.411482810974121, + "step": 1340, + "unclipped_grad_norm": 1.31504225730896 + }, + { + "bits_per_byte": 2.0528777913299496, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4229464530944824, + "step": 1350, + "unclipped_grad_norm": 1.3823131322860718 + }, + { + "bits_per_byte": 2.0989251142025616, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548640251159668, + "step": 1360, + "unclipped_grad_norm": 1.3448033332824707 + }, + { + "bits_per_byte": 2.081831758533307, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4430158138275146, + "step": 1370, + "unclipped_grad_norm": 1.1824978590011597 + }, + { + "bits_per_byte": 2.037543130248418, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4123172760009766, + "step": 1380, + "unclipped_grad_norm": 1.241772174835205 + }, + { + "bits_per_byte": 2.0465340392702243, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4185492992401123, + "step": 1390, + "unclipped_grad_norm": 1.1579062938690186 + }, + { + "bits_per_byte": 1.983804227281269, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3750683069229126, + "step": 1400, + "unclipped_grad_norm": 1.2093477249145508 + }, + { + "bits_per_byte": 2.004750510277803, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.389587163925171, + "step": 1410, + "unclipped_grad_norm": 1.0995537042617798 + }, + { + "bits_per_byte": 2.044002282666873, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4167944192886353, + "step": 1420, + "unclipped_grad_norm": 1.377259612083435 + }, + { + "bits_per_byte": 2.072302027866348, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4364103078842163, + "step": 1430, + "unclipped_grad_norm": 1.2491836547851562 + }, + { + "bits_per_byte": 2.010299358525241, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3934333324432373, + "step": 1440, + "unclipped_grad_norm": 1.1953747272491455 + }, + { + "bits_per_byte": 1.9653831655473537, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3622997999191284, + "step": 1450, + "unclipped_grad_norm": 1.1474745273590088 + }, + { + "bits_per_byte": 2.0178239434657903, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.398648977279663, + "step": 1460, + "unclipped_grad_norm": 1.309095859527588 + }, + { + "bits_per_byte": 1.9352492253764266, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3414125442504883, + "step": 1470, + "unclipped_grad_norm": 1.2327831983566284 + }, + { + "bits_per_byte": 2.02495382822261, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4035910367965698, + "step": 1480, + "unclipped_grad_norm": 1.2556331157684326 + }, + { + "bits_per_byte": 2.0395678819970264, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.413720726966858, + "step": 1490, + "unclipped_grad_norm": 1.1320099830627441 + }, + { + "bits_per_byte": 2.0169458000504847, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3980402946472168, + "step": 1500, + "unclipped_grad_norm": 1.2224371433258057 + }, + { + "bits_per_byte": 2.0172811662196675, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3982727527618408, + "step": 1510, + "unclipped_grad_norm": 1.1516478061676025 + }, + { + "bits_per_byte": 1.959494651564462, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3582181930541992, + "step": 1520, + "unclipped_grad_norm": 0.9697362780570984 + }, + { + "bits_per_byte": 2.05711750763902, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4258852005004883, + "step": 1530, + "unclipped_grad_norm": 1.1719471216201782 + }, + { + "bits_per_byte": 2.0050322178599163, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.389782428741455, + "step": 1540, + "unclipped_grad_norm": 1.2557746171951294 + }, + { + "bits_per_byte": 2.0500259750133427, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209697246551514, + "step": 1550, + "unclipped_grad_norm": 1.1871017217636108 + }, + { + "bits_per_byte": 1.9591589414299777, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.357985496520996, + "step": 1560, + "unclipped_grad_norm": 1.1881691217422485 + }, + { + "bits_per_byte": 2.048241998975943, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4197331666946411, + "step": 1570, + "unclipped_grad_norm": 1.3408379554748535 + }, + { + "bits_per_byte": 2.0003424229535387, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3865317106246948, + "step": 1580, + "unclipped_grad_norm": 1.160094141960144 + }, + { + "bits_per_byte": 1.9972685770346676, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3844010829925537, + "step": 1590, + "unclipped_grad_norm": 1.1695513725280762 + }, + { + "bits_per_byte": 2.0280931995314595, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4057670831680298, + "step": 1600, + "unclipped_grad_norm": 1.1151546239852905 + }, + { + "bits_per_byte": 1.9648931869750457, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3619601726531982, + "step": 1610, + "unclipped_grad_norm": 1.1052403450012207 + }, + { + "bits_per_byte": 1.934925382044852, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3411880731582642, + "step": 1620, + "unclipped_grad_norm": 1.0654687881469727 + }, + { + "bits_per_byte": 1.9529704897039815, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3536959886550903, + "step": 1630, + "unclipped_grad_norm": 1.1012754440307617 + }, + { + "bits_per_byte": 1.9931929321745208, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3815760612487793, + "step": 1640, + "unclipped_grad_norm": 1.143923282623291 + }, + { + "bits_per_byte": 2.059868542122221, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4277920722961426, + "step": 1650, + "unclipped_grad_norm": 1.2584717273712158 + }, + { + "bits_per_byte": 1.992070745377641, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3807982206344604, + "step": 1660, + "unclipped_grad_norm": 1.177032232284546 + }, + { + "bits_per_byte": 2.009216383772758, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.392682671546936, + "step": 1670, + "unclipped_grad_norm": 1.1299933195114136 + }, + { + "bits_per_byte": 2.027195106128654, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4051445722579956, + "step": 1680, + "unclipped_grad_norm": 1.2568069696426392 + }, + { + "bits_per_byte": 2.0647518175108197, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4311769008636475, + "step": 1690, + "unclipped_grad_norm": 1.1674096584320068 + }, + { + "bits_per_byte": 1.9602546428986247, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3587449789047241, + "step": 1700, + "unclipped_grad_norm": 1.178646206855774 + }, + { + "bits_per_byte": 1.8971180919403792, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3149820566177368, + "step": 1710, + "unclipped_grad_norm": 1.0156482458114624 + }, + { + "bits_per_byte": 1.8941140709777584, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3128998279571533, + "step": 1720, + "unclipped_grad_norm": 1.0928010940551758 + }, + { + "bits_per_byte": 1.8690832000232327, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.295549750328064, + "step": 1730, + "unclipped_grad_norm": 1.0898301601409912 + }, + { + "bits_per_byte": 2.0020670649763908, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.38772714138031, + "step": 1740, + "unclipped_grad_norm": 1.1916359663009644 + }, + { + "bits_per_byte": 1.9965376507685006, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.383894443511963, + "step": 1750, + "unclipped_grad_norm": 1.1260547637939453 + }, + { + "bits_per_byte": 2.010018682839033, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3932387828826904, + "step": 1760, + "unclipped_grad_norm": 1.2356810569763184 + }, + { + "bits_per_byte": 2.000046096846102, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3863263130187988, + "step": 1770, + "unclipped_grad_norm": 1.1323615312576294 + }, + { + "bits_per_byte": 1.8857641412957182, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3071120977401733, + "step": 1780, + "unclipped_grad_norm": 1.1074084043502808 + }, + { + "bits_per_byte": 1.98148022572016, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.373457431793213, + "step": 1790, + "unclipped_grad_norm": 1.1702499389648438 + }, + { + "bits_per_byte": 1.8942642118319617, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3130038976669312, + "step": 1800, + "unclipped_grad_norm": 1.2193527221679688 + }, + { + "bits_per_byte": 1.9583697130451683, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3574384450912476, + "step": 1810, + "unclipped_grad_norm": 1.4279322624206543 + }, + { + "bits_per_byte": 1.9039834673801714, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3197407722473145, + "step": 1820, + "unclipped_grad_norm": 1.2399523258209229 + }, + { + "bits_per_byte": 1.902950023631137, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3190244436264038, + "step": 1830, + "unclipped_grad_norm": 1.066636562347412 + }, + { + "bits_per_byte": 1.9139311158887273, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3266359567642212, + "step": 1840, + "unclipped_grad_norm": 1.129875898361206 + }, + { + "bits_per_byte": 1.9166544611651397, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3285236358642578, + "step": 1850, + "unclipped_grad_norm": 1.1898164749145508 + }, + { + "bits_per_byte": 1.9000117000411458, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3169877529144287, + "step": 1860, + "unclipped_grad_norm": 1.1322225332260132 + }, + { + "bits_per_byte": 1.9091849106728662, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3233461380004883, + "step": 1870, + "unclipped_grad_norm": 1.1632918119430542 + }, + { + "bits_per_byte": 1.9066440389890191, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.321584939956665, + "step": 1880, + "unclipped_grad_norm": 1.0685805082321167 + }, + { + "bits_per_byte": 1.915992327959318, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3280646800994873, + "step": 1890, + "unclipped_grad_norm": 1.1415059566497803 + }, + { + "bits_per_byte": 1.9305143710155228, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3381305932998657, + "step": 1900, + "unclipped_grad_norm": 1.0947742462158203 + }, + { + "bits_per_byte": 1.863847876148318, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2919209003448486, + "step": 1910, + "unclipped_grad_norm": 1.0498101711273193 + }, + { + "bits_per_byte": 1.8709641742757193, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2968535423278809, + "step": 1920, + "unclipped_grad_norm": 1.227995753288269 + }, + { + "bits_per_byte": 1.9694514151535134, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3651196956634521, + "step": 1930, + "unclipped_grad_norm": 1.1106438636779785 + }, + { + "bits_per_byte": 1.8401163341210607, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2754714488983154, + "step": 1940, + "unclipped_grad_norm": 1.1273587942123413 + }, + { + "bits_per_byte": 1.9228790292478175, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3328381776809692, + "step": 1950, + "unclipped_grad_norm": 1.0416061878204346 + }, + { + "bits_per_byte": 1.840930843955547, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.276036024093628, + "step": 1960, + "unclipped_grad_norm": 1.0370304584503174 + }, + { + "bits_per_byte": 1.9309775202942963, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.338451623916626, + "step": 1970, + "unclipped_grad_norm": 1.147440791130066 + }, + { + "bits_per_byte": 1.9639140897436835, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3612815141677856, + "step": 1980, + "unclipped_grad_norm": 1.2305821180343628 + }, + { + "bits_per_byte": 1.9535571225260744, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.354102611541748, + "step": 1990, + "unclipped_grad_norm": 1.3296585083007812 + }, + { + "bits_per_byte": 1.92549075778382, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3346484899520874, + "step": 2000, + "unclipped_grad_norm": 1.0916969776153564 + }, + { + "bits_per_byte": 1.880516778635243, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3034749031066895, + "step": 2010, + "unclipped_grad_norm": 1.0219805240631104 + }, + { + "bits_per_byte": 1.9332846675556208, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3400508165359497, + "step": 2020, + "unclipped_grad_norm": 1.1069045066833496 + }, + { + "bits_per_byte": 1.9929170720025369, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3813848495483398, + "step": 2030, + "unclipped_grad_norm": 1.0479846000671387 + }, + { + "bits_per_byte": 1.9184877962233373, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3297944068908691, + "step": 2040, + "unclipped_grad_norm": 1.0204336643218994 + }, + { + "bits_per_byte": 1.9692321372736634, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.364967703819275, + "step": 2050, + "unclipped_grad_norm": 1.0853239297866821 + }, + { + "bits_per_byte": 1.9303957029864274, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3380483388900757, + "step": 2060, + "unclipped_grad_norm": 1.0730464458465576 + }, + { + "bits_per_byte": 1.889977716241857, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3100327253341675, + "step": 2070, + "unclipped_grad_norm": 1.0691337585449219 + }, + { + "bits_per_byte": 1.8844801188243765, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3062220811843872, + "step": 2080, + "unclipped_grad_norm": 1.0554076433181763 + }, + { + "bits_per_byte": 1.8957609768424206, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.3140413761138916, + "step": 2090, + "unclipped_grad_norm": 1.1196725368499756 + }, + { + "bits_per_byte": 1.8758003264179814, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3002057075500488, + "step": 2100, + "unclipped_grad_norm": 1.0327236652374268 + }, + { + "bits_per_byte": 1.914194593309849, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.326818585395813, + "step": 2110, + "unclipped_grad_norm": 1.0782309770584106 + }, + { + "bits_per_byte": 1.8761728408397502, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3004639148712158, + "step": 2120, + "unclipped_grad_norm": 1.0318117141723633 + }, + { + "bits_per_byte": 1.8463771905430704, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.279811143875122, + "step": 2130, + "unclipped_grad_norm": 1.1898446083068848 + }, + { + "bits_per_byte": 1.984834747325239, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3757826089859009, + "step": 2140, + "unclipped_grad_norm": 1.1598726511001587 + }, + { + "bits_per_byte": 1.8562880347643413, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.286680817604065, + "step": 2150, + "unclipped_grad_norm": 1.0410785675048828 + }, + { + "bits_per_byte": 1.8990754264898486, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3163387775421143, + "step": 2160, + "unclipped_grad_norm": 1.0785906314849854 + }, + { + "bits_per_byte": 1.9583294691048665, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3574105501174927, + "step": 2170, + "unclipped_grad_norm": 1.1421548128128052 + }, + { + "bits_per_byte": 1.8472800994601, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2804369926452637, + "step": 2180, + "unclipped_grad_norm": 1.0929888486862183 + }, + { + "bits_per_byte": 1.8586782496460328, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.2883375883102417, + "step": 2190, + "unclipped_grad_norm": 1.058180570602417 + }, + { + "bits_per_byte": 1.8910505440179393, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3107763528823853, + "step": 2200, + "unclipped_grad_norm": 0.9802089333534241 + }, + { + "bits_per_byte": 1.8399546704292495, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2753593921661377, + "step": 2210, + "unclipped_grad_norm": 1.060943365097046 + }, + { + "bits_per_byte": 1.8476158095945843, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2806696891784668, + "step": 2220, + "unclipped_grad_norm": 1.0172474384307861 + }, + { + "bits_per_byte": 1.9309195621409554, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3384114503860474, + "step": 2230, + "unclipped_grad_norm": 1.1571786403656006 + }, + { + "bits_per_byte": 1.848552427111183, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2813189029693604, + "step": 2240, + "unclipped_grad_norm": 1.085301160812378 + }, + { + "bits_per_byte": 1.864913308670413, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2926594018936157, + "step": 2250, + "unclipped_grad_norm": 1.0694295167922974 + }, + { + "bits_per_byte": 1.8611869605741684, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.290076494216919, + "step": 2260, + "unclipped_grad_norm": 1.1977293491363525 + }, + { + "bits_per_byte": 1.8337584794839643, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2710645198822021, + "step": 2270, + "unclipped_grad_norm": 1.1360434293746948 + }, + { + "bits_per_byte": 1.8587281246147829, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2883721590042114, + "step": 2280, + "unclipped_grad_norm": 1.113946795463562 + }, + { + "bits_per_byte": 1.8907026231152437, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.310535192489624, + "step": 2290, + "unclipped_grad_norm": 1.1474530696868896 + }, + { + "bits_per_byte": 1.8085284526197394, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.253576397895813, + "step": 2300, + "unclipped_grad_norm": 0.96898353099823 + }, + { + "bits_per_byte": 1.8446171200941404, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2785911560058594, + "step": 2310, + "unclipped_grad_norm": 0.9442379474639893 + }, + { + "bits_per_byte": 1.8195852172437095, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2612403631210327, + "step": 2320, + "unclipped_grad_norm": 0.9633188843727112 + }, + { + "bits_per_byte": 1.8903022475040352, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.3102576732635498, + "step": 2330, + "unclipped_grad_norm": 1.0167381763458252 + }, + { + "bits_per_byte": 1.8286817236131452, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2675455808639526, + "step": 2340, + "unclipped_grad_norm": 1.1636918783187866 + }, + { + "bits_per_byte": 1.8687012265556664, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2952849864959717, + "step": 2350, + "unclipped_grad_norm": 1.0884090662002563 + }, + { + "bits_per_byte": 1.8708572010668825, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2967793941497803, + "step": 2360, + "unclipped_grad_norm": 1.0467369556427002 + }, + { + "bits_per_byte": 1.8979811008824086, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3155802488327026, + "step": 2370, + "unclipped_grad_norm": 1.0231177806854248 + }, + { + "bits_per_byte": 1.853912954355926, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2850345373153687, + "step": 2380, + "unclipped_grad_norm": 1.0531508922576904 + }, + { + "bits_per_byte": 1.909142430958103, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3233166933059692, + "step": 2390, + "unclipped_grad_norm": 1.0615708827972412 + }, + { + "bits_per_byte": 1.8518789155441715, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2836246490478516, + "step": 2400, + "unclipped_grad_norm": 1.0747002363204956 + }, + { + "bits_per_byte": 1.8430525939192415, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.277506709098816, + "step": 2410, + "unclipped_grad_norm": 1.035906434059143 + }, + { + "bits_per_byte": 1.8218237434273397, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2627919912338257, + "step": 2420, + "unclipped_grad_norm": 0.9908418655395508 + }, + { + "bits_per_byte": 1.8454151195941437, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.279144287109375, + "step": 2430, + "unclipped_grad_norm": 1.1172900199890137 + }, + { + "bits_per_byte": 1.9008382486611926, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3175606727600098, + "step": 2440, + "unclipped_grad_norm": 1.0292961597442627 + }, + { + "bits_per_byte": 1.8510478953752025, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2830486297607422, + "step": 2450, + "unclipped_grad_norm": 1.0116122961044312 + }, + { + "bits_per_byte": 1.8498311181203482, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2822052240371704, + "step": 2460, + "unclipped_grad_norm": 0.9883264303207397 + }, + { + "bits_per_byte": 1.849598081628429, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.282043695449829, + "step": 2470, + "unclipped_grad_norm": 1.0902615785598755 + }, + { + "bits_per_byte": 1.8701792454571813, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.296309471130371, + "step": 2480, + "unclipped_grad_norm": 1.4108390808105469 + }, + { + "bits_per_byte": 1.81769770765049, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.259932041168213, + "step": 2490, + "unclipped_grad_norm": 1.0036945343017578 + }, + { + "bits_per_byte": 1.8526576529872785, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2841644287109375, + "step": 2500, + "unclipped_grad_norm": 1.0499781370162964 + }, + { + "bits_per_byte": 1.8174889207123426, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2597873210906982, + "step": 2510, + "unclipped_grad_norm": 1.112528681755066 + }, + { + "bits_per_byte": 1.8559390819657409, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2864389419555664, + "step": 2520, + "unclipped_grad_norm": 1.043527603149414 + }, + { + "bits_per_byte": 1.8664243482408935, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2937067747116089, + "step": 2530, + "unclipped_grad_norm": 1.047589898109436 + }, + { + "bits_per_byte": 1.8385335777851701, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2743743658065796, + "step": 2540, + "unclipped_grad_norm": 1.114875078201294 + }, + { + "bits_per_byte": 1.8303919190933249, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2687309980392456, + "step": 2550, + "unclipped_grad_norm": 1.026410460472107 + }, + { + "bits_per_byte": 1.8414952910156785, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2764272689819336, + "step": 2560, + "unclipped_grad_norm": 1.0519986152648926 + }, + { + "bits_per_byte": 1.8285018297603426, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2674208879470825, + "step": 2570, + "unclipped_grad_norm": 0.9845532178878784 + }, + { + "bits_per_byte": 1.8621820521920607, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2907662391662598, + "step": 2580, + "unclipped_grad_norm": 1.1312247514724731 + }, + { + "bits_per_byte": 1.821957545929711, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2628847360610962, + "step": 2590, + "unclipped_grad_norm": 0.9848816990852356 + }, + { + "bits_per_byte": 1.8336463467956017, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.270986795425415, + "step": 2600, + "unclipped_grad_norm": 1.0540134906768799 + }, + { + "bits_per_byte": 1.762311382870726, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.221541166305542, + "step": 2610, + "unclipped_grad_norm": 0.9237543940544128 + }, + { + "bits_per_byte": 1.8236500271968519, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2640578746795654, + "step": 2620, + "unclipped_grad_norm": 0.9258099794387817 + }, + { + "bits_per_byte": 1.8442078013850869, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2783074378967285, + "step": 2630, + "unclipped_grad_norm": 1.1065376996994019 + }, + { + "bits_per_byte": 1.6990233151970702, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1776732206344604, + "step": 2640, + "unclipped_grad_norm": 0.975441038608551 + }, + { + "bits_per_byte": 1.7897238695744, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2405420541763306, + "step": 2650, + "unclipped_grad_norm": 1.024839997291565 + }, + { + "bits_per_byte": 1.829891621561965, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2683842182159424, + "step": 2660, + "unclipped_grad_norm": 1.0799146890640259 + }, + { + "bits_per_byte": 1.8374299651145836, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2736093997955322, + "step": 2670, + "unclipped_grad_norm": 0.9827171564102173 + }, + { + "bits_per_byte": 1.757754702536116, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.218382716178894, + "step": 2680, + "unclipped_grad_norm": 0.9623468518257141 + }, + { + "bits_per_byte": 1.9223347041578367, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.332460880279541, + "step": 2690, + "unclipped_grad_norm": 1.0313293933868408 + }, + { + "bits_per_byte": 1.8426768118271062, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2772462368011475, + "step": 2700, + "unclipped_grad_norm": 1.154735803604126 + }, + { + "bits_per_byte": 1.7467827253590213, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2107775211334229, + "step": 2710, + "unclipped_grad_norm": 0.9442266225814819 + }, + { + "bits_per_byte": 1.7668878412101854, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2247133255004883, + "step": 2720, + "unclipped_grad_norm": 0.9805414080619812 + }, + { + "bits_per_byte": 1.8017586995338266, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2488839626312256, + "step": 2730, + "unclipped_grad_norm": 1.027875542640686 + }, + { + "bits_per_byte": 1.7992866209103247, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2471704483032227, + "step": 2740, + "unclipped_grad_norm": 1.0912469625473022 + }, + { + "bits_per_byte": 1.8291928560515094, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2678998708724976, + "step": 2750, + "unclipped_grad_norm": 1.0327590703964233 + }, + { + "bits_per_byte": 1.831356913747316, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.269399881362915, + "step": 2760, + "unclipped_grad_norm": 1.0535632371902466 + }, + { + "bits_per_byte": 1.7175185014535075, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.190493106842041, + "step": 2770, + "unclipped_grad_norm": 0.9521624445915222 + }, + { + "bits_per_byte": 1.7419930085324922, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2074575424194336, + "step": 2780, + "unclipped_grad_norm": 0.9253908395767212 + }, + { + "bits_per_byte": 1.829489182158946, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2681052684783936, + "step": 2790, + "unclipped_grad_norm": 1.0019073486328125 + }, + { + "bits_per_byte": 1.900268298156233, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.3171656131744385, + "step": 2800, + "unclipped_grad_norm": 1.1026051044464111 + }, + { + "bits_per_byte": 1.8251356133350047, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.265087604522705, + "step": 2810, + "unclipped_grad_norm": 1.095402479171753 + }, + { + "bits_per_byte": 1.8078448215825598, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2531025409698486, + "step": 2820, + "unclipped_grad_norm": 1.0277965068817139 + }, + { + "bits_per_byte": 1.8903005276775264, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.3102564811706543, + "step": 2830, + "unclipped_grad_norm": 1.0477416515350342 + }, + { + "bits_per_byte": 1.7718302786306808, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2281391620635986, + "step": 2840, + "unclipped_grad_norm": 0.9842959642410278 + }, + { + "bits_per_byte": 1.7997145137456714, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.247467041015625, + "step": 2850, + "unclipped_grad_norm": 1.0457379817962646 + }, + { + "bits_per_byte": 1.7627234533021934, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2218267917633057, + "step": 2860, + "unclipped_grad_norm": 1.0218563079833984 + }, + { + "bits_per_byte": 1.8575106594293251, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2875282764434814, + "step": 2870, + "unclipped_grad_norm": 1.1067496538162231 + }, + { + "bits_per_byte": 1.8563188196588458, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2867021560668945, + "step": 2880, + "unclipped_grad_norm": 1.0256601572036743 + }, + { + "bits_per_byte": 1.76771198207312, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2252845764160156, + "step": 2890, + "unclipped_grad_norm": 1.0549262762069702 + }, + { + "bits_per_byte": 1.7222432088380106, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.19376802444458, + "step": 2900, + "unclipped_grad_norm": 1.0038959980010986 + }, + { + "bits_per_byte": 1.7887707417233183, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2398813962936401, + "step": 2910, + "unclipped_grad_norm": 1.020102620124817 + }, + { + "bits_per_byte": 1.7699561836842288, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2268401384353638, + "step": 2920, + "unclipped_grad_norm": 0.9710083603858948 + }, + { + "bits_per_byte": 1.7845655939270717, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.236966609954834, + "step": 2930, + "unclipped_grad_norm": 1.063584566116333 + }, + { + "bits_per_byte": 1.8044884081683212, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2507760524749756, + "step": 2940, + "unclipped_grad_norm": 1.1001172065734863 + }, + { + "bits_per_byte": 1.788769881810064, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2398808002471924, + "step": 2950, + "unclipped_grad_norm": 1.052484393119812 + }, + { + "bits_per_byte": 1.8266576597951407, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2661426067352295, + "step": 2960, + "unclipped_grad_norm": 1.0507400035858154 + }, + { + "bits_per_byte": 1.7942014378896132, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2436456680297852, + "step": 2970, + "unclipped_grad_norm": 1.016513466835022 + }, + { + "bits_per_byte": 1.776544667056132, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2314069271087646, + "step": 2980, + "unclipped_grad_norm": 1.0853108167648315 + }, + { + "bits_per_byte": 1.797620281006115, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2460154294967651, + "step": 2990, + "unclipped_grad_norm": 1.025924563407898 + }, + { + "bits_per_byte": 1.8018763356670169, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2489655017852783, + "step": 3000, + "unclipped_grad_norm": 1.066794514656067 + }, + { + "bits_per_byte": 1.8036527444677788, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2501968145370483, + "step": 3010, + "unclipped_grad_norm": 0.9892268180847168 + }, + { + "bits_per_byte": 1.7646690930314042, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2231754064559937, + "step": 3020, + "unclipped_grad_norm": 1.1210085153579712 + }, + { + "bits_per_byte": 1.8358253669820335, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2724971771240234, + "step": 3030, + "unclipped_grad_norm": 1.0379256010055542 + }, + { + "bits_per_byte": 1.763304066731506, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.222229242324829, + "step": 3040, + "unclipped_grad_norm": 1.0503100156784058 + }, + { + "bits_per_byte": 1.7804524568490367, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2341156005859375, + "step": 3050, + "unclipped_grad_norm": 1.0255935192108154 + }, + { + "bits_per_byte": 1.779026720673384, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2331273555755615, + "step": 3060, + "unclipped_grad_norm": 1.0667622089385986 + }, + { + "bits_per_byte": 1.7089128335696342, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.184528112411499, + "step": 3070, + "unclipped_grad_norm": 0.9528610110282898 + }, + { + "bits_per_byte": 1.7514905784437398, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.214040756225586, + "step": 3080, + "unclipped_grad_norm": 0.9759883880615234 + }, + { + "bits_per_byte": 1.8129533942437885, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.256643533706665, + "step": 3090, + "unclipped_grad_norm": 1.074378490447998 + }, + { + "bits_per_byte": 1.7348117009630646, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2024798393249512, + "step": 3100, + "unclipped_grad_norm": 1.0595066547393799 + }, + { + "bits_per_byte": 1.8111885082806345, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.255420207977295, + "step": 3110, + "unclipped_grad_norm": 1.0304162502288818 + }, + { + "bits_per_byte": 1.7739528885076294, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2296104431152344, + "step": 3120, + "unclipped_grad_norm": 0.9790826439857483 + }, + { + "bits_per_byte": 1.8333825254091782, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2708039283752441, + "step": 3130, + "unclipped_grad_norm": 1.0250012874603271 + }, + { + "bits_per_byte": 1.807222588351738, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.252671241760254, + "step": 3140, + "unclipped_grad_norm": 1.065406322479248 + }, + { + "bits_per_byte": 1.7667856835155729, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2246425151824951, + "step": 3150, + "unclipped_grad_norm": 1.1064149141311646 + }, + { + "bits_per_byte": 1.77101886448391, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.227576732635498, + "step": 3160, + "unclipped_grad_norm": 0.9769210815429688 + }, + { + "bits_per_byte": 1.6904848925296843, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1717548370361328, + "step": 3170, + "unclipped_grad_norm": 1.003720998764038 + }, + { + "bits_per_byte": 1.7897881910858227, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2405866384506226, + "step": 3180, + "unclipped_grad_norm": 0.9989861249923706 + }, + { + "bits_per_byte": 1.7613923075845153, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2209041118621826, + "step": 3190, + "unclipped_grad_norm": 1.0040642023086548 + }, + { + "bits_per_byte": 1.7417902409871249, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2073169946670532, + "step": 3200, + "unclipped_grad_norm": 0.9894367456436157 + }, + { + "bits_per_byte": 1.7268853645500994, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1969857215881348, + "step": 3210, + "unclipped_grad_norm": 1.0016205310821533 + }, + { + "bits_per_byte": 1.7365394386736324, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.2036774158477783, + "step": 3220, + "unclipped_grad_norm": 0.9552999138832092 + }, + { + "bits_per_byte": 1.8281382584364185, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2671688795089722, + "step": 3230, + "unclipped_grad_norm": 1.0519078969955444 + }, + { + "bits_per_byte": 1.7008661093010653, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.178950548171997, + "step": 3240, + "unclipped_grad_norm": 0.956534206867218 + }, + { + "bits_per_byte": 1.8618967329742793, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.290568470954895, + "step": 3250, + "unclipped_grad_norm": 1.118624210357666 + }, + { + "bits_per_byte": 1.7141249397866825, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.188140869140625, + "step": 3260, + "unclipped_grad_norm": 0.9860926270484924 + }, + { + "bits_per_byte": 1.7454612106697915, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2098615169525146, + "step": 3270, + "unclipped_grad_norm": 0.968685507774353 + }, + { + "bits_per_byte": 1.8017138120619516, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2488528490066528, + "step": 3280, + "unclipped_grad_norm": 1.0125019550323486 + }, + { + "bits_per_byte": 1.7954340375483469, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2445000410079956, + "step": 3290, + "unclipped_grad_norm": 1.0358432531356812 + }, + { + "bits_per_byte": 1.7138430602219183, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1879454851150513, + "step": 3300, + "unclipped_grad_norm": 1.0642298460006714 + }, + { + "bits_per_byte": 1.7570346111769533, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.217883586883545, + "step": 3310, + "unclipped_grad_norm": 1.017012596130371 + }, + { + "bits_per_byte": 1.753239298037713, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2152528762817383, + "step": 3320, + "unclipped_grad_norm": 0.9849429130554199 + }, + { + "bits_per_byte": 1.7669095110241941, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2247283458709717, + "step": 3330, + "unclipped_grad_norm": 0.9692639708518982 + }, + { + "bits_per_byte": 1.7337540076602584, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.2017467021942139, + "step": 3340, + "unclipped_grad_norm": 0.9712040424346924 + }, + { + "bits_per_byte": 1.7289699662612077, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1984306573867798, + "step": 3350, + "unclipped_grad_norm": 0.9344873428344727 + }, + { + "bits_per_byte": 1.771012673108479, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2275724411010742, + "step": 3360, + "unclipped_grad_norm": 1.0185227394104004 + }, + { + "bits_per_byte": 1.7121330367243894, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1867601871490479, + "step": 3370, + "unclipped_grad_norm": 1.017809271812439 + }, + { + "bits_per_byte": 1.7444269070075025, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2091445922851562, + "step": 3380, + "unclipped_grad_norm": 0.9594104886054993 + }, + { + "bits_per_byte": 1.782493202984175, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.235530138015747, + "step": 3390, + "unclipped_grad_norm": 1.0056157112121582 + }, + { + "bits_per_byte": 1.8303659497130447, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2687129974365234, + "step": 3400, + "unclipped_grad_norm": 0.9542431235313416 + }, + { + "bits_per_byte": 1.8696024156461875, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2959096431732178, + "step": 3410, + "unclipped_grad_norm": 1.0261633396148682 + }, + { + "bits_per_byte": 1.7306273630675726, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1995794773101807, + "step": 3420, + "unclipped_grad_norm": 1.0138797760009766 + }, + { + "bits_per_byte": 1.663148250175041, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1528065204620361, + "step": 3430, + "unclipped_grad_norm": 0.9656930565834045 + }, + { + "bits_per_byte": 1.7509053214828536, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2136350870132446, + "step": 3440, + "unclipped_grad_norm": 1.1262567043304443 + }, + { + "bits_per_byte": 1.8269866626062412, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2663706541061401, + "step": 3450, + "unclipped_grad_norm": 1.0454469919204712 + }, + { + "bits_per_byte": 1.7413009503454202, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2069778442382812, + "step": 3460, + "unclipped_grad_norm": 1.1041505336761475 + }, + { + "bits_per_byte": 1.7427603951206418, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2079894542694092, + "step": 3470, + "unclipped_grad_norm": 1.0516799688339233 + }, + { + "bits_per_byte": 1.6734345325231454, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1599364280700684, + "step": 3480, + "unclipped_grad_norm": 0.9458639621734619 + }, + { + "bits_per_byte": 1.8872512752777288, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.308142900466919, + "step": 3490, + "unclipped_grad_norm": 1.0662349462509155 + }, + { + "bits_per_byte": 1.7517492403506374, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.2142200469970703, + "step": 3500, + "unclipped_grad_norm": 1.0241658687591553 + }, + { + "bits_per_byte": 1.7728004327641977, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2288116216659546, + "step": 3510, + "unclipped_grad_norm": 1.0039124488830566 + }, + { + "bits_per_byte": 1.7731452579791778, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.229050636291504, + "step": 3520, + "unclipped_grad_norm": 0.9908767342567444 + }, + { + "bits_per_byte": 1.743414617124524, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.2084429264068604, + "step": 3530, + "unclipped_grad_norm": 0.9787898063659668 + }, + { + "bits_per_byte": 1.7481145590073028, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.211700677871704, + "step": 3540, + "unclipped_grad_norm": 1.0219446420669556 + }, + { + "bits_per_byte": 1.7531405799961175, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2151844501495361, + "step": 3550, + "unclipped_grad_norm": 1.2019073963165283 + }, + { + "bits_per_byte": 1.6952811446969462, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.175079345703125, + "step": 3560, + "unclipped_grad_norm": 0.9977114796638489 + }, + { + "bits_per_byte": 1.7182931113129936, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1910300254821777, + "step": 3570, + "unclipped_grad_norm": 1.0032203197479248 + }, + { + "bits_per_byte": 1.7239327664000867, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.194939136505127, + "step": 3580, + "unclipped_grad_norm": 1.0535842180252075 + }, + { + "bits_per_byte": 1.7655023489748345, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2237529754638672, + "step": 3590, + "unclipped_grad_norm": 1.055069923400879 + }, + { + "bits_per_byte": 1.7734857836278861, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2292866706848145, + "step": 3600, + "unclipped_grad_norm": 1.0577540397644043 + }, + { + "bits_per_byte": 1.7140950148054324, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1881201267242432, + "step": 3610, + "unclipped_grad_norm": 1.0167864561080933 + }, + { + "bits_per_byte": 1.6696412831757155, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1573071479797363, + "step": 3620, + "unclipped_grad_norm": 0.9244848489761353 + }, + { + "bits_per_byte": 1.7374531824976664, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2043107748031616, + "step": 3630, + "unclipped_grad_norm": 0.9926364421844482 + }, + { + "bits_per_byte": 1.7074227758825589, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.183495283126831, + "step": 3640, + "unclipped_grad_norm": 0.9468497633934021 + }, + { + "bits_per_byte": 1.7634653864580154, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2223410606384277, + "step": 3650, + "unclipped_grad_norm": 0.9865193367004395 + }, + { + "bits_per_byte": 1.6666551484087844, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1552373170852661, + "step": 3660, + "unclipped_grad_norm": 1.0027393102645874 + }, + { + "bits_per_byte": 1.749947206134897, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2129709720611572, + "step": 3670, + "unclipped_grad_norm": 1.0161325931549072 + }, + { + "bits_per_byte": 1.700321268263132, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1785728931427002, + "step": 3680, + "unclipped_grad_norm": 1.0996551513671875 + }, + { + "bits_per_byte": 1.6689437215438159, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1568236351013184, + "step": 3690, + "unclipped_grad_norm": 0.9738581776618958 + }, + { + "bits_per_byte": 1.6837737855277157, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1671030521392822, + "step": 3700, + "unclipped_grad_norm": 1.0792776346206665 + }, + { + "bits_per_byte": 1.7280950905162684, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.197824239730835, + "step": 3710, + "unclipped_grad_norm": 0.9678228497505188 + }, + { + "bits_per_byte": 1.705079684247204, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1818711757659912, + "step": 3720, + "unclipped_grad_norm": 0.9917497634887695 + }, + { + "bits_per_byte": 1.649159353336511, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1431101560592651, + "step": 3730, + "unclipped_grad_norm": 0.9426941275596619 + }, + { + "bits_per_byte": 1.6954601786364945, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1752034425735474, + "step": 3740, + "unclipped_grad_norm": 1.0086418390274048 + }, + { + "bits_per_byte": 1.721657263946521, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1933618783950806, + "step": 3750, + "unclipped_grad_norm": 0.9619976878166199 + }, + { + "bits_per_byte": 1.808596729732132, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2536237239837646, + "step": 3760, + "unclipped_grad_norm": 1.012074589729309 + }, + { + "bits_per_byte": 1.7638779728374354, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.22262704372406, + "step": 3770, + "unclipped_grad_norm": 1.050723910331726 + }, + { + "bits_per_byte": 1.589899463311364, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.102034330368042, + "step": 3780, + "unclipped_grad_norm": 0.9615285992622375 + }, + { + "bits_per_byte": 1.6942863970443558, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1743898391723633, + "step": 3790, + "unclipped_grad_norm": 0.9870805144309998 + }, + { + "bits_per_byte": 1.6994689222454558, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1779820919036865, + "step": 3800, + "unclipped_grad_norm": 0.9850614070892334 + }, + { + "bits_per_byte": 1.7199389852817506, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1921708583831787, + "step": 3810, + "unclipped_grad_norm": 0.9774236083030701 + }, + { + "bits_per_byte": 1.718822817877651, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1913971900939941, + "step": 3820, + "unclipped_grad_norm": 1.0395604372024536 + }, + { + "bits_per_byte": 1.762180848038721, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.221450686454773, + "step": 3830, + "unclipped_grad_norm": 1.0331096649169922 + }, + { + "bits_per_byte": 1.68565630762406, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.168407917022705, + "step": 3840, + "unclipped_grad_norm": 1.0491844415664673 + }, + { + "bits_per_byte": 1.6311636046981792, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.13063645362854, + "step": 3850, + "unclipped_grad_norm": 0.9598016738891602 + }, + { + "bits_per_byte": 1.7078809376644575, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1838128566741943, + "step": 3860, + "unclipped_grad_norm": 1.0586371421813965 + }, + { + "bits_per_byte": 1.778100422115837, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.232485294342041, + "step": 3870, + "unclipped_grad_norm": 0.9732621908187866 + }, + { + "bits_per_byte": 1.6634643542873266, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1530256271362305, + "step": 3880, + "unclipped_grad_norm": 0.9620565176010132 + }, + { + "bits_per_byte": 1.6362388127251406, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1341543197631836, + "step": 3890, + "unclipped_grad_norm": 0.9925653338432312 + }, + { + "bits_per_byte": 1.721069255263221, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1929543018341064, + "step": 3900, + "unclipped_grad_norm": 1.121357798576355 + }, + { + "bits_per_byte": 1.7172674067832479, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1903190612792969, + "step": 3910, + "unclipped_grad_norm": 1.0240309238433838 + }, + { + "bits_per_byte": 1.671385531220766, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1585161685943604, + "step": 3920, + "unclipped_grad_norm": 0.9621630311012268 + }, + { + "bits_per_byte": 1.7072636919305109, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1833850145339966, + "step": 3930, + "unclipped_grad_norm": 0.9956491589546204 + }, + { + "bits_per_byte": 1.7254322831329596, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1959785223007202, + "step": 3940, + "unclipped_grad_norm": 0.9730517268180847 + }, + { + "bits_per_byte": 1.7623370082857046, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.221558928489685, + "step": 3950, + "unclipped_grad_norm": 0.9878451228141785 + }, + { + "bits_per_byte": 1.7644881672826966, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2230499982833862, + "step": 3960, + "unclipped_grad_norm": 1.003400444984436 + }, + { + "bits_per_byte": 1.682019734471566, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1658872365951538, + "step": 3970, + "unclipped_grad_norm": 1.0887466669082642 + }, + { + "bits_per_byte": 1.7137744391442242, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1878979206085205, + "step": 3980, + "unclipped_grad_norm": 1.0592182874679565 + }, + { + "bits_per_byte": 1.7370323409510051, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2040190696716309, + "step": 3990, + "unclipped_grad_norm": 1.0668442249298096 + }, + { + "bits_per_byte": 1.620528197568823, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1232645511627197, + "step": 4000, + "unclipped_grad_norm": 0.979034423828125 + }, + { + "bits_per_byte": 1.7224021208074078, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.193878173828125, + "step": 4010, + "unclipped_grad_norm": 1.0592741966247559 + }, + { + "bits_per_byte": 1.6779786581242426, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.163086175918579, + "step": 4020, + "unclipped_grad_norm": 1.0186548233032227 + }, + { + "bits_per_byte": 1.7290010951210137, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1984522342681885, + "step": 4030, + "unclipped_grad_norm": 1.0059210062026978 + }, + { + "bits_per_byte": 1.6595417739864478, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1503067016601562, + "step": 4040, + "unclipped_grad_norm": 1.009366750717163 + }, + { + "bits_per_byte": 1.6963183720643, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1757982969284058, + "step": 4050, + "unclipped_grad_norm": 1.0469119548797607 + }, + { + "bits_per_byte": 1.706078903448717, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1825637817382812, + "step": 4060, + "unclipped_grad_norm": 1.0410232543945312 + }, + { + "bits_per_byte": 1.7226642223673228, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1940598487854004, + "step": 4070, + "unclipped_grad_norm": 1.0424585342407227 + }, + { + "bits_per_byte": 1.7834826191745887, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2362159490585327, + "step": 4080, + "unclipped_grad_norm": 1.0965845584869385 + }, + { + "bits_per_byte": 1.6886807945221334, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1705043315887451, + "step": 4090, + "unclipped_grad_norm": 1.0564258098602295 + }, + { + "bits_per_byte": 1.6871728506393684, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1694591045379639, + "step": 4100, + "unclipped_grad_norm": 1.032038688659668 + }, + { + "bits_per_byte": 1.6672990514536148, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1556836366653442, + "step": 4110, + "unclipped_grad_norm": 1.0212913751602173 + }, + { + "bits_per_byte": 1.7325631997856843, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.2009212970733643, + "step": 4120, + "unclipped_grad_norm": 1.1421489715576172 + }, + { + "bits_per_byte": 1.6623953101295634, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1522846221923828, + "step": 4130, + "unclipped_grad_norm": 1.0443917512893677 + }, + { + "bits_per_byte": 1.7315172013031366, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2001962661743164, + "step": 4140, + "unclipped_grad_norm": 1.0389643907546997 + }, + { + "bits_per_byte": 1.6632394009799982, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.152869701385498, + "step": 4150, + "unclipped_grad_norm": 0.9869729280471802 + }, + { + "bits_per_byte": 1.697900440469587, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1768949031829834, + "step": 4160, + "unclipped_grad_norm": 1.018755316734314 + }, + { + "bits_per_byte": 1.74134876152236, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.2070109844207764, + "step": 4170, + "unclipped_grad_norm": 1.0534698963165283 + }, + { + "bits_per_byte": 1.7750522016119445, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.230372428894043, + "step": 4180, + "unclipped_grad_norm": 1.1320860385894775 + }, + { + "bits_per_byte": 1.6878769476120008, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1699471473693848, + "step": 4190, + "unclipped_grad_norm": 1.0554869174957275 + }, + { + "bits_per_byte": 1.725129765650092, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1957688331604004, + "step": 4200, + "unclipped_grad_norm": 1.0452762842178345 + }, + { + "bits_per_byte": 1.7145272072070508, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1884196996688843, + "step": 4210, + "unclipped_grad_norm": 0.9850034117698669 + }, + { + "bits_per_byte": 1.7283826455085112, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1980235576629639, + "step": 4220, + "unclipped_grad_norm": 1.0109167098999023 + }, + { + "bits_per_byte": 1.7140664656853892, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1881003379821777, + "step": 4230, + "unclipped_grad_norm": 1.0496208667755127 + }, + { + "bits_per_byte": 1.659261958213494, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1501127481460571, + "step": 4240, + "unclipped_grad_norm": 0.9920905232429504 + }, + { + "bits_per_byte": 1.6850357942197471, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1679778099060059, + "step": 4250, + "unclipped_grad_norm": 1.0322012901306152 + }, + { + "bits_per_byte": 1.667805196395104, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1560344696044922, + "step": 4260, + "unclipped_grad_norm": 1.0430845022201538 + }, + { + "bits_per_byte": 1.6625848350108143, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1524159908294678, + "step": 4270, + "unclipped_grad_norm": 1.0265511274337769 + }, + { + "bits_per_byte": 1.6648945620119018, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1540169715881348, + "step": 4280, + "unclipped_grad_norm": 0.9929917454719543 + }, + { + "bits_per_byte": 1.72171487813456, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.19340181350708, + "step": 4290, + "unclipped_grad_norm": 1.0577914714813232 + }, + { + "bits_per_byte": 1.6424783432984433, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.138479232788086, + "step": 4300, + "unclipped_grad_norm": 1.0359195470809937 + }, + { + "bits_per_byte": 1.7237528725472844, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1948144435882568, + "step": 4310, + "unclipped_grad_norm": 1.124787449836731 + }, + { + "bits_per_byte": 1.7081155220002342, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1839754581451416, + "step": 4320, + "unclipped_grad_norm": 1.0496878623962402 + }, + { + "bits_per_byte": 1.6609448084521865, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1512792110443115, + "step": 4330, + "unclipped_grad_norm": 1.0365654230117798 + }, + { + "bits_per_byte": 1.6981388084236828, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1770601272583008, + "step": 4340, + "unclipped_grad_norm": 1.0712857246398926 + }, + { + "bits_per_byte": 1.679871155214337, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.164397954940796, + "step": 4350, + "unclipped_grad_norm": 1.0368938446044922 + }, + { + "bits_per_byte": 1.658068570599157, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1492855548858643, + "step": 4360, + "unclipped_grad_norm": 1.054489254951477 + }, + { + "bits_per_byte": 1.6791636185886873, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.163907527923584, + "step": 4370, + "unclipped_grad_norm": 1.05704927444458 + }, + { + "bits_per_byte": 1.733114060216398, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.2013031244277954, + "step": 4380, + "unclipped_grad_norm": 1.066345453262329 + }, + { + "bits_per_byte": 1.6866214742607022, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.169076919555664, + "step": 4390, + "unclipped_grad_norm": 1.039746880531311 + }, + { + "bits_per_byte": 1.656244522604106, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1480212211608887, + "step": 4400, + "unclipped_grad_norm": 1.0477714538574219 + }, + { + "bits_per_byte": 1.7190543065257124, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1915576457977295, + "step": 4410, + "unclipped_grad_norm": 1.0500718355178833 + }, + { + "bits_per_byte": 1.6606173534849438, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1510522365570068, + "step": 4420, + "unclipped_grad_norm": 1.0457862615585327 + }, + { + "bits_per_byte": 1.7152531459763425, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1889228820800781, + "step": 4430, + "unclipped_grad_norm": 1.0248607397079468 + }, + { + "bits_per_byte": 1.6480127450032085, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.14231538772583, + "step": 4440, + "unclipped_grad_norm": 1.026747703552246 + }, + { + "bits_per_byte": 1.64553137931656, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1405954360961914, + "step": 4450, + "unclipped_grad_norm": 1.0325950384140015 + }, + { + "bits_per_byte": 1.6368648295742814, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1345882415771484, + "step": 4460, + "unclipped_grad_norm": 1.0030871629714966 + }, + { + "bits_per_byte": 1.686227977955528, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1688041687011719, + "step": 4470, + "unclipped_grad_norm": 0.9844985604286194 + }, + { + "bits_per_byte": 1.7063241507088474, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1827337741851807, + "step": 4480, + "unclipped_grad_norm": 1.1298258304595947 + }, + { + "bits_per_byte": 1.6590796566035795, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1499863862991333, + "step": 4490, + "unclipped_grad_norm": 0.9567995071411133 + }, + { + "bits_per_byte": 1.699403568838128, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1779367923736572, + "step": 4500, + "unclipped_grad_norm": 1.0041329860687256 + }, + { + "bits_per_byte": 1.630450564627702, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1301422119140625, + "step": 4510, + "unclipped_grad_norm": 1.0097919702529907 + }, + { + "bits_per_byte": 1.677602188101504, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1628252267837524, + "step": 4520, + "unclipped_grad_norm": 1.0421855449676514 + }, + { + "bits_per_byte": 1.6955814264053528, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1752874851226807, + "step": 4530, + "unclipped_grad_norm": 1.0619962215423584 + }, + { + "bits_per_byte": 1.738972993183341, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.2053642272949219, + "step": 4540, + "unclipped_grad_norm": 1.0259038209915161 + }, + { + "bits_per_byte": 1.6138776284599583, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.118654727935791, + "step": 4550, + "unclipped_grad_norm": 1.1216750144958496 + }, + { + "bits_per_byte": 1.6847919228208237, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1678087711334229, + "step": 4560, + "unclipped_grad_norm": 1.1666667461395264 + }, + { + "bits_per_byte": 1.6566538413131595, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1483049392700195, + "step": 4570, + "unclipped_grad_norm": 0.9492276310920715 + }, + { + "bits_per_byte": 1.6127442627907724, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1178691387176514, + "step": 4580, + "unclipped_grad_norm": 1.0188902616500854 + }, + { + "bits_per_byte": 1.6539726317862082, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1464464664459229, + "step": 4590, + "unclipped_grad_norm": 1.0299646854400635 + }, + { + "bits_per_byte": 1.6561746976478557, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.147972822189331, + "step": 4600, + "unclipped_grad_norm": 1.0103821754455566 + }, + { + "bits_per_byte": 1.719434732149421, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1918213367462158, + "step": 4610, + "unclipped_grad_norm": 1.0808284282684326 + }, + { + "bits_per_byte": 1.7013596995090414, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1792926788330078, + "step": 4620, + "unclipped_grad_norm": 1.0917437076568604 + }, + { + "bits_per_byte": 1.683035292024911, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1665911674499512, + "step": 4630, + "unclipped_grad_norm": 1.0124032497406006 + }, + { + "bits_per_byte": 1.7275268598378177, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1974303722381592, + "step": 4640, + "unclipped_grad_norm": 1.0515177249908447 + }, + { + "bits_per_byte": 1.6549154406782383, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1470999717712402, + "step": 4650, + "unclipped_grad_norm": 1.1102732419967651 + }, + { + "bits_per_byte": 1.6181718632693518, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1216312646865845, + "step": 4660, + "unclipped_grad_norm": 1.066238522529602 + }, + { + "bits_per_byte": 1.7593243881905407, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.219470739364624, + "step": 4670, + "unclipped_grad_norm": 1.1023285388946533 + }, + { + "bits_per_byte": 1.7532637195741354, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.2152698040008545, + "step": 4680, + "unclipped_grad_norm": 1.0341681241989136 + }, + { + "bits_per_byte": 1.7266088164475122, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.196794033050537, + "step": 4690, + "unclipped_grad_norm": 1.0434808731079102 + }, + { + "bits_per_byte": 1.6483247215318737, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1425316333770752, + "step": 4700, + "unclipped_grad_norm": 1.045780897140503 + }, + { + "bits_per_byte": 1.7162458298371228, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1896109580993652, + "step": 4710, + "unclipped_grad_norm": 1.1512850522994995 + }, + { + "bits_per_byte": 1.6600910865733036, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1506874561309814, + "step": 4720, + "unclipped_grad_norm": 1.0870006084442139 + }, + { + "bits_per_byte": 1.579696248583626, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0949620008468628, + "step": 4730, + "unclipped_grad_norm": 1.066159725189209 + }, + { + "bits_per_byte": 1.6295564268258662, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.129522442817688, + "step": 4740, + "unclipped_grad_norm": 1.0375505685806274 + }, + { + "bits_per_byte": 1.6749376608916864, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1609783172607422, + "step": 4750, + "unclipped_grad_norm": 1.0134141445159912 + }, + { + "bits_per_byte": 1.6434221840863785, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1391334533691406, + "step": 4760, + "unclipped_grad_norm": 1.0448397397994995 + }, + { + "bits_per_byte": 1.6362174868764336, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1341395378112793, + "step": 4770, + "unclipped_grad_norm": 1.055105447769165 + }, + { + "bits_per_byte": 1.666649472981306, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.155233383178711, + "step": 4780, + "unclipped_grad_norm": 1.0775461196899414 + }, + { + "bits_per_byte": 1.6314788488972107, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1308549642562866, + "step": 4790, + "unclipped_grad_norm": 1.065148949623108 + }, + { + "bits_per_byte": 1.6269109896902945, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1276887655258179, + "step": 4800, + "unclipped_grad_norm": 1.1373785734176636 + }, + { + "bits_per_byte": 1.6901620810940148, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.171531081199646, + "step": 4810, + "unclipped_grad_norm": 1.1410114765167236 + }, + { + "bits_per_byte": 1.6112782826748178, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1168529987335205, + "step": 4820, + "unclipped_grad_norm": 1.0628036260604858 + }, + { + "bits_per_byte": 1.649350942009572, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1432429552078247, + "step": 4830, + "unclipped_grad_norm": 1.1659979820251465 + }, + { + "bits_per_byte": 1.6571300612733986, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1486350297927856, + "step": 4840, + "unclipped_grad_norm": 1.1067216396331787 + }, + { + "bits_per_byte": 1.668376866726572, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.156430721282959, + "step": 4850, + "unclipped_grad_norm": 0.9945695996284485 + }, + { + "bits_per_byte": 1.600552412671108, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1094183921813965, + "step": 4860, + "unclipped_grad_norm": 1.0181769132614136 + }, + { + "bits_per_byte": 1.6771526254521485, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1625136137008667, + "step": 4870, + "unclipped_grad_norm": 1.0462807416915894 + }, + { + "bits_per_byte": 1.539689988305221, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0672317743301392, + "step": 4880, + "unclipped_grad_norm": 1.0612690448760986 + }, + { + "bits_per_byte": 1.5967718900398415, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1067979335784912, + "step": 4890, + "unclipped_grad_norm": 1.0629737377166748 + }, + { + "bits_per_byte": 1.6387246499607118, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1358773708343506, + "step": 4900, + "unclipped_grad_norm": 1.0522006750106812 + }, + { + "bits_per_byte": 1.7368791044090865, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.2039128541946411, + "step": 4910, + "unclipped_grad_norm": 1.1116063594818115 + }, + { + "bits_per_byte": 1.6773633041994556, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1626596450805664, + "step": 4920, + "unclipped_grad_norm": 1.0861767530441284 + }, + { + "bits_per_byte": 1.6375162998557495, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1350398063659668, + "step": 4930, + "unclipped_grad_norm": 1.0190138816833496 + }, + { + "bits_per_byte": 1.6159926711002692, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1201207637786865, + "step": 4940, + "unclipped_grad_norm": 1.0380074977874756 + }, + { + "bits_per_byte": 1.5895295286293578, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1017779111862183, + "step": 4950, + "unclipped_grad_norm": 1.0411629676818848 + }, + { + "bits_per_byte": 1.5717379233976003, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.08944571018219, + "step": 4960, + "unclipped_grad_norm": 1.0460548400878906 + }, + { + "bits_per_byte": 1.6461665112461963, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1410356760025024, + "step": 4970, + "unclipped_grad_norm": 1.112047553062439 + }, + { + "bits_per_byte": 1.7016685803499911, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.179506778717041, + "step": 4980, + "unclipped_grad_norm": 1.062510371208191 + }, + { + "bits_per_byte": 1.636959248049605, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1346536874771118, + "step": 4990, + "unclipped_grad_norm": 1.1910533905029297 + }, + { + "bits_per_byte": 1.6019551031715449, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1103906631469727, + "step": 5000, + "unclipped_grad_norm": 1.0508692264556885 + }, + { + "bits_per_byte": 1.5837816964548719, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0977938175201416, + "step": 5010, + "unclipped_grad_norm": 1.0227967500686646 + }, + { + "bits_per_byte": 1.639255904369227, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.136245608329773, + "step": 5020, + "unclipped_grad_norm": 1.0390269756317139 + }, + { + "bits_per_byte": 1.6173587292960725, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1210676431655884, + "step": 5030, + "unclipped_grad_norm": 1.083649754524231 + }, + { + "bits_per_byte": 1.591486175248224, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.1031341552734375, + "step": 5040, + "unclipped_grad_norm": 1.0160164833068848 + }, + { + "bits_per_byte": 1.5612364907532663, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0821666717529297, + "step": 5050, + "unclipped_grad_norm": 0.9697359800338745 + }, + { + "bits_per_byte": 1.5996653261579574, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1088035106658936, + "step": 5060, + "unclipped_grad_norm": 1.0825191736221313 + }, + { + "bits_per_byte": 1.6612003746713688, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.151456356048584, + "step": 5070, + "unclipped_grad_norm": 1.051671028137207 + }, + { + "bits_per_byte": 1.6135078657606032, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1183984279632568, + "step": 5080, + "unclipped_grad_norm": 1.0758769512176514 + }, + { + "bits_per_byte": 1.6053440213067967, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1127396821975708, + "step": 5090, + "unclipped_grad_norm": 1.0736517906188965 + }, + { + "bits_per_byte": 1.620226196033908, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1230552196502686, + "step": 5100, + "unclipped_grad_norm": 1.0084763765335083 + }, + { + "bits_per_byte": 1.5469340695422136, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0722529888153076, + "step": 5110, + "unclipped_grad_norm": 1.001043438911438 + }, + { + "bits_per_byte": 1.627192525289757, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1278839111328125, + "step": 5120, + "unclipped_grad_norm": 1.0516539812088013 + }, + { + "bits_per_byte": 1.642070228467946, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.138196349143982, + "step": 5130, + "unclipped_grad_norm": 1.082858681678772 + }, + { + "bits_per_byte": 1.6022958008029042, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1106268167495728, + "step": 5140, + "unclipped_grad_norm": 1.0696102380752563 + }, + { + "bits_per_byte": 1.6803482350878305, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1647286415100098, + "step": 5150, + "unclipped_grad_norm": 1.045096755027771 + }, + { + "bits_per_byte": 1.5608273440268638, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0818830728530884, + "step": 5160, + "unclipped_grad_norm": 1.0852984189987183 + }, + { + "bits_per_byte": 1.7284662290768305, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1980814933776855, + "step": 5170, + "unclipped_grad_norm": 1.2249319553375244 + }, + { + "bits_per_byte": 1.6274811121779047, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1280839443206787, + "step": 5180, + "unclipped_grad_norm": 1.1194853782653809 + }, + { + "bits_per_byte": 1.567900990456851, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.086786150932312, + "step": 5190, + "unclipped_grad_norm": 1.0312306880950928 + }, + { + "bits_per_byte": 1.5985697966719614, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.108044147491455, + "step": 5200, + "unclipped_grad_norm": 1.137611985206604 + }, + { + "bits_per_byte": 1.6023676895509649, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.110676646232605, + "step": 5210, + "unclipped_grad_norm": 1.031370759010315 + }, + { + "bits_per_byte": 1.6946726700781938, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1746575832366943, + "step": 5220, + "unclipped_grad_norm": 1.113584280014038 + }, + { + "bits_per_byte": 1.5914005278880945, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1030747890472412, + "step": 5230, + "unclipped_grad_norm": 1.1469851732254028 + }, + { + "bits_per_byte": 1.6053940682581977, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.11277437210083, + "step": 5240, + "unclipped_grad_norm": 1.0413556098937988 + }, + { + "bits_per_byte": 1.5800591319769466, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.095213532447815, + "step": 5250, + "unclipped_grad_norm": 1.0926955938339233 + }, + { + "bits_per_byte": 1.5918442431273205, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.1033823490142822, + "step": 5260, + "unclipped_grad_norm": 1.0704180002212524 + }, + { + "bits_per_byte": 1.589662299235824, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1018699407577515, + "step": 5270, + "unclipped_grad_norm": 1.0476908683776855 + }, + { + "bits_per_byte": 1.6108980290337602, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1165894269943237, + "step": 5280, + "unclipped_grad_norm": 1.1081280708312988 + }, + { + "bits_per_byte": 1.608212003992585, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.1147276163101196, + "step": 5290, + "unclipped_grad_norm": 1.1043791770935059 + }, + { + "bits_per_byte": 1.614489714714379, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.1190789937973022, + "step": 5300, + "unclipped_grad_norm": 1.0849814414978027 + }, + { + "bits_per_byte": 1.6119760162893682, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.117336630821228, + "step": 5310, + "unclipped_grad_norm": 1.1325145959854126 + }, + { + "bits_per_byte": 1.5916898027068456, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1032752990722656, + "step": 5320, + "unclipped_grad_norm": 1.072629451751709 + }, + { + "bits_per_byte": 1.6015805249579658, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.110131025314331, + "step": 5330, + "unclipped_grad_norm": 1.1142079830169678 + }, + { + "bits_per_byte": 1.553969879789011, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.07712984085083, + "step": 5340, + "unclipped_grad_norm": 1.0742131471633911 + }, + { + "bits_per_byte": 1.626766180298268, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1275883913040161, + "step": 5350, + "unclipped_grad_norm": 1.127328634262085 + }, + { + "bits_per_byte": 1.5508274127924457, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0749516487121582, + "step": 5360, + "unclipped_grad_norm": 1.1100437641143799 + }, + { + "bits_per_byte": 1.5306884163590617, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0609923601150513, + "step": 5370, + "unclipped_grad_norm": 1.0785573720932007 + }, + { + "bits_per_byte": 1.633070548330946, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.131958246231079, + "step": 5380, + "unclipped_grad_norm": 1.0940150022506714 + }, + { + "bits_per_byte": 1.5790724675089465, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.094529628753662, + "step": 5390, + "unclipped_grad_norm": 1.1135237216949463 + }, + { + "bits_per_byte": 1.6121983898569339, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1174907684326172, + "step": 5400, + "unclipped_grad_norm": 1.1351183652877808 + }, + { + "bits_per_byte": 1.6233261833157102, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1252039670944214, + "step": 5410, + "unclipped_grad_norm": 1.1545859575271606 + }, + { + "bits_per_byte": 1.5857118577455054, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.09913170337677, + "step": 5420, + "unclipped_grad_norm": 1.1009039878845215 + }, + { + "bits_per_byte": 1.6290494219711227, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1291710138320923, + "step": 5430, + "unclipped_grad_norm": 1.1000012159347534 + }, + { + "bits_per_byte": 1.5753132707263873, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0919239521026611, + "step": 5440, + "unclipped_grad_norm": 1.0844659805297852 + }, + { + "bits_per_byte": 1.5681787424379945, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0869786739349365, + "step": 5450, + "unclipped_grad_norm": 1.1008447408676147 + }, + { + "bits_per_byte": 1.5540281819076536, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0771702527999878, + "step": 5460, + "unclipped_grad_norm": 1.077485203742981 + }, + { + "bits_per_byte": 1.5997471898997682, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1088602542877197, + "step": 5470, + "unclipped_grad_norm": 1.0731958150863647 + }, + { + "bits_per_byte": 1.6110182449067132, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.1166727542877197, + "step": 5480, + "unclipped_grad_norm": 1.098453164100647 + }, + { + "bits_per_byte": 1.627262694211309, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1279325485229492, + "step": 5490, + "unclipped_grad_norm": 1.2330873012542725 + }, + { + "bits_per_byte": 1.5907870657724668, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1026495695114136, + "step": 5500, + "unclipped_grad_norm": 1.1288633346557617 + }, + { + "bits_per_byte": 1.677719824234694, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1629067659378052, + "step": 5510, + "unclipped_grad_norm": 1.1779037714004517 + }, + { + "bits_per_byte": 1.6037174093949362, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1116122007369995, + "step": 5520, + "unclipped_grad_norm": 1.07362699508667 + }, + { + "bits_per_byte": 1.5749782485225066, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0916917324066162, + "step": 5530, + "unclipped_grad_norm": 1.0827255249023438 + }, + { + "bits_per_byte": 1.6475040203219564, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1419627666473389, + "step": 5540, + "unclipped_grad_norm": 1.2362996339797974 + }, + { + "bits_per_byte": 1.5801301608117528, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0952627658843994, + "step": 5550, + "unclipped_grad_norm": 1.0816401243209839 + }, + { + "bits_per_byte": 1.566390638816974, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.085739254951477, + "step": 5560, + "unclipped_grad_norm": 1.1727906465530396 + }, + { + "bits_per_byte": 1.545297310653952, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0711184740066528, + "step": 5570, + "unclipped_grad_norm": 1.096352458000183 + }, + { + "bits_per_byte": 1.6197793851069664, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1227455139160156, + "step": 5580, + "unclipped_grad_norm": 1.098443865776062 + }, + { + "bits_per_byte": 1.5488499562728253, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0735809803009033, + "step": 5590, + "unclipped_grad_norm": 1.106048822402954 + }, + { + "bits_per_byte": 1.6060310919969936, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1132159233093262, + "step": 5600, + "unclipped_grad_norm": 1.1421784162521362 + }, + { + "bits_per_byte": 1.6012358717256368, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1098921298980713, + "step": 5610, + "unclipped_grad_norm": 1.0477343797683716 + }, + { + "bits_per_byte": 1.612595669780427, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1177661418914795, + "step": 5620, + "unclipped_grad_norm": 1.0961613655090332 + }, + { + "bits_per_byte": 1.5887812321154537, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.1012592315673828, + "step": 5630, + "unclipped_grad_norm": 1.0803630352020264 + }, + { + "bits_per_byte": 1.5675833385007074, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0865659713745117, + "step": 5640, + "unclipped_grad_norm": 1.0926858186721802 + }, + { + "bits_per_byte": 1.598304599424331, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1078603267669678, + "step": 5650, + "unclipped_grad_norm": 1.1493103504180908 + }, + { + "bits_per_byte": 1.6562716958629424, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1480400562286377, + "step": 5660, + "unclipped_grad_norm": 1.1589452028274536 + }, + { + "bits_per_byte": 1.5621947780838739, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0828309059143066, + "step": 5670, + "unclipped_grad_norm": 1.118128776550293 + }, + { + "bits_per_byte": 1.5797031278896605, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0949667692184448, + "step": 5680, + "unclipped_grad_norm": 1.079695701599121 + }, + { + "bits_per_byte": 1.650425317629512, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1439876556396484, + "step": 5690, + "unclipped_grad_norm": 1.1217325925827026 + }, + { + "bits_per_byte": 1.5713879387030945, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.089203119277954, + "step": 5700, + "unclipped_grad_norm": 1.0890491008758545 + }, + { + "bits_per_byte": 1.5792038622542057, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.094620704650879, + "step": 5710, + "unclipped_grad_norm": 1.0754131078720093 + }, + { + "bits_per_byte": 1.6025209260928837, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1107828617095947, + "step": 5720, + "unclipped_grad_norm": 1.0796459913253784 + }, + { + "bits_per_byte": 1.5944050647986676, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.1051573753356934, + "step": 5730, + "unclipped_grad_norm": 1.0518229007720947 + }, + { + "bits_per_byte": 1.599884088089855, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1089551448822021, + "step": 5740, + "unclipped_grad_norm": 1.2038764953613281 + }, + { + "bits_per_byte": 1.5972181850188307, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.1071072816848755, + "step": 5750, + "unclipped_grad_norm": 1.1661580801010132 + }, + { + "bits_per_byte": 1.6206038699352028, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.123317003250122, + "step": 5760, + "unclipped_grad_norm": 1.1536771059036255 + }, + { + "bits_per_byte": 1.569340313261922, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0877838134765625, + "step": 5770, + "unclipped_grad_norm": 1.1234960556030273 + }, + { + "bits_per_byte": 1.548520437513772, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.073352575302124, + "step": 5780, + "unclipped_grad_norm": 1.0824602842330933 + }, + { + "bits_per_byte": 1.575600653735979, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0921231508255005, + "step": 5790, + "unclipped_grad_norm": 1.0697625875473022 + }, + { + "bits_per_byte": 1.5474534571478191, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.072613000869751, + "step": 5800, + "unclipped_grad_norm": 1.121078372001648 + }, + { + "bits_per_byte": 1.6281222635003212, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.128528356552124, + "step": 5810, + "unclipped_grad_norm": 1.163888931274414 + }, + { + "bits_per_byte": 1.5571622217543264, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0793426036834717, + "step": 5820, + "unclipped_grad_norm": 1.0646225214004517 + }, + { + "bits_per_byte": 1.5715148618994312, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0892910957336426, + "step": 5830, + "unclipped_grad_norm": 1.1465011835098267 + }, + { + "bits_per_byte": 1.5656843060698804, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.085249662399292, + "step": 5840, + "unclipped_grad_norm": 1.0994694232940674 + }, + { + "bits_per_byte": 1.529561070082656, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.060210943222046, + "step": 5850, + "unclipped_grad_norm": 1.123828649520874 + }, + { + "bits_per_byte": 1.5867750544931392, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.099868655204773, + "step": 5860, + "unclipped_grad_norm": 1.1375004053115845 + }, + { + "bits_per_byte": 1.5937161022993112, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.104679822921753, + "step": 5870, + "unclipped_grad_norm": 1.0862858295440674 + }, + { + "bits_per_byte": 1.6139739387444414, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1187214851379395, + "step": 5880, + "unclipped_grad_norm": 1.1279133558273315 + }, + { + "bits_per_byte": 1.6231863614205586, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1251070499420166, + "step": 5890, + "unclipped_grad_norm": 1.1251599788665771 + }, + { + "bits_per_byte": 1.6124319422968056, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1176526546478271, + "step": 5900, + "unclipped_grad_norm": 1.1988507509231567 + }, + { + "bits_per_byte": 1.6244132856518139, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.1259574890136719, + "step": 5910, + "unclipped_grad_norm": 1.1206265687942505 + }, + { + "bits_per_byte": 1.5302899325570125, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.060716152191162, + "step": 5920, + "unclipped_grad_norm": 1.1038360595703125 + }, + { + "bits_per_byte": 1.6585005910181243, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1495850086212158, + "step": 5930, + "unclipped_grad_norm": 1.2028963565826416 + }, + { + "bits_per_byte": 1.5036174872000014, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0422282218933105, + "step": 5940, + "unclipped_grad_norm": 1.0828943252563477 + }, + { + "bits_per_byte": 1.564500033536039, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0844287872314453, + "step": 5950, + "unclipped_grad_norm": 1.1633635759353638 + }, + { + "bits_per_byte": 1.6026038217305996, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1108403205871582, + "step": 5960, + "unclipped_grad_norm": 1.1734113693237305 + }, + { + "bits_per_byte": 1.582973034030515, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0972332954406738, + "step": 5970, + "unclipped_grad_norm": 1.255017638206482 + }, + { + "bits_per_byte": 1.6250308753510623, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.1263855695724487, + "step": 5980, + "unclipped_grad_norm": 1.1516860723495483 + }, + { + "bits_per_byte": 1.5794843659577629, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0948151350021362, + "step": 5990, + "unclipped_grad_norm": 1.1278661489486694 + }, + { + "bits_per_byte": 1.5522483334538744, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0759365558624268, + "step": 6000, + "unclipped_grad_norm": 1.067203402519226 + }, + { + "bits_per_byte": 1.577093635128119, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0931580066680908, + "step": 6010, + "unclipped_grad_norm": 1.1185256242752075 + }, + { + "bits_per_byte": 1.534402725669746, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0635669231414795, + "step": 6020, + "unclipped_grad_norm": 1.1662800312042236 + }, + { + "bits_per_byte": 1.5661140907143865, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0855475664138794, + "step": 6030, + "unclipped_grad_norm": 1.1327592134475708 + }, + { + "bits_per_byte": 1.6326177180112242, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.131644368171692, + "step": 6040, + "unclipped_grad_norm": 1.1918420791625977 + }, + { + "bits_per_byte": 1.5457630396724886, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0714412927627563, + "step": 6050, + "unclipped_grad_norm": 1.147464394569397 + }, + { + "bits_per_byte": 1.5419047808830324, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0687669515609741, + "step": 6060, + "unclipped_grad_norm": 1.1692534685134888 + }, + { + "bits_per_byte": 1.6009338701907216, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.1096827983856201, + "step": 6070, + "unclipped_grad_norm": 1.1858196258544922 + }, + { + "bits_per_byte": 1.5324048032146727, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.062182068824768, + "step": 6080, + "unclipped_grad_norm": 1.1669044494628906 + }, + { + "bits_per_byte": 1.5495642002218584, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0740760564804077, + "step": 6090, + "unclipped_grad_norm": 1.0681768655776978 + }, + { + "bits_per_byte": 1.5432428059067451, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0696943998336792, + "step": 6100, + "unclipped_grad_norm": 1.086612582206726 + }, + { + "bits_per_byte": 1.6119146184830102, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1172940731048584, + "step": 6110, + "unclipped_grad_norm": 1.1307899951934814 + }, + { + "bits_per_byte": 1.6174417969164392, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.1211252212524414, + "step": 6120, + "unclipped_grad_norm": 1.2087455987930298 + }, + { + "bits_per_byte": 1.5350297744147916, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0640015602111816, + "step": 6130, + "unclipped_grad_norm": 1.1170084476470947 + }, + { + "bits_per_byte": 1.5441861307467275, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0703482627868652, + "step": 6140, + "unclipped_grad_norm": 1.191177487373352 + }, + { + "bits_per_byte": 1.6015168913771465, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.1100869178771973, + "step": 6150, + "unclipped_grad_norm": 1.1631611585617065 + }, + { + "bits_per_byte": 1.4840940166740553, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0286955833435059, + "step": 6160, + "unclipped_grad_norm": 1.0670503377914429 + }, + { + "bits_per_byte": 1.5765931656141083, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.092811107635498, + "step": 6170, + "unclipped_grad_norm": 1.12460458278656 + }, + { + "bits_per_byte": 1.531391309453138, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0614795684814453, + "step": 6180, + "unclipped_grad_norm": 1.1588267087936401 + }, + { + "bits_per_byte": 1.5676526475090051, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0866140127182007, + "step": 6190, + "unclipped_grad_norm": 1.2090874910354614 + }, + { + "bits_per_byte": 1.5723472579296074, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0898680686950684, + "step": 6200, + "unclipped_grad_norm": 1.1369835138320923 + }, + { + "bits_per_byte": 1.5728810920778855, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.090238094329834, + "step": 6210, + "unclipped_grad_norm": 1.2529284954071045 + }, + { + "bits_per_byte": 1.5681563846933824, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.086963176727295, + "step": 6220, + "unclipped_grad_norm": 1.0888316631317139 + }, + { + "bits_per_byte": 1.5207944264375752, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0541343688964844, + "step": 6230, + "unclipped_grad_norm": 1.15282142162323 + }, + { + "bits_per_byte": 1.5617125387308546, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0824966430664062, + "step": 6240, + "unclipped_grad_norm": 1.227396845817566 + }, + { + "bits_per_byte": 1.595514008931431, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.1059260368347168, + "step": 6250, + "unclipped_grad_norm": 1.1354495286941528 + }, + { + "bits_per_byte": 1.4806825688115406, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0263309478759766, + "step": 6260, + "unclipped_grad_norm": 1.1806098222732544 + }, + { + "bits_per_byte": 1.561176984756068, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0821254253387451, + "step": 6270, + "unclipped_grad_norm": 1.1072839498519897 + }, + { + "bits_per_byte": 1.557884892853252, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.079843521118164, + "step": 6280, + "unclipped_grad_norm": 1.16572904586792 + }, + { + "bits_per_byte": 1.5084749651909706, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0455951690673828, + "step": 6290, + "unclipped_grad_norm": 1.1536295413970947 + }, + { + "bits_per_byte": 1.5632667459467018, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0835739374160767, + "step": 6300, + "unclipped_grad_norm": 1.1973258256912231 + }, + { + "bits_per_byte": 1.5420322200273218, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0688552856445312, + "step": 6310, + "unclipped_grad_norm": 1.146715760231018 + }, + { + "bits_per_byte": 1.4920311879940247, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.034197211265564, + "step": 6320, + "unclipped_grad_norm": 1.1094690561294556 + }, + { + "bits_per_byte": 1.5741831727275677, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.091140627861023, + "step": 6330, + "unclipped_grad_norm": 1.2368261814117432 + }, + { + "bits_per_byte": 1.5350222071781536, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0639963150024414, + "step": 6340, + "unclipped_grad_norm": 1.105275273323059 + }, + { + "bits_per_byte": 1.6462146663884378, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1410690546035767, + "step": 6350, + "unclipped_grad_norm": 1.3086775541305542 + }, + { + "bits_per_byte": 1.52795922367252, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.05910062789917, + "step": 6360, + "unclipped_grad_norm": 1.1954522132873535 + }, + { + "bits_per_byte": 1.5079418189732958, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0452256202697754, + "step": 6370, + "unclipped_grad_norm": 1.1107361316680908 + }, + { + "bits_per_byte": 1.5318044117805105, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0617659091949463, + "step": 6380, + "unclipped_grad_norm": 1.0837970972061157 + }, + { + "bits_per_byte": 1.5828646849604715, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0971581935882568, + "step": 6390, + "unclipped_grad_norm": 1.1240276098251343 + }, + { + "bits_per_byte": 1.5799186221511916, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.095116138458252, + "step": 6400, + "unclipped_grad_norm": 1.1799321174621582 + }, + { + "bits_per_byte": 1.552409309215082, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0760481357574463, + "step": 6410, + "unclipped_grad_norm": 1.1518590450286865 + }, + { + "bits_per_byte": 1.5442817531006072, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0704145431518555, + "step": 6420, + "unclipped_grad_norm": 1.1955360174179077 + }, + { + "bits_per_byte": 1.5805626971786728, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0955625772476196, + "step": 6430, + "unclipped_grad_norm": 1.156053066253662 + }, + { + "bits_per_byte": 1.5364046037257888, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0649545192718506, + "step": 6440, + "unclipped_grad_norm": 1.1582752466201782 + }, + { + "bits_per_byte": 1.5944274225432797, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.105172872543335, + "step": 6450, + "unclipped_grad_norm": 1.2264126539230347 + }, + { + "bits_per_byte": 1.645741886081216, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.1407413482666016, + "step": 6460, + "unclipped_grad_norm": 1.2444238662719727 + }, + { + "bits_per_byte": 1.6345733327341851, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.1329998970031738, + "step": 6470, + "unclipped_grad_norm": 1.2292388677597046 + }, + { + "bits_per_byte": 1.5111716531564994, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.047464370727539, + "step": 6480, + "unclipped_grad_norm": 1.2222721576690674 + }, + { + "bits_per_byte": 1.6197842006211907, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.122748851776123, + "step": 6490, + "unclipped_grad_norm": 1.2203607559204102 + }, + { + "bits_per_byte": 1.5564085937782457, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0788202285766602, + "step": 6500, + "unclipped_grad_norm": 1.1637120246887207 + }, + { + "bits_per_byte": 1.5288421826020495, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0597126483917236, + "step": 6510, + "unclipped_grad_norm": 1.1861555576324463 + }, + { + "bits_per_byte": 1.5068566084463515, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.04447340965271, + "step": 6520, + "unclipped_grad_norm": 1.1399540901184082 + }, + { + "bits_per_byte": 1.5271775625243484, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0585588216781616, + "step": 6530, + "unclipped_grad_norm": 1.200762152671814 + }, + { + "bits_per_byte": 1.5654438743239742, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0850830078125, + "step": 6540, + "unclipped_grad_norm": 1.1505160331726074 + }, + { + "bits_per_byte": 1.505103417303456, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0432581901550293, + "step": 6550, + "unclipped_grad_norm": 1.1259007453918457 + }, + { + "bits_per_byte": 1.5822820077393482, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0967543125152588, + "step": 6560, + "unclipped_grad_norm": 1.1489042043685913 + }, + { + "bits_per_byte": 1.5307443107205922, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0610311031341553, + "step": 6570, + "unclipped_grad_norm": 1.2110720872879028 + }, + { + "bits_per_byte": 1.5820749406277095, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0966107845306396, + "step": 6580, + "unclipped_grad_norm": 1.2832127809524536 + }, + { + "bits_per_byte": 1.555234640203456, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0780065059661865, + "step": 6590, + "unclipped_grad_norm": 1.1875557899475098 + }, + { + "bits_per_byte": 1.4893780116391642, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.032358169555664, + "step": 6600, + "unclipped_grad_norm": 1.1434928178787231 + }, + { + "bits_per_byte": 1.4980550523231455, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0383726358413696, + "step": 6610, + "unclipped_grad_norm": 1.1379001140594482 + }, + { + "bits_per_byte": 1.4935792038344409, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0352702140808105, + "step": 6620, + "unclipped_grad_norm": 1.1951154470443726 + }, + { + "bits_per_byte": 1.5329076804857955, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0625306367874146, + "step": 6630, + "unclipped_grad_norm": 1.144928216934204 + }, + { + "bits_per_byte": 1.5874349519244997, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.1003260612487793, + "step": 6640, + "unclipped_grad_norm": 1.2067164182662964 + }, + { + "bits_per_byte": 1.5760562357781147, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0924389362335205, + "step": 6650, + "unclipped_grad_norm": 1.1507362127304077 + }, + { + "bits_per_byte": 1.5807757836830918, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.095710277557373, + "step": 6660, + "unclipped_grad_norm": 1.134096622467041 + }, + { + "bits_per_byte": 1.5557022610311522, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.078330636024475, + "step": 6670, + "unclipped_grad_norm": 1.2659887075424194 + }, + { + "bits_per_byte": 1.5265064862206816, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0580936670303345, + "step": 6680, + "unclipped_grad_norm": 1.196817398071289 + }, + { + "bits_per_byte": 1.558342366704547, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0801606178283691, + "step": 6690, + "unclipped_grad_norm": 1.191389799118042 + }, + { + "bits_per_byte": 1.6202516494662358, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.123072862625122, + "step": 6700, + "unclipped_grad_norm": 1.2432118654251099 + }, + { + "bits_per_byte": 1.5747254340257382, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0915164947509766, + "step": 6710, + "unclipped_grad_norm": 1.1514227390289307 + }, + { + "bits_per_byte": 1.5896156919374402, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1018376350402832, + "step": 6720, + "unclipped_grad_norm": 1.1839321851730347 + }, + { + "bits_per_byte": 1.5521088555240246, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.075839877128601, + "step": 6730, + "unclipped_grad_norm": 1.2353639602661133 + }, + { + "bits_per_byte": 1.4976266435398462, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0380756855010986, + "step": 6740, + "unclipped_grad_norm": 1.1758193969726562 + }, + { + "bits_per_byte": 1.5596673210467942, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0810790061950684, + "step": 6750, + "unclipped_grad_norm": 1.1156134605407715 + }, + { + "bits_per_byte": 1.536689063030316, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0651516914367676, + "step": 6760, + "unclipped_grad_norm": 1.2122220993041992 + }, + { + "bits_per_byte": 1.443431470599958, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0005104541778564, + "step": 6770, + "unclipped_grad_norm": 1.1533452272415161 + }, + { + "bits_per_byte": 1.5402759331967106, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0676379203796387, + "step": 6780, + "unclipped_grad_norm": 1.1691333055496216 + }, + { + "bits_per_byte": 1.523325667092974, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0558888912200928, + "step": 6790, + "unclipped_grad_norm": 1.2405608892440796 + }, + { + "bits_per_byte": 1.57861671348416, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0942137241363525, + "step": 6800, + "unclipped_grad_norm": 1.2281029224395752 + }, + { + "bits_per_byte": 1.5738859867068768, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0909346342086792, + "step": 6810, + "unclipped_grad_norm": 1.2371455430984497 + }, + { + "bits_per_byte": 1.582858837550342, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.097154140472412, + "step": 6820, + "unclipped_grad_norm": 1.3370100259780884 + }, + { + "bits_per_byte": 1.5273318309621724, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0586657524108887, + "step": 6830, + "unclipped_grad_norm": 1.196358561515808 + }, + { + "bits_per_byte": 1.4954078953610652, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.036537766456604, + "step": 6840, + "unclipped_grad_norm": 1.1750952005386353 + }, + { + "bits_per_byte": 1.5476113372213112, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0727224349975586, + "step": 6850, + "unclipped_grad_norm": 1.1924985647201538 + }, + { + "bits_per_byte": 1.526412067745358, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.058028221130371, + "step": 6860, + "unclipped_grad_norm": 1.1877104043960571 + }, + { + "bits_per_byte": 1.4832949852781467, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.028141736984253, + "step": 6870, + "unclipped_grad_norm": 1.175843596458435 + }, + { + "bits_per_byte": 1.4908706490660026, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0333927869796753, + "step": 6880, + "unclipped_grad_norm": 1.1246671676635742 + }, + { + "bits_per_byte": 1.5010700801754213, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0404624938964844, + "step": 6890, + "unclipped_grad_norm": 1.1924952268600464 + }, + { + "bits_per_byte": 1.6045640799851337, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.112199068069458, + "step": 6900, + "unclipped_grad_norm": 1.2239227294921875 + }, + { + "bits_per_byte": 1.5736668808096776, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0907827615737915, + "step": 6910, + "unclipped_grad_norm": 1.2664575576782227 + }, + { + "bits_per_byte": 1.5080825007817016, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.045323133468628, + "step": 6920, + "unclipped_grad_norm": 1.1531251668930054 + }, + { + "bits_per_byte": 1.5085007625886, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0456130504608154, + "step": 6930, + "unclipped_grad_norm": 1.2472591400146484 + }, + { + "bits_per_byte": 1.484484417291514, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0289661884307861, + "step": 6940, + "unclipped_grad_norm": 1.187600016593933 + }, + { + "bits_per_byte": 1.4929882714460763, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.034860610961914, + "step": 6950, + "unclipped_grad_norm": 1.228901743888855 + }, + { + "bits_per_byte": 1.5665772399931601, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0858685970306396, + "step": 6960, + "unclipped_grad_norm": 1.205407738685608 + }, + { + "bits_per_byte": 1.5512240047853354, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0752265453338623, + "step": 6970, + "unclipped_grad_norm": 1.2078452110290527 + }, + { + "bits_per_byte": 1.5256931802647515, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0575299263000488, + "step": 6980, + "unclipped_grad_norm": 1.216407060623169 + }, + { + "bits_per_byte": 1.5495781308165784, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0740857124328613, + "step": 6990, + "unclipped_grad_norm": 1.2360340356826782 + }, + { + "bits_per_byte": 1.579722561929208, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.094980239868164, + "step": 7000, + "unclipped_grad_norm": 1.2281304597854614 + }, + { + "bits_per_byte": 1.5775139607268278, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.093449354171753, + "step": 7010, + "unclipped_grad_norm": 1.238048791885376 + }, + { + "bits_per_byte": 1.5315163408403152, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0615662336349487, + "step": 7020, + "unclipped_grad_norm": 1.2856208086013794 + }, + { + "bits_per_byte": 1.5716388613907033, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0893770456314087, + "step": 7030, + "unclipped_grad_norm": 1.228158950805664 + }, + { + "bits_per_byte": 1.5737668027298288, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0908520221710205, + "step": 7040, + "unclipped_grad_norm": 1.227662444114685 + }, + { + "bits_per_byte": 1.5448513596402649, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0708093643188477, + "step": 7050, + "unclipped_grad_norm": 1.2028948068618774 + }, + { + "bits_per_byte": 1.5624737339435735, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.083024263381958, + "step": 7060, + "unclipped_grad_norm": 1.259883999824524 + }, + { + "bits_per_byte": 1.539107139101447, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0668277740478516, + "step": 7070, + "unclipped_grad_norm": 1.2382662296295166 + }, + { + "bits_per_byte": 1.5001651074665814, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0398352146148682, + "step": 7080, + "unclipped_grad_norm": 1.1878225803375244 + }, + { + "bits_per_byte": 1.5461551601164558, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0717130899429321, + "step": 7090, + "unclipped_grad_norm": 1.3277226686477661 + }, + { + "bits_per_byte": 1.525824231044709, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0576207637786865, + "step": 7100, + "unclipped_grad_norm": 1.2164647579193115 + }, + { + "bits_per_byte": 1.5404795606553323, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0677790641784668, + "step": 7110, + "unclipped_grad_norm": 1.1725497245788574 + }, + { + "bits_per_byte": 1.5736082347257334, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0907421112060547, + "step": 7120, + "unclipped_grad_norm": 1.218058705329895 + }, + { + "bits_per_byte": 1.5565896915096042, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0789457559585571, + "step": 7130, + "unclipped_grad_norm": 1.3638907670974731 + }, + { + "bits_per_byte": 1.5273908210114182, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0587066411972046, + "step": 7140, + "unclipped_grad_norm": 1.2031904458999634 + }, + { + "bits_per_byte": 1.5130964829849558, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0487985610961914, + "step": 7150, + "unclipped_grad_norm": 1.1664750576019287 + }, + { + "bits_per_byte": 1.5860308855628558, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0993528366088867, + "step": 7160, + "unclipped_grad_norm": 1.2491129636764526 + }, + { + "bits_per_byte": 1.5206325907631133, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.054022192955017, + "step": 7170, + "unclipped_grad_norm": 1.20036780834198 + }, + { + "bits_per_byte": 1.5424085180674096, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0691161155700684, + "step": 7180, + "unclipped_grad_norm": 1.2460989952087402 + }, + { + "bits_per_byte": 1.5104255924170564, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0469472408294678, + "step": 7190, + "unclipped_grad_norm": 1.2477872371673584 + }, + { + "bits_per_byte": 1.550646659026389, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0748263597488403, + "step": 7200, + "unclipped_grad_norm": 1.26192045211792 + }, + { + "bits_per_byte": 1.5958559104413463, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.1061630249023438, + "step": 7210, + "unclipped_grad_norm": 1.2875406742095947 + }, + { + "bits_per_byte": 1.5889428958072649, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.1013712882995605, + "step": 7220, + "unclipped_grad_norm": 1.2469474077224731 + }, + { + "bits_per_byte": 1.512987102019007, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.048722743988037, + "step": 7230, + "unclipped_grad_norm": 1.2010514736175537 + }, + { + "bits_per_byte": 1.5378745394427134, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0659734010696411, + "step": 7240, + "unclipped_grad_norm": 1.201562762260437 + }, + { + "bits_per_byte": 1.5587365509403246, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0804338455200195, + "step": 7250, + "unclipped_grad_norm": 1.305864691734314 + }, + { + "bits_per_byte": 1.457526824699372, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0102806091308594, + "step": 7260, + "unclipped_grad_norm": 1.1730221509933472 + }, + { + "bits_per_byte": 1.4923226985872373, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0343992710113525, + "step": 7270, + "unclipped_grad_norm": 1.222469687461853 + }, + { + "bits_per_byte": 1.56038208094378, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0815744400024414, + "step": 7280, + "unclipped_grad_norm": 1.2304381132125854 + }, + { + "bits_per_byte": 1.4970130094415677, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0376503467559814, + "step": 7290, + "unclipped_grad_norm": 1.1906161308288574 + }, + { + "bits_per_byte": 1.5193676583660174, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.053145408630371, + "step": 7300, + "unclipped_grad_norm": 1.2887614965438843 + }, + { + "bits_per_byte": 1.5227811700203424, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.055511474609375, + "step": 7310, + "unclipped_grad_norm": 1.1584672927856445 + }, + { + "bits_per_byte": 1.4581731355013146, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0107285976409912, + "step": 7320, + "unclipped_grad_norm": 1.2041478157043457 + }, + { + "bits_per_byte": 1.4976147767369368, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0380674600601196, + "step": 7330, + "unclipped_grad_norm": 1.2121145725250244 + }, + { + "bits_per_byte": 1.456979231939025, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0099010467529297, + "step": 7340, + "unclipped_grad_norm": 1.236539363861084 + }, + { + "bits_per_byte": 1.5469846324415673, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0722880363464355, + "step": 7350, + "unclipped_grad_norm": 1.240252137184143 + }, + { + "bits_per_byte": 1.4978240796230369, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.038212537765503, + "step": 7360, + "unclipped_grad_norm": 1.2412164211273193 + }, + { + "bits_per_byte": 1.5397654886889498, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.067284107208252, + "step": 7370, + "unclipped_grad_norm": 1.2414754629135132 + }, + { + "bits_per_byte": 1.4664532402271044, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0164679288864136, + "step": 7380, + "unclipped_grad_norm": 1.196345567703247 + }, + { + "bits_per_byte": 1.5561741814251195, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0786577463150024, + "step": 7390, + "unclipped_grad_norm": 1.2357971668243408 + }, + { + "bits_per_byte": 1.4878109057245024, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0312719345092773, + "step": 7400, + "unclipped_grad_norm": 1.2395251989364624 + }, + { + "bits_per_byte": 1.4828184213526059, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0278114080429077, + "step": 7410, + "unclipped_grad_norm": 1.1855907440185547 + }, + { + "bits_per_byte": 1.5432890692398271, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0697264671325684, + "step": 7420, + "unclipped_grad_norm": 1.2337415218353271 + }, + { + "bits_per_byte": 1.5564147851536767, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.078824520111084, + "step": 7430, + "unclipped_grad_norm": 1.271388053894043 + }, + { + "bits_per_byte": 1.5798298791033463, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0950546264648438, + "step": 7440, + "unclipped_grad_norm": 1.3756663799285889 + }, + { + "bits_per_byte": 1.5198117175705452, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0534532070159912, + "step": 7450, + "unclipped_grad_norm": 1.2638351917266846 + }, + { + "bits_per_byte": 1.5153886677556554, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0503873825073242, + "step": 7460, + "unclipped_grad_norm": 1.2390333414077759 + }, + { + "bits_per_byte": 1.500396252149341, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0399954319000244, + "step": 7470, + "unclipped_grad_norm": 1.1914620399475098 + }, + { + "bits_per_byte": 1.5462138062004, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.071753740310669, + "step": 7480, + "unclipped_grad_norm": 1.2698674201965332 + }, + { + "bits_per_byte": 1.5915580639962847, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.1031839847564697, + "step": 7490, + "unclipped_grad_norm": 1.3026750087738037 + }, + { + "bits_per_byte": 1.5533068866699349, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0766702890396118, + "step": 7500, + "unclipped_grad_norm": 1.1910096406936646 + }, + { + "bits_per_byte": 1.5004994417398587, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0400669574737549, + "step": 7510, + "unclipped_grad_norm": 1.2149840593338013 + }, + { + "bits_per_byte": 1.5185469571561, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0525765419006348, + "step": 7520, + "unclipped_grad_norm": 1.25714111328125 + }, + { + "bits_per_byte": 1.5776331447038756, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0935319662094116, + "step": 7530, + "unclipped_grad_norm": 1.2656311988830566 + }, + { + "bits_per_byte": 1.5129728274589855, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0487128496170044, + "step": 7540, + "unclipped_grad_norm": 1.2008068561553955 + }, + { + "bits_per_byte": 1.542592195538531, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0692434310913086, + "step": 7550, + "unclipped_grad_norm": 1.3036167621612549 + }, + { + "bits_per_byte": 1.5036082001368547, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0422217845916748, + "step": 7560, + "unclipped_grad_norm": 1.2302430868148804 + }, + { + "bits_per_byte": 1.4793622580008667, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0254157781600952, + "step": 7570, + "unclipped_grad_norm": 1.2421420812606812 + }, + { + "bits_per_byte": 1.6626436530774094, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1524567604064941, + "step": 7580, + "unclipped_grad_norm": 1.2984310388565063 + }, + { + "bits_per_byte": 1.510733441362101, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0471606254577637, + "step": 7590, + "unclipped_grad_norm": 1.2202924489974976 + }, + { + "bits_per_byte": 1.5062574208907455, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.044058084487915, + "step": 7600, + "unclipped_grad_norm": 1.2830675840377808 + }, + { + "bits_per_byte": 1.5167008953817387, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0512969493865967, + "step": 7610, + "unclipped_grad_norm": 1.2556214332580566 + }, + { + "bits_per_byte": 1.4693692060724834, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.018489122390747, + "step": 7620, + "unclipped_grad_norm": 1.2434303760528564 + }, + { + "bits_per_byte": 1.482038652013594, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0272709131240845, + "step": 7630, + "unclipped_grad_norm": 1.2516939640045166 + }, + { + "bits_per_byte": 1.4886369383965965, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.0318444967269897, + "step": 7640, + "unclipped_grad_norm": 1.2500675916671753 + }, + { + "bits_per_byte": 1.61461749782397, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1191675662994385, + "step": 7650, + "unclipped_grad_norm": 1.2970722913742065 + }, + { + "bits_per_byte": 1.5327549598918293, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0624247789382935, + "step": 7660, + "unclipped_grad_norm": 1.313043236732483 + }, + { + "bits_per_byte": 1.560077327686451, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0813632011413574, + "step": 7670, + "unclipped_grad_norm": 1.2718157768249512 + }, + { + "bits_per_byte": 1.506822211916179, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0444495677947998, + "step": 7680, + "unclipped_grad_norm": 1.2939503192901611 + }, + { + "bits_per_byte": 1.5160353225228995, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0508356094360352, + "step": 7690, + "unclipped_grad_norm": 1.2968342304229736 + }, + { + "bits_per_byte": 1.4732078588397413, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0211498737335205, + "step": 7700, + "unclipped_grad_norm": 1.2705638408660889 + }, + { + "bits_per_byte": 1.539139815805111, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0668504238128662, + "step": 7710, + "unclipped_grad_norm": 1.2731654644012451 + }, + { + "bits_per_byte": 1.5645254869683667, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0844464302062988, + "step": 7720, + "unclipped_grad_norm": 1.281749963760376 + }, + { + "bits_per_byte": 1.5594863952980864, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.080953598022461, + "step": 7730, + "unclipped_grad_norm": 1.2457833290100098 + }, + { + "bits_per_byte": 1.5044648457208023, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0428155660629272, + "step": 7740, + "unclipped_grad_norm": 1.313395619392395 + }, + { + "bits_per_byte": 1.5058419108062608, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0437700748443604, + "step": 7750, + "unclipped_grad_norm": 1.252753496170044 + }, + { + "bits_per_byte": 1.5365542286320395, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0650582313537598, + "step": 7760, + "unclipped_grad_norm": 1.208566665649414 + }, + { + "bits_per_byte": 1.4627389309164203, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0138933658599854, + "step": 7770, + "unclipped_grad_norm": 1.2469086647033691 + }, + { + "bits_per_byte": 1.539320397588517, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0669755935668945, + "step": 7780, + "unclipped_grad_norm": 1.2808659076690674 + }, + { + "bits_per_byte": 1.4840185162903266, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.028643250465393, + "step": 7790, + "unclipped_grad_norm": 1.1823458671569824 + }, + { + "bits_per_byte": 1.5017442521668036, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0409297943115234, + "step": 7800, + "unclipped_grad_norm": 1.2036527395248413 + }, + { + "bits_per_byte": 1.4928728710873473, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0347806215286255, + "step": 7810, + "unclipped_grad_norm": 1.2650182247161865 + }, + { + "bits_per_byte": 1.4653472197994057, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0157012939453125, + "step": 7820, + "unclipped_grad_norm": 1.2730145454406738 + }, + { + "bits_per_byte": 1.5153749091435864, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0503778457641602, + "step": 7830, + "unclipped_grad_norm": 1.3102401494979858 + }, + { + "bits_per_byte": 1.5122207473267624, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0481915473937988, + "step": 7840, + "unclipped_grad_norm": 1.2572261095046997 + }, + { + "bits_per_byte": 1.458128591994741, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0106977224349976, + "step": 7850, + "unclipped_grad_norm": 1.253402829170227 + }, + { + "bits_per_byte": 1.5241797329371587, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.056480884552002, + "step": 7860, + "unclipped_grad_norm": 1.2006174325942993 + }, + { + "bits_per_byte": 1.539945898489705, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0674091577529907, + "step": 7870, + "unclipped_grad_norm": 1.2778089046478271 + }, + { + "bits_per_byte": 1.453407496245906, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.007425308227539, + "step": 7880, + "unclipped_grad_norm": 1.2744390964508057 + }, + { + "bits_per_byte": 1.5524588402185304, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.076082468032837, + "step": 7890, + "unclipped_grad_norm": 1.2452044486999512 + }, + { + "bits_per_byte": 1.5187442212566398, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0527132749557495, + "step": 7900, + "unclipped_grad_norm": 1.270271897315979 + }, + { + "bits_per_byte": 1.5143311464355, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0496543645858765, + "step": 7910, + "unclipped_grad_norm": 1.2356430292129517 + }, + { + "bits_per_byte": 1.5350039770171622, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.063983678817749, + "step": 7920, + "unclipped_grad_norm": 1.2489451169967651 + }, + { + "bits_per_byte": 1.4882054339255817, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0315454006195068, + "step": 7930, + "unclipped_grad_norm": 1.2472636699676514 + }, + { + "bits_per_byte": 1.4964919020094534, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0372891426086426, + "step": 7940, + "unclipped_grad_norm": 1.3003731966018677 + }, + { + "bits_per_byte": 1.5605986071012163, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0817245244979858, + "step": 7950, + "unclipped_grad_norm": 1.2693594694137573 + }, + { + "bits_per_byte": 1.4811777068633747, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0266741514205933, + "step": 7960, + "unclipped_grad_norm": 1.3084068298339844 + }, + { + "bits_per_byte": 1.5064964767754447, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0442237854003906, + "step": 7970, + "unclipped_grad_norm": 1.2679471969604492 + }, + { + "bits_per_byte": 1.5642893547887322, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0842827558517456, + "step": 7980, + "unclipped_grad_norm": 1.3388662338256836 + }, + { + "bits_per_byte": 1.598760697414419, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.1081764698028564, + "step": 7990, + "unclipped_grad_norm": 1.3723206520080566 + }, + { + "bits_per_byte": 1.4810289418703784, + "learning_rate": 3e-05, + "loss_nats": 1.0265710353851318, + "step": 8000, + "unclipped_grad_norm": 1.1791528463363647 + } + ] + }, + "depth-16-block-seed-2026073002": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "6db370af4c7fc867e59305adb946f528981bb02fea76beb10dfc1fa42360d383", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00026696090935729444, + 0.00023596938990522176, + 0.00020828393462579697, + 0.00018593396816868335, + 0.00016940363275352865, + 0.00015857137623243034, + 0.00015041495498735458, + 0.00014485203428193927, + 0.00013822126493323594, + 0.00013424422650132328, + 0.00013012201816309243, + 0.00012678767961915582, + 0.00012444572348613292, + 0.00012238229101058096, + 0.00012059078289894387, + 0.00011903457198059186 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022428705051424913, + "first_to_last_ratio": 1.84426351740015, + "imbalance_abs_log_ratio": 0.6120800202390253, + "last_quartile_mean": 0.0001216133423440624, + "mean": 0.00015851367243158165, + "normalized": [ + 1.6841506808980231, + 1.488637454961949, + 1.3139808789408836, + 1.1729837894515815, + 1.0687004480742655, + 1.0003640304332333, + 0.9489083981211609, + 0.9138141338846396, + 0.8719832353445448, + 0.8468936744826624, + 0.8208882941580717, + 0.7998532724291052, + 0.785078798422557, + 0.772061419896786, + 0.7607595045215659, + 0.7509419859789702 + ], + "population_cv": 0.27150931413441437 + }, + "activation_output_rms_by_block": [ + 0.0048858714289963245, + 0.006843581795692444, + 0.004983869381248951, + 0.007334618363529444, + 0.005787155125290155, + 0.007929109036922455, + 0.005876626819372177, + 0.008892860263586044, + 0.006918553728610277, + 0.009880779311060905, + 0.007473294623196125, + 0.010543949902057648, + 0.008116732351481915, + 0.011967957951128483, + 0.008228716440498829, + 0.012017112225294113 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006011985242366791, + "first_to_last_ratio": 0.5962715478148781, + "imbalance_abs_log_ratio": 0.5170590985397028, + "last_quartile_mean": 0.010082629742100835, + "mean": 0.007980049296747893, + "normalized": [ + 0.612260807835794, + 0.8575864059488212, + 0.6245411771177937, + 0.9191194303171183, + 0.725202929216063, + 0.9936165482278163, + 0.736414851693595, + 1.114386632575083, + 0.866981326973731, + 1.2381852471873345, + 0.9364973003665171, + 1.321288817896729, + 1.017128096538166, + 1.499734839483483, + 1.031161103710509, + 1.5058944849114457 + ], + "population_cv": 0.2698221312597705 + }, + "bits_per_byte": 8.029409055280222, + "branch_output_rms_by_sublayer": [ + 0.0029694242402911186, + 0.0038970725145190954, + 0.003062367672100663, + 0.0038208949845284224, + 0.003184171626344323, + 0.0038178148679435253, + 0.003822639584541321, + 0.0037665588315576315, + 0.004263176582753658, + 0.0038892142474651337, + 0.004122398793697357, + 0.003805827582255006, + 0.004464023746550083, + 0.0037649995647370815, + 0.005492920521646738, + 0.0037115272134542465, + 0.005817505065351725, + 0.003727257251739502, + 0.006086217239499092, + 0.00368275074288249, + 0.0065266890451312065, + 0.0036771725863218307, + 0.0064827753230929375, + 0.003755199955776334, + 0.007209060713648796, + 0.0037714343052357435, + 0.007561341859400272, + 0.0037067290395498276, + 0.007512317039072514, + 0.0035454558674246073, + 0.007687372621148825, + 0.0036056456156075 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006887647168374071, + 0.005982369825085796, + 0.005174924477158116, + 0.004946754810206586, + 0.004967935165757302, + 0.004123664448940147, + 0.00409438714185196, + 0.004247982317789705, + 0.0044010147653324285, + 0.004095379628913903, + 0.00386485833459634, + 0.0038097011691932423, + 0.0037333099925226194, + 0.004058250822051008, + 0.003851730608686184, + 0.00381805442027398 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005747924070206143, + "first_to_last_ratio": 1.487043658003427, + "imbalance_abs_log_ratio": 0.39679002683413417, + "last_quartile_mean": 0.003865336460883448, + "mean": 0.004503622818545837, + "normalized": [ + 1.5293570189783356, + 1.3283461040410636, + 1.1490581440008412, + 1.0983945613375836, + 1.10309752080024, + 0.9156327283801325, + 0.909131893770355, + 0.9432366983079914, + 0.9772165526849029, + 0.9093522690331003, + 0.858166522889289, + 0.8459192349556811, + 0.8289570736583262, + 0.9011080602352411, + 0.8552515971863425, + 0.8477740197405745 + ], + "population_cv": 0.191198476656264 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02822229079902172, + 0.014168336987495422, + 0.01429277565330267, + 0.014353902079164982, + 0.014476437121629715, + 0.009725152514874935, + 0.00981744285672903, + 0.009922475554049015, + 0.010004729963839054, + 0.0075368694961071014, + 0.007599710486829281, + 0.007675632368773222, + 0.007730280980467796, + 0.006251458544284105, + 0.0062932223081588745, + 0.006393968593329191, + 0.006436701864004135, + 0.005461040884256363, + 0.005505258683115244, + 0.005599391181021929, + 0.005641042720526457, + 0.004922671243548393, + 0.004954405128955841, + 0.005046301055699587, + 0.005088083446025848, + 0.00459245452657342, + 0.0046256110072135925, + 0.0047518787905573845, + 0.004772109445184469, + 0.004340849816799164, + 0.004358765669167042, + 0.004422077443450689 + ], + "loss_nats": 5.5655622482299805, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0029694242402911186, + 0.0048858714289963245, + 0.005668045952916145, + 0.006843581795692444, + 0.003184171626344323, + 0.004983869381248951, + 0.006258165929466486, + 0.007334618363529444, + 0.004263176582753658, + 0.005787155125290155, + 0.006935684010386467, + 0.007929109036922455, + 0.004464023746550083, + 0.005876626819372177, + 0.008052977733314037, + 0.008892860263586044, + 0.005817505065351725, + 0.006918553728610277, + 0.009139667265117168, + 0.009880779311060905, + 0.0065266890451312065, + 0.007473294623196125, + 0.00979399960488081, + 0.010543949902057648, + 0.007209060713648796, + 0.008116732351481915, + 0.0113961361348629, + 0.011967957951128483, + 0.007512317039072514, + 0.008228716440498829, + 0.011409307830035686, + 0.012017112225294113 + ] + }, + { + "activation_grad_rms_by_block": [ + 8.836213237373158e-05, + 8.102934953058138e-05, + 6.889242649776861e-05, + 6.090942770242691e-05, + 5.060216426500119e-05, + 4.3459844164317474e-05, + 3.709086377057247e-05, + 3.2571624615229666e-05, + 2.8896074582007714e-05, + 2.6415440515847877e-05, + 2.491714622010477e-05, + 2.3614533347426914e-05, + 2.2646127035841346e-05, + 2.2026059014024213e-05, + 2.118862721545156e-05, + 2.081490310956724e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.479833402612712e-05, + "first_to_last_ratio": 3.4518703579034327, + "imbalance_abs_log_ratio": 1.2389162168680456, + "last_quartile_mean": 2.166892909372109e-05, + "mean": 4.083979649749381e-05, + "normalized": [ + 2.1636281262849595, + 1.984078190388482, + 1.6868944609456193, + 1.4914233890995197, + 1.2390405585910906, + 1.0641542782169457, + 0.9082039322318514, + 0.7975462026905171, + 0.7075469777079071, + 0.6468063697983631, + 0.6101192551641105, + 0.578223580249134, + 0.5545112605355675, + 0.5393283244047475, + 0.518823036171393, + 0.5096720575197914 + ], + "population_cv": 0.5343944992100478 + }, + "activation_output_rms_by_block": [ + 0.009077239781618118, + 0.016407418996095657, + 0.010880433954298496, + 0.01978805474936962, + 0.012077964842319489, + 0.02216547541320324, + 0.013358108699321747, + 0.02644582837820053, + 0.017588114365935326, + 0.03464006260037422, + 0.023338599130511284, + 0.0432722382247448, + 0.023765092715620995, + 0.04745635390281677, + 0.02874108962714672, + 0.05685042589902878 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014038286870345473, + "first_to_last_ratio": 0.3580899608898232, + "imbalance_abs_log_ratio": 1.0269710367470868, + "last_quartile_mean": 0.039203240536153316, + "mean": 0.025365781330037862, + "normalized": [ + 0.35785374254838964, + 0.6468327855789793, + 0.4289414078254319, + 0.7801082289523973, + 0.4761518947581916, + 0.8738337339112493, + 0.5266192484086122, + 1.042578899270242, + 0.6933795626909267, + 1.3656217464649458, + 0.9200820123318648, + 1.7059296404760187, + 0.9368957496877358, + 1.8708808250514843, + 1.133065418060348, + 2.2412251039831825 + ], + "population_cv": 0.5292067698975893 + }, + "bits_per_byte": 5.216988460540972, + "branch_output_rms_by_sublayer": [ + 0.004302801564335823, + 0.007946929894387722, + 0.004027083050459623, + 0.008643880486488342, + 0.004623814020305872, + 0.0097366226837039, + 0.005764479748904705, + 0.009610678069293499, + 0.0056845624931156635, + 0.010435172356665134, + 0.006739616859704256, + 0.010610987432301044, + 0.006770467385649681, + 0.010895426385104656, + 0.009306530468165874, + 0.011636263690888882, + 0.010160733014345169, + 0.012227990664541721, + 0.012430583126842976, + 0.012838989496231079, + 0.015689926221966743, + 0.013102410361170769, + 0.015432302840054035, + 0.013170000165700912, + 0.015551909804344177, + 0.012761367484927177, + 0.018755123019218445, + 0.013125414028763771, + 0.019000543281435966, + 0.01451444998383522, + 0.022471774369478226, + 0.013329307548701763 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00035471167919676877, + 0.00030778414793465906, + 0.0002905596100869004, + 0.0002669019974562734, + 0.0002519467909729747, + 0.0002484952213280207, + 0.00022573236044592232, + 0.00021694664140763627, + 0.00020872399951088957, + 0.00018641204799172754, + 0.0001823569017298164, + 0.0001572320679108608, + 0.00014731276753985554, + 0.00014544881518036755, + 0.00014011786536140868, + 0.0001387938590477502 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003049893586686504, + "first_to_last_ratio": 2.1340115402633257, + "imbalance_abs_log_ratio": 0.7580035606734332, + "last_quartile_mean": 0.0001429183267823455, + "mean": 0.00021684229831886448, + "normalized": [ + 1.6358048311919693, + 1.4193916515405378, + 1.339958173933553, + 1.230857630294052, + 1.1618895064582344, + 1.1459720878009276, + 1.0409978228232257, + 1.0004811934275772, + 0.9625612766931799, + 0.8596664462466195, + 0.840965545668873, + 0.7250986967480513, + 0.6793543911033149, + 0.6707585019528177, + 0.6461740465200508, + 0.6400681975970168 + ], + "population_cv": 0.294333589312772 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6924017667770386, + "mean_weights": [ + 0.4816230535507202, + 0.5183769464492798 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925060749053955, + "mean_weights": [ + 0.5170555114746094, + 0.4829444885253906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922789812088013, + "mean_weights": [ + 0.5104873776435852, + 0.4895126223564148 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927056312561035, + "mean_weights": [ + 0.5133966207504272, + 0.48660340905189514 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097900390625, + "mean_weights": [ + 0.3491937518119812, + 0.32761698961257935, + 0.32318928837776184 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982853174209595, + "mean_weights": [ + 0.3437661826610565, + 0.3295633792877197, + 0.32667046785354614 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097551703453064, + "mean_weights": [ + 0.3531244695186615, + 0.3277324438095093, + 0.3191430866718292 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098252296447754, + "mean_weights": [ + 0.3423447012901306, + 0.3336608111858368, + 0.323994517326355 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3856292963027954, + "mean_weights": [ + 0.2598890960216522, + 0.25528842210769653, + 0.2465861737728119, + 0.23823627829551697 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862417936325073, + "mean_weights": [ + 0.2525826096534729, + 0.2512264847755432, + 0.24808365106582642, + 0.24810725450515747 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854787349700928, + "mean_weights": [ + 0.2609134912490845, + 0.2559056282043457, + 0.24643474817276, + 0.23674613237380981 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386183738708496, + "mean_weights": [ + 0.25091099739074707, + 0.2541857361793518, + 0.24937650561332703, + 0.24552680552005768 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6086668968200684, + "mean_weights": [ + 0.20840778946876526, + 0.2072581648826599, + 0.20017966628074646, + 0.19401825964450836, + 0.1901361048221588 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093640327453613, + "mean_weights": [ + 0.20082800090312958, + 0.20345230400562286, + 0.19957157969474792, + 0.1978151500225067, + 0.19833296537399292 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085686683654785, + "mean_weights": [ + 0.2064795196056366, + 0.20873931050300598, + 0.20255593955516815, + 0.1941128820180893, + 0.18811233341693878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093556880950928, + "mean_weights": [ + 0.2007957398891449, + 0.20384852588176727, + 0.2003384232521057, + 0.19736894965171814, + 0.19764836132526398 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7904106378555298, + "mean_weights": [ + 0.17261356115341187, + 0.1785532385110855, + 0.16990220546722412, + 0.16298523545265198, + 0.1574374884366989, + 0.1585082709789276 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791741132736206, + "mean_weights": [ + 0.16599833965301514, + 0.16723501682281494, + 0.16713106632232666, + 0.1660757064819336, + 0.16713276505470276, + 0.1664271056652069 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893474102020264, + "mean_weights": [ + 0.17279052734375, + 0.18296070396900177, + 0.1714734584093094, + 0.16615013778209686, + 0.1573297083377838, + 0.14929547905921936 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916802167892456, + "mean_weights": [ + 0.1656593233346939, + 0.16598697006702423, + 0.1664597988128662, + 0.16510574519634247, + 0.16595518589019775, + 0.17083296179771423 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9424470663070679, + "mean_weights": [ + 0.14923666417598724, + 0.16057875752449036, + 0.15159368515014648, + 0.14453238248825073, + 0.13740569353103638, + 0.1286274939775467, + 0.12802532315254211 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9457793235778809, + "mean_weights": [ + 0.1414237767457962, + 0.14120322465896606, + 0.14217954874038696, + 0.14177344739437103, + 0.14084486663341522, + 0.145671084523201, + 0.14690405130386353 + ], + "sources": 7 + }, + { + "entropy_mean": 1.940324306488037, + "mean_weights": [ + 0.15125462412834167, + 0.16445259749889374, + 0.15269356966018677, + 0.1458570957183838, + 0.1393839716911316, + 0.12596309185028076, + 0.12039506435394287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458446502685547, + "mean_weights": [ + 0.1414341926574707, + 0.14160503447055817, + 0.14392146468162537, + 0.1416645050048828, + 0.14250624179840088, + 0.1432597041130066, + 0.1456088423728943 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073972225189209, + "mean_weights": [ + 0.13393926620483398, + 0.1460031270980835, + 0.13552871346473694, + 0.12954440712928772, + 0.12087934464216232, + 0.11472145467996597, + 0.10819212347269058, + 0.11119155585765839 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0792579650878906, + "mean_weights": [ + 0.12344250082969666, + 0.1223982647061348, + 0.12281741201877594, + 0.12404577434062958, + 0.1245848536491394, + 0.1274431347846985, + 0.1261020004749298, + 0.12916606664657593 + ], + "sources": 8 + }, + { + "entropy_mean": 2.070988178253174, + "mean_weights": [ + 0.13450230658054352, + 0.1483522206544876, + 0.13837148249149323, + 0.13302284479141235, + 0.12319637835025787, + 0.11490452289581299, + 0.10592599213123322, + 0.10172426700592041 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0791521072387695, + "mean_weights": [ + 0.12388300150632858, + 0.12212474644184113, + 0.12333755195140839, + 0.12223256379365921, + 0.12343418598175049, + 0.12649230659008026, + 0.12949144840240479, + 0.12900421023368835 + ], + "sources": 8 + }, + { + "entropy_mean": 2.18794584274292, + "mean_weights": [ + 0.12259161472320557, + 0.13473162055015564, + 0.1250339150428772, + 0.11940491199493408, + 0.11259052157402039, + 0.10314525663852692, + 0.09605535864830017, + 0.09219000488519669, + 0.09425680339336395 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197138786315918, + "mean_weights": [ + 0.11026664823293686, + 0.11047966778278351, + 0.11075460910797119, + 0.10965213924646378, + 0.10944131761789322, + 0.111859992146492, + 0.11317223310470581, + 0.1125045120716095, + 0.11186887323856354 + ], + "sources": 9 + }, + { + "entropy_mean": 2.189896583557129, + "mean_weights": [ + 0.12157853692770004, + 0.13215823471546173, + 0.1242145225405693, + 0.11826580762863159, + 0.11163709312677383, + 0.1045699343085289, + 0.09787097573280334, + 0.09515326470136642, + 0.09455163776874542 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028616169467568398, + 0.013977974653244019, + 0.015653911978006363, + 0.015701081603765488, + 0.01704590953886509, + 0.011780167929828167, + 0.012960037216544151, + 0.013451050966978073, + 0.014845823869109154, + 0.011557077057659626, + 0.012834185734391212, + 0.013193685561418533, + 0.014761172235012054, + 0.012218841351568699, + 0.013451457023620605, + 0.01399444043636322, + 0.015514986589550972, + 0.013508559204638004, + 0.014893320389091969, + 0.015537911094725132, + 0.017302624881267548, + 0.015652775764465332, + 0.017217284068465233, + 0.017610477283596992, + 0.019684365019202232, + 0.017625300213694572, + 0.019488012418150902, + 0.019592516124248505, + 0.022200310602784157, + 0.019619757309556007, + 0.022100279107689857, + 0.021992051973938942 + ], + "loss_nats": 3.616140842437744, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1926791667938232, + "mean_weights": [ + 0.09771968424320221, + 0.09518586099147797, + 0.10263510048389435, + 0.10697779804468155, + 0.11385782063007355, + 0.11843808740377426, + 0.1225852519273758, + 0.12233239412307739, + 0.12026800215244293 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004302801564335823, + 0.009077239781618118, + 0.010217261500656605, + 0.016407418996095657, + 0.004623814020305872, + 0.010880433954298496, + 0.012616241350769997, + 0.01978805474936962, + 0.0056845624931156635, + 0.012077964842319489, + 0.014198417775332928, + 0.02216547541320324, + 0.006770467385649681, + 0.013358108699321747, + 0.017934804782271385, + 0.02644582837820053, + 0.010160733014345169, + 0.017588114365935326, + 0.02522732876241207, + 0.03464006260037422, + 0.015689926221966743, + 0.023338599130511284, + 0.03383738547563553, + 0.0432722382247448, + 0.015551909804344177, + 0.023765092715620995, + 0.03784027695655823, + 0.04745635390281677, + 0.019000543281435966, + 0.02874108962714672, + 0.04672664403915405, + 0.05685042589902878 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015875036478973925, + 0.00016262078133877367, + 0.00014673685654997826, + 0.00013248690811451524, + 0.00010898245091084391, + 9.717086504679173e-05, + 8.911665645428002e-05, + 7.969579746713862e-05, + 7.858985918574035e-05, + 6.784927973058075e-05, + 6.598868640139699e-05, + 5.512806092156097e-05, + 4.8474703362444416e-05, + 4.3001487938454375e-05, + 3.610895510064438e-05, + 3.5184959415346384e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001501487276982516, + "first_to_last_ratio": 3.689835475493601, + "imbalance_abs_log_ratio": 1.3055818704713245, + "last_quartile_mean": 4.069252645422239e-05, + "mean": 8.786791704551433e-05, + "normalized": [ + 1.806693160912291, + 1.8507412808538344, + 1.6699708094135275, + 1.5077961623454543, + 1.2402985591930278, + 1.1058742293442396, + 1.0142115512777987, + 0.9069954102344013, + 0.8944090383413996, + 0.7721735305895072, + 0.7509986422827776, + 0.6273969245567232, + 0.5516769372982315, + 0.489387815079083, + 0.4109458413807676, + 0.4004301068969357 + ], + "population_cv": 0.4737558240032668 + }, + "activation_output_rms_by_block": [ + 0.02468283288180828, + 0.04329179599881172, + 0.023283371701836586, + 0.03508218377828598, + 0.025414658710360527, + 0.03976430371403694, + 0.02767801284790039, + 0.03781749680638313, + 0.03550800308585167, + 0.04360951855778694, + 0.040389809757471085, + 0.05188606306910515, + 0.03616104647517204, + 0.06982707977294922, + 0.04418032616376877, + 0.09354322403669357 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03158504609018564, + "first_to_last_ratio": 0.5184002104527677, + "imbalance_abs_log_ratio": 0.6570077279781673, + "last_quartile_mean": 0.0609279191121459, + "mean": 0.042007482959888875, + "normalized": [ + 0.5875818102545408, + 1.0305734347413573, + 0.5542672414833535, + 0.8351412964157943, + 0.6050031308618963, + 0.9466004842995747, + 0.6588829155588524, + 0.9002561958423794, + 0.8452780453367492, + 1.038136910022145, + 0.9614908323842579, + 1.2351623904400295, + 0.8608239277202269, + 1.662253361850413, + 1.0517251463496313, + 2.2268228764387987 + ], + "population_cv": 0.4130647858768022 + }, + "bits_per_byte": 3.137961051112634, + "branch_output_rms_by_sublayer": [ + 0.013650949113070965, + 0.01843169331550598, + 0.019044609740376472, + 0.014011893421411514, + 0.018698271363973618, + 0.012105143629014492, + 0.021805183961987495, + 0.010238726623356342, + 0.020464032888412476, + 0.011298555880784988, + 0.021483227610588074, + 0.011180058121681213, + 0.022897671908140182, + 0.01260110642760992, + 0.023534495383501053, + 0.012234007939696312, + 0.028167055919766426, + 0.01504397764801979, + 0.03861295431852341, + 0.015801267698407173, + 0.035523753613233566, + 0.0176701582968235, + 0.020830485969781876, + 0.024693600833415985, + 0.022914253175258636, + 0.028566768392920494, + 0.02199499122798443, + 0.03558572381734848, + 0.013380669057369232, + 0.04152822494506836, + 0.013844908215105534, + 0.04695267602801323 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0008395536207241178, + 0.0006692968987216388, + 0.0005333404576068953, + 0.0004419910350210091, + 0.0004333787114417237, + 0.0004609433225170186, + 0.0004705681902150354, + 0.0005107414594492524, + 0.0005339389183678795, + 0.0004946080122888242, + 0.000448918054550765, + 0.00035668617352665593, + 0.00032572588316331664, + 0.000267052972482006, + 0.0002153120435976381, + 0.0002101978579413297 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0006210455030184152, + "first_to_last_ratio": 2.439565392966498, + "imbalance_abs_log_ratio": 0.8918199058035992, + "last_quartile_mean": 0.00025457218929607263, + "mean": 0.0004507658507259441, + "normalized": [ + 1.8625049332642285, + 1.484799475478804, + 1.183187361571351, + 0.980533539329114, + 0.9614275587730993, + 1.022578178392804, + 1.04393043407614, + 1.1330526893878927, + 1.1845150148530281, + 1.0972614972768537, + 0.9959007627303548, + 0.7912892535053935, + 0.7226055004804499, + 0.5924427772244185, + 0.47765828589473863, + 0.46631273776132937 + ], + "population_cv": 0.34661834917809303 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6546831130981445, + "mean_weights": [ + 0.3866303563117981, + 0.6133696436882019 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6832369565963745, + "mean_weights": [ + 0.48925086855888367, + 0.510749101638794 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6867785453796387, + "mean_weights": [ + 0.5347544550895691, + 0.4652456045150757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6725893020629883, + "mean_weights": [ + 0.4844614565372467, + 0.5155385732650757 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0954453945159912, + "mean_weights": [ + 0.332688570022583, + 0.3308390974998474, + 0.3364723324775696 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0907697677612305, + "mean_weights": [ + 0.3209903836250305, + 0.33586689829826355, + 0.34314271807670593 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0933208465576172, + "mean_weights": [ + 0.33219069242477417, + 0.312181293964386, + 0.35562801361083984 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0861705541610718, + "mean_weights": [ + 0.319164514541626, + 0.32808834314346313, + 0.3527471721172333 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3827753067016602, + "mean_weights": [ + 0.24703660607337952, + 0.23925676941871643, + 0.2577625513076782, + 0.25594407320022583 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3791232109069824, + "mean_weights": [ + 0.24712958931922913, + 0.2454403042793274, + 0.24896711111068726, + 0.25846296548843384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3816200494766235, + "mean_weights": [ + 0.2438572347164154, + 0.2418915331363678, + 0.2618292272090912, + 0.252422034740448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765439987182617, + "mean_weights": [ + 0.24315571784973145, + 0.25071609020233154, + 0.23800383508205414, + 0.26812440156936646 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6069204807281494, + "mean_weights": [ + 0.1977769434452057, + 0.19419988989830017, + 0.19346478581428528, + 0.20617546141147614, + 0.20838290452957153 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6016991138458252, + "mean_weights": [ + 0.19928976893424988, + 0.19587931036949158, + 0.19414740800857544, + 0.18995173275470734, + 0.22073179483413696 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6063685417175293, + "mean_weights": [ + 0.19720253348350525, + 0.19060829281806946, + 0.18910765647888184, + 0.20065464079380035, + 0.2224268764257431 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6036955118179321, + "mean_weights": [ + 0.19818633794784546, + 0.20014071464538574, + 0.19487890601158142, + 0.18811310827732086, + 0.2186809629201889 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7846078872680664, + "mean_weights": [ + 0.15616348385810852, + 0.1693512499332428, + 0.1684132218360901, + 0.14385344088077545, + 0.16338428854942322, + 0.19883427023887634 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7881443500518799, + "mean_weights": [ + 0.16932255029678345, + 0.1670200079679489, + 0.16782812774181366, + 0.15219610929489136, + 0.1653819978237152, + 0.1782512068748474 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7859947681427002, + "mean_weights": [ + 0.15620459616184235, + 0.18192027509212494, + 0.16651901602745056, + 0.14659695327281952, + 0.16306468844413757, + 0.18569447100162506 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7886135578155518, + "mean_weights": [ + 0.17137612402439117, + 0.16813044250011444, + 0.16593416035175323, + 0.15192663669586182, + 0.16282179951667786, + 0.17981085181236267 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937737226486206, + "mean_weights": [ + 0.12751945853233337, + 0.15645764768123627, + 0.14123380184173584, + 0.1275305449962616, + 0.13446006178855896, + 0.15007954835891724, + 0.16271892189979553 + ], + "sources": 7 + }, + { + "entropy_mean": 1.943496823310852, + "mean_weights": [ + 0.14288359880447388, + 0.1535882204771042, + 0.14250940084457397, + 0.1380811333656311, + 0.1316976547241211, + 0.13936954736709595, + 0.15187042951583862 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273982048034668, + "mean_weights": [ + 0.12311284244060516, + 0.16783711314201355, + 0.141703262925148, + 0.11705678701400757, + 0.12322613596916199, + 0.14240297675132751, + 0.184660866856575 + ], + "sources": 7 + }, + { + "entropy_mean": 1.937564730644226, + "mean_weights": [ + 0.13963916897773743, + 0.16193091869354248, + 0.14834126830101013, + 0.12877905368804932, + 0.1304946392774582, + 0.13670659065246582, + 0.15410837531089783 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0582637786865234, + "mean_weights": [ + 0.10422320663928986, + 0.14683584868907928, + 0.12450192868709564, + 0.10290674865245819, + 0.1043052226305008, + 0.1136581152677536, + 0.13712583482265472, + 0.1664431095123291 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073244333267212, + "mean_weights": [ + 0.11922207474708557, + 0.13852310180664062, + 0.1310172826051712, + 0.1193084791302681, + 0.1166592389345169, + 0.11741948127746582, + 0.12622521817684174, + 0.13162514567375183 + ], + "sources": 8 + }, + { + "entropy_mean": 2.05503249168396, + "mean_weights": [ + 0.10400840640068054, + 0.1620255410671234, + 0.13802370429039001, + 0.10328710824251175, + 0.10247690975666046, + 0.11085900664329529, + 0.13948379456996918, + 0.13983553647994995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0654377937316895, + "mean_weights": [ + 0.12162524461746216, + 0.14755912125110626, + 0.1359378546476364, + 0.11907999217510223, + 0.10864339768886566, + 0.11126887798309326, + 0.12606029212474823, + 0.12982521951198578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.158114194869995, + "mean_weights": [ + 0.08706305176019669, + 0.1523345708847046, + 0.1240597814321518, + 0.09008736908435822, + 0.08798909187316895, + 0.09299354255199432, + 0.11455917358398438, + 0.09811963140964508, + 0.15279380977153778 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1800241470336914, + "mean_weights": [ + 0.10917439311742783, + 0.13041174411773682, + 0.1256810426712036, + 0.10147396475076675, + 0.10038554668426514, + 0.10393467545509338, + 0.11099927127361298, + 0.11076852679252625, + 0.10717083513736725 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1717891693115234, + "mean_weights": [ + 0.09730195999145508, + 0.15644855797290802, + 0.1323733627796173, + 0.10013202577829361, + 0.09525948762893677, + 0.09674295783042908, + 0.12173393368721008, + 0.10994285345077515, + 0.09006483852863312 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04337252303957939, + 0.018801197409629822, + 0.025790294632315636, + 0.029389970004558563, + 0.031753405928611755, + 0.022941799834370613, + 0.02458134852349758, + 0.02529502473771572, + 0.026913003996014595, + 0.020798955112695694, + 0.022398613393306732, + 0.022611256688833237, + 0.02451939694583416, + 0.01929222047328949, + 0.020593678578734398, + 0.020593887194991112, + 0.021762654185295105, + 0.01771371066570282, + 0.019139636307954788, + 0.0189701896160841, + 0.02020873688161373, + 0.016272971406579018, + 0.017277248203754425, + 0.01622883230447769, + 0.018368225544691086, + 0.014602466486394405, + 0.017397841438651085, + 0.017420755699276924, + 0.02048002928495407, + 0.016635844483971596, + 0.021542873233556747, + 0.021214226260781288 + ], + "loss_nats": 2.1750688552856445, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.173111915588379, + "mean_weights": [ + 0.1217261254787445, + 0.09291160106658936, + 0.09640607982873917, + 0.10597392916679382, + 0.09402626007795334, + 0.09312883019447327, + 0.11353092640638351, + 0.147413432598114, + 0.13488279283046722 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.013650949113070965, + 0.02468283288180828, + 0.0356261320412159, + 0.04329179599881172, + 0.018698271363973618, + 0.023283371701836586, + 0.03009101189672947, + 0.03508218377828598, + 0.020464032888412476, + 0.025414658710360527, + 0.034230802208185196, + 0.03976430371403694, + 0.022897671908140182, + 0.02767801284790039, + 0.031892068684101105, + 0.03781749680638313, + 0.028167055919766426, + 0.03550800308585167, + 0.035372670739889145, + 0.04360951855778694, + 0.035523753613233566, + 0.040389809757471085, + 0.03893519565463066, + 0.05188606306910515, + 0.022914253175258636, + 0.03616104647517204, + 0.04390736669301987, + 0.06982707977294922, + 0.013380669057369232, + 0.04418032616376877, + 0.051131755113601685, + 0.09354322403669357 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.532809104304761e-05, + 7.798708247719333e-05, + 7.679135887883604e-05, + 7.304617611225694e-05, + 7.868888496886939e-05, + 7.320931035792455e-05, + 8.902617992134765e-05, + 7.941434887470677e-05, + 8.246234938269481e-05, + 7.884583465056494e-05, + 0.0001634695363463834, + 0.00012072100798832253, + 0.000141287746373564, + 8.843849354889244e-05, + 5.323962977854535e-05, + 4.3788353650597855e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.578817712783348e-05, + "first_to_last_ratio": 0.9277698246768501, + "imbalance_abs_log_ratio": 0.07497161071478954, + "last_quartile_mean": 8.168855583789991e-05, + "mean": 8.723402402210922e-05, + "normalized": [ + 0.8635173246617158, + 0.8939984524550617, + 0.8802913741474712, + 0.8373587820933674, + 0.9020435071174282, + 0.8392288579897432, + 1.0205442305262016, + 0.9103598024387758, + 0.9453003034893238, + 0.9038426867776004, + 1.8739194732659872, + 1.383875263598136, + 1.6196403634636298, + 1.0138073365327953, + 0.6103080807673381, + 0.5019641606754243 + ], + "population_cv": 0.3374467083690701 + }, + "activation_output_rms_by_block": [ + 0.09266375005245209, + 0.20113667845726013, + 0.14358533918857574, + 0.17041823267936707, + 0.05199930816888809, + 0.07249397784471512, + 0.03721517324447632, + 0.07351206243038177, + 0.05156216770410538, + 0.08937478810548782, + 0.062482453882694244, + 0.09533937275409698, + 0.05472978577017784, + 0.08922836184501648, + 0.08201298862695694, + 0.20069743692874908 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15195100009441376, + "first_to_last_ratio": 1.4245342605399758, + "imbalance_abs_log_ratio": 0.353844925589721, + "last_quartile_mean": 0.10666714329272509, + "mean": 0.09802824235521257, + "normalized": [ + 0.9452760533712127, + 2.0518237767482, + 1.4647344044819621, + 1.7384605557023458, + 0.5304523157771692, + 0.739521347144304, + 0.37963725912400215, + 0.7499069723601224, + 0.5259929839123919, + 0.9117248861979151, + 0.6373923716420872, + 0.9725704599356962, + 0.5583063049509795, + 0.9102311711526044, + 0.8366261258645934, + 2.0473430116344136 + ], + "population_cv": 0.5182803671925184 + }, + "bits_per_byte": 2.0687232408845437, + "branch_output_rms_by_sublayer": [ + 0.04482239484786987, + 0.07626598328351974, + 0.10980217158794403, + 0.05728650838136673, + 0.14111557602882385, + 0.014705370180308819, + 0.04088159278035164, + 0.014925715513527393, + 0.04827577620744705, + 0.015019712969660759, + 0.04055904969573021, + 0.02146836742758751, + 0.028491884469985962, + 0.0222648773342371, + 0.0389801524579525, + 0.027360456064343452, + 0.036527831107378006, + 0.03173159062862396, + 0.03612164035439491, + 0.0401885062456131, + 0.04509544372558594, + 0.040587812662124634, + 0.02791302651166916, + 0.05573032796382904, + 0.02739257924258709, + 0.044091735035181046, + 0.017023129388689995, + 0.05625532940030098, + 0.02216578833758831, + 0.07780560851097107, + 0.0160520039498806, + 0.13296744227409363 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003765937729119843, + 0.0003149609092301355, + 0.00027693435492303714, + 0.0001635333627570268, + 0.000180917699642147, + 0.00017837059454152616, + 0.000200147742296221, + 0.00024611271555124715, + 0.0002957900398296376, + 0.0002986151161329657, + 0.0009593337211615513, + 0.0007552262449127961, + 0.0007605792150238727, + 0.0005098371211048817, + 0.00029447700210242914, + 0.0002817503499702034 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00028300559995554593, + "first_to_last_ratio": 0.6130161476493403, + "imbalance_abs_log_ratio": 0.48936400138742103, + "last_quartile_mean": 0.00046166092205034674, + "mean": 0.00038082374763072894, + "normalized": [ + 0.9888925657996352, + 0.827051651031862, + 0.727198229222749, + 0.4294200762805349, + 0.47506937465878935, + 0.46838096534485474, + 0.5255652872002538, + 0.6462640974530133, + 0.7767111207477916, + 0.7841294509357175, + 2.5191016241239836, + 1.9831385243472572, + 1.997194817171378, + 1.3387744968027897, + 0.7732632324914067, + 0.7398444863879827 + ], + "population_cv": 0.6091669692594756 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.573702335357666, + "mean_weights": [ + 0.7102901935577393, + 0.28970980644226074 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6441590189933777, + "mean_weights": [ + 0.564466118812561, + 0.43553388118743896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3999374806880951, + "mean_weights": [ + 0.8575316667556763, + 0.14246836304664612 + ], + "sources": 2 + }, + { + "entropy_mean": 0.676960825920105, + "mean_weights": [ + 0.46661365032196045, + 0.5333863496780396 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9920796155929565, + "mean_weights": [ + 0.5238763093948364, + 0.2048875242471695, + 0.2712361812591553 + ], + "sources": 3 + }, + { + "entropy_mean": 1.092700719833374, + "mean_weights": [ + 0.2899855673313141, + 0.34371256828308105, + 0.36630189418792725 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9979273080825806, + "mean_weights": [ + 0.49421679973602295, + 0.18891796469688416, + 0.3168652057647705 + ], + "sources": 3 + }, + { + "entropy_mean": 1.091429591178894, + "mean_weights": [ + 0.3115330636501312, + 0.3673863708972931, + 0.3210805654525757 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2927134037017822, + "mean_weights": [ + 0.35746461153030396, + 0.11402665078639984, + 0.22846516966819763, + 0.3000435531139374 + ], + "sources": 4 + }, + { + "entropy_mean": 1.378767490386963, + "mean_weights": [ + 0.2549898624420166, + 0.23654252290725708, + 0.21668663620948792, + 0.2917810082435608 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2672696113586426, + "mean_weights": [ + 0.3839718699455261, + 0.12064628303050995, + 0.1715085655450821, + 0.32387328147888184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.375225305557251, + "mean_weights": [ + 0.23853997886180878, + 0.24968335032463074, + 0.2089165896177292, + 0.3028600811958313 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5798927545547485, + "mean_weights": [ + 0.22726407647132874, + 0.12221817672252655, + 0.18549029529094696, + 0.23680329322814941, + 0.22822415828704834 + ], + "sources": 5 + }, + { + "entropy_mean": 1.594498634338379, + "mean_weights": [ + 0.21032124757766724, + 0.19536492228507996, + 0.1600019335746765, + 0.1794220358133316, + 0.2548898458480835 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5424928665161133, + "mean_weights": [ + 0.23003876209259033, + 0.11985675245523453, + 0.1379462629556656, + 0.19240085780620575, + 0.319757342338562 + ], + "sources": 5 + }, + { + "entropy_mean": 1.595255732536316, + "mean_weights": [ + 0.2046268880367279, + 0.19130583107471466, + 0.17092418670654297, + 0.17953911423683167, + 0.253603994846344 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7457678318023682, + "mean_weights": [ + 0.17887650430202484, + 0.11389653384685516, + 0.11488981544971466, + 0.1474636197090149, + 0.20503896474838257, + 0.23983454704284668 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7712774276733398, + "mean_weights": [ + 0.16810789704322815, + 0.16247668862342834, + 0.1383969932794571, + 0.13383737206459045, + 0.1661752164363861, + 0.23100584745407104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.675686001777649, + "mean_weights": [ + 0.16882894933223724, + 0.11588791012763977, + 0.08043210208415985, + 0.1250714659690857, + 0.19613179564476013, + 0.3136477470397949 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7837879657745361, + "mean_weights": [ + 0.17464125156402588, + 0.18035069108009338, + 0.1619444340467453, + 0.15330034494400024, + 0.15676669776439667, + 0.17299658060073853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6526988744735718, + "mean_weights": [ + 0.09642067551612854, + 0.07173497974872589, + 0.05561239272356033, + 0.09171071648597717, + 0.11456114053726196, + 0.12951162457466125, + 0.44044846296310425 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8849880695343018, + "mean_weights": [ + 0.13606244325637817, + 0.13500544428825378, + 0.12342768162488937, + 0.11247340589761734, + 0.11233137547969818, + 0.11334770917892456, + 0.2673519253730774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.644601583480835, + "mean_weights": [ + 0.10291393101215363, + 0.08116044104099274, + 0.05550362914800644, + 0.08702410012483597, + 0.11089185625314713, + 0.11528079956769943, + 0.4472252428531647 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9121408462524414, + "mean_weights": [ + 0.14125728607177734, + 0.1494411677122116, + 0.12786200642585754, + 0.11922307312488556, + 0.1195257231593132, + 0.11599892377853394, + 0.2266918122768402 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7500674724578857, + "mean_weights": [ + 0.07664570212364197, + 0.0604618638753891, + 0.041497331112623215, + 0.06405743956565857, + 0.0716431736946106, + 0.07514288276433945, + 0.270142525434494, + 0.3404091000556946 + ], + "sources": 8 + }, + { + "entropy_mean": 1.952926754951477, + "mean_weights": [ + 0.10562807321548462, + 0.10351885855197906, + 0.07600137591362, + 0.08402621746063232, + 0.08850346505641937, + 0.09127135574817657, + 0.15880608558654785, + 0.2922445833683014 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7527103424072266, + "mean_weights": [ + 0.08054207265377045, + 0.06648321449756622, + 0.04810335487127304, + 0.06408451497554779, + 0.07014857977628708, + 0.07217346131801605, + 0.20459653437137604, + 0.39386826753616333 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9524379968643188, + "mean_weights": [ + 0.10777557641267776, + 0.09555133432149887, + 0.06806209683418274, + 0.08129536360502243, + 0.0918373167514801, + 0.10075562447309494, + 0.2000727653503418, + 0.25464993715286255 + ], + "sources": 8 + }, + { + "entropy_mean": 2.000408887863159, + "mean_weights": [ + 0.07838036119937897, + 0.07984089851379395, + 0.04941036179661751, + 0.06521518528461456, + 0.06765615195035934, + 0.06713232398033142, + 0.18892306089401245, + 0.23950761556625366, + 0.16393405199050903 + ], + "sources": 9 + }, + { + "entropy_mean": 2.096604347229004, + "mean_weights": [ + 0.09755367040634155, + 0.08867941796779633, + 0.060870569199323654, + 0.06980546563863754, + 0.08883253484964371, + 0.09541568160057068, + 0.17317476868629456, + 0.19020704925060272, + 0.13546085357666016 + ], + "sources": 9 + }, + { + "entropy_mean": 2.027402877807617, + "mean_weights": [ + 0.08629745244979858, + 0.08307671546936035, + 0.05121873319149017, + 0.0663638487458229, + 0.07480055093765259, + 0.08018200099468231, + 0.15043827891349792, + 0.25634968280792236, + 0.1512727439403534 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.055510263890028, + 0.04334520921111107, + 0.05630175396800041, + 0.058260008692741394, + 0.11489111185073853, + 0.08273965120315552, + 0.11550594866275787, + 0.09142354130744934, + 0.12066955119371414, + 0.0711931437253952, + 0.08880246430635452, + 0.06823744624853134, + 0.09300445020198822, + 0.06510575115680695, + 0.07136023044586182, + 0.05894701927900314, + 0.07342072576284409, + 0.05002941936254501, + 0.059311795979738235, + 0.04801420122385025, + 0.06730704009532928, + 0.02817549556493759, + 0.046817973256111145, + 0.03746112436056137, + 0.05246857553720474, + 0.03106791339814663, + 0.03552677109837532, + 0.03383694216609001, + 0.03949129208922386, + 0.03412646800279617, + 0.039247769862413406, + 0.04047730192542076 + ], + "loss_nats": 1.433929681777954, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.913235068321228, + "mean_weights": [ + 0.09498223662376404, + 0.06303347647190094, + 0.06038350611925125, + 0.06479769200086594, + 0.05589054524898529, + 0.06028280779719353, + 0.07675591856241226, + 0.2476816177368164, + 0.27619218826293945 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04482239484786987, + 0.09266375005245209, + 0.17245197296142578, + 0.20113667845726013, + 0.14111557602882385, + 0.14358533918857574, + 0.16786204278469086, + 0.17041823267936707, + 0.04827577620744705, + 0.05199930816888809, + 0.06626670062541962, + 0.07249397784471512, + 0.028491884469985962, + 0.03721517324447632, + 0.05938867852091789, + 0.07351206243038177, + 0.036527831107378006, + 0.05156216770410538, + 0.066385917365551, + 0.08937478810548782, + 0.04509544372558594, + 0.062482453882694244, + 0.07032289355993271, + 0.09533937275409698, + 0.02739257924258709, + 0.05472978577017784, + 0.058526892215013504, + 0.08922836184501648, + 0.02216578833758831, + 0.08201298862695694, + 0.0856289193034172, + 0.20069743692874908 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.492530908668414e-05, + 5.6656335800653324e-05, + 6.325786671368405e-05, + 5.9555983170866966e-05, + 6.814395601395518e-05, + 6.255660991882905e-05, + 7.046976679703221e-05, + 6.272071914281696e-05, + 7.296914554899558e-05, + 6.464503530878574e-05, + 0.00014864247350487858, + 0.00010719253623392433, + 0.0001445752423023805, + 9.124213829636574e-05, + 6.016199040459469e-05, + 4.548503056867048e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.859887369297212e-05, + "first_to_last_ratio": 0.6864419649392262, + "imbalance_abs_log_ratio": 0.37623359494524666, + "last_quartile_mean": 8.536610039300285e-05, + "mean": 7.707500867581984e-05, + "normalized": [ + 0.7126215102706234, + 0.7350804985173837, + 0.8207312305308824, + 0.772701608395031, + 0.8841251812318441, + 0.8116328625007921, + 0.9143011205283218, + 0.8137620769739057, + 0.9467289955931933, + 0.8387288748897194, + 1.9285430654970663, + 1.3907560709435642, + 1.8757732861305145, + 1.18380964029642, + 0.7805641729816484, + 0.5901398047190899 + ], + "population_cv": 0.3863401678553132 + }, + "activation_output_rms_by_block": [ + 0.14486487209796906, + 0.30995267629623413, + 0.2574450671672821, + 0.28713259100914, + 0.05978552997112274, + 0.08792455494403839, + 0.04697949066758156, + 0.09852420538663864, + 0.07244286686182022, + 0.13140366971492767, + 0.08212646842002869, + 0.12985831499099731, + 0.06840891391038895, + 0.11060241609811783, + 0.0924442708492279, + 0.23933660984039307 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24984880164265633, + "first_to_last_ratio": 1.9565592145673032, + "imbalance_abs_log_ratio": 0.6711874277699096, + "last_quartile_mean": 0.12769805267453194, + "mean": 0.13870203238911927, + "normalized": [ + 1.0444322235420485, + 2.234665714390418, + 1.8561016210998063, + 2.0701397525568246, + 0.43103571693454673, + 0.6339096365752733, + 0.3387080193301259, + 0.7103299330916477, + 0.5222913147990983, + 0.9473809968860692, + 0.5921071739571081, + 0.9362394534110972, + 0.4932077254533107, + 0.7974102051210767, + 0.6664954309384717, + 1.725545081913077 + ], + "population_cv": 0.5979328649415788 + }, + "bits_per_byte": 1.9415087059372296, + "branch_output_rms_by_sublayer": [ + 0.06868160516023636, + 0.12181960046291351, + 0.16538235545158386, + 0.0929274708032608, + 0.2537953853607178, + 0.01320737786591053, + 0.04932497814297676, + 0.013867277652025223, + 0.055594079196453094, + 0.0172379482537508, + 0.04591496288776398, + 0.035032741725444794, + 0.03727728873491287, + 0.025992874056100845, + 0.0513935461640358, + 0.04214895889163017, + 0.05297093465924263, + 0.04459983855485916, + 0.05040198564529419, + 0.06986947357654572, + 0.054249413311481476, + 0.05820845440030098, + 0.03775457665324211, + 0.08249213546514511, + 0.03362919017672539, + 0.05606101080775261, + 0.02498369663953781, + 0.07096737623214722, + 0.02770768664777279, + 0.08668208867311478, + 0.022235019132494926, + 0.16932900249958038 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003274625120219605, + 0.0002950920627246982, + 0.0002675536202763891, + 0.00011636112532418193, + 0.00013949143839672265, + 0.0001705676155207084, + 0.00017763939173179324, + 0.00023071980568273914, + 0.00033685005198124334, + 0.0003711494248343089, + 0.0010020275600187679, + 0.0008489747430264115, + 0.0009447094037588545, + 0.0006663788253529681, + 0.0003910019531118643, + 0.00034755869426738895 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00025161733008680743, + "first_to_last_ratio": 0.4283488185904071, + "imbalance_abs_log_ratio": 0.8478174186232621, + "last_quartile_mean": 0.000587412219122769, + "mean": 0.00041459613925193755, + "normalized": [ + 0.789834928547107, + 0.7117578645501561, + 0.6453355324512677, + 0.2806613817826046, + 0.3364513684290676, + 0.4114066663246461, + 0.4284636901161473, + 0.5564928947457894, + 0.8124775416120065, + 0.895207141832, + 2.4168762444984226, + 2.047715023488232, + 2.278625665601732, + 1.607296263190791, + 0.9430911581023291, + 0.8383066347277007 + ], + "population_cv": 0.6738437321539833 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4194117784500122, + "mean_weights": [ + 0.8254866600036621, + 0.1745133101940155 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5809611082077026, + "mean_weights": [ + 0.6678193211555481, + 0.3321806788444519 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31704896688461304, + "mean_weights": [ + 0.8999109268188477, + 0.10008908808231354 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6732906103134155, + "mean_weights": [ + 0.4609490633010864, + 0.5390509366989136 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9927794337272644, + "mean_weights": [ + 0.4832823872566223, + 0.17606672644615173, + 0.34065088629722595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832945108413696, + "mean_weights": [ + 0.2762538492679596, + 0.3156396150588989, + 0.40810656547546387 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9905548095703125, + "mean_weights": [ + 0.4605642259120941, + 0.16346418857574463, + 0.37597161531448364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.09372079372406, + "mean_weights": [ + 0.32242417335510254, + 0.34579548239707947, + 0.3317803740501404 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2746227979660034, + "mean_weights": [ + 0.37224799394607544, + 0.09596331417560577, + 0.2427763193845749, + 0.2890123426914215 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3707606792449951, + "mean_weights": [ + 0.27432072162628174, + 0.2153838574886322, + 0.20746882259845734, + 0.30282658338546753 + ], + "sources": 4 + }, + { + "entropy_mean": 1.198214054107666, + "mean_weights": [ + 0.44253695011138916, + 0.10026887059211731, + 0.13153521716594696, + 0.325658917427063 + ], + "sources": 4 + }, + { + "entropy_mean": 1.372360110282898, + "mean_weights": [ + 0.24982473254203796, + 0.2343549281358719, + 0.2051662802696228, + 0.31065404415130615 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5764262676239014, + "mean_weights": [ + 0.2228686511516571, + 0.11472160369157791, + 0.20744341611862183, + 0.22938773036003113, + 0.22557863593101501 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5863620042800903, + "mean_weights": [ + 0.2179623544216156, + 0.17305074632167816, + 0.15634828805923462, + 0.18247520923614502, + 0.2701634168624878 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4903430938720703, + "mean_weights": [ + 0.2547823488712311, + 0.09995707869529724, + 0.10266982018947601, + 0.1966245472431183, + 0.3459661602973938 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5923073291778564, + "mean_weights": [ + 0.22257764637470245, + 0.17832019925117493, + 0.1619993895292282, + 0.1890055239200592, + 0.2480972558259964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.691253662109375, + "mean_weights": [ + 0.18423792719841003, + 0.09657466411590576, + 0.08509969711303711, + 0.15257084369659424, + 0.18859976530075073, + 0.29291707277297974 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7595478296279907, + "mean_weights": [ + 0.1724337935447693, + 0.14752474427223206, + 0.13264453411102295, + 0.13973915576934814, + 0.15476800501346588, + 0.2528897523880005 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5512144565582275, + "mean_weights": [ + 0.16704371571540833, + 0.08849328011274338, + 0.04574323818087578, + 0.12566779553890228, + 0.1718127727508545, + 0.40123921632766724 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7799327373504639, + "mean_weights": [ + 0.18437626957893372, + 0.17012780904769897, + 0.17119532823562622, + 0.15378496050834656, + 0.1488233208656311, + 0.17169234156608582 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5764116048812866, + "mean_weights": [ + 0.09582588821649551, + 0.05901939421892166, + 0.040849827229976654, + 0.09081389009952545, + 0.10021352022886276, + 0.13485172390937805, + 0.4784257411956787 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8431057929992676, + "mean_weights": [ + 0.13672879338264465, + 0.12035658955574036, + 0.11983893811702728, + 0.10482897609472275, + 0.10095320641994476, + 0.10664592683315277, + 0.3106476068496704 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5526533126831055, + "mean_weights": [ + 0.10306400060653687, + 0.06465981900691986, + 0.04082629829645157, + 0.08607284724712372, + 0.09878605604171753, + 0.11178258061408997, + 0.4948084056377411 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8949611186981201, + "mean_weights": [ + 0.1451701521873474, + 0.13929708302021027, + 0.1330569088459015, + 0.11840799450874329, + 0.10893036425113678, + 0.10698798298835754, + 0.24814949929714203 + ], + "sources": 7 + }, + { + "entropy_mean": 1.676889419555664, + "mean_weights": [ + 0.07736113667488098, + 0.04877929389476776, + 0.032506294548511505, + 0.06437233090400696, + 0.06250663101673126, + 0.06869478523731232, + 0.2701185345649719, + 0.3756609857082367 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8798964023590088, + "mean_weights": [ + 0.10042576491832733, + 0.08721707761287689, + 0.06748545169830322, + 0.0845440924167633, + 0.07678259164094925, + 0.08073122054338455, + 0.1554347574710846, + 0.34737905859947205 + ], + "sources": 8 + }, + { + "entropy_mean": 1.663856029510498, + "mean_weights": [ + 0.07920834422111511, + 0.05299394577741623, + 0.03748705983161926, + 0.06209969520568848, + 0.060945168137550354, + 0.06926221400499344, + 0.19488413631916046, + 0.44311946630477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9090958833694458, + "mean_weights": [ + 0.1065661758184433, + 0.0803120881319046, + 0.06028372794389725, + 0.08741366118192673, + 0.08287163078784943, + 0.09473372995853424, + 0.1887454390525818, + 0.29907354712486267 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8632590770721436, + "mean_weights": [ + 0.0691547840833664, + 0.05765335261821747, + 0.03405343368649483, + 0.05519368499517441, + 0.05223262310028076, + 0.057386673986911774, + 0.1615200936794281, + 0.25146448612213135, + 0.2613408863544464 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0701708793640137, + "mean_weights": [ + 0.09429951012134552, + 0.07856893539428711, + 0.04940951615571976, + 0.076503224670887, + 0.08159374445676804, + 0.0895766168832779, + 0.16711056232452393, + 0.2026878148317337, + 0.16025006771087646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9422186613082886, + "mean_weights": [ + 0.07884211093187332, + 0.06442099809646606, + 0.037228696048259735, + 0.06615173816680908, + 0.06501315534114838, + 0.0695233941078186, + 0.13319489359855652, + 0.2805612087249756, + 0.20506379008293152 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05865398049354553, + 0.05170911177992821, + 0.06799201667308807, + 0.06424286961555481, + 0.17415167391300201, + 0.13562268018722534, + 0.1871119737625122, + 0.1505064219236374, + 0.18585707247257233, + 0.1044568344950676, + 0.12573961913585663, + 0.08286763727664948, + 0.13412274420261383, + 0.0987003743648529, + 0.09964247792959213, + 0.07276491075754166, + 0.1041010394692421, + 0.06199917569756508, + 0.0827898308634758, + 0.06073001027107239, + 0.10040723532438278, + 0.036688607186079025, + 0.0660671666264534, + 0.05293381214141846, + 0.07866564393043518, + 0.04279600828886032, + 0.04821427911520004, + 0.04622510448098183, + 0.053889211267232895, + 0.04172775521874428, + 0.05102568119764328, + 0.053032174706459045 + ], + "loss_nats": 1.3457512855529785, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7949464321136475, + "mean_weights": [ + 0.0834353119134903, + 0.05303390324115753, + 0.048570070415735245, + 0.05988207459449768, + 0.04784438759088516, + 0.05165069177746773, + 0.06509792059659958, + 0.2334408313035965, + 0.3570447862148285 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06868160516023636, + 0.14486487209796906, + 0.2633455693721771, + 0.30995267629623413, + 0.2537953853607178, + 0.2574450671672821, + 0.28461524844169617, + 0.28713259100914, + 0.055594079196453094, + 0.05978552997112274, + 0.07737205922603607, + 0.08792455494403839, + 0.03727728873491287, + 0.04697949066758156, + 0.07717122882604599, + 0.09852420538663864, + 0.05297093465924263, + 0.07244286686182022, + 0.09201814234256744, + 0.13140366971492767, + 0.054249413311481476, + 0.08212646842002869, + 0.09215818345546722, + 0.12985831499099731, + 0.03362919017672539, + 0.06840891391038895, + 0.07437368482351303, + 0.11060241609811783, + 0.02770768664777279, + 0.0924442708492279, + 0.09714528173208237, + 0.23933660984039307 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.899879372213036e-05, + 7.050983549561352e-05, + 8.400688238907605e-05, + 8.067955786827952e-05, + 9.6201452834066e-05, + 8.90634983079508e-05, + 0.00010480848868610337, + 9.241609222954139e-05, + 0.00012084992340533063, + 0.00010282631410518661, + 0.00023664520995225757, + 0.0001637248496990651, + 0.00023947370937094092, + 0.0001411983830621466, + 0.00010364947956986725, + 6.421437865355983e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.604876736877486e-05, + "first_to_last_ratio": 0.5545581271583456, + "imbalance_abs_log_ratio": 0.5895836496274593, + "last_quartile_mean": 0.00013713398766412865, + "mean": 0.00011620417808444472, + "normalized": [ + 0.5937720558721171, + 0.6067753901617422, + 0.7229248016196876, + 0.6942913688494942, + 0.8278656965686476, + 0.7664397251123708, + 0.9019339098972828, + 0.7952906147865294, + 1.0399791590755874, + 0.8848762221825061, + 2.0364604255476015, + 1.4089411619958057, + 2.060801197672231, + 1.2150886946555293, + 0.891960007621636, + 0.5525995683812308 + ], + "population_cv": 0.4517546138899352 + }, + "activation_output_rms_by_block": [ + 0.15090593695640564, + 0.3367030918598175, + 0.29793402552604675, + 0.3255155086517334, + 0.05378299206495285, + 0.08782409131526947, + 0.044081781059503555, + 0.09709836542606354, + 0.0669482871890068, + 0.12247904390096664, + 0.07360157370567322, + 0.12167256325483322, + 0.06349270045757294, + 0.10058031976222992, + 0.07551935315132141, + 0.19356463849544525 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2777646407485008, + "first_to_last_ratio": 2.565024996839381, + "imbalance_abs_log_ratio": 0.941968223931731, + "last_quartile_mean": 0.10828925296664238, + "mean": 0.13823151704855263, + "normalized": [ + 1.0916897982346618, + 2.4357910485895444, + 2.155326309711265, + 2.3548573842056504, + 0.38907908422984344, + 0.6353405734845695, + 0.3188981934128684, + 0.7024329002477675, + 0.4843199916954669, + 0.8860428252259355, + 0.5324514645948746, + 0.8802085505008006, + 0.45932144718683565, + 0.7276221943430017, + 0.5463251417894509, + 1.400293092547464 + ], + "population_cv": 0.6866316887805146 + }, + "bits_per_byte": 1.8676321823979032, + "branch_output_rms_by_sublayer": [ + 0.07169820368289948, + 0.12625248730182648, + 0.19025787711143494, + 0.09543261677026749, + 0.2934682369232178, + 0.013759235851466656, + 0.04595690965652466, + 0.014787186868488789, + 0.048878561705350876, + 0.017967192456126213, + 0.039782870560884476, + 0.03611106425523758, + 0.03420666605234146, + 0.025706099346280098, + 0.046541497111320496, + 0.04672740399837494, + 0.04840917885303497, + 0.04226908087730408, + 0.0452793650329113, + 0.07270561158657074, + 0.04624715819954872, + 0.05458534136414528, + 0.0338163822889328, + 0.0825076773762703, + 0.03031807579100132, + 0.0527048334479332, + 0.022609181702136993, + 0.06718777120113373, + 0.02396680787205696, + 0.07009968906641006, + 0.018549330532550812, + 0.1417386531829834 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00046247293295990515, + 0.0003824975682884986, + 0.00041690386457328876, + 0.00021732383526524047, + 0.00022655054548791778, + 0.0002917516444309179, + 0.000280629308661355, + 0.0003847878626551635, + 0.0005123546145807317, + 0.0005899763687075028, + 0.0015400377759773427, + 0.0012824907104132966, + 0.001537981149104749, + 0.001057571910144345, + 0.0006563117026664665, + 0.0005021677227557722 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036979955027173326, + "first_to_last_ratio": 0.394029142562531, + "imbalance_abs_log_ratio": 0.9313304065247964, + "last_quartile_mean": 0.0009385081211678332, + "mean": 0.0006463630947920308, + "normalized": [ + 0.7155002144865761, + 0.5917688855852271, + 0.6449994870258314, + 0.3362256246006202, + 0.3505004343739815, + 0.45137423033842894, + 0.434166664097133, + 0.595312241301416, + 0.792673063652533, + 0.9127630792369563, + 2.3826202151483513, + 1.9841645055956383, + 2.3794383706260316, + 1.6361885736756394, + 1.0153916706485797, + 0.7769127396070564 + ], + "population_cv": 0.675897465620864 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.43482720851898193, + "mean_weights": [ + 0.8150075674057007, + 0.18499243259429932 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611247420310974, + "mean_weights": [ + 0.6986296772956848, + 0.3013703525066376 + ], + "sources": 2 + }, + { + "entropy_mean": 0.31192272901535034, + "mean_weights": [ + 0.9031083583831787, + 0.09689167141914368 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6764674186706543, + "mean_weights": [ + 0.4619978070259094, + 0.5380021929740906 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9888706207275391, + "mean_weights": [ + 0.48646074533462524, + 0.1758481115102768, + 0.33769115805625916 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0781972408294678, + "mean_weights": [ + 0.276906281709671, + 0.2993447184562683, + 0.42374899983406067 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9839773178100586, + "mean_weights": [ + 0.46118396520614624, + 0.15739500522613525, + 0.3814210295677185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0957449674606323, + "mean_weights": [ + 0.3292894959449768, + 0.32970887422561646, + 0.3410016596317291 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2506439685821533, + "mean_weights": [ + 0.3648650050163269, + 0.0835169106721878, + 0.212466761469841, + 0.3391513228416443 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655824661254883, + "mean_weights": [ + 0.2861315608024597, + 0.20811524987220764, + 0.20031729340553284, + 0.3054359257221222 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1652674674987793, + "mean_weights": [ + 0.43905186653137207, + 0.08880466222763062, + 0.10791443288326263, + 0.3642290234565735 + ], + "sources": 4 + }, + { + "entropy_mean": 1.363007664680481, + "mean_weights": [ + 0.2524264454841614, + 0.21885012090206146, + 0.19597390294075012, + 0.33274954557418823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5649902820587158, + "mean_weights": [ + 0.22229184210300446, + 0.10384112596511841, + 0.1934116780757904, + 0.2539443373680115, + 0.22651101648807526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5797945261001587, + "mean_weights": [ + 0.21991753578186035, + 0.1605391800403595, + 0.15625253319740295, + 0.18265283107757568, + 0.2806379497051239 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4139833450317383, + "mean_weights": [ + 0.24990326166152954, + 0.08050692081451416, + 0.06835947930812836, + 0.20686665177345276, + 0.394363671541214 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5895863771438599, + "mean_weights": [ + 0.2244223803281784, + 0.16505929827690125, + 0.1643591672182083, + 0.19251328706741333, + 0.2536458969116211 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6177464723587036, + "mean_weights": [ + 0.1738494634628296, + 0.07661910355091095, + 0.06195354461669922, + 0.14467598497867584, + 0.19003543257713318, + 0.35286644101142883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7444759607315063, + "mean_weights": [ + 0.172578826546669, + 0.1303727626800537, + 0.13036483526229858, + 0.13626831769943237, + 0.1541597843170166, + 0.27625542879104614 + ], + "sources": 6 + }, + { + "entropy_mean": 1.470452904701233, + "mean_weights": [ + 0.15901926159858704, + 0.07292591035366058, + 0.03418249264359474, + 0.11740352213382721, + 0.16174395382404327, + 0.45472484827041626 + ], + "sources": 6 + }, + { + "entropy_mean": 1.778969645500183, + "mean_weights": [ + 0.18775978684425354, + 0.15501554310321808, + 0.17432880401611328, + 0.15116485953330994, + 0.14906319975852966, + 0.1826678216457367 + ], + "sources": 6 + }, + { + "entropy_mean": 1.531482219696045, + "mean_weights": [ + 0.09255115687847137, + 0.04949861764907837, + 0.03270748257637024, + 0.08605319261550903, + 0.09672325849533081, + 0.1471216380596161, + 0.4953446388244629 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8260389566421509, + "mean_weights": [ + 0.13673919439315796, + 0.10493747889995575, + 0.11646746844053268, + 0.09983028471469879, + 0.10169579833745956, + 0.1137751042842865, + 0.32655465602874756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4906136989593506, + "mean_weights": [ + 0.10088150203227997, + 0.05257141590118408, + 0.03239753842353821, + 0.08061188459396362, + 0.09571854770183563, + 0.11411523073911667, + 0.523703932762146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8863525390625, + "mean_weights": [ + 0.14810651540756226, + 0.12629631161689758, + 0.13384857773780823, + 0.11124427616596222, + 0.1082591712474823, + 0.11263860762119293, + 0.2596065402030945 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5897703170776367, + "mean_weights": [ + 0.07248121500015259, + 0.03783262521028519, + 0.02471475675702095, + 0.053565122187137604, + 0.058412034064531326, + 0.06658737361431122, + 0.26026394963264465, + 0.42614293098449707 + ], + "sources": 8 + }, + { + "entropy_mean": 1.814478874206543, + "mean_weights": [ + 0.09743618965148926, + 0.07339455187320709, + 0.06428516656160355, + 0.07672879844903946, + 0.07112542539834976, + 0.07841956615447998, + 0.14423531293869019, + 0.3943749666213989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5432119369506836, + "mean_weights": [ + 0.07432855665683746, + 0.04178062081336975, + 0.029931148514151573, + 0.0517733134329319, + 0.053720228374004364, + 0.06251072138547897, + 0.17899465560913086, + 0.5069607496261597 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8694422245025635, + "mean_weights": [ + 0.10448876023292542, + 0.07235300540924072, + 0.05752243846654892, + 0.08492527902126312, + 0.07930260896682739, + 0.08949960768222809, + 0.1757010817527771, + 0.33620721101760864 + ], + "sources": 8 + }, + { + "entropy_mean": 1.746185302734375, + "mean_weights": [ + 0.06162667274475098, + 0.04293470457196236, + 0.02593991532921791, + 0.04546971619129181, + 0.044149380177259445, + 0.052234895527362823, + 0.14164786040782928, + 0.2587880492210388, + 0.3272087574005127 + ], + "sources": 9 + }, + { + "entropy_mean": 2.061832904815674, + "mean_weights": [ + 0.09089995175600052, + 0.06807379424571991, + 0.04443599283695221, + 0.08077649772167206, + 0.08381937444210052, + 0.09205536544322968, + 0.15836367011070251, + 0.20683875679969788, + 0.17473658919334412 + ], + "sources": 9 + }, + { + "entropy_mean": 1.849208116531372, + "mean_weights": [ + 0.07359056174755096, + 0.050991274416446686, + 0.029770921915769577, + 0.0592363178730011, + 0.055861424654722214, + 0.06031768023967743, + 0.11290363222360611, + 0.2822529673576355, + 0.275075227022171 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.06120532751083374, + 0.05362993851304054, + 0.06824863702058792, + 0.06727377325296402, + 0.1878596842288971, + 0.1534913182258606, + 0.21103329956531525, + 0.1696806252002716, + 0.20562554895877838, + 0.10344218462705612, + 0.13388557732105255, + 0.08117230981588364, + 0.14070458710193634, + 0.10246162861585617, + 0.10507974773645401, + 0.06554406136274338, + 0.11050253361463547, + 0.05469200015068054, + 0.08388946205377579, + 0.05699421092867851, + 0.10618287324905396, + 0.03451571613550186, + 0.06762637197971344, + 0.050157804042100906, + 0.08278355002403259, + 0.03850048407912254, + 0.04651740565896034, + 0.04368532821536064, + 0.05266447737812996, + 0.036846842616796494, + 0.04717044159770012, + 0.048093993216753006 + ], + "loss_nats": 1.294543981552124, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.636631727218628, + "mean_weights": [ + 0.07672396302223206, + 0.03940216451883316, + 0.03699969872832298, + 0.05207572132349014, + 0.04068697616457939, + 0.04208337888121605, + 0.051315970718860626, + 0.2188674658536911, + 0.4418446719646454 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07169820368289948, + 0.15090593695640564, + 0.29125523567199707, + 0.3367030918598175, + 0.2934682369232178, + 0.29793402552604675, + 0.3226769268512726, + 0.3255155086517334, + 0.048878561705350876, + 0.05378299206495285, + 0.0762643963098526, + 0.08782409131526947, + 0.03420666605234146, + 0.044081781059503555, + 0.07245275378227234, + 0.09709836542606354, + 0.04840917885303497, + 0.0669482871890068, + 0.08269722014665604, + 0.12247904390096664, + 0.04624715819954872, + 0.07360157370567322, + 0.08215674012899399, + 0.12167256325483322, + 0.03031807579100132, + 0.06349270045757294, + 0.06805067509412766, + 0.10058031976222992, + 0.02396680787205696, + 0.07551935315132141, + 0.07906875014305115, + 0.19356463849544525 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.030247040746552, + "cross_entropy_nats": 5.566143095493317, + "step": 0 + }, + { + "bits_per_byte": 5.230785295754151, + "cross_entropy_nats": 3.6257040798664093, + "step": 100 + }, + { + "bits_per_byte": 3.16099769723908, + "cross_entropy_nats": 2.191036641597748, + "step": 500 + }, + { + "bits_per_byte": 1.983385019569791, + "cross_entropy_nats": 1.3747777342796326, + "step": 2000 + }, + { + "bits_per_byte": 1.818564586202164, + "cross_entropy_nats": 1.2605329155921936, + "step": 4000 + }, + { + "bits_per_byte": 1.7311772990915377, + "cross_entropy_nats": 1.1999606639146805, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "7517e8197003ce4ba5267d82e82d6a41d49e7ce8744b4095b5a5160f1d4d01c2", + "final_model_state": "9867cf3c0d2372bb731808e8fd98ce5f38db0a40bd3cc08683bc5f66c9472428", + "final_optimizer_state": "fa42da28c86335ac8bab241bdce833e1a77d2508025d738b471764f2dd16cf93", + "final_public_parameters": "8c73d93d872090c7e7d7cdebca989ac8c083cf6d1cb5fb16509c6e6993d3c9a0", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.937352295237, + "measured_steps": 7980, + "median_ms": 54.89947448950261, + "p95_ms": 55.28610254696105, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.033595800932828, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.568464279174805, + "step": 1, + "unclipped_grad_norm": 13.626673698425293 + }, + { + "bits_per_byte": 7.5882983213188435, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.259807586669922, + "step": 10, + "unclipped_grad_norm": 10.73694896697998 + }, + { + "bits_per_byte": 6.986202883052029, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.842466831207275, + "step": 20, + "unclipped_grad_norm": 4.4299492835998535 + }, + { + "bits_per_byte": 6.739012218966915, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.6711273193359375, + "step": 30, + "unclipped_grad_norm": 3.205341339111328 + }, + { + "bits_per_byte": 6.623832685961678, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.5912909507751465, + "step": 40, + "unclipped_grad_norm": 3.10516619682312 + }, + { + "bits_per_byte": 6.3865585415253125, + "learning_rate": 3.75e-05, + "loss_nats": 4.426825046539307, + "step": 50, + "unclipped_grad_norm": 3.1714048385620117 + }, + { + "bits_per_byte": 6.103358837933146, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.230525970458984, + "step": 60, + "unclipped_grad_norm": 2.932384490966797 + }, + { + "bits_per_byte": 5.8558104499342365, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 4.058938503265381, + "step": 70, + "unclipped_grad_norm": 2.6460657119750977 + }, + { + "bits_per_byte": 5.640669096802708, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.90981388092041, + "step": 80, + "unclipped_grad_norm": 2.5319507122039795 + }, + { + "bits_per_byte": 5.451491620506636, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.778686046600342, + "step": 90, + "unclipped_grad_norm": 2.4583022594451904 + }, + { + "bits_per_byte": 5.270375658987107, + "learning_rate": 7.5e-05, + "loss_nats": 3.6531460285186768, + "step": 100, + "unclipped_grad_norm": 2.433866262435913 + }, + { + "bits_per_byte": 5.10413206918359, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.537914752960205, + "step": 110, + "unclipped_grad_norm": 2.285804271697998 + }, + { + "bits_per_byte": 4.823540998439711, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.343423843383789, + "step": 120, + "unclipped_grad_norm": 2.0681240558624268 + }, + { + "bits_per_byte": 4.663901886394628, + "learning_rate": 9.75e-05, + "loss_nats": 3.2327704429626465, + "step": 130, + "unclipped_grad_norm": 1.8398888111114502 + }, + { + "bits_per_byte": 4.463271741412299, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0937042236328125, + "step": 140, + "unclipped_grad_norm": 1.681030035018921 + }, + { + "bits_per_byte": 4.327779681478958, + "learning_rate": 0.0001125, + "loss_nats": 2.999788284301758, + "step": 150, + "unclipped_grad_norm": 1.4562726020812988 + }, + { + "bits_per_byte": 4.208912840439223, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.917396068572998, + "step": 160, + "unclipped_grad_norm": 1.1446112394332886 + }, + { + "bits_per_byte": 4.0496410833363825, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.806997299194336, + "step": 170, + "unclipped_grad_norm": 1.193884253501892 + }, + { + "bits_per_byte": 3.9053046435997705, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7069509029388428, + "step": 180, + "unclipped_grad_norm": 1.1651051044464111 + }, + { + "bits_per_byte": 3.9164302012840855, + "learning_rate": 0.0001425, + "loss_nats": 2.714662551879883, + "step": 190, + "unclipped_grad_norm": 1.1523211002349854 + }, + { + "bits_per_byte": 3.7885273917680182, + "learning_rate": 0.00015, + "loss_nats": 2.626007080078125, + "step": 200, + "unclipped_grad_norm": 0.7202922701835632 + }, + { + "bits_per_byte": 3.72972377171561, + "learning_rate": 0.0001575, + "loss_nats": 2.58524751663208, + "step": 210, + "unclipped_grad_norm": 0.606606662273407 + }, + { + "bits_per_byte": 3.7010143638417796, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.565347671508789, + "step": 220, + "unclipped_grad_norm": 0.6799013018608093 + }, + { + "bits_per_byte": 3.715506997864687, + "learning_rate": 0.0001725, + "loss_nats": 2.5753931999206543, + "step": 230, + "unclipped_grad_norm": 0.6519911289215088 + }, + { + "bits_per_byte": 3.7068112110717615, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5693657398223877, + "step": 240, + "unclipped_grad_norm": 0.912699818611145 + }, + { + "bits_per_byte": 3.5753026132977044, + "learning_rate": 0.0001875, + "loss_nats": 2.478210926055908, + "step": 250, + "unclipped_grad_norm": 0.6829943060874939 + }, + { + "bits_per_byte": 3.5693214006659977, + "learning_rate": 0.000195, + "loss_nats": 2.474065065383911, + "step": 260, + "unclipped_grad_norm": 0.640023410320282 + }, + { + "bits_per_byte": 3.539509240034835, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4534008502960205, + "step": 270, + "unclipped_grad_norm": 0.6751542687416077 + }, + { + "bits_per_byte": 3.5394008909647914, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4533257484436035, + "step": 280, + "unclipped_grad_norm": 0.7841543555259705 + }, + { + "bits_per_byte": 3.4452169999768865, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.388042449951172, + "step": 290, + "unclipped_grad_norm": 0.8627198934555054 + }, + { + "bits_per_byte": 3.5355663657811545, + "learning_rate": 0.000225, + "loss_nats": 2.4506678581237793, + "step": 300, + "unclipped_grad_norm": 1.1167978048324585 + }, + { + "bits_per_byte": 3.4429038333227817, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.386439085006714, + "step": 310, + "unclipped_grad_norm": 0.8460172414779663 + }, + { + "bits_per_byte": 3.4221950583670893, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.372084856033325, + "step": 320, + "unclipped_grad_norm": 0.821152925491333 + }, + { + "bits_per_byte": 3.429671832130699, + "learning_rate": 0.0002475, + "loss_nats": 2.377267360687256, + "step": 330, + "unclipped_grad_norm": 0.9970561265945435 + }, + { + "bits_per_byte": 3.425132178078524, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3741207122802734, + "step": 340, + "unclipped_grad_norm": 1.0991977453231812 + }, + { + "bits_per_byte": 3.393254505810499, + "learning_rate": 0.0002625, + "loss_nats": 2.352024793624878, + "step": 350, + "unclipped_grad_norm": 0.9725445508956909 + }, + { + "bits_per_byte": 3.382284076477262, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3444206714630127, + "step": 360, + "unclipped_grad_norm": 1.0039488077163696 + }, + { + "bits_per_byte": 3.4119693139120884, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.364996910095215, + "step": 370, + "unclipped_grad_norm": 1.7546980381011963 + }, + { + "bits_per_byte": 3.446912748914394, + "learning_rate": 0.000285, + "loss_nats": 2.3892178535461426, + "step": 380, + "unclipped_grad_norm": 0.9759950041770935 + }, + { + "bits_per_byte": 3.3737516732026567, + "learning_rate": 0.0002925, + "loss_nats": 2.3385064601898193, + "step": 390, + "unclipped_grad_norm": 0.9621731638908386 + }, + { + "bits_per_byte": 3.3279967208365115, + "learning_rate": 0.0003, + "loss_nats": 2.3067915439605713, + "step": 400, + "unclipped_grad_norm": 0.9455732107162476 + }, + { + "bits_per_byte": 3.307004174506895, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.292240619659424, + "step": 410, + "unclipped_grad_norm": 1.4261541366577148 + }, + { + "bits_per_byte": 3.3082469211420302, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2931020259857178, + "step": 420, + "unclipped_grad_norm": 0.9587261080741882 + }, + { + "bits_per_byte": 3.315426508884949, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2980785369873047, + "step": 430, + "unclipped_grad_norm": 2.1751644611358643 + }, + { + "bits_per_byte": 3.299565580891777, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2870845794677734, + "step": 440, + "unclipped_grad_norm": 1.11757493019104 + }, + { + "bits_per_byte": 3.305290883339, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.291053056716919, + "step": 450, + "unclipped_grad_norm": 1.8777321577072144 + }, + { + "bits_per_byte": 3.227380334741036, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2370495796203613, + "step": 460, + "unclipped_grad_norm": 1.1160534620285034 + }, + { + "bits_per_byte": 3.2510836716135514, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.253479480743408, + "step": 470, + "unclipped_grad_norm": 1.6702216863632202 + }, + { + "bits_per_byte": 3.198324209843066, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.216909408569336, + "step": 480, + "unclipped_grad_norm": 1.0277327299118042 + }, + { + "bits_per_byte": 3.1569424323230604, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.188225746154785, + "step": 490, + "unclipped_grad_norm": 1.0115255117416382 + }, + { + "bits_per_byte": 3.1652801512368893, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.194005012512207, + "step": 500, + "unclipped_grad_norm": 1.3175867795944214 + }, + { + "bits_per_byte": 3.134954794375552, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.172985076904297, + "step": 510, + "unclipped_grad_norm": 0.9790136814117432 + }, + { + "bits_per_byte": 3.0134504274021845, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0887646675109863, + "step": 520, + "unclipped_grad_norm": 1.0963760614395142 + }, + { + "bits_per_byte": 3.0906245462891544, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1422576904296875, + "step": 530, + "unclipped_grad_norm": 1.0304492712020874 + }, + { + "bits_per_byte": 3.024995622754605, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0967671871185303, + "step": 540, + "unclipped_grad_norm": 1.323768973350525 + }, + { + "bits_per_byte": 2.9805828229957934, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0659825801849365, + "step": 550, + "unclipped_grad_norm": 1.1477609872817993 + }, + { + "bits_per_byte": 2.9872017472968992, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.070570468902588, + "step": 560, + "unclipped_grad_norm": 1.223684310913086 + }, + { + "bits_per_byte": 2.9579880862907393, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.050321102142334, + "step": 570, + "unclipped_grad_norm": 1.308420181274414 + }, + { + "bits_per_byte": 2.8546603937044113, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.978699803352356, + "step": 580, + "unclipped_grad_norm": 1.0794485807418823 + }, + { + "bits_per_byte": 2.911534540396789, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0181219577789307, + "step": 590, + "unclipped_grad_norm": 1.4487859010696411 + }, + { + "bits_per_byte": 2.825803768681442, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9586979150772095, + "step": 600, + "unclipped_grad_norm": 1.2767362594604492 + }, + { + "bits_per_byte": 2.883143472409706, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9984427690505981, + "step": 610, + "unclipped_grad_norm": 1.1309611797332764 + }, + { + "bits_per_byte": 2.7721532647968363, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9215102195739746, + "step": 620, + "unclipped_grad_norm": 1.533046007156372 + }, + { + "bits_per_byte": 2.7733690101557857, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.922352910041809, + "step": 630, + "unclipped_grad_norm": 0.9842169880867004 + }, + { + "bits_per_byte": 2.7997499449195846, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.940638780593872, + "step": 640, + "unclipped_grad_norm": 1.2905892133712769 + }, + { + "bits_per_byte": 2.715926288819655, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8825366497039795, + "step": 650, + "unclipped_grad_norm": 1.1535414457321167 + }, + { + "bits_per_byte": 2.6764600540649592, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8551807403564453, + "step": 660, + "unclipped_grad_norm": 1.1525585651397705 + }, + { + "bits_per_byte": 2.696502224265908, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8690729141235352, + "step": 670, + "unclipped_grad_norm": 1.264070749282837 + }, + { + "bits_per_byte": 2.7320018511611033, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8936793804168701, + "step": 680, + "unclipped_grad_norm": 1.6425000429153442 + }, + { + "bits_per_byte": 2.602245068668735, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8037388324737549, + "step": 690, + "unclipped_grad_norm": 1.3727850914001465 + }, + { + "bits_per_byte": 2.5536510267021466, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7700560092926025, + "step": 700, + "unclipped_grad_norm": 2.108612298965454 + }, + { + "bits_per_byte": 2.623769213372168, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8186582326889038, + "step": 710, + "unclipped_grad_norm": 1.441254734992981 + }, + { + "bits_per_byte": 2.6835147824033516, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8600707054138184, + "step": 720, + "unclipped_grad_norm": 1.4497488737106323 + }, + { + "bits_per_byte": 2.4942898388956416, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.728909969329834, + "step": 730, + "unclipped_grad_norm": 1.7889822721481323 + }, + { + "bits_per_byte": 2.518159998939494, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7454555034637451, + "step": 740, + "unclipped_grad_norm": 1.1689488887786865 + }, + { + "bits_per_byte": 2.5159042744907776, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743891954421997, + "step": 750, + "unclipped_grad_norm": 1.466252088546753 + }, + { + "bits_per_byte": 2.5144026939660944, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7428511381149292, + "step": 760, + "unclipped_grad_norm": 1.454863429069519 + }, + { + "bits_per_byte": 2.485389736713491, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.722740888595581, + "step": 770, + "unclipped_grad_norm": 1.2171542644500732 + }, + { + "bits_per_byte": 2.4154798210336645, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.6742830276489258, + "step": 780, + "unclipped_grad_norm": 1.3978826999664307 + }, + { + "bits_per_byte": 2.492438617641754, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7276268005371094, + "step": 790, + "unclipped_grad_norm": 1.4353379011154175 + }, + { + "bits_per_byte": 2.4424274386320564, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6929616928100586, + "step": 800, + "unclipped_grad_norm": 1.3198370933532715 + }, + { + "bits_per_byte": 2.473405641636069, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7144341468811035, + "step": 810, + "unclipped_grad_norm": 1.3822002410888672 + }, + { + "bits_per_byte": 2.441873482513627, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6925777196884155, + "step": 820, + "unclipped_grad_norm": 1.3033113479614258 + }, + { + "bits_per_byte": 2.4253691674235776, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6811378002166748, + "step": 830, + "unclipped_grad_norm": 1.450317621231079 + }, + { + "bits_per_byte": 2.4107564895103684, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6710090637207031, + "step": 840, + "unclipped_grad_norm": 1.322706699371338 + }, + { + "bits_per_byte": 2.347782118191886, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6273585557937622, + "step": 850, + "unclipped_grad_norm": 1.2753863334655762 + }, + { + "bits_per_byte": 2.3967822111971615, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.661322832107544, + "step": 860, + "unclipped_grad_norm": 2.4916832447052 + }, + { + "bits_per_byte": 2.3462526764777634, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.626298427581787, + "step": 870, + "unclipped_grad_norm": 1.3712787628173828 + }, + { + "bits_per_byte": 2.4080783756711326, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6691527366638184, + "step": 880, + "unclipped_grad_norm": 1.4296884536743164 + }, + { + "bits_per_byte": 2.363901364126654, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6385315656661987, + "step": 890, + "unclipped_grad_norm": 1.3781569004058838 + }, + { + "bits_per_byte": 2.2957977822288393, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.5913257598876953, + "step": 900, + "unclipped_grad_norm": 1.3345599174499512 + }, + { + "bits_per_byte": 2.3445476404771095, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6251165866851807, + "step": 910, + "unclipped_grad_norm": 1.6203302145004272 + }, + { + "bits_per_byte": 2.297230053745225, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5923185348510742, + "step": 920, + "unclipped_grad_norm": 1.685173511505127 + }, + { + "bits_per_byte": 2.297947909329926, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5928161144256592, + "step": 930, + "unclipped_grad_norm": 1.4890943765640259 + }, + { + "bits_per_byte": 2.2609902135207154, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5671989917755127, + "step": 940, + "unclipped_grad_norm": 1.3654035329818726 + }, + { + "bits_per_byte": 2.2430326450308753, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5547517538070679, + "step": 950, + "unclipped_grad_norm": 1.1998120546340942 + }, + { + "bits_per_byte": 2.2479709548677502, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.558174729347229, + "step": 960, + "unclipped_grad_norm": 1.40827214717865 + }, + { + "bits_per_byte": 2.2631277858882894, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5686806440353394, + "step": 970, + "unclipped_grad_norm": 1.4645981788635254 + }, + { + "bits_per_byte": 2.2528444272452495, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5615527629852295, + "step": 980, + "unclipped_grad_norm": 1.594014048576355 + }, + { + "bits_per_byte": 2.177915369882643, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5096158981323242, + "step": 990, + "unclipped_grad_norm": 1.2862826585769653 + }, + { + "bits_per_byte": 2.2127330855671676, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5337496995925903, + "step": 1000, + "unclipped_grad_norm": 1.4609711170196533 + }, + { + "bits_per_byte": 2.208716086790965, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5309653282165527, + "step": 1010, + "unclipped_grad_norm": 1.2298572063446045 + }, + { + "bits_per_byte": 2.233356385145381, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5480446815490723, + "step": 1020, + "unclipped_grad_norm": 1.469051718711853 + }, + { + "bits_per_byte": 2.1593615375422615, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4967553615570068, + "step": 1030, + "unclipped_grad_norm": 1.3269134759902954 + }, + { + "bits_per_byte": 2.236164345886018, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5499910116195679, + "step": 1040, + "unclipped_grad_norm": 1.3589463233947754 + }, + { + "bits_per_byte": 2.2039167389359875, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5276386737823486, + "step": 1050, + "unclipped_grad_norm": 1.4478572607040405 + }, + { + "bits_per_byte": 2.1642855728191144, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5001684427261353, + "step": 1060, + "unclipped_grad_norm": 1.4022979736328125 + }, + { + "bits_per_byte": 2.2672321518511303, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5715255737304688, + "step": 1070, + "unclipped_grad_norm": 1.4223567247390747 + }, + { + "bits_per_byte": 2.187999916581285, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5166059732437134, + "step": 1080, + "unclipped_grad_norm": 1.188625454902649 + }, + { + "bits_per_byte": 2.1147346193831256, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.4658223390579224, + "step": 1090, + "unclipped_grad_norm": 1.1601507663726807 + }, + { + "bits_per_byte": 2.1969277079702247, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.522794246673584, + "step": 1100, + "unclipped_grad_norm": 1.1728529930114746 + }, + { + "bits_per_byte": 2.2198211785398274, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5386627912521362, + "step": 1110, + "unclipped_grad_norm": 1.3392254114151 + }, + { + "bits_per_byte": 2.126032503683605, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4736534357070923, + "step": 1120, + "unclipped_grad_norm": 1.3545775413513184 + }, + { + "bits_per_byte": 2.1170070261489755, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4673974514007568, + "step": 1130, + "unclipped_grad_norm": 1.2578480243682861 + }, + { + "bits_per_byte": 2.127665650936198, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4747854471206665, + "step": 1140, + "unclipped_grad_norm": 1.6450976133346558 + }, + { + "bits_per_byte": 2.1180404698980104, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4681137800216675, + "step": 1150, + "unclipped_grad_norm": 1.3040683269500732 + }, + { + "bits_per_byte": 2.0764574726764953, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4392906427383423, + "step": 1160, + "unclipped_grad_norm": 1.3104010820388794 + }, + { + "bits_per_byte": 2.1839928927988326, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5138285160064697, + "step": 1170, + "unclipped_grad_norm": 1.3359254598617554 + }, + { + "bits_per_byte": 2.091760488950268, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4498978853225708, + "step": 1180, + "unclipped_grad_norm": 1.3903101682662964 + }, + { + "bits_per_byte": 2.1040587963308166, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4584224224090576, + "step": 1190, + "unclipped_grad_norm": 1.4786474704742432 + }, + { + "bits_per_byte": 2.081396986391926, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4427144527435303, + "step": 1200, + "unclipped_grad_norm": 1.1696821451187134 + }, + { + "bits_per_byte": 2.2277741722636764, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.544175386428833, + "step": 1210, + "unclipped_grad_norm": 1.6089588403701782 + }, + { + "bits_per_byte": 2.071474963298349, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4358370304107666, + "step": 1220, + "unclipped_grad_norm": 1.219665765762329 + }, + { + "bits_per_byte": 2.020156544159443, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4002658128738403, + "step": 1230, + "unclipped_grad_norm": 1.1330406665802002 + }, + { + "bits_per_byte": 2.0202031514578267, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4002981185913086, + "step": 1240, + "unclipped_grad_norm": 1.4843499660491943 + }, + { + "bits_per_byte": 2.1912978558942307, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5188919305801392, + "step": 1250, + "unclipped_grad_norm": 1.2303693294525146 + }, + { + "bits_per_byte": 2.124901717754182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.472869634628296, + "step": 1260, + "unclipped_grad_norm": 1.2569689750671387 + }, + { + "bits_per_byte": 2.0056606424661694, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3902180194854736, + "step": 1270, + "unclipped_grad_norm": 1.3274110555648804 + }, + { + "bits_per_byte": 2.135876962601643, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4804770946502686, + "step": 1280, + "unclipped_grad_norm": 1.203521966934204 + }, + { + "bits_per_byte": 2.0353620462701763, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4108054637908936, + "step": 1290, + "unclipped_grad_norm": 1.1280517578125 + }, + { + "bits_per_byte": 2.0816712987200523, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4429045915603638, + "step": 1300, + "unclipped_grad_norm": 1.2584275007247925 + }, + { + "bits_per_byte": 2.144767433755345, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4866394996643066, + "step": 1310, + "unclipped_grad_norm": 1.2562772035598755 + }, + { + "bits_per_byte": 2.0818640912716693, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4430382251739502, + "step": 1320, + "unclipped_grad_norm": 1.1640011072158813 + }, + { + "bits_per_byte": 1.9798470784675668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3723254203796387, + "step": 1330, + "unclipped_grad_norm": 1.20253324508667 + }, + { + "bits_per_byte": 2.0020558861040847, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3877193927764893, + "step": 1340, + "unclipped_grad_norm": 1.1452504396438599 + }, + { + "bits_per_byte": 2.176502532405805, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.5086365938186646, + "step": 1350, + "unclipped_grad_norm": 1.2560323476791382 + }, + { + "bits_per_byte": 2.009120589436227, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3926162719726562, + "step": 1360, + "unclipped_grad_norm": 1.1015381813049316 + }, + { + "bits_per_byte": 2.0924102394052277, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4503482580184937, + "step": 1370, + "unclipped_grad_norm": 1.1648008823394775 + }, + { + "bits_per_byte": 2.0633229856474515, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4301865100860596, + "step": 1380, + "unclipped_grad_norm": 1.124997854232788 + }, + { + "bits_per_byte": 2.0840274610368725, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4445377588272095, + "step": 1390, + "unclipped_grad_norm": 1.2330824136734009 + }, + { + "bits_per_byte": 2.084845582507027, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4451048374176025, + "step": 1400, + "unclipped_grad_norm": 1.2093770503997803 + }, + { + "bits_per_byte": 2.0117663705371007, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.3944501876831055, + "step": 1410, + "unclipped_grad_norm": 1.1717233657836914 + }, + { + "bits_per_byte": 2.0551273244032355, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4245057106018066, + "step": 1420, + "unclipped_grad_norm": 1.118058204650879 + }, + { + "bits_per_byte": 2.04396186674392, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4167664051055908, + "step": 1430, + "unclipped_grad_norm": 1.282050371170044 + }, + { + "bits_per_byte": 2.01288047414939, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3952224254608154, + "step": 1440, + "unclipped_grad_norm": 1.2009295225143433 + }, + { + "bits_per_byte": 2.0606357567277205, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4283238649368286, + "step": 1450, + "unclipped_grad_norm": 1.1897740364074707 + }, + { + "bits_per_byte": 2.0059322030718816, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3904062509536743, + "step": 1460, + "unclipped_grad_norm": 1.0764319896697998 + }, + { + "bits_per_byte": 2.00337052148728, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3886306285858154, + "step": 1470, + "unclipped_grad_norm": 1.102418065071106 + }, + { + "bits_per_byte": 1.9153828214446602, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3276422023773193, + "step": 1480, + "unclipped_grad_norm": 1.2223329544067383 + }, + { + "bits_per_byte": 2.0269326606034372, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4049626588821411, + "step": 1490, + "unclipped_grad_norm": 1.1502348184585571 + }, + { + "bits_per_byte": 1.9137665284918515, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.326521873474121, + "step": 1500, + "unclipped_grad_norm": 1.104032039642334 + }, + { + "bits_per_byte": 1.9998878728073082, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.386216640472412, + "step": 1510, + "unclipped_grad_norm": 1.2259501218795776 + }, + { + "bits_per_byte": 1.9900984483375461, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.379431128501892, + "step": 1520, + "unclipped_grad_norm": 1.192402958869934 + }, + { + "bits_per_byte": 1.9577958069392392, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3570406436920166, + "step": 1530, + "unclipped_grad_norm": 1.1483442783355713 + }, + { + "bits_per_byte": 2.0275844748502077, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4054144620895386, + "step": 1540, + "unclipped_grad_norm": 1.132032871246338 + }, + { + "bits_per_byte": 2.0123937632474482, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3948850631713867, + "step": 1550, + "unclipped_grad_norm": 1.1288913488388062 + }, + { + "bits_per_byte": 2.071048790289511, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4355416297912598, + "step": 1560, + "unclipped_grad_norm": 1.146075963973999 + }, + { + "bits_per_byte": 2.0218080935556784, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4014105796813965, + "step": 1570, + "unclipped_grad_norm": 1.1695427894592285 + }, + { + "bits_per_byte": 2.0099278759993773, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3931758403778076, + "step": 1580, + "unclipped_grad_norm": 1.1285966634750366 + }, + { + "bits_per_byte": 1.999800161655368, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3861558437347412, + "step": 1590, + "unclipped_grad_norm": 1.136389136314392 + }, + { + "bits_per_byte": 2.019440580383901, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3997695446014404, + "step": 1600, + "unclipped_grad_norm": 1.4346710443496704 + }, + { + "bits_per_byte": 2.0397723693689023, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4138624668121338, + "step": 1610, + "unclipped_grad_norm": 1.3147592544555664 + }, + { + "bits_per_byte": 2.0494484572717453, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4205694198608398, + "step": 1620, + "unclipped_grad_norm": 1.2143282890319824 + }, + { + "bits_per_byte": 1.921803965697274, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3320930004119873, + "step": 1630, + "unclipped_grad_norm": 1.3924999237060547 + }, + { + "bits_per_byte": 2.0487386848716347, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4200774431228638, + "step": 1640, + "unclipped_grad_norm": 1.1033058166503906 + }, + { + "bits_per_byte": 1.9803865880433233, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3726993799209595, + "step": 1650, + "unclipped_grad_norm": 1.1198365688323975 + }, + { + "bits_per_byte": 1.9604881953384965, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.358906865119934, + "step": 1660, + "unclipped_grad_norm": 1.0561511516571045 + }, + { + "bits_per_byte": 2.1180337625746266, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.468109130859375, + "step": 1670, + "unclipped_grad_norm": 1.1442371606826782 + }, + { + "bits_per_byte": 2.035602821981384, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4109723567962646, + "step": 1680, + "unclipped_grad_norm": 1.0968410968780518 + }, + { + "bits_per_byte": 1.9074798746722124, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3221642971038818, + "step": 1690, + "unclipped_grad_norm": 1.049187183380127 + }, + { + "bits_per_byte": 1.9460256582794904, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3488821983337402, + "step": 1700, + "unclipped_grad_norm": 1.1202492713928223 + }, + { + "bits_per_byte": 2.030805537918217, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.4076471328735352, + "step": 1710, + "unclipped_grad_norm": 1.272226095199585 + }, + { + "bits_per_byte": 2.0120939974869945, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3946772813796997, + "step": 1720, + "unclipped_grad_norm": 1.1422768831253052 + }, + { + "bits_per_byte": 1.9049754633103482, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3204283714294434, + "step": 1730, + "unclipped_grad_norm": 1.1759874820709229 + }, + { + "bits_per_byte": 1.9628171843964806, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3605211973190308, + "step": 1740, + "unclipped_grad_norm": 1.0964117050170898 + }, + { + "bits_per_byte": 2.0712307479341234, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4356677532196045, + "step": 1750, + "unclipped_grad_norm": 1.1146889925003052 + }, + { + "bits_per_byte": 1.9217773083863903, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.332074522972107, + "step": 1760, + "unclipped_grad_norm": 1.044190764427185 + }, + { + "bits_per_byte": 1.9951038314082574, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.382900595664978, + "step": 1770, + "unclipped_grad_norm": 1.0358567237854004 + }, + { + "bits_per_byte": 1.9346029145744843, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3409645557403564, + "step": 1780, + "unclipped_grad_norm": 1.1725130081176758 + }, + { + "bits_per_byte": 1.9131714685198662, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3261094093322754, + "step": 1790, + "unclipped_grad_norm": 1.2142103910446167 + }, + { + "bits_per_byte": 1.9953318804033016, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3830586671829224, + "step": 1800, + "unclipped_grad_norm": 1.2420480251312256 + }, + { + "bits_per_byte": 1.9723242133535257, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3671109676361084, + "step": 1810, + "unclipped_grad_norm": 1.1430673599243164 + }, + { + "bits_per_byte": 2.0342909383206025, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4100630283355713, + "step": 1820, + "unclipped_grad_norm": 1.1687225103378296 + }, + { + "bits_per_byte": 1.8908906001526369, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.310665488243103, + "step": 1830, + "unclipped_grad_norm": 1.146242380142212 + }, + { + "bits_per_byte": 1.978836336428446, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3716248273849487, + "step": 1840, + "unclipped_grad_norm": 1.0820521116256714 + }, + { + "bits_per_byte": 1.8676517884201014, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2945575714111328, + "step": 1850, + "unclipped_grad_norm": 1.0944465398788452 + }, + { + "bits_per_byte": 2.0069171477133727, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3910889625549316, + "step": 1860, + "unclipped_grad_norm": 1.2028480768203735 + }, + { + "bits_per_byte": 2.0278259384920188, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4055818319320679, + "step": 1870, + "unclipped_grad_norm": 1.1793227195739746 + }, + { + "bits_per_byte": 1.8922862393643884, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3116328716278076, + "step": 1880, + "unclipped_grad_norm": 1.123720645904541 + }, + { + "bits_per_byte": 1.9751340659033223, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.369058609008789, + "step": 1890, + "unclipped_grad_norm": 1.006156086921692 + }, + { + "bits_per_byte": 1.8741764662285347, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2990801334381104, + "step": 1900, + "unclipped_grad_norm": 1.0260387659072876 + }, + { + "bits_per_byte": 1.8761183223394267, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3004261255264282, + "step": 1910, + "unclipped_grad_norm": 1.11003839969635 + }, + { + "bits_per_byte": 1.9401261374069434, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3447929620742798, + "step": 1920, + "unclipped_grad_norm": 1.1032140254974365 + }, + { + "bits_per_byte": 1.9031926911515042, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.31919264793396, + "step": 1930, + "unclipped_grad_norm": 1.1424028873443604 + }, + { + "bits_per_byte": 1.8781860697507502, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3018593788146973, + "step": 1940, + "unclipped_grad_norm": 1.0988123416900635 + }, + { + "bits_per_byte": 1.9390512458390508, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3440479040145874, + "step": 1950, + "unclipped_grad_norm": 1.28093421459198 + }, + { + "bits_per_byte": 1.9095305958011006, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3235857486724854, + "step": 1960, + "unclipped_grad_norm": 1.052490234375 + }, + { + "bits_per_byte": 1.8582639434401043, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2880504131317139, + "step": 1970, + "unclipped_grad_norm": 1.0047657489776611 + }, + { + "bits_per_byte": 1.8448327863383223, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.278740644454956, + "step": 1980, + "unclipped_grad_norm": 1.101292610168457 + }, + { + "bits_per_byte": 1.9462698736437154, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3490514755249023, + "step": 1990, + "unclipped_grad_norm": 1.17987859249115 + }, + { + "bits_per_byte": 1.896080176642422, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3142626285552979, + "step": 2000, + "unclipped_grad_norm": 1.1739213466644287 + }, + { + "bits_per_byte": 1.9557370026257606, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3556135892868042, + "step": 2010, + "unclipped_grad_norm": 1.0457618236541748 + }, + { + "bits_per_byte": 1.9291909645171335, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3372132778167725, + "step": 2020, + "unclipped_grad_norm": 1.1008185148239136 + }, + { + "bits_per_byte": 1.9305355248815788, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3381452560424805, + "step": 2030, + "unclipped_grad_norm": 1.1080964803695679 + }, + { + "bits_per_byte": 1.9343679862734056, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.34080171585083, + "step": 2040, + "unclipped_grad_norm": 1.104992389678955 + }, + { + "bits_per_byte": 1.8835350741578853, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3055670261383057, + "step": 2050, + "unclipped_grad_norm": 1.1497511863708496 + }, + { + "bits_per_byte": 1.7856179557677012, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2376960515975952, + "step": 2060, + "unclipped_grad_norm": 1.0708798170089722 + }, + { + "bits_per_byte": 1.8782170266279055, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3018808364868164, + "step": 2070, + "unclipped_grad_norm": 1.0489474534988403 + }, + { + "bits_per_byte": 1.9968320850667778, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3840985298156738, + "step": 2080, + "unclipped_grad_norm": 1.0985883474349976 + }, + { + "bits_per_byte": 1.8512350124993413, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2831783294677734, + "step": 2090, + "unclipped_grad_norm": 0.9845348000526428 + }, + { + "bits_per_byte": 1.9251153196569866, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.334388256072998, + "step": 2100, + "unclipped_grad_norm": 1.009676456451416 + }, + { + "bits_per_byte": 1.8359577936231979, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2725889682769775, + "step": 2110, + "unclipped_grad_norm": 1.0760974884033203 + }, + { + "bits_per_byte": 1.8868116876221235, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3078382015228271, + "step": 2120, + "unclipped_grad_norm": 0.9828676581382751 + }, + { + "bits_per_byte": 1.910013867050025, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.323920726776123, + "step": 2130, + "unclipped_grad_norm": 1.0712695121765137 + }, + { + "bits_per_byte": 1.8022548694815659, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2492278814315796, + "step": 2140, + "unclipped_grad_norm": 1.0451561212539673 + }, + { + "bits_per_byte": 1.8551622363317934, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2859004735946655, + "step": 2150, + "unclipped_grad_norm": 1.0957838296890259 + }, + { + "bits_per_byte": 1.9088151479735114, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.323089838027954, + "step": 2160, + "unclipped_grad_norm": 1.0826293230056763 + }, + { + "bits_per_byte": 1.871177432762789, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2970013618469238, + "step": 2170, + "unclipped_grad_norm": 1.0311518907546997 + }, + { + "bits_per_byte": 1.8055650195627224, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2515223026275635, + "step": 2180, + "unclipped_grad_norm": 1.040400505065918 + }, + { + "bits_per_byte": 1.9270800494604432, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3357501029968262, + "step": 2190, + "unclipped_grad_norm": 1.10823392868042 + }, + { + "bits_per_byte": 1.980437322925328, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.372734546661377, + "step": 2200, + "unclipped_grad_norm": 1.11663019657135 + }, + { + "bits_per_byte": 1.9995734885215308, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3859987258911133, + "step": 2210, + "unclipped_grad_norm": 1.0498319864273071 + }, + { + "bits_per_byte": 1.8802302555389057, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.3032763004302979, + "step": 2220, + "unclipped_grad_norm": 1.0160832405090332 + }, + { + "bits_per_byte": 1.8937652901618087, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3126580715179443, + "step": 2230, + "unclipped_grad_norm": 1.0416548252105713 + }, + { + "bits_per_byte": 1.8469913405893015, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.280236840248108, + "step": 2240, + "unclipped_grad_norm": 1.0407441854476929 + }, + { + "bits_per_byte": 1.8529231942002105, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.284348487854004, + "step": 2250, + "unclipped_grad_norm": 0.9867598414421082 + }, + { + "bits_per_byte": 1.8825788506190881, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3049042224884033, + "step": 2260, + "unclipped_grad_norm": 0.9744513034820557 + }, + { + "bits_per_byte": 1.9268210435882438, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.3355705738067627, + "step": 2270, + "unclipped_grad_norm": 1.0475958585739136 + }, + { + "bits_per_byte": 1.8280555347813536, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2671115398406982, + "step": 2280, + "unclipped_grad_norm": 1.0294238328933716 + }, + { + "bits_per_byte": 1.8388550133596329, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.27459716796875, + "step": 2290, + "unclipped_grad_norm": 1.0671623945236206 + }, + { + "bits_per_byte": 1.8867002428643644, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3077609539031982, + "step": 2300, + "unclipped_grad_norm": 0.9974833726882935 + }, + { + "bits_per_byte": 1.8601865374940996, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.289383053779602, + "step": 2310, + "unclipped_grad_norm": 1.0144871473312378 + }, + { + "bits_per_byte": 1.874428248829398, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2992546558380127, + "step": 2320, + "unclipped_grad_norm": 1.0221670866012573 + }, + { + "bits_per_byte": 1.8178058847378828, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2600070238113403, + "step": 2330, + "unclipped_grad_norm": 1.0877128839492798 + }, + { + "bits_per_byte": 1.8362632348111303, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2728006839752197, + "step": 2340, + "unclipped_grad_norm": 1.050228238105774 + }, + { + "bits_per_byte": 1.780467247357011, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2341258525848389, + "step": 2350, + "unclipped_grad_norm": 0.9972993731498718 + }, + { + "bits_per_byte": 1.8791837411084054, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.3025509119033813, + "step": 2360, + "unclipped_grad_norm": 1.0690455436706543 + }, + { + "bits_per_byte": 1.8768631792003134, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3009424209594727, + "step": 2370, + "unclipped_grad_norm": 1.0405195951461792 + }, + { + "bits_per_byte": 1.8106722163627444, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2550623416900635, + "step": 2380, + "unclipped_grad_norm": 1.000722050666809 + }, + { + "bits_per_byte": 1.7959209204329396, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2448375225067139, + "step": 2390, + "unclipped_grad_norm": 1.0424110889434814 + }, + { + "bits_per_byte": 1.9177011475782908, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3292491436004639, + "step": 2400, + "unclipped_grad_norm": 1.0584142208099365 + }, + { + "bits_per_byte": 1.8792028311826512, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.3025641441345215, + "step": 2410, + "unclipped_grad_norm": 1.109039068222046 + }, + { + "bits_per_byte": 1.9615544877738458, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.359645962715149, + "step": 2420, + "unclipped_grad_norm": 1.0733245611190796 + }, + { + "bits_per_byte": 1.8745142401548294, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.299314260482788, + "step": 2430, + "unclipped_grad_norm": 1.0562880039215088 + }, + { + "bits_per_byte": 1.7973332419618249, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2458164691925049, + "step": 2440, + "unclipped_grad_norm": 1.0554269552230835 + }, + { + "bits_per_byte": 1.8932479663480133, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3122994899749756, + "step": 2450, + "unclipped_grad_norm": 1.1328672170639038 + }, + { + "bits_per_byte": 1.9065160838967772, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3214962482452393, + "step": 2460, + "unclipped_grad_norm": 1.1163077354431152 + }, + { + "bits_per_byte": 1.8460499075584784, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.279584288597107, + "step": 2470, + "unclipped_grad_norm": 1.0860918760299683 + }, + { + "bits_per_byte": 1.8485799443353212, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2813379764556885, + "step": 2480, + "unclipped_grad_norm": 0.99190354347229 + }, + { + "bits_per_byte": 1.7724624868552525, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2285773754119873, + "step": 2490, + "unclipped_grad_norm": 1.025232195854187 + }, + { + "bits_per_byte": 1.8512375922391042, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2831801176071167, + "step": 2500, + "unclipped_grad_norm": 1.0759034156799316 + }, + { + "bits_per_byte": 1.8193396260182773, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2610701322555542, + "step": 2510, + "unclipped_grad_norm": 1.0067415237426758 + }, + { + "bits_per_byte": 1.8146814759196581, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2578413486480713, + "step": 2520, + "unclipped_grad_norm": 0.990001916885376 + }, + { + "bits_per_byte": 1.8138671380678228, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2572768926620483, + "step": 2530, + "unclipped_grad_norm": 0.9393356442451477 + }, + { + "bits_per_byte": 1.8368911434694304, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2732359170913696, + "step": 2540, + "unclipped_grad_norm": 1.1141552925109863 + }, + { + "bits_per_byte": 1.8603834576293374, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2895195484161377, + "step": 2550, + "unclipped_grad_norm": 1.0569089651107788 + }, + { + "bits_per_byte": 1.816235855118156, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2589187622070312, + "step": 2560, + "unclipped_grad_norm": 0.9373295903205872 + }, + { + "bits_per_byte": 1.8059492288047498, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.25178861618042, + "step": 2570, + "unclipped_grad_norm": 1.143198847770691 + }, + { + "bits_per_byte": 1.8450266107858446, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2788749933242798, + "step": 2580, + "unclipped_grad_norm": 1.057027816772461 + }, + { + "bits_per_byte": 1.8154227211448768, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2583551406860352, + "step": 2590, + "unclipped_grad_norm": 1.0370285511016846 + }, + { + "bits_per_byte": 1.8561616275159571, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2865931987762451, + "step": 2600, + "unclipped_grad_norm": 0.9653204083442688 + }, + { + "bits_per_byte": 1.7605767658541238, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2203388214111328, + "step": 2610, + "unclipped_grad_norm": 1.0564771890640259 + }, + { + "bits_per_byte": 1.812449141111459, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2562940120697021, + "step": 2620, + "unclipped_grad_norm": 1.0267692804336548 + }, + { + "bits_per_byte": 1.8349191903946374, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2718690633773804, + "step": 2630, + "unclipped_grad_norm": 0.9602619409561157 + }, + { + "bits_per_byte": 1.8363758834474455, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2728787660598755, + "step": 2640, + "unclipped_grad_norm": 1.0029752254486084 + }, + { + "bits_per_byte": 1.771385015547597, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2278305292129517, + "step": 2650, + "unclipped_grad_norm": 0.9146384000778198 + }, + { + "bits_per_byte": 1.779632271587072, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2335470914840698, + "step": 2660, + "unclipped_grad_norm": 0.9666176438331604 + }, + { + "bits_per_byte": 1.8136729696549987, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2571423053741455, + "step": 2670, + "unclipped_grad_norm": 1.0540763139724731 + }, + { + "bits_per_byte": 1.7953530337197905, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2444438934326172, + "step": 2680, + "unclipped_grad_norm": 1.0946050882339478 + }, + { + "bits_per_byte": 1.8769845989518226, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3010265827178955, + "step": 2690, + "unclipped_grad_norm": 1.068537712097168 + }, + { + "bits_per_byte": 1.8642107595416384, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2921724319458008, + "step": 2700, + "unclipped_grad_norm": 1.1353117227554321 + }, + { + "bits_per_byte": 1.7364708175959382, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2036298513412476, + "step": 2710, + "unclipped_grad_norm": 0.9047647714614868 + }, + { + "bits_per_byte": 1.8594279220211436, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2888572216033936, + "step": 2720, + "unclipped_grad_norm": 1.0444105863571167 + }, + { + "bits_per_byte": 1.7814004252205926, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2347726821899414, + "step": 2730, + "unclipped_grad_norm": 0.9630917310714722 + }, + { + "bits_per_byte": 1.7840532576101513, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2366114854812622, + "step": 2740, + "unclipped_grad_norm": 1.0223243236541748 + }, + { + "bits_per_byte": 1.8198583256932797, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2614296674728394, + "step": 2750, + "unclipped_grad_norm": 1.0687127113342285 + }, + { + "bits_per_byte": 1.9021738659277931, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3184864521026611, + "step": 2760, + "unclipped_grad_norm": 1.0979900360107422 + }, + { + "bits_per_byte": 1.7971648709466301, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2456997632980347, + "step": 2770, + "unclipped_grad_norm": 0.965108335018158 + }, + { + "bits_per_byte": 1.8086140999798692, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2536357641220093, + "step": 2780, + "unclipped_grad_norm": 1.0402190685272217 + }, + { + "bits_per_byte": 1.7938860217079309, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.243427038192749, + "step": 2790, + "unclipped_grad_norm": 1.0398236513137817 + }, + { + "bits_per_byte": 1.7550451158717724, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2165045738220215, + "step": 2800, + "unclipped_grad_norm": 1.0063706636428833 + }, + { + "bits_per_byte": 1.828329675126829, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2673015594482422, + "step": 2810, + "unclipped_grad_norm": 0.9668177366256714 + }, + { + "bits_per_byte": 1.8254919613875926, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2653346061706543, + "step": 2820, + "unclipped_grad_norm": 0.9527478814125061 + }, + { + "bits_per_byte": 1.7516809632382448, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2141727209091187, + "step": 2830, + "unclipped_grad_norm": 0.9219958186149597 + }, + { + "bits_per_byte": 1.8254823303591443, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2653279304504395, + "step": 2840, + "unclipped_grad_norm": 1.0127615928649902 + }, + { + "bits_per_byte": 1.7530225998976257, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2151026725769043, + "step": 2850, + "unclipped_grad_norm": 1.0182650089263916 + }, + { + "bits_per_byte": 1.7287747659524784, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1982953548431396, + "step": 2860, + "unclipped_grad_norm": 0.978410542011261 + }, + { + "bits_per_byte": 1.855343334063152, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2860260009765625, + "step": 2870, + "unclipped_grad_norm": 1.018862009048462 + }, + { + "bits_per_byte": 1.768852571013642, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2260751724243164, + "step": 2880, + "unclipped_grad_norm": 0.9639438390731812 + }, + { + "bits_per_byte": 1.8105554401428086, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2549813985824585, + "step": 2890, + "unclipped_grad_norm": 0.9698073267936707 + }, + { + "bits_per_byte": 1.7403103302764504, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2062911987304688, + "step": 2900, + "unclipped_grad_norm": 0.9920547604560852 + }, + { + "bits_per_byte": 1.7643693272709504, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2229676246643066, + "step": 2910, + "unclipped_grad_norm": 1.0128531455993652 + }, + { + "bits_per_byte": 1.8473500963990013, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2804855108261108, + "step": 2920, + "unclipped_grad_norm": 1.0619620084762573 + }, + { + "bits_per_byte": 1.8003603085996611, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2479146718978882, + "step": 2930, + "unclipped_grad_norm": 1.0764249563217163 + }, + { + "bits_per_byte": 1.8426398355571707, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.277220606803894, + "step": 2940, + "unclipped_grad_norm": 1.0425148010253906 + }, + { + "bits_per_byte": 1.7862247105599451, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2381166219711304, + "step": 2950, + "unclipped_grad_norm": 1.004768967628479 + }, + { + "bits_per_byte": 1.7811238771180051, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2345809936523438, + "step": 2960, + "unclipped_grad_norm": 0.9997458457946777 + }, + { + "bits_per_byte": 1.8373286673332254, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2735391855239868, + "step": 2970, + "unclipped_grad_norm": 1.039349913597107 + }, + { + "bits_per_byte": 1.7631166056420657, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2220993041992188, + "step": 2980, + "unclipped_grad_norm": 0.9808847904205322 + }, + { + "bits_per_byte": 1.9032267437163752, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.319216251373291, + "step": 2990, + "unclipped_grad_norm": 1.1331971883773804 + }, + { + "bits_per_byte": 1.81958332543455, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2612390518188477, + "step": 3000, + "unclipped_grad_norm": 0.944084107875824 + }, + { + "bits_per_byte": 1.7894560925870064, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2403564453125, + "step": 3010, + "unclipped_grad_norm": 1.057902216911316 + }, + { + "bits_per_byte": 1.7654951257034983, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.223747968673706, + "step": 3020, + "unclipped_grad_norm": 0.971477746963501 + }, + { + "bits_per_byte": 1.791092679492617, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2414908409118652, + "step": 3030, + "unclipped_grad_norm": 1.017361044883728 + }, + { + "bits_per_byte": 1.760655361925568, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2203933000564575, + "step": 3040, + "unclipped_grad_norm": 0.9738672971725464 + }, + { + "bits_per_byte": 1.7859287284178103, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2379114627838135, + "step": 3050, + "unclipped_grad_norm": 1.0311107635498047 + }, + { + "bits_per_byte": 1.7871233199107035, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2387394905090332, + "step": 3060, + "unclipped_grad_norm": 1.097463607788086 + }, + { + "bits_per_byte": 1.7530494291911602, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.2151212692260742, + "step": 3070, + "unclipped_grad_norm": 0.95722895860672 + }, + { + "bits_per_byte": 1.7753578147825277, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2305842638015747, + "step": 3080, + "unclipped_grad_norm": 1.0311440229415894 + }, + { + "bits_per_byte": 1.7406597990230035, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.206533432006836, + "step": 3090, + "unclipped_grad_norm": 0.9659063220024109 + }, + { + "bits_per_byte": 1.8326863396384856, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2703213691711426, + "step": 3100, + "unclipped_grad_norm": 1.0265647172927856 + }, + { + "bits_per_byte": 1.8128345542320425, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2565611600875854, + "step": 3110, + "unclipped_grad_norm": 1.0288156270980835 + }, + { + "bits_per_byte": 1.6999466900495526, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1783132553100586, + "step": 3120, + "unclipped_grad_norm": 0.9754294753074646 + }, + { + "bits_per_byte": 1.7980240962703409, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2462953329086304, + "step": 3130, + "unclipped_grad_norm": 1.0256381034851074 + }, + { + "bits_per_byte": 1.7337763654048706, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2017621994018555, + "step": 3140, + "unclipped_grad_norm": 0.9683999419212341 + }, + { + "bits_per_byte": 1.7391683654747212, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2054996490478516, + "step": 3150, + "unclipped_grad_norm": 0.9884417057037354 + }, + { + "bits_per_byte": 1.7859572775378536, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.237931251525879, + "step": 3160, + "unclipped_grad_norm": 1.0163674354553223 + }, + { + "bits_per_byte": 1.8529599984874952, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2843739986419678, + "step": 3170, + "unclipped_grad_norm": 1.0605453252792358 + }, + { + "bits_per_byte": 1.7071207743476438, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1832859516143799, + "step": 3180, + "unclipped_grad_norm": 1.0476000308990479 + }, + { + "bits_per_byte": 1.8011364663030052, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2484526634216309, + "step": 3190, + "unclipped_grad_norm": 0.9400752782821655 + }, + { + "bits_per_byte": 1.7046130953154133, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.18154776096344, + "step": 3200, + "unclipped_grad_norm": 0.9428834915161133 + }, + { + "bits_per_byte": 1.7343896555378473, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2021872997283936, + "step": 3210, + "unclipped_grad_norm": 0.982049822807312 + }, + { + "bits_per_byte": 1.7352428614687776, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.202778697013855, + "step": 3220, + "unclipped_grad_norm": 0.9146644473075867 + }, + { + "bits_per_byte": 1.7184315573469382, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1911259889602661, + "step": 3230, + "unclipped_grad_norm": 0.9842981696128845 + }, + { + "bits_per_byte": 1.7760342225483712, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.231053113937378, + "step": 3240, + "unclipped_grad_norm": 1.0283819437026978 + }, + { + "bits_per_byte": 1.780553754630395, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.234185814857483, + "step": 3250, + "unclipped_grad_norm": 0.9948496222496033 + }, + { + "bits_per_byte": 1.8669033199235465, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2940387725830078, + "step": 3260, + "unclipped_grad_norm": 1.0557905435562134 + }, + { + "bits_per_byte": 1.7253210103578513, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1959013938903809, + "step": 3270, + "unclipped_grad_norm": 1.0562386512756348 + }, + { + "bits_per_byte": 1.6636741731213793, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1531710624694824, + "step": 3280, + "unclipped_grad_norm": 0.9674620628356934 + }, + { + "bits_per_byte": 1.7215127985197962, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.193261742591858, + "step": 3290, + "unclipped_grad_norm": 1.081762671470642 + }, + { + "bits_per_byte": 1.784075099406811, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2366266250610352, + "step": 3300, + "unclipped_grad_norm": 0.9871881604194641 + }, + { + "bits_per_byte": 1.7790666206483843, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2331550121307373, + "step": 3310, + "unclipped_grad_norm": 1.018639326095581 + }, + { + "bits_per_byte": 1.7964894950766919, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2452316284179688, + "step": 3320, + "unclipped_grad_norm": 1.0376429557800293 + }, + { + "bits_per_byte": 1.7442502808250664, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2090221643447876, + "step": 3330, + "unclipped_grad_norm": 0.9751812815666199 + }, + { + "bits_per_byte": 1.740043757167613, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.206106424331665, + "step": 3340, + "unclipped_grad_norm": 1.0885974168777466 + }, + { + "bits_per_byte": 1.7459623681144056, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2102088928222656, + "step": 3350, + "unclipped_grad_norm": 1.028318166732788 + }, + { + "bits_per_byte": 1.7533509147781228, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2153302431106567, + "step": 3360, + "unclipped_grad_norm": 1.038756251335144 + }, + { + "bits_per_byte": 1.6755667734285427, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.161414384841919, + "step": 3370, + "unclipped_grad_norm": 1.0451469421386719 + }, + { + "bits_per_byte": 1.794013632834871, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2435154914855957, + "step": 3380, + "unclipped_grad_norm": 1.1703273057937622 + }, + { + "bits_per_byte": 1.7554886591283476, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.216812014579773, + "step": 3390, + "unclipped_grad_norm": 1.036725401878357 + }, + { + "bits_per_byte": 1.7445490146896152, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2092292308807373, + "step": 3400, + "unclipped_grad_norm": 1.0196164846420288 + }, + { + "bits_per_byte": 1.794997717563108, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2441976070404053, + "step": 3410, + "unclipped_grad_norm": 1.202675700187683 + }, + { + "bits_per_byte": 1.7549782146205868, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.2164582014083862, + "step": 3420, + "unclipped_grad_norm": 0.9739375710487366 + }, + { + "bits_per_byte": 1.7093242160704982, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1848132610321045, + "step": 3430, + "unclipped_grad_norm": 1.0012179613113403 + }, + { + "bits_per_byte": 1.7178566193451037, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1907274723052979, + "step": 3440, + "unclipped_grad_norm": 1.0250310897827148 + }, + { + "bits_per_byte": 1.7050728049411694, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1818664073944092, + "step": 3450, + "unclipped_grad_norm": 1.0344855785369873 + }, + { + "bits_per_byte": 1.706558735044624, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.182896375656128, + "step": 3460, + "unclipped_grad_norm": 0.9980728626251221 + }, + { + "bits_per_byte": 1.7653320861504804, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.223634958267212, + "step": 3470, + "unclipped_grad_norm": 1.0550833940505981 + }, + { + "bits_per_byte": 1.8114688400015408, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2556145191192627, + "step": 3480, + "unclipped_grad_norm": 1.0545322895050049 + }, + { + "bits_per_byte": 1.740959048835505, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2067408561706543, + "step": 3490, + "unclipped_grad_norm": 0.9756637811660767 + }, + { + "bits_per_byte": 1.704705106033625, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1816115379333496, + "step": 3500, + "unclipped_grad_norm": 0.9728481769561768 + }, + { + "bits_per_byte": 1.7154590092094253, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.1890655755996704, + "step": 3510, + "unclipped_grad_norm": 0.9984562993049622 + }, + { + "bits_per_byte": 1.7629102264610303, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2219562530517578, + "step": 3520, + "unclipped_grad_norm": 1.0547406673431396 + }, + { + "bits_per_byte": 1.7070648799861134, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1832472085952759, + "step": 3530, + "unclipped_grad_norm": 0.9733150601387024 + }, + { + "bits_per_byte": 1.7245068444886669, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1953370571136475, + "step": 3540, + "unclipped_grad_norm": 1.0125547647476196 + }, + { + "bits_per_byte": 1.7698511022845516, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2267673015594482, + "step": 3550, + "unclipped_grad_norm": 1.0299363136291504 + }, + { + "bits_per_byte": 1.7181577609667646, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1909362077713013, + "step": 3560, + "unclipped_grad_norm": 0.9789594411849976 + }, + { + "bits_per_byte": 1.7459088815099875, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2101718187332153, + "step": 3570, + "unclipped_grad_norm": 1.0010499954223633 + }, + { + "bits_per_byte": 1.7705113436812139, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2272249460220337, + "step": 3580, + "unclipped_grad_norm": 1.082968831062317 + }, + { + "bits_per_byte": 1.7425258107848651, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.207826852798462, + "step": 3590, + "unclipped_grad_norm": 1.0055863857269287 + }, + { + "bits_per_byte": 1.7755588625013865, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2307236194610596, + "step": 3600, + "unclipped_grad_norm": 1.0473418235778809 + }, + { + "bits_per_byte": 1.7693408297594415, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.226413607597351, + "step": 3610, + "unclipped_grad_norm": 1.1090856790542603 + }, + { + "bits_per_byte": 1.7112244523798812, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.186130404472351, + "step": 3620, + "unclipped_grad_norm": 0.9742531776428223 + }, + { + "bits_per_byte": 1.7137644641504741, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1878910064697266, + "step": 3630, + "unclipped_grad_norm": 1.0429282188415527 + }, + { + "bits_per_byte": 1.6520834023664803, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1451369524002075, + "step": 3640, + "unclipped_grad_norm": 0.9745917320251465 + }, + { + "bits_per_byte": 1.7256807980634563, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.196150779724121, + "step": 3650, + "unclipped_grad_norm": 1.0385748147964478 + }, + { + "bits_per_byte": 1.7126971398192194, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1871511936187744, + "step": 3660, + "unclipped_grad_norm": 0.9310101270675659 + }, + { + "bits_per_byte": 1.7674923602279682, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.2251323461532593, + "step": 3670, + "unclipped_grad_norm": 1.0780647993087769 + }, + { + "bits_per_byte": 1.6993364956042913, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1778903007507324, + "step": 3680, + "unclipped_grad_norm": 0.9666725397109985 + }, + { + "bits_per_byte": 1.7697671747509305, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.2267091274261475, + "step": 3690, + "unclipped_grad_norm": 1.0202223062515259 + }, + { + "bits_per_byte": 1.6641650116069417, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1535112857818604, + "step": 3700, + "unclipped_grad_norm": 1.0115612745285034 + }, + { + "bits_per_byte": 1.726182815421325, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1964987516403198, + "step": 3710, + "unclipped_grad_norm": 0.9808560013771057 + }, + { + "bits_per_byte": 1.7616059100368868, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.2210521697998047, + "step": 3720, + "unclipped_grad_norm": 1.0511683225631714 + }, + { + "bits_per_byte": 1.7329405297216773, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2011828422546387, + "step": 3730, + "unclipped_grad_norm": 1.0690499544143677 + }, + { + "bits_per_byte": 1.7000292417219667, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.178370475769043, + "step": 3740, + "unclipped_grad_norm": 0.9935287237167358 + }, + { + "bits_per_byte": 1.7278594742845865, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1976609230041504, + "step": 3750, + "unclipped_grad_norm": 1.010900855064392 + }, + { + "bits_per_byte": 1.743004610484867, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2081587314605713, + "step": 3760, + "unclipped_grad_norm": 1.0119491815567017 + }, + { + "bits_per_byte": 1.751619221466585, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.21412992477417, + "step": 3770, + "unclipped_grad_norm": 1.011305570602417 + }, + { + "bits_per_byte": 1.8012363882231563, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2485219240188599, + "step": 3780, + "unclipped_grad_norm": 1.0745822191238403 + }, + { + "bits_per_byte": 1.698407961272283, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1772466897964478, + "step": 3790, + "unclipped_grad_norm": 1.0157029628753662 + }, + { + "bits_per_byte": 1.7299170747195092, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.199087142944336, + "step": 3800, + "unclipped_grad_norm": 1.0643014907836914 + }, + { + "bits_per_byte": 1.6600432753963636, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1506543159484863, + "step": 3810, + "unclipped_grad_norm": 0.9955329895019531 + }, + { + "bits_per_byte": 1.661928377232471, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1519609689712524, + "step": 3820, + "unclipped_grad_norm": 1.0222501754760742 + }, + { + "bits_per_byte": 1.7482129330835965, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.2117688655853271, + "step": 3830, + "unclipped_grad_norm": 1.0880961418151855 + }, + { + "bits_per_byte": 1.6885180989344173, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.17039155960083, + "step": 3840, + "unclipped_grad_norm": 0.9980700016021729 + }, + { + "bits_per_byte": 1.661674014891845, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1517846584320068, + "step": 3850, + "unclipped_grad_norm": 0.961553156375885 + }, + { + "bits_per_byte": 1.6409110654011305, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1373928785324097, + "step": 3860, + "unclipped_grad_norm": 1.019968032836914 + }, + { + "bits_per_byte": 1.6069470715954888, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1138508319854736, + "step": 3870, + "unclipped_grad_norm": 0.9610836505889893 + }, + { + "bits_per_byte": 1.7034871249002144, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.180767297744751, + "step": 3880, + "unclipped_grad_norm": 0.9531551003456116 + }, + { + "bits_per_byte": 1.6934515932570677, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1738111972808838, + "step": 3890, + "unclipped_grad_norm": 1.0675544738769531 + }, + { + "bits_per_byte": 1.6830007235120874, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1665672063827515, + "step": 3900, + "unclipped_grad_norm": 1.0058032274246216 + }, + { + "bits_per_byte": 1.7034874688655162, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.18076753616333, + "step": 3910, + "unclipped_grad_norm": 0.9943428039550781 + }, + { + "bits_per_byte": 1.743468619676895, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.2084803581237793, + "step": 3920, + "unclipped_grad_norm": 1.2279266119003296 + }, + { + "bits_per_byte": 1.721944302990811, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1935608386993408, + "step": 3930, + "unclipped_grad_norm": 1.008062720298767 + }, + { + "bits_per_byte": 1.6512047430032224, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1445279121398926, + "step": 3940, + "unclipped_grad_norm": 0.9797955751419067 + }, + { + "bits_per_byte": 1.750776850442659, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2135460376739502, + "step": 3950, + "unclipped_grad_norm": 1.0508832931518555 + }, + { + "bits_per_byte": 1.7346926889686676, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.202397346496582, + "step": 3960, + "unclipped_grad_norm": 1.0876473188400269 + }, + { + "bits_per_byte": 1.6890646597988592, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1707704067230225, + "step": 3970, + "unclipped_grad_norm": 1.0190049409866333 + }, + { + "bits_per_byte": 1.6854375456921626, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1682562828063965, + "step": 3980, + "unclipped_grad_norm": 1.0094109773635864 + }, + { + "bits_per_byte": 1.710786068602832, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1858265399932861, + "step": 3990, + "unclipped_grad_norm": 1.082020878791809 + }, + { + "bits_per_byte": 1.6440261871562087, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.139552116394043, + "step": 4000, + "unclipped_grad_norm": 0.9749544262886047 + }, + { + "bits_per_byte": 1.706837174956371, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1830893754959106, + "step": 4010, + "unclipped_grad_norm": 1.1478910446166992 + }, + { + "bits_per_byte": 1.7296654641012967, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1989127397537231, + "step": 4020, + "unclipped_grad_norm": 1.0259206295013428 + }, + { + "bits_per_byte": 1.6863480218458304, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1688873767852783, + "step": 4030, + "unclipped_grad_norm": 1.0554102659225464 + }, + { + "bits_per_byte": 1.6602591136231966, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1508039236068726, + "step": 4040, + "unclipped_grad_norm": 1.00386643409729 + }, + { + "bits_per_byte": 1.6813582891963477, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1654287576675415, + "step": 4050, + "unclipped_grad_norm": 0.9927709698677063 + }, + { + "bits_per_byte": 1.6720621109692604, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1589851379394531, + "step": 4060, + "unclipped_grad_norm": 1.0185434818267822 + }, + { + "bits_per_byte": 1.7963911210003984, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2451634407043457, + "step": 4070, + "unclipped_grad_norm": 1.0195153951644897 + }, + { + "bits_per_byte": 1.6278679011596953, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1283520460128784, + "step": 4080, + "unclipped_grad_norm": 0.9588671326637268 + }, + { + "bits_per_byte": 1.6697190193339053, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1573610305786133, + "step": 4090, + "unclipped_grad_norm": 1.0382872819900513 + }, + { + "bits_per_byte": 1.786499366853373, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.238306999206543, + "step": 4100, + "unclipped_grad_norm": 1.0833542346954346 + }, + { + "bits_per_byte": 1.6685632959201073, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.156559944152832, + "step": 4110, + "unclipped_grad_norm": 1.0042530298233032 + }, + { + "bits_per_byte": 1.6813281922324468, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1654078960418701, + "step": 4120, + "unclipped_grad_norm": 1.0854146480560303 + }, + { + "bits_per_byte": 1.7532864212840493, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.2152855396270752, + "step": 4130, + "unclipped_grad_norm": 1.0470608472824097 + }, + { + "bits_per_byte": 1.7313932018118645, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.2001103162765503, + "step": 4140, + "unclipped_grad_norm": 1.0418859720230103 + }, + { + "bits_per_byte": 1.6694201134867057, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.157153844833374, + "step": 4150, + "unclipped_grad_norm": 1.0297493934631348 + }, + { + "bits_per_byte": 1.6896791538103921, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1711963415145874, + "step": 4160, + "unclipped_grad_norm": 1.0267698764801025 + }, + { + "bits_per_byte": 1.7146273011098527, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1884890794754028, + "step": 4170, + "unclipped_grad_norm": 1.1191542148590088 + }, + { + "bits_per_byte": 1.6875347021367837, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1697099208831787, + "step": 4180, + "unclipped_grad_norm": 1.0307927131652832 + }, + { + "bits_per_byte": 1.7039927538937512, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1811177730560303, + "step": 4190, + "unclipped_grad_norm": 1.089546799659729 + }, + { + "bits_per_byte": 1.7112641803722306, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1861579418182373, + "step": 4200, + "unclipped_grad_norm": 1.0752745866775513 + }, + { + "bits_per_byte": 1.7401058429045744, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.2061494588851929, + "step": 4210, + "unclipped_grad_norm": 1.0523611307144165 + }, + { + "bits_per_byte": 1.693671731050172, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1739637851715088, + "step": 4220, + "unclipped_grad_norm": 0.997297465801239 + }, + { + "bits_per_byte": 1.7078159282224312, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1837677955627441, + "step": 4230, + "unclipped_grad_norm": 1.0999088287353516 + }, + { + "bits_per_byte": 1.7184320732948908, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1911263465881348, + "step": 4240, + "unclipped_grad_norm": 0.9742965698242188 + }, + { + "bits_per_byte": 1.6537339198668108, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1462810039520264, + "step": 4250, + "unclipped_grad_norm": 0.9919567108154297 + }, + { + "bits_per_byte": 1.6890791063415318, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1707804203033447, + "step": 4260, + "unclipped_grad_norm": 1.0240992307662964 + }, + { + "bits_per_byte": 1.6052090149258693, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1126461029052734, + "step": 4270, + "unclipped_grad_norm": 0.9614320993423462 + }, + { + "bits_per_byte": 1.677325124050964, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1626331806182861, + "step": 4280, + "unclipped_grad_norm": 1.0358916521072388 + }, + { + "bits_per_byte": 1.6570518091672561, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.14858078956604, + "step": 4290, + "unclipped_grad_norm": 1.0068124532699585 + }, + { + "bits_per_byte": 1.6268774530733763, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1276655197143555, + "step": 4300, + "unclipped_grad_norm": 1.0985108613967896 + }, + { + "bits_per_byte": 1.6144661530912108, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1190626621246338, + "step": 4310, + "unclipped_grad_norm": 0.9936105608940125 + }, + { + "bits_per_byte": 1.7397008237617926, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.2058687210083008, + "step": 4320, + "unclipped_grad_norm": 1.1129765510559082 + }, + { + "bits_per_byte": 1.6818943591190871, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1658003330230713, + "step": 4330, + "unclipped_grad_norm": 1.0247257947921753 + }, + { + "bits_per_byte": 1.6456326770979182, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1406656503677368, + "step": 4340, + "unclipped_grad_norm": 1.0460948944091797 + }, + { + "bits_per_byte": 1.6091236840248087, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1153595447540283, + "step": 4350, + "unclipped_grad_norm": 1.0163395404815674 + }, + { + "bits_per_byte": 1.686970427059303, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1693187952041626, + "step": 4360, + "unclipped_grad_norm": 1.063033938407898 + }, + { + "bits_per_byte": 1.65933092325649, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.150160551071167, + "step": 4370, + "unclipped_grad_norm": 0.9919126033782959 + }, + { + "bits_per_byte": 1.6024796502566765, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1107542514801025, + "step": 4380, + "unclipped_grad_norm": 1.0177253484725952 + }, + { + "bits_per_byte": 1.7322169987094975, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.2006813287734985, + "step": 4390, + "unclipped_grad_norm": 0.9770697951316833 + }, + { + "bits_per_byte": 1.6952770171133256, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1750764846801758, + "step": 4400, + "unclipped_grad_norm": 1.0351864099502563 + }, + { + "bits_per_byte": 1.6960017520040616, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1755788326263428, + "step": 4410, + "unclipped_grad_norm": 1.1030160188674927 + }, + { + "bits_per_byte": 1.7127384156554266, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1871798038482666, + "step": 4420, + "unclipped_grad_norm": 1.0353890657424927 + }, + { + "bits_per_byte": 1.6726172709662455, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.159369945526123, + "step": 4430, + "unclipped_grad_norm": 1.0875210762023926 + }, + { + "bits_per_byte": 1.680560461678995, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1648757457733154, + "step": 4440, + "unclipped_grad_norm": 0.9928231239318848 + }, + { + "bits_per_byte": 1.6836972532480818, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1670500040054321, + "step": 4450, + "unclipped_grad_norm": 0.9873722791671753 + }, + { + "bits_per_byte": 1.6829482688035744, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1665308475494385, + "step": 4460, + "unclipped_grad_norm": 1.0991053581237793 + }, + { + "bits_per_byte": 1.6608082542274016, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1511845588684082, + "step": 4470, + "unclipped_grad_norm": 1.0593136548995972 + }, + { + "bits_per_byte": 1.698860447626703, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1775603294372559, + "step": 4480, + "unclipped_grad_norm": 1.0394423007965088 + }, + { + "bits_per_byte": 1.740063707155113, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.206120252609253, + "step": 4490, + "unclipped_grad_norm": 1.0491414070129395 + }, + { + "bits_per_byte": 1.6564844384020596, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.148187518119812, + "step": 4500, + "unclipped_grad_norm": 1.0240522623062134 + }, + { + "bits_per_byte": 1.6424668204608355, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.138471245765686, + "step": 4510, + "unclipped_grad_norm": 1.025274634361267 + }, + { + "bits_per_byte": 1.71951625192593, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.191877841949463, + "step": 4520, + "unclipped_grad_norm": 1.063368320465088 + }, + { + "bits_per_byte": 1.680697359869082, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1649706363677979, + "step": 4530, + "unclipped_grad_norm": 1.0728412866592407 + }, + { + "bits_per_byte": 1.6916681331676204, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1725749969482422, + "step": 4540, + "unclipped_grad_norm": 1.1416125297546387 + }, + { + "bits_per_byte": 1.6884355472620032, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1703343391418457, + "step": 4550, + "unclipped_grad_norm": 1.0188941955566406 + }, + { + "bits_per_byte": 1.6889325771229966, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1706788539886475, + "step": 4560, + "unclipped_grad_norm": 1.0978926420211792 + }, + { + "bits_per_byte": 1.6867642198609183, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1691758632659912, + "step": 4570, + "unclipped_grad_norm": 1.043805480003357 + }, + { + "bits_per_byte": 1.6647294586670736, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.153902530670166, + "step": 4580, + "unclipped_grad_norm": 1.061089277267456 + }, + { + "bits_per_byte": 1.6460370083100966, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1409459114074707, + "step": 4590, + "unclipped_grad_norm": 1.0722659826278687 + }, + { + "bits_per_byte": 1.6445638049228057, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1399247646331787, + "step": 4600, + "unclipped_grad_norm": 1.0390986204147339 + }, + { + "bits_per_byte": 1.6450002968906956, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1402273178100586, + "step": 4610, + "unclipped_grad_norm": 1.0198664665222168 + }, + { + "bits_per_byte": 1.7089066421942032, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1845238208770752, + "step": 4620, + "unclipped_grad_norm": 1.0599243640899658 + }, + { + "bits_per_byte": 1.69811834248823, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1770459413528442, + "step": 4630, + "unclipped_grad_norm": 1.1346435546875 + }, + { + "bits_per_byte": 1.7044230544862098, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1814160346984863, + "step": 4640, + "unclipped_grad_norm": 1.0656381845474243 + }, + { + "bits_per_byte": 1.7338580571640303, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.201818823814392, + "step": 4650, + "unclipped_grad_norm": 1.0118787288665771 + }, + { + "bits_per_byte": 1.652913906587497, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1457126140594482, + "step": 4660, + "unclipped_grad_norm": 1.0275835990905762 + }, + { + "bits_per_byte": 1.66807348933045, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1562204360961914, + "step": 4670, + "unclipped_grad_norm": 1.0584690570831299 + }, + { + "bits_per_byte": 1.6481818039490068, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1424325704574585, + "step": 4680, + "unclipped_grad_norm": 1.1420236825942993 + }, + { + "bits_per_byte": 1.658637833173513, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1496801376342773, + "step": 4690, + "unclipped_grad_norm": 1.080548644065857 + }, + { + "bits_per_byte": 1.6754130209386713, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1613078117370605, + "step": 4700, + "unclipped_grad_norm": 1.063712239265442 + }, + { + "bits_per_byte": 1.6541807307937524, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1465907096862793, + "step": 4710, + "unclipped_grad_norm": 1.065613865852356 + }, + { + "bits_per_byte": 1.6715305125954434, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1586166620254517, + "step": 4720, + "unclipped_grad_norm": 1.041472315788269 + }, + { + "bits_per_byte": 1.7163307892666488, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1896698474884033, + "step": 4730, + "unclipped_grad_norm": 1.0371977090835571 + }, + { + "bits_per_byte": 1.6468645888260485, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.141519546508789, + "step": 4740, + "unclipped_grad_norm": 1.118260383605957 + }, + { + "bits_per_byte": 1.6331114802018514, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1319866180419922, + "step": 4750, + "unclipped_grad_norm": 1.1252490282058716 + }, + { + "bits_per_byte": 1.6043413624522664, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1120446920394897, + "step": 4760, + "unclipped_grad_norm": 1.0504546165466309 + }, + { + "bits_per_byte": 1.6005166402797284, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.10939359664917, + "step": 4770, + "unclipped_grad_norm": 0.984404981136322 + }, + { + "bits_per_byte": 1.6404936635074865, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.13710355758667, + "step": 4780, + "unclipped_grad_norm": 1.0672088861465454 + }, + { + "bits_per_byte": 1.6438352864137509, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1394197940826416, + "step": 4790, + "unclipped_grad_norm": 1.0523202419281006 + }, + { + "bits_per_byte": 1.6719128300283113, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.158881664276123, + "step": 4800, + "unclipped_grad_norm": 1.1081420183181763 + }, + { + "bits_per_byte": 1.6392536685947656, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1362440586090088, + "step": 4810, + "unclipped_grad_norm": 1.0507419109344482 + }, + { + "bits_per_byte": 1.648858383697501, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1429015398025513, + "step": 4820, + "unclipped_grad_norm": 1.1054445505142212 + }, + { + "bits_per_byte": 1.6256828615804833, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1268374919891357, + "step": 4830, + "unclipped_grad_norm": 1.0212544202804565 + }, + { + "bits_per_byte": 1.5951660880287355, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1056848764419556, + "step": 4840, + "unclipped_grad_norm": 1.0648442506790161 + }, + { + "bits_per_byte": 1.6564994008926848, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.148197889328003, + "step": 4850, + "unclipped_grad_norm": 1.047589898109436 + }, + { + "bits_per_byte": 1.641553764567405, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.137838363647461, + "step": 4860, + "unclipped_grad_norm": 1.0177857875823975 + }, + { + "bits_per_byte": 1.6551041056462348, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1472307443618774, + "step": 4870, + "unclipped_grad_norm": 1.0283644199371338 + }, + { + "bits_per_byte": 1.6180980827121316, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1215801239013672, + "step": 4880, + "unclipped_grad_norm": 1.0304375886917114 + }, + { + "bits_per_byte": 1.5903827345602883, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.1023693084716797, + "step": 4890, + "unclipped_grad_norm": 1.0921995639801025 + }, + { + "bits_per_byte": 1.567706994026678, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0866516828536987, + "step": 4900, + "unclipped_grad_norm": 1.0604755878448486 + }, + { + "bits_per_byte": 1.6439632415059928, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1395084857940674, + "step": 4910, + "unclipped_grad_norm": 1.0503017902374268 + }, + { + "bits_per_byte": 1.642120275419347, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1382310390472412, + "step": 4920, + "unclipped_grad_norm": 1.1088591814041138 + }, + { + "bits_per_byte": 1.622944209848144, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.124939203262329, + "step": 4930, + "unclipped_grad_norm": 1.099297046661377 + }, + { + "bits_per_byte": 1.6933755769253864, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1737585067749023, + "step": 4940, + "unclipped_grad_norm": 1.08061945438385 + }, + { + "bits_per_byte": 1.5670388414280758, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.086188554763794, + "step": 4950, + "unclipped_grad_norm": 1.1187851428985596 + }, + { + "bits_per_byte": 1.6577792957804058, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1490850448608398, + "step": 4960, + "unclipped_grad_norm": 1.095150351524353 + }, + { + "bits_per_byte": 1.7221861105979241, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1937284469604492, + "step": 4970, + "unclipped_grad_norm": 1.2121388912200928 + }, + { + "bits_per_byte": 1.5930121773093298, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1041918992996216, + "step": 4980, + "unclipped_grad_norm": 1.1045044660568237 + }, + { + "bits_per_byte": 1.665033523993799, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1541132926940918, + "step": 4990, + "unclipped_grad_norm": 1.1089035272598267 + }, + { + "bits_per_byte": 1.6784994215910551, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1634471416473389, + "step": 5000, + "unclipped_grad_norm": 1.0390889644622803 + }, + { + "bits_per_byte": 1.642436207548982, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.138450026512146, + "step": 5010, + "unclipped_grad_norm": 1.1114050149917603 + }, + { + "bits_per_byte": 1.6556807634745778, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1476304531097412, + "step": 5020, + "unclipped_grad_norm": 1.0220290422439575 + }, + { + "bits_per_byte": 1.636998288111351, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.1346807479858398, + "step": 5030, + "unclipped_grad_norm": 1.1270323991775513 + }, + { + "bits_per_byte": 1.6052772920382619, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.112693428993225, + "step": 5040, + "unclipped_grad_norm": 1.009588360786438 + }, + { + "bits_per_byte": 1.6583182894082098, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.149458646774292, + "step": 5050, + "unclipped_grad_norm": 1.060039758682251 + }, + { + "bits_per_byte": 1.609498090255737, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1156190633773804, + "step": 5060, + "unclipped_grad_norm": 1.0725975036621094 + }, + { + "bits_per_byte": 1.6927082842400387, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1732959747314453, + "step": 5070, + "unclipped_grad_norm": 1.120245099067688 + }, + { + "bits_per_byte": 1.690623166580978, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1718506813049316, + "step": 5080, + "unclipped_grad_norm": 1.1778441667556763 + }, + { + "bits_per_byte": 1.67411111227164, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1604053974151611, + "step": 5090, + "unclipped_grad_norm": 1.3122999668121338 + }, + { + "bits_per_byte": 1.5883206625764432, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1009399890899658, + "step": 5100, + "unclipped_grad_norm": 1.0812361240386963 + }, + { + "bits_per_byte": 1.6343635139001327, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1328544616699219, + "step": 5110, + "unclipped_grad_norm": 1.054042935371399 + }, + { + "bits_per_byte": 1.6096652573923755, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.1157349348068237, + "step": 5120, + "unclipped_grad_norm": 1.183487892150879 + }, + { + "bits_per_byte": 1.6704048861455463, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1578364372253418, + "step": 5130, + "unclipped_grad_norm": 1.1154927015304565 + }, + { + "bits_per_byte": 1.6086259662332116, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1150145530700684, + "step": 5140, + "unclipped_grad_norm": 1.1280518770217896 + }, + { + "bits_per_byte": 1.607841897327928, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1144710779190063, + "step": 5150, + "unclipped_grad_norm": 1.1053173542022705 + }, + { + "bits_per_byte": 1.6377722100402334, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1352171897888184, + "step": 5160, + "unclipped_grad_norm": 1.0772578716278076 + }, + { + "bits_per_byte": 1.6387738369988585, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.135911464691162, + "step": 5170, + "unclipped_grad_norm": 1.08511221408844 + }, + { + "bits_per_byte": 1.6731652076918944, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1597497463226318, + "step": 5180, + "unclipped_grad_norm": 1.0842925310134888 + }, + { + "bits_per_byte": 1.6308702022958073, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.1304330825805664, + "step": 5190, + "unclipped_grad_norm": 1.1379977464675903 + }, + { + "bits_per_byte": 1.5988843529403896, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1082621812820435, + "step": 5200, + "unclipped_grad_norm": 1.1327811479568481 + }, + { + "bits_per_byte": 1.6538405491103456, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1463549137115479, + "step": 5210, + "unclipped_grad_norm": 1.131410837173462 + }, + { + "bits_per_byte": 1.6579122383695226, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1491771936416626, + "step": 5220, + "unclipped_grad_norm": 1.111109972000122 + }, + { + "bits_per_byte": 1.6745703059494434, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1607236862182617, + "step": 5230, + "unclipped_grad_norm": 1.1052273511886597 + }, + { + "bits_per_byte": 1.6223493218588094, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.124526858329773, + "step": 5240, + "unclipped_grad_norm": 1.0814346075057983 + }, + { + "bits_per_byte": 1.5820224859191963, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0965744256973267, + "step": 5250, + "unclipped_grad_norm": 1.0832712650299072 + }, + { + "bits_per_byte": 1.5670176875620196, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0861738920211792, + "step": 5260, + "unclipped_grad_norm": 1.0310815572738647 + }, + { + "bits_per_byte": 1.6681894056371316, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1563007831573486, + "step": 5270, + "unclipped_grad_norm": 1.1380984783172607 + }, + { + "bits_per_byte": 1.6634579909292448, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.153021216392517, + "step": 5280, + "unclipped_grad_norm": 1.1410281658172607 + }, + { + "bits_per_byte": 1.627283848077365, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.127947211265564, + "step": 5290, + "unclipped_grad_norm": 1.0713688135147095 + }, + { + "bits_per_byte": 1.57530983107337, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0919215679168701, + "step": 5300, + "unclipped_grad_norm": 1.103183388710022 + }, + { + "bits_per_byte": 1.6444668067077193, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.139857530593872, + "step": 5310, + "unclipped_grad_norm": 1.1310855150222778 + }, + { + "bits_per_byte": 1.6034007893346978, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1113927364349365, + "step": 5320, + "unclipped_grad_norm": 1.0775246620178223 + }, + { + "bits_per_byte": 1.6848084331553064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1678202152252197, + "step": 5330, + "unclipped_grad_norm": 1.1329472064971924 + }, + { + "bits_per_byte": 1.5891140185448733, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1014899015426636, + "step": 5340, + "unclipped_grad_norm": 1.0905873775482178 + }, + { + "bits_per_byte": 1.6324792719772796, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1315484046936035, + "step": 5350, + "unclipped_grad_norm": 1.13417649269104 + }, + { + "bits_per_byte": 1.556942083961222, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0791900157928467, + "step": 5360, + "unclipped_grad_norm": 1.11483633518219 + }, + { + "bits_per_byte": 1.585282417066301, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0988340377807617, + "step": 5370, + "unclipped_grad_norm": 1.0698448419570923 + }, + { + "bits_per_byte": 1.6547109533063624, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1469582319259644, + "step": 5380, + "unclipped_grad_norm": 1.1219947338104248 + }, + { + "bits_per_byte": 1.5562904416971028, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0787383317947388, + "step": 5390, + "unclipped_grad_norm": 1.115169644355774 + }, + { + "bits_per_byte": 1.6265400231123834, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1274316310882568, + "step": 5400, + "unclipped_grad_norm": 1.079226016998291 + }, + { + "bits_per_byte": 1.5897746039068374, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.1019477844238281, + "step": 5410, + "unclipped_grad_norm": 1.0649707317352295 + }, + { + "bits_per_byte": 1.5705084194265821, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0885934829711914, + "step": 5420, + "unclipped_grad_norm": 1.115608811378479 + }, + { + "bits_per_byte": 1.6107165873370999, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1164636611938477, + "step": 5430, + "unclipped_grad_norm": 1.1144897937774658 + }, + { + "bits_per_byte": 1.5986757379848928, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1081175804138184, + "step": 5440, + "unclipped_grad_norm": 1.0857055187225342 + }, + { + "bits_per_byte": 1.5965995634236771, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1066784858703613, + "step": 5450, + "unclipped_grad_norm": 1.0530359745025635 + }, + { + "bits_per_byte": 1.5899876904112564, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.1020954847335815, + "step": 5460, + "unclipped_grad_norm": 1.1017006635665894 + }, + { + "bits_per_byte": 1.611714258694755, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1171551942825317, + "step": 5470, + "unclipped_grad_norm": 1.11431884765625 + }, + { + "bits_per_byte": 1.5834313677950644, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0975509881973267, + "step": 5480, + "unclipped_grad_norm": 1.0543001890182495 + }, + { + "bits_per_byte": 1.6635541292310771, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.153087854385376, + "step": 5490, + "unclipped_grad_norm": 1.154997706413269 + }, + { + "bits_per_byte": 1.62576197359988, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.126892328262329, + "step": 5500, + "unclipped_grad_norm": 1.1008871793746948 + }, + { + "bits_per_byte": 1.5806631350467766, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0956321954727173, + "step": 5510, + "unclipped_grad_norm": 1.0707919597625732 + }, + { + "bits_per_byte": 1.5578388015028208, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0798115730285645, + "step": 5520, + "unclipped_grad_norm": 1.0223692655563354 + }, + { + "bits_per_byte": 1.573020741990386, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0903348922729492, + "step": 5530, + "unclipped_grad_norm": 1.1420265436172485 + }, + { + "bits_per_byte": 1.6833484724321321, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1668082475662231, + "step": 5540, + "unclipped_grad_norm": 1.1607446670532227 + }, + { + "bits_per_byte": 1.6178791487975832, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.121428370475769, + "step": 5550, + "unclipped_grad_norm": 1.1443098783493042 + }, + { + "bits_per_byte": 1.6447739677221602, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1400704383850098, + "step": 5560, + "unclipped_grad_norm": 1.1718472242355347 + }, + { + "bits_per_byte": 1.59615963180277, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1063735485076904, + "step": 5570, + "unclipped_grad_norm": 1.093506097793579 + }, + { + "bits_per_byte": 1.624393163681663, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.1259435415267944, + "step": 5580, + "unclipped_grad_norm": 1.0647783279418945 + }, + { + "bits_per_byte": 1.6292150412639035, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.1292858123779297, + "step": 5590, + "unclipped_grad_norm": 1.1650937795639038 + }, + { + "bits_per_byte": 1.6101528282075717, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1160728931427002, + "step": 5600, + "unclipped_grad_norm": 1.1434698104858398 + }, + { + "bits_per_byte": 1.5665055232277503, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.085818886756897, + "step": 5610, + "unclipped_grad_norm": 1.1304930448532104 + }, + { + "bits_per_byte": 1.6202220684502873, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.1230523586273193, + "step": 5620, + "unclipped_grad_norm": 1.1116455793380737 + }, + { + "bits_per_byte": 1.538340096478599, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.066296100616455, + "step": 5630, + "unclipped_grad_norm": 1.058684229850769 + }, + { + "bits_per_byte": 1.5357409226761094, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0644944906234741, + "step": 5640, + "unclipped_grad_norm": 1.033333420753479 + }, + { + "bits_per_byte": 1.6143326945541414, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1189701557159424, + "step": 5650, + "unclipped_grad_norm": 1.1008256673812866 + }, + { + "bits_per_byte": 1.5570710709493691, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0792794227600098, + "step": 5660, + "unclipped_grad_norm": 1.1397353410720825 + }, + { + "bits_per_byte": 1.6224767610030988, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.12461519241333, + "step": 5670, + "unclipped_grad_norm": 1.1256861686706543 + }, + { + "bits_per_byte": 1.6278986860541997, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.128373384475708, + "step": 5680, + "unclipped_grad_norm": 1.2497283220291138 + }, + { + "bits_per_byte": 1.6402167714395974, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1369116306304932, + "step": 5690, + "unclipped_grad_norm": 1.099176049232483 + }, + { + "bits_per_byte": 1.6346018818542285, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1330196857452393, + "step": 5700, + "unclipped_grad_norm": 1.132989525794983 + }, + { + "bits_per_byte": 1.6142821316547877, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1189351081848145, + "step": 5710, + "unclipped_grad_norm": 1.169878363609314 + }, + { + "bits_per_byte": 1.6389509791292471, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.1360342502593994, + "step": 5720, + "unclipped_grad_norm": 1.1713305711746216 + }, + { + "bits_per_byte": 1.5731861893005161, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.090449571609497, + "step": 5730, + "unclipped_grad_norm": 1.0991312265396118 + }, + { + "bits_per_byte": 1.642317195554585, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1383675336837769, + "step": 5740, + "unclipped_grad_norm": 1.074776530265808 + }, + { + "bits_per_byte": 1.5380594207923908, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0661015510559082, + "step": 5750, + "unclipped_grad_norm": 1.1045373678207397 + }, + { + "bits_per_byte": 1.574528857855802, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.09138023853302, + "step": 5760, + "unclipped_grad_norm": 1.1376982927322388 + }, + { + "bits_per_byte": 1.5630001728378644, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.083389163017273, + "step": 5770, + "unclipped_grad_norm": 1.1196208000183105 + }, + { + "bits_per_byte": 1.658872245526639, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.149842619895935, + "step": 5780, + "unclipped_grad_norm": 1.1231145858764648 + }, + { + "bits_per_byte": 1.4960170579104213, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0369600057601929, + "step": 5790, + "unclipped_grad_norm": 1.1306540966033936 + }, + { + "bits_per_byte": 1.632525363327711, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.1315803527832031, + "step": 5800, + "unclipped_grad_norm": 1.1228681802749634 + }, + { + "bits_per_byte": 1.5271961366506415, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.058571696281433, + "step": 5810, + "unclipped_grad_norm": 1.0998179912567139 + }, + { + "bits_per_byte": 1.544303422914616, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0704295635223389, + "step": 5820, + "unclipped_grad_norm": 1.1354378461837769 + }, + { + "bits_per_byte": 1.5973088198758354, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.1071701049804688, + "step": 5830, + "unclipped_grad_norm": 1.112725853919983 + }, + { + "bits_per_byte": 1.5755839714188453, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.092111587524414, + "step": 5840, + "unclipped_grad_norm": 1.105390191078186 + }, + { + "bits_per_byte": 1.551858620767019, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0756664276123047, + "step": 5850, + "unclipped_grad_norm": 1.0705221891403198 + }, + { + "bits_per_byte": 1.58110633433805, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0959393978118896, + "step": 5860, + "unclipped_grad_norm": 1.1176000833511353 + }, + { + "bits_per_byte": 1.6256357383341469, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1268048286437988, + "step": 5870, + "unclipped_grad_norm": 1.1718623638153076 + }, + { + "bits_per_byte": 1.5672180473502748, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0863127708435059, + "step": 5880, + "unclipped_grad_norm": 1.1114801168441772 + }, + { + "bits_per_byte": 1.6374564498932493, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1349983215332031, + "step": 5890, + "unclipped_grad_norm": 1.2559460401535034 + }, + { + "bits_per_byte": 1.6402117839427224, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1369081735610962, + "step": 5900, + "unclipped_grad_norm": 1.1818214654922485 + }, + { + "bits_per_byte": 1.5692250848858438, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0877039432525635, + "step": 5910, + "unclipped_grad_norm": 1.0690149068832397 + }, + { + "bits_per_byte": 1.5417845650100792, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0686836242675781, + "step": 5920, + "unclipped_grad_norm": 1.1236122846603394 + }, + { + "bits_per_byte": 1.6662781624380931, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1549760103225708, + "step": 5930, + "unclipped_grad_norm": 1.2468876838684082 + }, + { + "bits_per_byte": 1.596861664983592, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.1068601608276367, + "step": 5940, + "unclipped_grad_norm": 1.2251354455947876 + }, + { + "bits_per_byte": 1.5910101272706358, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.102804183959961, + "step": 5950, + "unclipped_grad_norm": 1.11275053024292 + }, + { + "bits_per_byte": 1.5416916943786134, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0686192512512207, + "step": 5960, + "unclipped_grad_norm": 1.1575579643249512 + }, + { + "bits_per_byte": 1.5659371205666486, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0854249000549316, + "step": 5970, + "unclipped_grad_norm": 1.102811336517334 + }, + { + "bits_per_byte": 1.5505245513442762, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0747417211532593, + "step": 5980, + "unclipped_grad_norm": 1.1679552793502808 + }, + { + "bits_per_byte": 1.5609080038901184, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0819389820098877, + "step": 5990, + "unclipped_grad_norm": 1.1159348487854004 + }, + { + "bits_per_byte": 1.6522259759840456, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1452357769012451, + "step": 6000, + "unclipped_grad_norm": 1.1946688890457153 + }, + { + "bits_per_byte": 1.610257393659296, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.116145372390747, + "step": 6010, + "unclipped_grad_norm": 1.2001700401306152 + }, + { + "bits_per_byte": 1.5550891428808262, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0779056549072266, + "step": 6020, + "unclipped_grad_norm": 1.11931574344635 + }, + { + "bits_per_byte": 1.5522581364649735, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0759433507919312, + "step": 6030, + "unclipped_grad_norm": 1.0901384353637695 + }, + { + "bits_per_byte": 1.5778765001548465, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.093700647354126, + "step": 6040, + "unclipped_grad_norm": 1.1465646028518677 + }, + { + "bits_per_byte": 1.585459043248737, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0989564657211304, + "step": 6050, + "unclipped_grad_norm": 1.093497395515442 + }, + { + "bits_per_byte": 1.580860743112618, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0957691669464111, + "step": 6060, + "unclipped_grad_norm": 1.1618996858596802 + }, + { + "bits_per_byte": 1.4692863104347675, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0184316635131836, + "step": 6070, + "unclipped_grad_norm": 1.095646619796753 + }, + { + "bits_per_byte": 1.5593341906520728, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0808480978012085, + "step": 6080, + "unclipped_grad_norm": 1.2226048707962036 + }, + { + "bits_per_byte": 1.5548665973306097, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0777513980865479, + "step": 6090, + "unclipped_grad_norm": 1.160715103149414 + }, + { + "bits_per_byte": 1.6437169623499572, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.1393377780914307, + "step": 6100, + "unclipped_grad_norm": 1.216336965560913 + }, + { + "bits_per_byte": 1.5953482176559992, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.1058111190795898, + "step": 6110, + "unclipped_grad_norm": 1.1638948917388916 + }, + { + "bits_per_byte": 1.5315409343593887, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0615832805633545, + "step": 6120, + "unclipped_grad_norm": 1.1416045427322388 + }, + { + "bits_per_byte": 1.6181004904692438, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.121581792831421, + "step": 6130, + "unclipped_grad_norm": 1.1373443603515625 + }, + { + "bits_per_byte": 1.5591868015202834, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0807459354400635, + "step": 6140, + "unclipped_grad_norm": 1.0817177295684814 + }, + { + "bits_per_byte": 1.5489426549216403, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0736452341079712, + "step": 6150, + "unclipped_grad_norm": 1.156706690788269 + }, + { + "bits_per_byte": 1.6171114182441317, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1208962202072144, + "step": 6160, + "unclipped_grad_norm": 1.2064051628112793 + }, + { + "bits_per_byte": 1.5388670513208427, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0666613578796387, + "step": 6170, + "unclipped_grad_norm": 1.0822783708572388 + }, + { + "bits_per_byte": 1.5490690621700245, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.073732852935791, + "step": 6180, + "unclipped_grad_norm": 1.109446406364441 + }, + { + "bits_per_byte": 1.5625440748477764, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0830730199813843, + "step": 6190, + "unclipped_grad_norm": 1.1676242351531982 + }, + { + "bits_per_byte": 1.5465116801516945, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.071960210800171, + "step": 6200, + "unclipped_grad_norm": 1.094281792640686 + }, + { + "bits_per_byte": 1.5807004552820139, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0956580638885498, + "step": 6210, + "unclipped_grad_norm": 1.2013437747955322 + }, + { + "bits_per_byte": 1.592342648849521, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.1037278175354004, + "step": 6220, + "unclipped_grad_norm": 1.1159899234771729 + }, + { + "bits_per_byte": 1.550343453612918, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0746161937713623, + "step": 6230, + "unclipped_grad_norm": 1.156030535697937 + }, + { + "bits_per_byte": 1.5936101609863798, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.1046063899993896, + "step": 6240, + "unclipped_grad_norm": 1.2167954444885254 + }, + { + "bits_per_byte": 1.5477754086702344, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.07283616065979, + "step": 6250, + "unclipped_grad_norm": 1.1424623727798462 + }, + { + "bits_per_byte": 1.5145149958892723, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0497817993164062, + "step": 6260, + "unclipped_grad_norm": 1.159928798675537 + }, + { + "bits_per_byte": 1.558105374611658, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0799963474273682, + "step": 6270, + "unclipped_grad_norm": 1.1427968740463257 + }, + { + "bits_per_byte": 1.5868821996846267, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.099942922592163, + "step": 6280, + "unclipped_grad_norm": 1.2086946964263916 + }, + { + "bits_per_byte": 1.5572024656946284, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0793704986572266, + "step": 6290, + "unclipped_grad_norm": 1.26358163356781 + }, + { + "bits_per_byte": 1.525056844456559, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.057088851928711, + "step": 6300, + "unclipped_grad_norm": 1.1142038106918335 + }, + { + "bits_per_byte": 1.585295831713068, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0988433361053467, + "step": 6310, + "unclipped_grad_norm": 1.202506422996521 + }, + { + "bits_per_byte": 1.5589988244828903, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0806156396865845, + "step": 6320, + "unclipped_grad_norm": 1.1080162525177002 + }, + { + "bits_per_byte": 1.5721742433828394, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0897481441497803, + "step": 6330, + "unclipped_grad_norm": 1.1463266611099243 + }, + { + "bits_per_byte": 1.4897746036320538, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0326330661773682, + "step": 6340, + "unclipped_grad_norm": 1.0811859369277954 + }, + { + "bits_per_byte": 1.5979289893148465, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1075999736785889, + "step": 6350, + "unclipped_grad_norm": 1.1469974517822266 + }, + { + "bits_per_byte": 1.6121533304024078, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.1174595355987549, + "step": 6360, + "unclipped_grad_norm": 1.2663604021072388 + }, + { + "bits_per_byte": 1.522211219515383, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0551164150238037, + "step": 6370, + "unclipped_grad_norm": 1.136039137840271 + }, + { + "bits_per_byte": 1.5209657211578345, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.054253101348877, + "step": 6380, + "unclipped_grad_norm": 1.1138842105865479 + }, + { + "bits_per_byte": 1.6340888576067047, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.1326640844345093, + "step": 6390, + "unclipped_grad_norm": 1.227125883102417 + }, + { + "bits_per_byte": 1.5932569086215076, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.1043615341186523, + "step": 6400, + "unclipped_grad_norm": 1.1935430765151978 + }, + { + "bits_per_byte": 1.6221133616618257, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.1243633031845093, + "step": 6410, + "unclipped_grad_norm": 1.289966106414795 + }, + { + "bits_per_byte": 1.5341647016809516, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0634019374847412, + "step": 6420, + "unclipped_grad_norm": 1.2202966213226318 + }, + { + "bits_per_byte": 1.5828500664351481, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.097148060798645, + "step": 6430, + "unclipped_grad_norm": 1.1407660245895386 + }, + { + "bits_per_byte": 1.518648426920109, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0526468753814697, + "step": 6440, + "unclipped_grad_norm": 1.1754008531570435 + }, + { + "bits_per_byte": 1.5227140967865058, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0554649829864502, + "step": 6450, + "unclipped_grad_norm": 1.1674573421478271 + }, + { + "bits_per_byte": 1.5506523344538674, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0748302936553955, + "step": 6460, + "unclipped_grad_norm": 1.1624079942703247 + }, + { + "bits_per_byte": 1.5647150118496176, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0845777988433838, + "step": 6470, + "unclipped_grad_norm": 1.1915463209152222 + }, + { + "bits_per_byte": 1.5303090226312583, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0607293844223022, + "step": 6480, + "unclipped_grad_norm": 1.1169558763504028 + }, + { + "bits_per_byte": 1.610437975442702, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.1162705421447754, + "step": 6490, + "unclipped_grad_norm": 1.212812900543213 + }, + { + "bits_per_byte": 1.6629645727039195, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.152679204940796, + "step": 6500, + "unclipped_grad_norm": 1.249900221824646 + }, + { + "bits_per_byte": 1.6085310318099353, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.1149487495422363, + "step": 6510, + "unclipped_grad_norm": 1.2940583229064941 + }, + { + "bits_per_byte": 1.5857588090091908, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0991642475128174, + "step": 6520, + "unclipped_grad_norm": 1.1899638175964355 + }, + { + "bits_per_byte": 1.557970024265429, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0799025297164917, + "step": 6530, + "unclipped_grad_norm": 1.18255615234375 + }, + { + "bits_per_byte": 1.5442463246745295, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.070389986038208, + "step": 6540, + "unclipped_grad_norm": 1.1805294752120972 + }, + { + "bits_per_byte": 1.5888496812104973, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.101306676864624, + "step": 6550, + "unclipped_grad_norm": 1.1853868961334229 + }, + { + "bits_per_byte": 1.598041465968511, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.107677936553955, + "step": 6560, + "unclipped_grad_norm": 1.167456865310669 + }, + { + "bits_per_byte": 1.5763221209563485, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.092623233795166, + "step": 6570, + "unclipped_grad_norm": 1.2579692602157593 + }, + { + "bits_per_byte": 1.5598688847136053, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0812187194824219, + "step": 6580, + "unclipped_grad_norm": 1.218462347984314 + }, + { + "bits_per_byte": 1.5745766690327418, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0914133787155151, + "step": 6590, + "unclipped_grad_norm": 1.1964205503463745 + }, + { + "bits_per_byte": 1.483007602268555, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0279425382614136, + "step": 6600, + "unclipped_grad_norm": 1.1824510097503662 + }, + { + "bits_per_byte": 1.5325055850480782, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0622519254684448, + "step": 6610, + "unclipped_grad_norm": 1.1718802452087402 + }, + { + "bits_per_byte": 1.648197798335537, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1424436569213867, + "step": 6620, + "unclipped_grad_norm": 1.2758471965789795 + }, + { + "bits_per_byte": 1.5936395700196773, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.1046267747879028, + "step": 6630, + "unclipped_grad_norm": 1.1752488613128662 + }, + { + "bits_per_byte": 1.550369594975849, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.074634313583374, + "step": 6640, + "unclipped_grad_norm": 1.1907187700271606 + }, + { + "bits_per_byte": 1.539030778804464, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.066774845123291, + "step": 6650, + "unclipped_grad_norm": 1.172040343284607 + }, + { + "bits_per_byte": 1.5824082430050816, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.096841812133789, + "step": 6660, + "unclipped_grad_norm": 1.2395251989364624 + }, + { + "bits_per_byte": 1.55066678099654, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0748403072357178, + "step": 6670, + "unclipped_grad_norm": 1.1819844245910645 + }, + { + "bits_per_byte": 1.567913545190364, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0867948532104492, + "step": 6680, + "unclipped_grad_norm": 1.207129955291748 + }, + { + "bits_per_byte": 1.5601953077849429, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0814449787139893, + "step": 6690, + "unclipped_grad_norm": 1.284845232963562 + }, + { + "bits_per_byte": 1.5747911313983678, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.091562032699585, + "step": 6700, + "unclipped_grad_norm": 1.1885749101638794 + }, + { + "bits_per_byte": 1.5878793550943293, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.1006340980529785, + "step": 6710, + "unclipped_grad_norm": 1.178633689880371 + }, + { + "bits_per_byte": 1.6366173465396898, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.1344166994094849, + "step": 6720, + "unclipped_grad_norm": 1.265533685684204 + }, + { + "bits_per_byte": 1.5154492056487592, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.050429344177246, + "step": 6730, + "unclipped_grad_norm": 1.138245940208435 + }, + { + "bits_per_byte": 1.547962697777024, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0729659795761108, + "step": 6740, + "unclipped_grad_norm": 1.167292833328247 + }, + { + "bits_per_byte": 1.5245414124519232, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0567315816879272, + "step": 6750, + "unclipped_grad_norm": 1.1478474140167236 + }, + { + "bits_per_byte": 1.5232747602283185, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0558536052703857, + "step": 6760, + "unclipped_grad_norm": 1.185871958732605 + }, + { + "bits_per_byte": 1.5451932611501802, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0710463523864746, + "step": 6770, + "unclipped_grad_norm": 1.2135790586471558 + }, + { + "bits_per_byte": 1.5261805790972967, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0578677654266357, + "step": 6780, + "unclipped_grad_norm": 1.2295221090316772 + }, + { + "bits_per_byte": 1.5602444948230898, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0814790725708008, + "step": 6790, + "unclipped_grad_norm": 1.174469232559204 + }, + { + "bits_per_byte": 1.5335944072106906, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0630066394805908, + "step": 6800, + "unclipped_grad_norm": 1.1970314979553223 + }, + { + "bits_per_byte": 1.5348973477736272, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0639097690582275, + "step": 6810, + "unclipped_grad_norm": 1.162747859954834 + }, + { + "bits_per_byte": 1.5590328770477613, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0806392431259155, + "step": 6820, + "unclipped_grad_norm": 1.2186238765716553 + }, + { + "bits_per_byte": 1.5711721004762618, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0890535116195679, + "step": 6830, + "unclipped_grad_norm": 1.1893293857574463 + }, + { + "bits_per_byte": 1.526197777362383, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0578796863555908, + "step": 6840, + "unclipped_grad_norm": 1.2071653604507446 + }, + { + "bits_per_byte": 1.624858204769596, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.1262658834457397, + "step": 6850, + "unclipped_grad_norm": 1.2998149394989014 + }, + { + "bits_per_byte": 1.574837222748799, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0915939807891846, + "step": 6860, + "unclipped_grad_norm": 1.2379318475723267 + }, + { + "bits_per_byte": 1.5010148737444944, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0404242277145386, + "step": 6870, + "unclipped_grad_norm": 1.1776995658874512 + }, + { + "bits_per_byte": 1.5287346934452601, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0596381425857544, + "step": 6880, + "unclipped_grad_norm": 1.2149008512496948 + }, + { + "bits_per_byte": 1.5161942344922967, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.05094575881958, + "step": 6890, + "unclipped_grad_norm": 1.1354871988296509 + }, + { + "bits_per_byte": 1.5760503883679853, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0924348831176758, + "step": 6900, + "unclipped_grad_norm": 1.2098510265350342 + }, + { + "bits_per_byte": 1.5705858116194704, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0886471271514893, + "step": 6910, + "unclipped_grad_norm": 1.2706652879714966 + }, + { + "bits_per_byte": 1.523291786510754, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0558654069900513, + "step": 6920, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 1.5043128130574397, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0427101850509644, + "step": 6930, + "unclipped_grad_norm": 1.1997718811035156 + }, + { + "bits_per_byte": 1.5789452003473081, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0944414138793945, + "step": 6940, + "unclipped_grad_norm": 1.2278647422790527 + }, + { + "bits_per_byte": 1.4992432804579567, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.039196252822876, + "step": 6950, + "unclipped_grad_norm": 1.2732465267181396 + }, + { + "bits_per_byte": 1.4942526878952196, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0357370376586914, + "step": 6960, + "unclipped_grad_norm": 1.1388111114501953 + }, + { + "bits_per_byte": 1.4775614276636824, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.024167537689209, + "step": 6970, + "unclipped_grad_norm": 1.2496774196624756 + }, + { + "bits_per_byte": 1.5972561931846714, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1071336269378662, + "step": 6980, + "unclipped_grad_norm": 1.2329373359680176 + }, + { + "bits_per_byte": 1.5648880263963856, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0846977233886719, + "step": 6990, + "unclipped_grad_norm": 1.225949764251709 + }, + { + "bits_per_byte": 1.5748815942727217, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0916247367858887, + "step": 7000, + "unclipped_grad_norm": 1.2655893564224243 + }, + { + "bits_per_byte": 1.4699690815586928, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0189049243927002, + "step": 7010, + "unclipped_grad_norm": 1.1935946941375732 + }, + { + "bits_per_byte": 1.5527095909234883, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.076256275177002, + "step": 7020, + "unclipped_grad_norm": 1.2986186742782593 + }, + { + "bits_per_byte": 1.5324563980099315, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0622178316116333, + "step": 7030, + "unclipped_grad_norm": 1.2135052680969238 + }, + { + "bits_per_byte": 1.5215922539549278, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0546873807907104, + "step": 7040, + "unclipped_grad_norm": 1.1919881105422974 + }, + { + "bits_per_byte": 1.5129705916845244, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0487112998962402, + "step": 7050, + "unclipped_grad_norm": 1.177833914756775 + }, + { + "bits_per_byte": 1.5002927185935215, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0399236679077148, + "step": 7060, + "unclipped_grad_norm": 1.2103830575942993 + }, + { + "bits_per_byte": 1.5379128915738558, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.065999984741211, + "step": 7070, + "unclipped_grad_norm": 1.2223129272460938 + }, + { + "bits_per_byte": 1.5494862920810175, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0740220546722412, + "step": 7080, + "unclipped_grad_norm": 1.2125953435897827 + }, + { + "bits_per_byte": 1.5050088268454815, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0431926250457764, + "step": 7090, + "unclipped_grad_norm": 1.274321436882019 + }, + { + "bits_per_byte": 1.5086218383748073, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0456969738006592, + "step": 7100, + "unclipped_grad_norm": 1.244695782661438 + }, + { + "bits_per_byte": 1.480645764524256, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0263054370880127, + "step": 7110, + "unclipped_grad_norm": 1.1544733047485352 + }, + { + "bits_per_byte": 1.5449419944972695, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.070872187614441, + "step": 7120, + "unclipped_grad_norm": 1.25104558467865 + }, + { + "bits_per_byte": 1.5693447848108444, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0877869129180908, + "step": 7130, + "unclipped_grad_norm": 1.2125219106674194 + }, + { + "bits_per_byte": 1.5401648324042534, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0675609111785889, + "step": 7140, + "unclipped_grad_norm": 1.2392476797103882 + }, + { + "bits_per_byte": 1.581065058501843, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0959107875823975, + "step": 7150, + "unclipped_grad_norm": 1.2786449193954468 + }, + { + "bits_per_byte": 1.568793580414829, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0874048471450806, + "step": 7160, + "unclipped_grad_norm": 1.2588245868682861 + }, + { + "bits_per_byte": 1.5794673396753276, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0948033332824707, + "step": 7170, + "unclipped_grad_norm": 1.2507553100585938 + }, + { + "bits_per_byte": 1.519351835962138, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0531344413757324, + "step": 7180, + "unclipped_grad_norm": 1.194468379020691 + }, + { + "bits_per_byte": 1.5739864245749806, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0910042524337769, + "step": 7190, + "unclipped_grad_norm": 1.2753334045410156 + }, + { + "bits_per_byte": 1.5095894127685614, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0463676452636719, + "step": 7200, + "unclipped_grad_norm": 1.1532927751541138 + }, + { + "bits_per_byte": 1.5104264523303108, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0469478368759155, + "step": 7210, + "unclipped_grad_norm": 1.1723830699920654 + }, + { + "bits_per_byte": 1.5078061246617651, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0451315641403198, + "step": 7220, + "unclipped_grad_norm": 1.254270076751709 + }, + { + "bits_per_byte": 1.5518125294165879, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.075634479522705, + "step": 7230, + "unclipped_grad_norm": 1.2554386854171753 + }, + { + "bits_per_byte": 1.5582962753541159, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0801286697387695, + "step": 7240, + "unclipped_grad_norm": 1.2507786750793457 + }, + { + "bits_per_byte": 1.571778683285855, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0894739627838135, + "step": 7250, + "unclipped_grad_norm": 1.2601356506347656 + }, + { + "bits_per_byte": 1.5335579468887077, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.062981367111206, + "step": 7260, + "unclipped_grad_norm": 1.203548789024353 + }, + { + "bits_per_byte": 1.5322409037484004, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0620684623718262, + "step": 7270, + "unclipped_grad_norm": 1.2632365226745605 + }, + { + "bits_per_byte": 1.522663877852454, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0554301738739014, + "step": 7280, + "unclipped_grad_norm": 1.226701259613037 + }, + { + "bits_per_byte": 1.5219511817472784, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.054936170578003, + "step": 7290, + "unclipped_grad_norm": 1.2358945608139038 + }, + { + "bits_per_byte": 1.5307491262348163, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0610344409942627, + "step": 7300, + "unclipped_grad_norm": 1.1983592510223389 + }, + { + "bits_per_byte": 1.5145411372522035, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.049799919128418, + "step": 7310, + "unclipped_grad_norm": 1.224237084388733 + }, + { + "bits_per_byte": 1.4925153191562035, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0345327854156494, + "step": 7320, + "unclipped_grad_norm": 1.2309412956237793 + }, + { + "bits_per_byte": 1.55153787312316, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0754441022872925, + "step": 7330, + "unclipped_grad_norm": 1.2790515422821045 + }, + { + "bits_per_byte": 1.5015385609163716, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0407872200012207, + "step": 7340, + "unclipped_grad_norm": 1.2610660791397095 + }, + { + "bits_per_byte": 1.5181352306899345, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0522911548614502, + "step": 7350, + "unclipped_grad_norm": 1.2389053106307983 + }, + { + "bits_per_byte": 1.5177651240252776, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.052034616470337, + "step": 7360, + "unclipped_grad_norm": 1.2198708057403564 + }, + { + "bits_per_byte": 1.4782792832483838, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.024665117263794, + "step": 7370, + "unclipped_grad_norm": 1.2335501909255981 + }, + { + "bits_per_byte": 1.4776752801785535, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0242464542388916, + "step": 7380, + "unclipped_grad_norm": 1.1802135705947876 + }, + { + "bits_per_byte": 1.4876678161589845, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.031172752380371, + "step": 7390, + "unclipped_grad_norm": 1.2134222984313965 + }, + { + "bits_per_byte": 1.508026778402822, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0452845096588135, + "step": 7400, + "unclipped_grad_norm": 1.23512601852417 + }, + { + "bits_per_byte": 1.5449191208047048, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0708563327789307, + "step": 7410, + "unclipped_grad_norm": 1.250113606452942 + }, + { + "bits_per_byte": 1.5275736385692853, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.058833360671997, + "step": 7420, + "unclipped_grad_norm": 1.2959712743759155 + }, + { + "bits_per_byte": 1.4779425412179943, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0244317054748535, + "step": 7430, + "unclipped_grad_norm": 1.2527908086776733 + }, + { + "bits_per_byte": 1.468023269846831, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0175561904907227, + "step": 7440, + "unclipped_grad_norm": 1.2249972820281982 + }, + { + "bits_per_byte": 1.5643605556061893, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0843321084976196, + "step": 7450, + "unclipped_grad_norm": 1.2696515321731567 + }, + { + "bits_per_byte": 1.5085103936170483, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0456197261810303, + "step": 7460, + "unclipped_grad_norm": 1.2498137950897217 + }, + { + "bits_per_byte": 1.4827952036947394, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0277953147888184, + "step": 7470, + "unclipped_grad_norm": 1.2631312608718872 + }, + { + "bits_per_byte": 1.5577199614910746, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0797291994094849, + "step": 7480, + "unclipped_grad_norm": 1.1877262592315674 + }, + { + "bits_per_byte": 1.530344623039987, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0607540607452393, + "step": 7490, + "unclipped_grad_norm": 1.268275499343872 + }, + { + "bits_per_byte": 1.5391859071555423, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0668823719024658, + "step": 7500, + "unclipped_grad_norm": 1.3035756349563599 + }, + { + "bits_per_byte": 1.5412744644676202, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0683300495147705, + "step": 7510, + "unclipped_grad_norm": 1.2108540534973145 + }, + { + "bits_per_byte": 1.5426088778556648, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.069254994392395, + "step": 7520, + "unclipped_grad_norm": 1.265081524848938 + }, + { + "bits_per_byte": 1.5294915890917073, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0601627826690674, + "step": 7530, + "unclipped_grad_norm": 1.2194392681121826 + }, + { + "bits_per_byte": 1.5287204188852386, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0596282482147217, + "step": 7540, + "unclipped_grad_norm": 1.2046699523925781 + }, + { + "bits_per_byte": 1.5311642923539992, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0613222122192383, + "step": 7550, + "unclipped_grad_norm": 1.2151964902877808 + }, + { + "bits_per_byte": 1.4454386801181778, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0019017457962036, + "step": 7560, + "unclipped_grad_norm": 1.2129703760147095 + }, + { + "bits_per_byte": 1.4692137337561033, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0183813571929932, + "step": 7570, + "unclipped_grad_norm": 1.2061578035354614 + }, + { + "bits_per_byte": 1.4652694836412157, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0156474113464355, + "step": 7580, + "unclipped_grad_norm": 1.1768887042999268 + }, + { + "bits_per_byte": 1.5398244787381958, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0673249959945679, + "step": 7590, + "unclipped_grad_norm": 1.2310807704925537 + }, + { + "bits_per_byte": 1.5671884663343265, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0862922668457031, + "step": 7600, + "unclipped_grad_norm": 1.2284492254257202 + }, + { + "bits_per_byte": 1.5395315922837765, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.067121982574463, + "step": 7610, + "unclipped_grad_norm": 1.2651493549346924 + }, + { + "bits_per_byte": 1.5411196800818436, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0682227611541748, + "step": 7620, + "unclipped_grad_norm": 1.2471308708190918 + }, + { + "bits_per_byte": 1.5165897945892812, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0512199401855469, + "step": 7630, + "unclipped_grad_norm": 1.253797173500061 + }, + { + "bits_per_byte": 1.497296092884888, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.037846565246582, + "step": 7640, + "unclipped_grad_norm": 1.318769931793213 + }, + { + "bits_per_byte": 1.4637429656321574, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0145893096923828, + "step": 7650, + "unclipped_grad_norm": 1.2420055866241455 + }, + { + "bits_per_byte": 1.4840704550508872, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0286792516708374, + "step": 7660, + "unclipped_grad_norm": 1.299368977546692 + }, + { + "bits_per_byte": 1.5379041204586616, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0659939050674438, + "step": 7670, + "unclipped_grad_norm": 1.2423371076583862 + }, + { + "bits_per_byte": 1.5171365274363742, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0515989065170288, + "step": 7680, + "unclipped_grad_norm": 1.2243461608886719 + }, + { + "bits_per_byte": 1.5051288707357837, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0432758331298828, + "step": 7690, + "unclipped_grad_norm": 1.245628833770752 + }, + { + "bits_per_byte": 1.544223278999314, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0703740119934082, + "step": 7700, + "unclipped_grad_norm": 1.2915515899658203 + }, + { + "bits_per_byte": 1.4916550619365878, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0339365005493164, + "step": 7710, + "unclipped_grad_norm": 1.201813817024231 + }, + { + "bits_per_byte": 1.4940050328779773, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0355653762817383, + "step": 7720, + "unclipped_grad_norm": 1.2110096216201782 + }, + { + "bits_per_byte": 1.4993191248069873, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0392488241195679, + "step": 7730, + "unclipped_grad_norm": 1.2087814807891846 + }, + { + "bits_per_byte": 1.5356127956012164, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0644056797027588, + "step": 7740, + "unclipped_grad_norm": 1.3031466007232666 + }, + { + "bits_per_byte": 1.4696118735928507, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0186573266983032, + "step": 7750, + "unclipped_grad_norm": 1.249332070350647 + }, + { + "bits_per_byte": 1.4759148657643217, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0230262279510498, + "step": 7760, + "unclipped_grad_norm": 1.2120082378387451 + }, + { + "bits_per_byte": 1.5360676897127488, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0647209882736206, + "step": 7770, + "unclipped_grad_norm": 1.2807056903839111 + }, + { + "bits_per_byte": 1.529660132089553, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0602796077728271, + "step": 7780, + "unclipped_grad_norm": 1.2968212366104126 + }, + { + "bits_per_byte": 1.5315347429839576, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0615789890289307, + "step": 7790, + "unclipped_grad_norm": 1.2608845233917236 + }, + { + "bits_per_byte": 1.5176234103209667, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.051936388015747, + "step": 7800, + "unclipped_grad_norm": 1.2756648063659668 + }, + { + "bits_per_byte": 1.5054119541791038, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0434720516204834, + "step": 7810, + "unclipped_grad_norm": 1.2882802486419678 + }, + { + "bits_per_byte": 1.5431284374439214, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.069615125656128, + "step": 7820, + "unclipped_grad_norm": 1.3333698511123657 + }, + { + "bits_per_byte": 1.5271158207526887, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.058516025543213, + "step": 7830, + "unclipped_grad_norm": 1.229974389076233 + }, + { + "bits_per_byte": 1.501238623173267, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0405793190002441, + "step": 7840, + "unclipped_grad_norm": 1.2474827766418457 + }, + { + "bits_per_byte": 1.4945333635814277, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0359315872192383, + "step": 7850, + "unclipped_grad_norm": 1.2343677282333374 + }, + { + "bits_per_byte": 1.534035198744852, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0633121728897095, + "step": 7860, + "unclipped_grad_norm": 1.256583571434021 + }, + { + "bits_per_byte": 1.5241123157380205, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.056434154510498, + "step": 7870, + "unclipped_grad_norm": 1.3308851718902588 + }, + { + "bits_per_byte": 1.498756569556015, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0388588905334473, + "step": 7880, + "unclipped_grad_norm": 1.301889181137085 + }, + { + "bits_per_byte": 1.516417124007815, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.051100254058838, + "step": 7890, + "unclipped_grad_norm": 1.2672597169876099 + }, + { + "bits_per_byte": 1.4738032627770283, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0215625762939453, + "step": 7900, + "unclipped_grad_norm": 1.2230888605117798 + }, + { + "bits_per_byte": 1.5374244608454053, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0656614303588867, + "step": 7910, + "unclipped_grad_norm": 1.3816039562225342 + }, + { + "bits_per_byte": 1.5497416863175488, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0741990804672241, + "step": 7920, + "unclipped_grad_norm": 1.2244949340820312 + }, + { + "bits_per_byte": 1.5130077399371107, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0487370491027832, + "step": 7930, + "unclipped_grad_norm": 1.2602943181991577 + }, + { + "bits_per_byte": 1.5360826522033737, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0647313594818115, + "step": 7940, + "unclipped_grad_norm": 1.278623342514038 + }, + { + "bits_per_byte": 1.479992574416279, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0258526802062988, + "step": 7950, + "unclipped_grad_norm": 1.2290197610855103 + }, + { + "bits_per_byte": 1.5513340736818877, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0753028392791748, + "step": 7960, + "unclipped_grad_norm": 1.3293349742889404 + }, + { + "bits_per_byte": 1.5048769161522697, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.043101191520691, + "step": 7970, + "unclipped_grad_norm": 1.2828478813171387 + }, + { + "bits_per_byte": 1.5449772509406965, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0708966255187988, + "step": 7980, + "unclipped_grad_norm": 1.325577974319458 + }, + { + "bits_per_byte": 1.5031080745881458, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0418751239776611, + "step": 7990, + "unclipped_grad_norm": 1.3532280921936035 + }, + { + "bits_per_byte": 1.5250520289423348, + "learning_rate": 3e-05, + "loss_nats": 1.0570855140686035, + "step": 8000, + "unclipped_grad_norm": 1.3144081830978394 + } + ] + }, + "depth-16-block-seed-2026073003": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "a4c4ddb026a74a7ed70ffc58516bae9bdba5550dc232a1cb003063f2c7f9ca1d", + "depth": 16, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0003044046170543879, + 0.0002627764770295471, + 0.0002327302354387939, + 0.0002064981235889718, + 0.0001888095575850457, + 0.00017376731557305902, + 0.00016044666699599475, + 0.0001515327749075368, + 0.000144502759212628, + 0.00014044572890270501, + 0.00013478762411978096, + 0.00013080857752356678, + 0.00012769536988344043, + 0.00012482800229918212, + 0.00012209055421408266, + 0.0001202532512252219 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002516023632779252, + "first_to_last_ratio": 2.0336961080101905, + "imbalance_abs_log_ratio": 0.7098548803728145, + "last_quartile_mean": 0.00012371679440548178, + "mean": 0.00017039860222212155, + "normalized": [ + 1.7864267258341946, + 1.5421281254819637, + 1.3657989702017657, + 1.2118533890306977, + 1.1080463989893807, + 1.0197696067162934, + 0.941596145177564, + 0.8892841427772256, + 0.8480278437041564, + 0.8242187850791655, + 0.7910136724248463, + 0.7676622684560079, + 0.7493921206993488, + 0.7325647081098923, + 0.7164997401500549, + 0.7057173571674424 + ], + "population_cv": 0.31342321744292045 + }, + "activation_output_rms_by_block": [ + 0.005092422943562269, + 0.007266214583069086, + 0.00560376700013876, + 0.007757550571113825, + 0.005751485005021095, + 0.008497623726725578, + 0.006417509168386459, + 0.009013078175485134, + 0.006685016211122274, + 0.010579346679151058, + 0.007517023477703333, + 0.010370824486017227, + 0.008143824525177479, + 0.011447666212916374, + 0.009210918098688126, + 0.013363312929868698 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.006429988774470985, + "first_to_last_ratio": 0.6099730781372781, + "imbalance_abs_log_ratio": 0.4943404569899061, + "last_quartile_mean": 0.01054143044166267, + "mean": 0.008294848987134174, + "normalized": [ + 0.6139259378273111, + 0.8759911837261217, + 0.675571913223562, + 0.93522505149235, + 0.6933803151741528, + 1.0244458627162376, + 0.7736740208701105, + 1.0865873736176486, + 0.8059237993954019, + 1.275411607319227, + 0.9062278878569946, + 1.250272850307826, + 0.9817929823447126, + 1.3800933845417096, + 1.1104383109294493, + 1.6110375186571844 + ], + "population_cv": 0.2672227042610828 + }, + "bits_per_byte": 8.019773211317682, + "branch_output_rms_by_sublayer": [ + 0.0034366261679679155, + 0.003824356012046337, + 0.0035537455696612597, + 0.003800130682066083, + 0.004073208663612604, + 0.0038295022677630186, + 0.004064237233251333, + 0.0037886372301727533, + 0.004280691966414452, + 0.0038349912501871586, + 0.0050888159312307835, + 0.0038198749534785748, + 0.005177160259336233, + 0.0037978272885084152, + 0.005646859761327505, + 0.003781275823712349, + 0.0056319693103432655, + 0.003643249161541462, + 0.006857539992779493, + 0.0037886397913098335, + 0.006555364932864904, + 0.003700864966958761, + 0.0065165311098098755, + 0.0036778782960027456, + 0.0071639628149569035, + 0.003695491934195161, + 0.007599730510264635, + 0.0037103460635989904, + 0.00840203370898962, + 0.0037034295964986086, + 0.008103356696665287, + 0.0036014309152960777 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006980641236819168, + 0.006790815718474037, + 0.006855853249106197, + 0.00601204909014858, + 0.005899153963056259, + 0.006300783956032697, + 0.005704077288608638, + 0.005731973180528048, + 0.005353243010845193, + 0.00629832195028187, + 0.005348066742481173, + 0.005021425359765903, + 0.004977505888369926, + 0.004996266682376707, + 0.0052721592596541, + 0.004713786845169383 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.006659839823636995, + "first_to_last_ratio": 1.3346560503958143, + "imbalance_abs_log_ratio": 0.2886736185043388, + "last_quartile_mean": 0.004989929668892529, + "mean": 0.005766007713857367, + "normalized": [ + 1.2106541619850226, + 1.1777326801269727, + 1.189012153526888, + 1.0426710106023453, + 1.0230915836062624, + 1.092746362598529, + 0.9892593925776594, + 0.9940973833164453, + 0.9284141257702132, + 1.0923193763936871, + 0.9275164043967956, + 0.8708669167573224, + 0.8632499530667561, + 0.8665036417431853, + 0.914351752770568, + 0.8175131007613471 + ], + "population_cv": 0.12132893681311732 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028142472729086876, + 0.014130397699773312, + 0.014246499165892601, + 0.01435180939733982, + 0.01446386519819498, + 0.009747256524860859, + 0.009819178842008114, + 0.00989843625575304, + 0.009975779801607132, + 0.007541276980191469, + 0.007608718704432249, + 0.007703919429332018, + 0.0077462210319936275, + 0.00631322804838419, + 0.006355294957756996, + 0.006443085614591837, + 0.006486334837973118, + 0.005488897208124399, + 0.00551867438480258, + 0.005632160697132349, + 0.0056700571440160275, + 0.004926848225295544, + 0.004952999297529459, + 0.005005215294659138, + 0.005028663668781519, + 0.004523467272520065, + 0.004554289858788252, + 0.004648423288017511, + 0.004657648038119078, + 0.004240449517965317, + 0.0042638350278139114, + 0.004358324687927961 + ], + "loss_nats": 5.558883190155029, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0034366261679679155, + 0.005092422943562269, + 0.006235890556126833, + 0.007266214583069086, + 0.004073208663612604, + 0.00560376700013876, + 0.006768420804291964, + 0.007757550571113825, + 0.004280691966414452, + 0.005751485005021095, + 0.00762521754950285, + 0.008497623726725578, + 0.005177160259336233, + 0.006417509168386459, + 0.008189847692847252, + 0.009013078175485134, + 0.0056319693103432655, + 0.006685016211122274, + 0.009837556630373001, + 0.010579346679151058, + 0.006555364932864904, + 0.007517023477703333, + 0.009742370806634426, + 0.010370824486017227, + 0.0071639628149569035, + 0.008143824525177479, + 0.01089458167552948, + 0.011447666212916374, + 0.00840203370898962, + 0.009210918098688126, + 0.012791652232408524, + 0.013363312929868698 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.495960016967729e-05, + 6.749085150659084e-05, + 5.70677402720321e-05, + 5.0705322792055085e-05, + 4.370361421024427e-05, + 3.821287100436166e-05, + 3.268615546403453e-05, + 2.8849746740888804e-05, + 2.5781664589885622e-05, + 2.37614458455937e-05, + 2.279552791151218e-05, + 2.1788309823023155e-05, + 2.0940320609952323e-05, + 2.0449986550374888e-05, + 1.9821660316665657e-05, + 1.9541532310540788e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.255587868508883e-05, + "first_to_last_ratio": 3.098608919721194, + "imbalance_abs_log_ratio": 1.1309532752053548, + "last_quartile_mean": 2.0188374946883414e-05, + "mean": 3.5534771882339555e-05, + "normalized": [ + 2.1094718271409953, + 1.8992904113768394, + 1.605968949540217, + 1.4269212972563121, + 1.229883066506037, + 1.0753655920710468, + 0.9198358039911673, + 0.811873700397296, + 0.7255334204832441, + 0.6686815360535046, + 0.6414992049756577, + 0.6131546276747519, + 0.5892909817822544, + 0.575492270446749, + 0.5578102592665534, + 0.549927051037374 + ], + "population_cv": 0.4936925451995582 + }, + "activation_output_rms_by_block": [ + 0.009570118971168995, + 0.01747945137321949, + 0.011283415369689465, + 0.020727796480059624, + 0.012305145151913166, + 0.023130038753151894, + 0.015205477364361286, + 0.028237100690603256, + 0.017268717288970947, + 0.03324708342552185, + 0.021605173125863075, + 0.04313390702009201, + 0.02657386288046837, + 0.05039682611823082, + 0.03066091425716877, + 0.0630461722612381 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.014765195548534393, + "first_to_last_ratio": 0.34603674681838265, + "imbalance_abs_log_ratio": 1.0612103048863237, + "last_quartile_mean": 0.042669443879276514, + "mean": 0.02649195003323257, + "normalized": [ + 0.3612463015808142, + 0.6598023683153774, + 0.42591864153205383, + 0.7824186763925585, + 0.46448619812724606, + 0.8730968737347248, + 0.5739659536307038, + 1.0658747527147492, + 0.6518477223197376, + 1.2549881523940432, + 0.8155372895827101, + 1.628189203361138, + 1.0030919901001265, + 1.9023449030747457, + 1.1573672084805566, + 2.3798237646587146 + ], + "population_cv": 0.544699846184418 + }, + "bits_per_byte": 5.090461856197109, + "branch_output_rms_by_sublayer": [ + 0.004518123809248209, + 0.008520865812897682, + 0.004510615952312946, + 0.009082728065550327, + 0.005324283614754677, + 0.0099419467151165, + 0.004656728357076645, + 0.010662632063031197, + 0.00614966033026576, + 0.010214399546384811, + 0.0060766469687223434, + 0.011413711123168468, + 0.0087440712377429, + 0.011133860796689987, + 0.009014265611767769, + 0.011885596439242363, + 0.010757530108094215, + 0.011522865854203701, + 0.011306028813123703, + 0.012577465735375881, + 0.014357471838593483, + 0.01213979721069336, + 0.01672077551484108, + 0.013535024598240852, + 0.01905498467385769, + 0.012188552878797054, + 0.018409499898552895, + 0.013168000616133213, + 0.021850280463695526, + 0.012649565935134888, + 0.02609730139374733, + 0.013150178827345371 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00025280350140969286, + 0.00022135664689195577, + 0.00022214031151724322, + 0.0001810293778015987, + 0.00018338803861517842, + 0.00016050318916432303, + 0.0001626860438404638, + 0.00015310192441761306, + 0.00014075054270255738, + 0.00013076915091262195, + 0.00013639133469956174, + 0.0001377615870880476, + 0.00013681686670328103, + 0.0001370125878547912, + 0.0001387461931789955, + 0.0001464231652858264 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021933245940512265, + "first_to_last_ratio": 1.5694663694832551, + "imbalance_abs_log_ratio": 0.45073566953494, + "last_quartile_mean": 0.00013974970325572353, + "mean": 0.0001651050288802345, + "normalized": [ + 1.531167785283354, + 1.3407020270262369, + 1.3454484882976008, + 1.0964498115494445, + 1.1107356320939579, + 0.9721278040582907, + 0.9853488106560013, + 0.9273001885888748, + 0.8524909486836794, + 0.7920361469273026, + 0.8260883125401264, + 0.8343875896595401, + 0.8286656538034743, + 0.8298510880257848, + 0.8403511032946221, + 0.8868486095117084 + ], + "population_cv": 0.21861370307121916 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6929093599319458, + "mean_weights": [ + 0.4913179874420166, + 0.5086820125579834 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6919252872467041, + "mean_weights": [ + 0.5235024690628052, + 0.4764975309371948 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930201053619385, + "mean_weights": [ + 0.4991857707500458, + 0.5008142590522766 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6926943063735962, + "mean_weights": [ + 0.5129889249801636, + 0.48701101541519165 + ], + "sources": 2 + }, + { + "entropy_mean": 1.097741961479187, + "mean_weights": [ + 0.3480756878852844, + 0.33060726523399353, + 0.32131701707839966 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982271432876587, + "mean_weights": [ + 0.3435906767845154, + 0.327891081571579, + 0.328518271446228 + ], + "sources": 3 + }, + { + "entropy_mean": 1.097235918045044, + "mean_weights": [ + 0.35132721066474915, + 0.33742234110832214, + 0.3112504482269287 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098421573638916, + "mean_weights": [ + 0.340750515460968, + 0.3319299519062042, + 0.32731953263282776 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3854012489318848, + "mean_weights": [ + 0.26307183504104614, + 0.2547699213027954, + 0.2451326996088028, + 0.23702551424503326 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862006664276123, + "mean_weights": [ + 0.25451934337615967, + 0.25035756826400757, + 0.24778643250465393, + 0.24733665585517883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3854570388793945, + "mean_weights": [ + 0.26253488659858704, + 0.2546469569206238, + 0.2452046275138855, + 0.2376135289669037 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861933946609497, + "mean_weights": [ + 0.2505362629890442, + 0.254558801651001, + 0.2493266463279724, + 0.24557828903198242 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6085797548294067, + "mean_weights": [ + 0.2100008726119995, + 0.20838744938373566, + 0.19692982733249664, + 0.19323000311851501, + 0.19145184755325317 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093010902404785, + "mean_weights": [ + 0.19938474893569946, + 0.20374906063079834, + 0.20229801535606384, + 0.19861868023872375, + 0.1959494650363922 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085379123687744, + "mean_weights": [ + 0.20589761435985565, + 0.21123698353767395, + 0.20005744695663452, + 0.1941205859184265, + 0.18868738412857056 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093740463256836, + "mean_weights": [ + 0.19956088066101074, + 0.2027459442615509, + 0.20118315517902374, + 0.1998024433851242, + 0.1967075765132904 + ], + "sources": 5 + }, + { + "entropy_mean": 1.789860725402832, + "mean_weights": [ + 0.17310243844985962, + 0.1810513734817505, + 0.17217648029327393, + 0.1649792194366455, + 0.1559385061264038, + 0.15275199711322784 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791734218597412, + "mean_weights": [ + 0.16535353660583496, + 0.16847901046276093, + 0.1662793755531311, + 0.16693073511123657, + 0.16613470017910004, + 0.1668226420879364 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890782356262207, + "mean_weights": [ + 0.17188119888305664, + 0.18350960314273834, + 0.17409591376781464, + 0.16590778529644012, + 0.15644267201423645, + 0.148162841796875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791684627532959, + "mean_weights": [ + 0.16539111733436584, + 0.16517111659049988, + 0.16458021104335785, + 0.16693061590194702, + 0.16811248660087585, + 0.16981446743011475 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9417986869812012, + "mean_weights": [ + 0.14831508696079254, + 0.16164575517177582, + 0.15476199984550476, + 0.14618372917175293, + 0.13434207439422607, + 0.1278611123561859, + 0.12689024209976196 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9456918239593506, + "mean_weights": [ + 0.14120136201381683, + 0.1386352777481079, + 0.14049293100833893, + 0.1421712040901184, + 0.14506074786186218, + 0.1458473801612854, + 0.14659109711647034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9406665563583374, + "mean_weights": [ + 0.14782890677452087, + 0.1637636125087738, + 0.15569329261779785, + 0.14784488081932068, + 0.13445764780044556, + 0.12887068092823029, + 0.12154098600149155 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9455351829528809, + "mean_weights": [ + 0.14214086532592773, + 0.1382882297039032, + 0.1401882916688919, + 0.1396511197090149, + 0.14402973651885986, + 0.14592906832695007, + 0.14977267384529114 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0713937282562256, + "mean_weights": [ + 0.1321096122264862, + 0.1488187313079834, + 0.13977529108524323, + 0.13292929530143738, + 0.12093928456306458, + 0.11470195651054382, + 0.10486428439617157, + 0.10586154460906982 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0790562629699707, + "mean_weights": [ + 0.12411445379257202, + 0.1212267130613327, + 0.12224400043487549, + 0.12172792106866837, + 0.12399555742740631, + 0.12670958042144775, + 0.1301731914281845, + 0.12980858981609344 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068070888519287, + "mean_weights": [ + 0.13267414271831512, + 0.15216735005378723, + 0.1437777280807495, + 0.13539820909500122, + 0.1199566125869751, + 0.11211173236370087, + 0.10534092038869858, + 0.09857328981161118 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079178810119629, + "mean_weights": [ + 0.12415283918380737, + 0.12161671370267868, + 0.12301617860794067, + 0.12307952344417572, + 0.12411122024059296, + 0.12585631012916565, + 0.12885767221450806, + 0.1293095499277115 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1886847019195557, + "mean_weights": [ + 0.11892519891262054, + 0.13333868980407715, + 0.12725916504859924, + 0.12129361927509308, + 0.10894724726676941, + 0.10413059592247009, + 0.09661625325679779, + 0.09465864300727844, + 0.09483058750629425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969594955444336, + "mean_weights": [ + 0.1101110577583313, + 0.10734203457832336, + 0.10833269357681274, + 0.10946638882160187, + 0.11091698706150055, + 0.11324633657932281, + 0.11330728977918625, + 0.11495929211378098, + 0.11231791973114014 + ], + "sources": 9 + }, + { + "entropy_mean": 2.186551570892334, + "mean_weights": [ + 0.11992302536964417, + 0.13721907138824463, + 0.12899263203144073, + 0.12160105258226395, + 0.10788539052009583, + 0.10353681445121765, + 0.09616796672344208, + 0.09295656532049179, + 0.09171747416257858 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02856776863336563, + 0.014094225130975246, + 0.01573058031499386, + 0.015321774408221245, + 0.017058780416846275, + 0.011773223988711834, + 0.013085381127893925, + 0.013459180481731892, + 0.015175317414104939, + 0.011872655712068081, + 0.013221408240497112, + 0.013565259985625744, + 0.015276176854968071, + 0.012822598218917847, + 0.014198821038007736, + 0.014604844152927399, + 0.0162027720361948, + 0.014113145880401134, + 0.015384308062493801, + 0.015927067026495934, + 0.017611440271139145, + 0.01576142944395542, + 0.017285119742155075, + 0.01772329770028591, + 0.019923174753785133, + 0.017852960154414177, + 0.019824320450425148, + 0.01966804265975952, + 0.022481828927993774, + 0.020090021193027496, + 0.02255004271864891, + 0.022322168573737144 + ], + "loss_nats": 3.5284392833709717, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.193685531616211, + "mean_weights": [ + 0.10035791993141174, + 0.09579961001873016, + 0.10334210842847824, + 0.108812615275383, + 0.11718772351741791, + 0.11711248755455017, + 0.12047697603702545, + 0.11920724809169769, + 0.11770330369472504 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.004518123809248209, + 0.009570118971168995, + 0.010795549489557743, + 0.01747945137321949, + 0.005324283614754677, + 0.011283415369689465, + 0.012478979304432869, + 0.020727796480059624, + 0.00614966033026576, + 0.012305145151913166, + 0.014650273136794567, + 0.023130038753151894, + 0.0087440712377429, + 0.015205477364361286, + 0.019528985023498535, + 0.028237100690603256, + 0.010757530108094215, + 0.017268717288970947, + 0.02412688545882702, + 0.03324708342552185, + 0.014357471838593483, + 0.021605173125863075, + 0.03339937701821327, + 0.04313390702009201, + 0.01905498467385769, + 0.02657386288046837, + 0.04070562124252319, + 0.05039682611823082, + 0.021850280463695526, + 0.03066091425716877, + 0.052838243544101715, + 0.0630461722612381 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00015504666953347623, + 0.00015570323739666492, + 0.00013642148405779153, + 0.00012308824807405472, + 0.00010802906763274223, + 9.803942521102726e-05, + 8.508376777172089e-05, + 7.592589827254415e-05, + 7.549385918537155e-05, + 6.5492422436364e-05, + 6.254414620343596e-05, + 5.1708771934499964e-05, + 4.589578747982159e-05, + 4.085309410584159e-05, + 3.8330261304508895e-05, + 3.7662506656488404e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00014256490976549685, + "first_to_last_ratio": 3.504079260905398, + "imbalance_abs_log_ratio": 1.2539277929397175, + "last_quartile_mean": 4.068541238666512e-05, + "mean": 8.470741545352212e-05, + "normalized": [ + 1.8303789426623265, + 1.8381299507608906, + 1.610502260367562, + 1.4530988510869118, + 1.2753200774023898, + 1.1573889332607517, + 1.0044429677872213, + 0.8963311873705286, + 0.8912308182368528, + 0.7731604380291695, + 0.7383550291149319, + 0.6104397313700582, + 0.5418154624845569, + 0.4822847431610892, + 0.45250183941145294, + 0.44461876749330576 + ], + "population_cv": 0.46442228143345987 + }, + "activation_output_rms_by_block": [ + 0.02604864165186882, + 0.04358876869082451, + 0.022780675441026688, + 0.03670342266559601, + 0.027025330811738968, + 0.05019363388419151, + 0.040244098752737045, + 0.04879867285490036, + 0.029798155650496483, + 0.04518449306488037, + 0.02838377095758915, + 0.051337067037820816, + 0.03289651870727539, + 0.0653979703783989, + 0.04346369579434395, + 0.08895192295312881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.032280377112329006, + "first_to_last_ratio": 0.5596699237065876, + "imbalance_abs_log_ratio": 0.5804080909828722, + "last_quartile_mean": 0.05767752695828676, + "mean": 0.04254980245605111, + "normalized": [ + 0.6121918351741814, + 1.0244176511946566, + 0.5353885124274412, + 0.8625991320055194, + 0.6351458585419503, + 1.1796443458470947, + 0.9458116472880084, + 1.1468601506506073, + 0.7003124322674456, + 1.0619201607704427, + 0.6670717446198772, + 1.2065171651700595, + 0.7731297634402319, + 1.5369747120670472, + 1.0214782040230805, + 2.0905366845123567 + ], + "population_cv": 0.38343105399742256 + }, + "bits_per_byte": 3.1146838872489564, + "branch_output_rms_by_sublayer": [ + 0.01352347619831562, + 0.020455393940210342, + 0.016984812915325165, + 0.0142062958329916, + 0.01867547444999218, + 0.011138904839754105, + 0.019149528816342354, + 0.0099713783711195, + 0.023481043055653572, + 0.011000796221196651, + 0.02560650371015072, + 0.010696547105908394, + 0.0353584885597229, + 0.012510747648775578, + 0.023770872503519058, + 0.012255984358489513, + 0.023732667788863182, + 0.014518127776682377, + 0.021041536703705788, + 0.015058746561408043, + 0.021396929398179054, + 0.018767673522233963, + 0.022012433037161827, + 0.023688357323408127, + 0.020202528685331345, + 0.02677428349852562, + 0.015811076387763023, + 0.03653915971517563, + 0.014067054726183414, + 0.0391332171857357, + 0.010850721038877964, + 0.043844256550073624 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006567095640112879, + 0.00045868421249221203, + 0.00039439577454117915, + 0.0003702348543320087, + 0.0003691077163774564, + 0.00041151853265381964, + 0.0004306774133910072, + 0.00035317417015185023, + 0.00038561746855580054, + 0.0003833308661148803, + 0.00032325470855348767, + 0.0002967607909467712, + 0.00021968458301266372, + 0.00017220390788738176, + 0.00016255203232502606, + 0.0001597780110258732 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00047000610134417196, + "first_to_last_ratio": 2.6322817390176136, + "imbalance_abs_log_ratio": 0.967851051434004, + "last_quartile_mean": 0.0001785546335627362, + "mean": 0.00034673028789829414, + "normalized": [ + 1.8940069181493584, + 1.3228847565424027, + 1.137471366957324, + 1.0677891930819992, + 1.0645384301867686, + 1.1868548754371575, + 1.242110736854166, + 1.0185847111673332, + 1.1121539767789572, + 1.105559218487778, + 0.9322944081778846, + 0.8558836689623711, + 0.6335892498583898, + 0.49665089522809175, + 0.46881405518489694, + 0.46081353894511995 + ], + "population_cv": 0.3565995806724142 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6422993540763855, + "mean_weights": [ + 0.370745986700058, + 0.6292540431022644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6814000606536865, + "mean_weights": [ + 0.4577118158340454, + 0.5422881841659546 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6871446371078491, + "mean_weights": [ + 0.49052417278289795, + 0.5094758868217468 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6793373227119446, + "mean_weights": [ + 0.4676857590675354, + 0.5323142409324646 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0915337800979614, + "mean_weights": [ + 0.29751554131507874, + 0.351987361907959, + 0.3504970669746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0892891883850098, + "mean_weights": [ + 0.3054355978965759, + 0.34222692251205444, + 0.35233747959136963 + ], + "sources": 3 + }, + { + "entropy_mean": 1.093106746673584, + "mean_weights": [ + 0.30644628405570984, + 0.34674984216690063, + 0.3468038737773895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0885257720947266, + "mean_weights": [ + 0.3052448034286499, + 0.3325393795967102, + 0.3622158169746399 + ], + "sources": 3 + }, + { + "entropy_mean": 1.381313443183899, + "mean_weights": [ + 0.2415536344051361, + 0.24482353031635284, + 0.25347256660461426, + 0.2601502537727356 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3802430629730225, + "mean_weights": [ + 0.24572397768497467, + 0.2491092085838318, + 0.25083041191101074, + 0.254336416721344 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3813848495483398, + "mean_weights": [ + 0.2464662790298462, + 0.23372246325016022, + 0.2595708668231964, + 0.260240375995636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772881031036377, + "mean_weights": [ + 0.24203142523765564, + 0.2443695217370987, + 0.2359469085931778, + 0.27765214443206787 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6055738925933838, + "mean_weights": [ + 0.20364060997962952, + 0.18423235416412354, + 0.18416231870651245, + 0.20781230926513672, + 0.22015240788459778 + ], + "sources": 5 + }, + { + "entropy_mean": 1.602186679840088, + "mean_weights": [ + 0.19954648613929749, + 0.1888481080532074, + 0.18637022376060486, + 0.21049518883228302, + 0.21474000811576843 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6062731742858887, + "mean_weights": [ + 0.19444647431373596, + 0.1968603879213333, + 0.18490161001682281, + 0.20200669765472412, + 0.2217848002910614 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6028468608856201, + "mean_weights": [ + 0.20196445286273956, + 0.19006873667240143, + 0.1860634982585907, + 0.1993759572505951, + 0.22252735495567322 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7881723642349243, + "mean_weights": [ + 0.15337401628494263, + 0.17425060272216797, + 0.1601826399564743, + 0.1535373479127884, + 0.17563557624816895, + 0.18301980197429657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7856398820877075, + "mean_weights": [ + 0.1621149629354477, + 0.15582670271396637, + 0.15492847561836243, + 0.1618887484073639, + 0.1784553974866867, + 0.1867857277393341 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7870616912841797, + "mean_weights": [ + 0.1564463973045349, + 0.18363508582115173, + 0.15811879932880402, + 0.15924012660980225, + 0.15980733931064606, + 0.18275225162506104 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7877882719039917, + "mean_weights": [ + 0.17034202814102173, + 0.15881556272506714, + 0.1534808874130249, + 0.16709929704666138, + 0.16602207720279694, + 0.1842401623725891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9344044923782349, + "mean_weights": [ + 0.12371677905321121, + 0.1564711332321167, + 0.13837161660194397, + 0.1230623871088028, + 0.12980599701404572, + 0.152274027466774, + 0.17629806697368622 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9417431354522705, + "mean_weights": [ + 0.14328667521476746, + 0.1450044810771942, + 0.14678549766540527, + 0.13414078950881958, + 0.1328907310962677, + 0.1347782164812088, + 0.16311360895633698 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9288355112075806, + "mean_weights": [ + 0.12207633256912231, + 0.16755375266075134, + 0.13929706811904907, + 0.12109649926424026, + 0.12036697566509247, + 0.15116539597511292, + 0.1784439980983734 + ], + "sources": 7 + }, + { + "entropy_mean": 1.938415288925171, + "mean_weights": [ + 0.1377446949481964, + 0.1613207459449768, + 0.156096950173378, + 0.13170479238033295, + 0.12767548859119415, + 0.13129547238349915, + 0.15416184067726135 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0616660118103027, + "mean_weights": [ + 0.10379506647586823, + 0.1403447985649109, + 0.12122530490159988, + 0.10852085053920746, + 0.10464349389076233, + 0.1186104342341423, + 0.13291975855827332, + 0.1699402928352356 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0691847801208496, + "mean_weights": [ + 0.11521603912115097, + 0.14423911273479462, + 0.13052678108215332, + 0.12005230039358139, + 0.10937757790088654, + 0.12122473865747452, + 0.12272199243307114, + 0.1366414725780487 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054232597351074, + "mean_weights": [ + 0.10057584941387177, + 0.16718831658363342, + 0.1352493166923523, + 0.11277885735034943, + 0.09778623282909393, + 0.11311028897762299, + 0.13487182557582855, + 0.1384393274784088 + ], + "sources": 8 + }, + { + "entropy_mean": 2.060153007507324, + "mean_weights": [ + 0.11611463129520416, + 0.15179042518138885, + 0.14188171923160553, + 0.1159345731139183, + 0.10906632244586945, + 0.12251802533864975, + 0.12206383049488068, + 0.12063045799732208 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1772594451904297, + "mean_weights": [ + 0.09167024493217468, + 0.14425000548362732, + 0.1316833794116974, + 0.09630900621414185, + 0.09528810530900955, + 0.1022176742553711, + 0.12638677656650543, + 0.10320886224508286, + 0.10898594558238983 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191457986831665, + "mean_weights": [ + 0.10587693750858307, + 0.12391996383666992, + 0.11278913170099258, + 0.11095453053712845, + 0.10569566488265991, + 0.11266011744737625, + 0.11442379653453827, + 0.11124123632907867, + 0.10243862122297287 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1550254821777344, + "mean_weights": [ + 0.08980270475149155, + 0.17290523648262024, + 0.13377593457698822, + 0.10608047246932983, + 0.09299036115407944, + 0.1071130782365799, + 0.12428411841392517, + 0.09728670865297318, + 0.07576138526201248 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04350041598081589, + 0.017946822568774223, + 0.025110680609941483, + 0.028106264770030975, + 0.031104078516364098, + 0.02212802693247795, + 0.02379882149398327, + 0.02537565305829048, + 0.026546044275164604, + 0.02055767923593521, + 0.022140897810459137, + 0.02340756356716156, + 0.02546003647148609, + 0.020307816565036774, + 0.02165406011044979, + 0.021843569353222847, + 0.023044398054480553, + 0.018649185076355934, + 0.019831404089927673, + 0.019260063767433167, + 0.02048473060131073, + 0.01540753897279501, + 0.01695212349295616, + 0.01553493645042181, + 0.017758244648575783, + 0.013987960293889046, + 0.016796667128801346, + 0.01665738970041275, + 0.01972099393606186, + 0.01747863180935383, + 0.020752927288413048, + 0.019990377128124237 + ], + "loss_nats": 2.1589343547821045, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1796395778656006, + "mean_weights": [ + 0.12411575019359589, + 0.09159804880619049, + 0.10466736555099487, + 0.10315616428852081, + 0.09333926439285278, + 0.09817550331354141, + 0.11351706087589264, + 0.1361408531665802, + 0.1352899968624115 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01352347619831562, + 0.02604864165186882, + 0.03525395691394806, + 0.04358876869082451, + 0.01867547444999218, + 0.022780675441026688, + 0.03279132395982742, + 0.03670342266559601, + 0.023481043055653572, + 0.027025330811738968, + 0.04530210420489311, + 0.05019363388419151, + 0.0353584885597229, + 0.040244098752737045, + 0.04290904477238655, + 0.04879867285490036, + 0.023732667788863182, + 0.029798155650496483, + 0.03790274262428284, + 0.04518449306488037, + 0.021396929398179054, + 0.02838377095758915, + 0.03953101858496666, + 0.051337067037820816, + 0.020202528685331345, + 0.03289651870727539, + 0.039050742983818054, + 0.0653979703783989, + 0.014067054726183414, + 0.04346369579434395, + 0.048713527619838715, + 0.08895192295312881 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.91841812315397e-05, + 8.04664014140144e-05, + 6.887180643389001e-05, + 6.264931289479136e-05, + 6.203571683727205e-05, + 6.256609776755795e-05, + 7.726731564616784e-05, + 7.684857700951397e-05, + 0.0001297412090934813, + 0.00010362039029132575, + 0.0001698000414762646, + 0.00012187293032184243, + 0.00013834609126206487, + 8.384700049646199e-05, + 4.917434125673026e-05, + 4.311763041187078e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.279292549355887e-05, + "first_to_last_ratio": 0.9258681439467011, + "imbalance_abs_log_ratio": 0.07702344762093526, + "last_quartile_mean": 7.862126585678197e-05, + "mean": 8.808806524029933e-05, + "normalized": [ + 0.8989206541831708, + 0.91347677116652, + 0.7818517326497246, + 0.7112122706281212, + 0.7042465590320558, + 0.7102675895637067, + 0.8771598676323168, + 0.8724062311946045, + 1.4728579716168646, + 1.1763272358026606, + 1.9276168799151117, + 1.3835351019389504, + 1.570542966117651, + 0.9518542638861694, + 0.5582406779236824, + 0.4894832267486894 + ], + "population_cv": 0.3870844002085187 + }, + "activation_output_rms_by_block": [ + 0.10205306112766266, + 0.1637924462556839, + 0.1347782015800476, + 0.18643726408481598, + 0.08823998272418976, + 0.14544272422790527, + 0.051831819117069244, + 0.08038940280675888, + 0.045988935977220535, + 0.08743251860141754, + 0.060310039669275284, + 0.09679152816534042, + 0.05415637046098709, + 0.0906868502497673, + 0.09274918586015701, + 0.19966518878936768 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14676524326205254, + "first_to_last_ratio": 1.3425975426784762, + "imbalance_abs_log_ratio": 0.2946062022500781, + "last_quartile_mean": 0.10931439884006977, + "mean": 0.10504659498110414, + "normalized": [ + 0.9715027997434856, + 1.5592361302634037, + 1.2830325590685887, + 1.7748054005782146, + 0.840008024439654, + 1.3845543899263715, + 0.49341741278137374, + 0.7652737608603057, + 0.4377955895238019, + 0.8323213010107083, + 0.5741265547933648, + 0.9214151889716308, + 0.5155461771105363, + 0.8633011880687815, + 0.8829337674090323, + 1.9007297554507465 + ], + "population_cv": 0.43941183630186603 + }, + "bits_per_byte": 2.0898630043632984, + "branch_output_rms_by_sublayer": [ + 0.04098131135106087, + 0.09075035154819489, + 0.05343177169561386, + 0.04652838781476021, + 0.13196392357349396, + 0.018610643222928047, + 0.06760989129543304, + 0.01781350187957287, + 0.0839577317237854, + 0.0164200272411108, + 0.07349737733602524, + 0.016965076327323914, + 0.04464862495660782, + 0.016964683309197426, + 0.0346895232796669, + 0.021388454362750053, + 0.03376077860593796, + 0.026654686778783798, + 0.04212235286831856, + 0.042033009231090546, + 0.0419253334403038, + 0.040410157293081284, + 0.028147248551249504, + 0.05647630989551544, + 0.025301706045866013, + 0.04489820450544357, + 0.015376480296254158, + 0.058749374002218246, + 0.025178087875247, + 0.0875716358423233, + 0.01239082682877779, + 0.1223064437508583 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040335839120789034, + 0.00023897821552144955, + 0.00022638555923562337, + 0.00017534070905568056, + 0.00017358160175308344, + 0.00015895432307312814, + 0.00019058997613466408, + 0.00021161798453125184, + 0.00037807554491806804, + 0.0004740713531015684, + 0.0009894587672613622, + 0.0007545475239678793, + 0.0006624146206615605, + 0.0004575991184432586, + 0.0002840295434450018, + 0.00025831777406547436 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00026101571875516093, + "first_to_last_ratio": 0.6280602344874717, + "imbalance_abs_log_ratio": 0.46511920232449994, + "last_quartile_mean": 0.00041559026415382384, + "mean": 0.00037733256289855906, + "normalized": [ + 1.0689731840512475, + 0.633335786568984, + 0.5999629544203535, + 0.464684806709403, + 0.460022852042453, + 0.4212579000658921, + 0.5050981412009801, + 0.5608261924326488, + 1.001969037640965, + 1.2563754091613246, + 2.6222459033501577, + 1.9996883337384523, + 1.7555193635372572, + 1.2127209878949095, + 0.7527300089426935, + 0.6845891382422771 + ], + "population_cv": 0.6186471183066214 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4914427399635315, + "mean_weights": [ + 0.7962706685066223, + 0.2037293016910553 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6456852555274963, + "mean_weights": [ + 0.6051734089851379, + 0.39482662081718445 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5276303291320801, + "mean_weights": [ + 0.7698293924331665, + 0.2301705777645111 + ], + "sources": 2 + }, + { + "entropy_mean": 0.667390763759613, + "mean_weights": [ + 0.5093090534210205, + 0.4906909465789795 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9880077838897705, + "mean_weights": [ + 0.5180364847183228, + 0.2465059459209442, + 0.23545756936073303 + ], + "sources": 3 + }, + { + "entropy_mean": 1.08414626121521, + "mean_weights": [ + 0.2851155400276184, + 0.3203214704990387, + 0.3945629596710205 + ], + "sources": 3 + }, + { + "entropy_mean": 0.995106041431427, + "mean_weights": [ + 0.5032885074615479, + 0.23357318341732025, + 0.2631382942199707 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0889081954956055, + "mean_weights": [ + 0.2929039001464844, + 0.3457888960838318, + 0.36130720376968384 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3091163635253906, + "mean_weights": [ + 0.3480933904647827, + 0.15371239185333252, + 0.20259280502796173, + 0.29560136795043945 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3786191940307617, + "mean_weights": [ + 0.22527548670768738, + 0.26118987798690796, + 0.22951865196228027, + 0.284015953540802 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308091163635254, + "mean_weights": [ + 0.28280675411224365, + 0.12650004029273987, + 0.24999231100082397, + 0.3407008647918701 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3780848979949951, + "mean_weights": [ + 0.23603014647960663, + 0.2631712555885315, + 0.21569286286830902, + 0.28510570526123047 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5637545585632324, + "mean_weights": [ + 0.21651385724544525, + 0.1117275133728981, + 0.18637539446353912, + 0.22181779146194458, + 0.2635654807090759 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585724115371704, + "mean_weights": [ + 0.18968582153320312, + 0.1962759643793106, + 0.1558789312839508, + 0.1798120141029358, + 0.2783472537994385 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5276048183441162, + "mean_weights": [ + 0.2346682846546173, + 0.15222865343093872, + 0.12689977884292603, + 0.14891111850738525, + 0.3372921645641327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.59938383102417, + "mean_weights": [ + 0.20912373065948486, + 0.21754522621631622, + 0.17016828060150146, + 0.17554700374603271, + 0.22761575877666473 + ], + "sources": 5 + }, + { + "entropy_mean": 1.762988805770874, + "mean_weights": [ + 0.15954776108264923, + 0.12860070168972015, + 0.13993129134178162, + 0.14435188472270966, + 0.1894117146730423, + 0.23815666139125824 + ], + "sources": 6 + }, + { + "entropy_mean": 1.730720043182373, + "mean_weights": [ + 0.17182672023773193, + 0.16310763359069824, + 0.12390270084142685, + 0.11688461899757385, + 0.1326894462108612, + 0.2915888726711273 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4424301385879517, + "mean_weights": [ + 0.1671508103609085, + 0.1365666687488556, + 0.058271970599889755, + 0.05150005966424942, + 0.10222888737916946, + 0.48428159952163696 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7727878093719482, + "mean_weights": [ + 0.1808200478553772, + 0.17958280444145203, + 0.1469326913356781, + 0.13797307014465332, + 0.14092227816581726, + 0.2137690931558609 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5771737098693848, + "mean_weights": [ + 0.09523316472768784, + 0.08390435576438904, + 0.04673727601766586, + 0.04574791342020035, + 0.07810044288635254, + 0.19550111889839172, + 0.45477572083473206 + ], + "sources": 7 + }, + { + "entropy_mean": 1.866133689880371, + "mean_weights": [ + 0.13646650314331055, + 0.12685352563858032, + 0.11049237847328186, + 0.10180146992206573, + 0.1026906818151474, + 0.1334708034992218, + 0.28822463750839233 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755802392959595, + "mean_weights": [ + 0.09840903431177139, + 0.08739377558231354, + 0.04908968508243561, + 0.048939045518636703, + 0.07940414547920227, + 0.16869762539863586, + 0.46806666254997253 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8938008546829224, + "mean_weights": [ + 0.14332836866378784, + 0.14485859870910645, + 0.1094474121928215, + 0.10118386149406433, + 0.10821370780467987, + 0.14364898204803467, + 0.24931909143924713 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7736961841583252, + "mean_weights": [ + 0.08116129040718079, + 0.07111318409442902, + 0.04280973970890045, + 0.041376810520887375, + 0.06107718497514725, + 0.1160595640540123, + 0.31332647800445557, + 0.2730756998062134 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9035927057266235, + "mean_weights": [ + 0.10087604820728302, + 0.09542917460203171, + 0.06220542639493942, + 0.057604290544986725, + 0.0788167417049408, + 0.11659102141857147, + 0.18421337008476257, + 0.3042638897895813 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7131891250610352, + "mean_weights": [ + 0.0740746557712555, + 0.07079474627971649, + 0.04342404007911682, + 0.04123558849096298, + 0.055328819900751114, + 0.09829520434141159, + 0.21916092932224274, + 0.39768603444099426 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9703106880187988, + "mean_weights": [ + 0.11310406029224396, + 0.11765976250171661, + 0.07340675592422485, + 0.0660770833492279, + 0.08407266438007355, + 0.12691649794578552, + 0.20392963290214539, + 0.2148335576057434 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9610494375228882, + "mean_weights": [ + 0.07792562991380692, + 0.0730937123298645, + 0.040804408490657806, + 0.03727761656045914, + 0.05315954610705376, + 0.0937754213809967, + 0.19250980019569397, + 0.26475197076797485, + 0.16670189797878265 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9782097339630127, + "mean_weights": [ + 0.08397573232650757, + 0.08872128278017044, + 0.0407399907708168, + 0.036753468215465546, + 0.058452628552913666, + 0.11618129909038544, + 0.22768989205360413, + 0.21488600969314575, + 0.13259968161582947 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0315680503845215, + "mean_weights": [ + 0.08375388383865356, + 0.10381080955266953, + 0.0493716336786747, + 0.043391548097133636, + 0.06576120853424072, + 0.10571172833442688, + 0.18509769439697266, + 0.2181759774684906, + 0.14492549002170563 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05570797994732857, + 0.04578034579753876, + 0.058472853153944016, + 0.058589257299900055, + 0.09159071743488312, + 0.0729803666472435, + 0.0950615182518959, + 0.08442825824022293, + 0.11022263020277023, + 0.07856440544128418, + 0.0938955619931221, + 0.10307082533836365, + 0.10586344450712204, + 0.08232943713665009, + 0.08149705082178116, + 0.07412237673997879, + 0.0869104191660881, + 0.0654435008764267, + 0.0601484477519989, + 0.04590349644422531, + 0.06966738402843475, + 0.028740333393216133, + 0.04708963632583618, + 0.03821079805493355, + 0.0513334795832634, + 0.034749191254377365, + 0.034582361578941345, + 0.034875962883234024, + 0.041408468037843704, + 0.03493491932749748, + 0.0412786528468132, + 0.04175883159041405 + ], + "loss_nats": 1.448582649230957, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.893564224243164, + "mean_weights": [ + 0.09584413468837738, + 0.060246460139751434, + 0.05342919006943703, + 0.050279051065444946, + 0.05668472498655319, + 0.06780324131250381, + 0.0837026908993721, + 0.2589869201183319, + 0.2730236053466797 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04098131135106087, + 0.10205306112766266, + 0.13058459758758545, + 0.1637924462556839, + 0.13196392357349396, + 0.1347782015800476, + 0.18347561359405518, + 0.18643726408481598, + 0.0839577317237854, + 0.08823998272418976, + 0.14121752977371216, + 0.14544272422790527, + 0.04464862495660782, + 0.051831819117069244, + 0.07186394184827805, + 0.08038940280675888, + 0.03376077860593796, + 0.045988935977220535, + 0.06799308955669403, + 0.08743251860141754, + 0.0419253334403038, + 0.060310039669275284, + 0.07043248414993286, + 0.09679152816534042, + 0.025301706045866013, + 0.05415637046098709, + 0.05762659013271332, + 0.0906868502497673, + 0.025178087875247, + 0.09274918586015701, + 0.0948956087231636, + 0.19966518878936768 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.6704131566220894e-05, + 5.7463053963147104e-05, + 5.5973152484511957e-05, + 4.9604332161834463e-05, + 4.9622754886513576e-05, + 5.175438491278328e-05, + 6.153700815048069e-05, + 5.993711238261312e-05, + 0.00010211551852989942, + 8.014418563107029e-05, + 0.00014815112808719277, + 0.00010670819756342098, + 0.00014386174734681845, + 8.734197035664693e-05, + 5.547845285036601e-05, + 4.4348533265292645e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.4936167543928605e-05, + "first_to_last_ratio": 0.6638196023526066, + "imbalance_abs_log_ratio": 0.4097448496221764, + "last_quartile_mean": 8.275767595478101e-05, + "mean": 7.567160400867579e-05, + "normalized": [ + 0.7493449135784109, + 0.7593740705768433, + 0.7396850273993744, + 0.6555210876215509, + 0.6557645438680578, + 0.6839340277080636, + 0.8132113618660104, + 0.792068744515331, + 1.3494562440911366, + 1.0591051515424692, + 1.9578166741411636, + 1.4101484825296797, + 1.9011325216566657, + 1.1542238531990563, + 0.7331475733487212, + 0.5860657223574653 + ], + "population_cv": 0.42524285954419916 + }, + "activation_output_rms_by_block": [ + 0.16565655171871185, + 0.24343863129615784, + 0.21182885766029358, + 0.28418517112731934, + 0.12311498820781708, + 0.18273214995861053, + 0.06521740555763245, + 0.10707039386034012, + 0.06429450213909149, + 0.1264011114835739, + 0.0799982100725174, + 0.12911832332611084, + 0.06644809991121292, + 0.1084718257188797, + 0.10412788391113281, + 0.23786598443984985 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22627730295062065, + "first_to_last_ratio": 1.750986764798193, + "imbalance_abs_log_ratio": 0.5601794946219173, + "last_quartile_mean": 0.12922844849526882, + "mean": 0.14374813064932823, + "normalized": [ + 1.152408389385104, + 1.6935081534383452, + 1.4736112163924233, + 1.976966029705009, + 0.8564632294812553, + 1.2711967044940802, + 0.4536921994257406, + 0.7448472086328344, + 0.4472719182410596, + 0.8793235147657527, + 0.5565165245012613, + 0.8982261038308273, + 0.4622536627854468, + 0.7545964266032466, + 0.7243773080093228, + 1.654741410308291 + ], + "population_cv": 0.4691623243633533 + }, + "bits_per_byte": 1.9440733112268958, + "branch_output_rms_by_sublayer": [ + 0.06033274158835411, + 0.15077131986618042, + 0.0633837878704071, + 0.06536085903644562, + 0.20718541741371155, + 0.025066526606678963, + 0.08910981565713882, + 0.0229792483150959, + 0.11816111207008362, + 0.01873566210269928, + 0.06798963993787766, + 0.025694282725453377, + 0.05406087636947632, + 0.025319905951619148, + 0.05011621490120888, + 0.03549748659133911, + 0.04759044572710991, + 0.038602542132139206, + 0.05848398804664612, + 0.0661846399307251, + 0.052182719111442566, + 0.057186078280210495, + 0.03716399520635605, + 0.08089981228113174, + 0.03087303601205349, + 0.055707309395074844, + 0.02278325706720352, + 0.07142244279384613, + 0.0293489471077919, + 0.09790552407503128, + 0.01774938404560089, + 0.15747468173503876 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00039487297360913916, + 0.00021664788561845028, + 0.0001999236891396226, + 0.000155700993556381, + 0.00014909408964546676, + 0.0001300025003667294, + 0.0001780845905877945, + 0.00021758503566474802, + 0.0003627569164977402, + 0.0004938025952783663, + 0.0010348495424230683, + 0.0008427402552292494, + 0.0008322795274123101, + 0.0005665088497243569, + 0.00036612876328907985, + 0.0003194282437589591 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00024178638548089825, + "first_to_last_ratio": 0.464004453994026, + "imbalance_abs_log_ratio": 0.7678611276768971, + "last_quartile_mean": 0.0005210863460461766, + "mean": 0.00040377540323759136, + "normalized": [ + 0.9779520259107666, + 0.5365554312652605, + 0.49513587884892185, + 0.3856128736617538, + 0.36925005448570164, + 0.3219673593892314, + 0.44104863535485134, + 0.5388763998997622, + 0.8984126164918598, + 1.2229635369537375, + 2.5629335866556917, + 2.087151046031797, + 2.061243752687373, + 1.4030296179061932, + 0.9067634143965939, + 0.7911037700605048 + ], + "population_cv": 0.6734220009206653 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2750788629055023, + "mean_weights": [ + 0.9180313944816589, + 0.08196862041950226 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5785284042358398, + "mean_weights": [ + 0.7040213346481323, + 0.2959786057472229 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4888499975204468, + "mean_weights": [ + 0.7967724800109863, + 0.20322750508785248 + ], + "sources": 2 + }, + { + "entropy_mean": 0.651995062828064, + "mean_weights": [ + 0.5629274845123291, + 0.4370725154876709 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9925327301025391, + "mean_weights": [ + 0.4965394139289856, + 0.22532734274864197, + 0.27813321352005005 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0594394207000732, + "mean_weights": [ + 0.2700359523296356, + 0.27400293946266174, + 0.455961138010025 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9841134548187256, + "mean_weights": [ + 0.49647465348243713, + 0.21238623559474945, + 0.2911390960216522 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0744149684906006, + "mean_weights": [ + 0.26788532733917236, + 0.3177185654640198, + 0.41439610719680786 + ], + "sources": 3 + }, + { + "entropy_mean": 1.307446002960205, + "mean_weights": [ + 0.33086591958999634, + 0.13986191153526306, + 0.23523268103599548, + 0.2940394878387451 + ], + "sources": 4 + }, + { + "entropy_mean": 1.376603603363037, + "mean_weights": [ + 0.21549805998802185, + 0.245200514793396, + 0.24268285930156708, + 0.29661858081817627 + ], + "sources": 4 + }, + { + "entropy_mean": 1.270529866218567, + "mean_weights": [ + 0.2375539094209671, + 0.10673755407333374, + 0.29245367646217346, + 0.3632548451423645 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3762142658233643, + "mean_weights": [ + 0.22679615020751953, + 0.24722415208816528, + 0.2286011129617691, + 0.2973785698413849 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5536365509033203, + "mean_weights": [ + 0.20543108880519867, + 0.10043682903051376, + 0.22657056152820587, + 0.26188796758651733, + 0.20567354559898376 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5861575603485107, + "mean_weights": [ + 0.1991366446018219, + 0.19079142808914185, + 0.16120794415473938, + 0.1751076579093933, + 0.27375632524490356 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4829623699188232, + "mean_weights": [ + 0.2499978393316269, + 0.13764306902885437, + 0.10969669371843338, + 0.13450437784194946, + 0.3681580424308777 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5994092226028442, + "mean_weights": [ + 0.2161637395620346, + 0.20309701561927795, + 0.17752361297607422, + 0.17640036344528198, + 0.22681528329849243 + ], + "sources": 5 + }, + { + "entropy_mean": 1.757448434829712, + "mean_weights": [ + 0.1632193624973297, + 0.11841119825839996, + 0.1406504064798355, + 0.1471654176712036, + 0.19204986095428467, + 0.23850372433662415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.719214677810669, + "mean_weights": [ + 0.17631781101226807, + 0.15250492095947266, + 0.11967972666025162, + 0.11400137841701508, + 0.13607722520828247, + 0.3014189600944519 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3921294212341309, + "mean_weights": [ + 0.18369951844215393, + 0.1281489133834839, + 0.04485193267464638, + 0.042511094361543655, + 0.10842494666576385, + 0.4923636019229889 + ], + "sources": 6 + }, + { + "entropy_mean": 1.769790768623352, + "mean_weights": [ + 0.1860106885433197, + 0.17010365426540375, + 0.1509144902229309, + 0.1396978795528412, + 0.13775232434272766, + 0.215520977973938 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5072013139724731, + "mean_weights": [ + 0.09493429213762283, + 0.07214362919330597, + 0.03675283491611481, + 0.03880048543214798, + 0.07758605480194092, + 0.19188526272773743, + 0.48789745569229126 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8300648927688599, + "mean_weights": [ + 0.13595148921012878, + 0.11550325155258179, + 0.1085444986820221, + 0.09914489090442657, + 0.09707978367805481, + 0.12102536857128143, + 0.3227507174015045 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5086650848388672, + "mean_weights": [ + 0.10266698896884918, + 0.07744058966636658, + 0.03928206115961075, + 0.0407833606004715, + 0.07880876958370209, + 0.16221223771572113, + 0.4988059997558594 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8771061897277832, + "mean_weights": [ + 0.1470566689968109, + 0.13075211644172668, + 0.10456536710262299, + 0.09934695065021515, + 0.1088101863861084, + 0.1398048996925354, + 0.26966381072998047 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6780993938446045, + "mean_weights": [ + 0.08037982881069183, + 0.05826176330447197, + 0.03183364123106003, + 0.03276165574789047, + 0.053941305726766586, + 0.10113498568534851, + 0.30644336342811584, + 0.33524346351623535 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294007778167725, + "mean_weights": [ + 0.09819990396499634, + 0.08131108433008194, + 0.05535554885864258, + 0.052511729300022125, + 0.07309524714946747, + 0.1031290739774704, + 0.1768367737531662, + 0.35956063866615295 + ], + "sources": 8 + }, + { + "entropy_mean": 1.620237112045288, + "mean_weights": [ + 0.07192602753639221, + 0.0598294772207737, + 0.03551005944609642, + 0.03596649691462517, + 0.052555397152900696, + 0.08522239327430725, + 0.2067522257566452, + 0.45223796367645264 + ], + "sources": 8 + }, + { + "entropy_mean": 1.922456979751587, + "mean_weights": [ + 0.11432191729545593, + 0.10134892165660858, + 0.06334333121776581, + 0.05966266989707947, + 0.07981826364994049, + 0.11505833268165588, + 0.20316767692565918, + 0.26327887177467346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8568212985992432, + "mean_weights": [ + 0.07325166463851929, + 0.05971681326627731, + 0.03186998516321182, + 0.03226577863097191, + 0.04611574858427048, + 0.07693300396203995, + 0.17066776752471924, + 0.3028550446033478, + 0.20632418990135193 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9784047603607178, + "mean_weights": [ + 0.08589711785316467, + 0.07969395816326141, + 0.03702075034379959, + 0.037476278841495514, + 0.0731067955493927, + 0.12099474668502808, + 0.22441011667251587, + 0.20659996569156647, + 0.1348002701997757 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9687256813049316, + "mean_weights": [ + 0.07957327365875244, + 0.0893080085515976, + 0.03895219415426254, + 0.03689703717827797, + 0.0598301999270916, + 0.09126800298690796, + 0.16929352283477783, + 0.23949456214904785, + 0.1953832060098648 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05895887315273285, + 0.05461842566728592, + 0.07232892513275146, + 0.07006148993968964, + 0.12035266309976578, + 0.10343272238969803, + 0.14036840200424194, + 0.12118023633956909, + 0.16676878929138184, + 0.11889537423849106, + 0.13824602961540222, + 0.15437477827072144, + 0.1504182070493698, + 0.12633125483989716, + 0.11159109324216843, + 0.09292281419038773, + 0.11971895396709442, + 0.08979442715644836, + 0.08137664943933487, + 0.06381704658269882, + 0.09776479750871658, + 0.03986503183841705, + 0.06484787911176682, + 0.05485111102461815, + 0.07206991314888, + 0.046776797622442245, + 0.04656966030597687, + 0.04702001437544823, + 0.055061157792806625, + 0.04538688063621521, + 0.05328623950481415, + 0.053778134286403656 + ], + "loss_nats": 1.3475289344787598, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7764334678649902, + "mean_weights": [ + 0.08629432320594788, + 0.05034743621945381, + 0.046479545533657074, + 0.04277106374502182, + 0.04524604231119156, + 0.05579439178109169, + 0.07300648093223572, + 0.2586928606033325, + 0.3413679003715515 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06033274158835411, + 0.16565655171871185, + 0.19572967290878296, + 0.24343863129615784, + 0.20718541741371155, + 0.21182885766029358, + 0.280988872051239, + 0.28418517112731934, + 0.11816111207008362, + 0.12311498820781708, + 0.17678114771842957, + 0.18273214995861053, + 0.05406087636947632, + 0.06521740555763245, + 0.0933222770690918, + 0.10707039386034012, + 0.04759044572710991, + 0.06429450213909149, + 0.09436589479446411, + 0.1264011114835739, + 0.052182719111442566, + 0.0799982100725174, + 0.0912039652466774, + 0.12911832332611084, + 0.03087303601205349, + 0.06644809991121292, + 0.07178688794374466, + 0.1084718257188797, + 0.0293489471077919, + 0.10412788391113281, + 0.10697942227125168, + 0.23786598443984985 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.570489106001332e-05, + 7.69118414609693e-05, + 7.822692714398727e-05, + 6.853767990833148e-05, + 6.88084473949857e-05, + 7.10485182935372e-05, + 8.91982635948807e-05, + 8.665756467962638e-05, + 0.000171030915225856, + 0.00012665522808674723, + 0.00023480529489461333, + 0.00016227969899773598, + 0.00023323859204538167, + 0.000136202055728063, + 9.528713417239487e-05, + 6.303648115135729e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 7.484533489332534e-05, + "first_to_last_ratio": 0.5672633797074002, + "imbalance_abs_log_ratio": 0.5669315686327029, + "last_quartile_mean": 0.0001319410657742992, + "mean": 0.00011485184586490504, + "normalized": [ + 0.6591525847051819, + 0.6696613439842941, + 0.6811116230208613, + 0.5967486146365397, + 0.5991061517280438, + 0.6186101560536372, + 0.7766376145125305, + 0.7545160813658816, + 1.4891438090340474, + 1.1027705051926284, + 2.0444189914962623, + 1.4129481139434028, + 2.0307779147035174, + 1.1858934847966764, + 0.8296526142424977, + 0.5488503965839976 + ], + "population_cv": 0.4839100882418841 + }, + "activation_output_rms_by_block": [ + 0.16869333386421204, + 0.24958865344524384, + 0.23942448198795319, + 0.31786683201789856, + 0.1331564038991928, + 0.19950589537620544, + 0.06165311112999916, + 0.10097606480121613, + 0.05752932280302048, + 0.11994150280952454, + 0.07362622767686844, + 0.12106145173311234, + 0.06137929856777191, + 0.09901073575019836, + 0.0852273628115654, + 0.19495683908462524 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2438933253288269, + "first_to_last_ratio": 2.2143221757549307, + "imbalance_abs_log_ratio": 0.7949463412042855, + "last_quartile_mean": 0.11014355905354023, + "mean": 0.142724844859913, + "normalized": [ + 1.1819479224502754, + 1.7487400577679353, + 1.6775249062134392, + 2.2271303383085868, + 0.9329588273848762, + 1.3978357837559683, + 0.43197182095740094, + 0.7074876480007805, + 0.4030785450107621, + 0.8403687734062649, + 0.5158613256797294, + 0.8482156827841455, + 0.430053356358641, + 0.6937175923882012, + 0.5971445468742151, + 1.3659628726587785 + ], + "population_cv": 0.529368044298578 + }, + "bits_per_byte": 1.883020158101202, + "branch_output_rms_by_sublayer": [ + 0.06029115617275238, + 0.1536618322134018, + 0.06518179178237915, + 0.06745198369026184, + 0.23389145731925964, + 0.026561493054032326, + 0.09204838424921036, + 0.022710271179676056, + 0.12803514301776886, + 0.017566412687301636, + 0.07296492159366608, + 0.024595333263278008, + 0.05156426504254341, + 0.023977436125278473, + 0.0450289212167263, + 0.03369171544909477, + 0.04221063107252121, + 0.034665971994400024, + 0.05162743479013443, + 0.07154636830091476, + 0.04606551304459572, + 0.05490076169371605, + 0.03287900984287262, + 0.08022980391979218, + 0.027681373059749603, + 0.05210135132074356, + 0.019983114674687386, + 0.06804364919662476, + 0.02240605279803276, + 0.08063697069883347, + 0.016830744221806526, + 0.13339926302433014 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 16, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005343193187437567, + 0.0003255019836390766, + 0.00031277964721318437, + 0.00022314651478922115, + 0.0002396348782003974, + 0.00021519463019459167, + 0.00025456643186586784, + 0.0003008996254324453, + 0.00059437697364117, + 0.0008147654749400623, + 0.001560733102975404, + 0.001320101058102649, + 0.0015032932063515777, + 0.0009169853654825589, + 0.0006174439085696804, + 0.00045539974927406244 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003489368660963097, + "first_to_last_ratio": 0.3995701760811125, + "imbalance_abs_log_ratio": 0.9173658694246768, + "last_quartile_mean": 0.0008732805574194699, + "mean": 0.0006368213668384816, + "normalized": [ + 0.8390411292202721, + 0.5111354621391565, + 0.4911575890834005, + 0.3504067646113047, + 0.3762984263390404, + 0.33791992762889184, + 0.39974543117117817, + 0.4725024019314401, + 0.9333496088423855, + 1.2794254674352237, + 2.450817739869046, + 2.072953463631928, + 2.3606199236290095, + 1.4399412664731395, + 0.9695715953046595, + 0.7151138026899253 + ], + "population_cv": 0.7016607196207993 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2716493308544159, + "mean_weights": [ + 0.9192699193954468, + 0.0807301327586174 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5622239112854004, + "mean_weights": [ + 0.72159743309021, + 0.27840256690979004 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48637402057647705, + "mean_weights": [ + 0.7995253801345825, + 0.20047461986541748 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6475558280944824, + "mean_weights": [ + 0.5858595371246338, + 0.41414040327072144 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9933564066886902, + "mean_weights": [ + 0.4998217821121216, + 0.2229493409395218, + 0.2772288918495178 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489425659179688, + "mean_weights": [ + 0.2563193440437317, + 0.269240140914917, + 0.4744405150413513 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9815441966056824, + "mean_weights": [ + 0.5070900917053223, + 0.19802016019821167, + 0.29488980770111084 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071545958518982, + "mean_weights": [ + 0.2598530650138855, + 0.3191677927970886, + 0.4209791421890259 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3049530982971191, + "mean_weights": [ + 0.3487145006656647, + 0.13604819774627686, + 0.23624856770038605, + 0.27898871898651123 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3723504543304443, + "mean_weights": [ + 0.2135777771472931, + 0.23673927783966064, + 0.23828449845314026, + 0.311398446559906 + ], + "sources": 4 + }, + { + "entropy_mean": 1.265662670135498, + "mean_weights": [ + 0.24546673893928528, + 0.10275381803512573, + 0.2848597466945648, + 0.36691969633102417 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3751296997070312, + "mean_weights": [ + 0.23200613260269165, + 0.24026888608932495, + 0.22661332786083221, + 0.30111163854599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5548532009124756, + "mean_weights": [ + 0.2040066421031952, + 0.09975787997245789, + 0.22992464900016785, + 0.259371280670166, + 0.20693954825401306 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5716381072998047, + "mean_weights": [ + 0.1984836459159851, + 0.17718908190727234, + 0.15249167382717133, + 0.17045220732688904, + 0.301383376121521 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4117064476013184, + "mean_weights": [ + 0.25685301423072815, + 0.12795110046863556, + 0.08630626648664474, + 0.10788753628730774, + 0.4210020899772644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5975096225738525, + "mean_weights": [ + 0.2208021879196167, + 0.19316574931144714, + 0.17618897557258606, + 0.17661380767822266, + 0.23322927951812744 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7283259630203247, + "mean_weights": [ + 0.1650083363056183, + 0.10661210119724274, + 0.12329171597957611, + 0.13269363343715668, + 0.19600069522857666, + 0.27639350295066833 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7028545141220093, + "mean_weights": [ + 0.17663681507110596, + 0.13835734128952026, + 0.11904168128967285, + 0.11434897780418396, + 0.12915222346782684, + 0.3224629759788513 + ], + "sources": 6 + }, + { + "entropy_mean": 1.27894127368927, + "mean_weights": [ + 0.17268770933151245, + 0.10967432707548141, + 0.03322022780776024, + 0.031795307993888855, + 0.09717383980751038, + 0.555448591709137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7646939754486084, + "mean_weights": [ + 0.1884503960609436, + 0.1533900499343872, + 0.15274858474731445, + 0.14330559968948364, + 0.1338358074426651, + 0.2282695174217224 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4344873428344727, + "mean_weights": [ + 0.09210026264190674, + 0.06219030171632767, + 0.029036182910203934, + 0.0315997451543808, + 0.07091090828180313, + 0.1949847936630249, + 0.5191777944564819 + ], + "sources": 7 + }, + { + "entropy_mean": 1.802895426750183, + "mean_weights": [ + 0.13523127138614655, + 0.10325458645820618, + 0.10260490328073502, + 0.09636874496936798, + 0.09404134750366211, + 0.12264730036258698, + 0.3458518385887146 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4361302852630615, + "mean_weights": [ + 0.10137400776147842, + 0.06668317317962646, + 0.03099493309855461, + 0.033589236438274384, + 0.07286111265420914, + 0.16290563344955444, + 0.5315918922424316 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8628063201904297, + "mean_weights": [ + 0.14754432439804077, + 0.11970964819192886, + 0.10172373056411743, + 0.09891469776630402, + 0.10695713013410568, + 0.14022418856620789, + 0.28492626547813416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6038438081741333, + "mean_weights": [ + 0.07592754065990448, + 0.048247914761304855, + 0.024802275002002716, + 0.026826059445738792, + 0.049207836389541626, + 0.09689367562532425, + 0.29891306161880493, + 0.3791816532611847 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7748796939849854, + "mean_weights": [ + 0.09652430564165115, + 0.07308399677276611, + 0.04972345381975174, + 0.050231896340847015, + 0.0702696442604065, + 0.09669448435306549, + 0.16264159977436066, + 0.40083062648773193 + ], + "sources": 8 + }, + { + "entropy_mean": 1.526896357536316, + "mean_weights": [ + 0.06852421164512634, + 0.05069870874285698, + 0.02793668583035469, + 0.029434790834784508, + 0.046449482440948486, + 0.08027796447277069, + 0.19853916764259338, + 0.49813902378082275 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9032639265060425, + "mean_weights": [ + 0.11537295579910278, + 0.09104456007480621, + 0.05989263206720352, + 0.057612620294094086, + 0.07744243741035461, + 0.11209231615066528, + 0.20047283172607422, + 0.2860696315765381 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7657151222229004, + "mean_weights": [ + 0.06711487472057343, + 0.04881071671843529, + 0.02479729615151882, + 0.025201791897416115, + 0.040198031812906265, + 0.06934815645217896, + 0.15364184975624084, + 0.3216337561607361, + 0.24925348162651062 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9778118133544922, + "mean_weights": [ + 0.08567827194929123, + 0.08225203305482864, + 0.030744101852178574, + 0.03173041343688965, + 0.07107861340045929, + 0.128017395734787, + 0.2095082402229309, + 0.20655101537704468, + 0.15443992614746094 + ], + "sources": 9 + }, + { + "entropy_mean": 1.88002610206604, + "mean_weights": [ + 0.07342863082885742, + 0.07307932525873184, + 0.030178282409906387, + 0.029849819839000702, + 0.050495631992816925, + 0.08296258747577667, + 0.14574018120765686, + 0.2550249397754669, + 0.25924059748649597 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.061662707477808, + 0.0571652427315712, + 0.07283993810415268, + 0.07206818461418152, + 0.1187756210565567, + 0.10985641181468964, + 0.15539979934692383, + 0.12964090704917908, + 0.18306857347488403, + 0.129257932305336, + 0.14960098266601562, + 0.16845102608203888, + 0.16322582960128784, + 0.13869591057300568, + 0.11515536159276962, + 0.08842873573303223, + 0.1271047443151474, + 0.08747127652168274, + 0.08377104997634888, + 0.05908897519111633, + 0.10309968143701553, + 0.03653678670525551, + 0.06588983535766602, + 0.052161477506160736, + 0.07435949891805649, + 0.04259056970477104, + 0.04427788779139519, + 0.043995074927806854, + 0.053472407162189484, + 0.04156629368662834, + 0.04980214312672615, + 0.0496271476149559 + ], + "loss_nats": 1.3052101135253906, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.609398603439331, + "mean_weights": [ + 0.0809592753648758, + 0.04013277217745781, + 0.03442784771323204, + 0.03263507038354874, + 0.03443540260195732, + 0.047240644693374634, + 0.05792279541492462, + 0.2393622249364853, + 0.43288397789001465 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.06029115617275238, + 0.16869333386421204, + 0.20056062936782837, + 0.24958865344524384, + 0.23389145731925964, + 0.23942448198795319, + 0.31467917561531067, + 0.31786683201789856, + 0.12803514301776886, + 0.1331564038991928, + 0.19432871043682098, + 0.19950589537620544, + 0.05156426504254341, + 0.06165311112999916, + 0.08753261715173721, + 0.10097606480121613, + 0.04221063107252121, + 0.05752932280302048, + 0.08379971981048584, + 0.11994150280952454, + 0.04606551304459572, + 0.07362622767686844, + 0.08261217921972275, + 0.12106145173311234, + 0.027681373059749603, + 0.06137929856777191, + 0.06537999957799911, + 0.09901073575019836, + 0.02240605279803276, + 0.0852273628115654, + 0.08763451874256134, + 0.19495683908462524 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.015965601418904, + "cross_entropy_nats": 5.55624395608902, + "step": 0 + }, + { + "bits_per_byte": 5.1077262486126465, + "cross_entropy_nats": 3.540406048297882, + "step": 100 + }, + { + "bits_per_byte": 3.1449978072640112, + "cross_entropy_nats": 2.1799463629722595, + "step": 500 + }, + { + "bits_per_byte": 1.978178932745523, + "cross_entropy_nats": 1.3711691498756409, + "step": 2000 + }, + { + "bits_per_byte": 1.8301036761704788, + "cross_entropy_nats": 1.2685312032699585, + "step": 4000 + }, + { + "bits_per_byte": 1.72661337398776, + "cross_entropy_nats": 1.1967971920967102, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "ad1807a65c19f598a4bde3ce03f75bb64bba14b79befbf1740b5aaddab905279", + "final_model_state": "6df5cb12d848445089dff93c7b22def426897ab70173e1fcca78c3db41e914dc", + "final_optimizer_state": "a8e9a566e68720356f0b3f878d687f8c4cac616f59f729e02056853409f8d145", + "final_public_parameters": "16b3a7903fb4172275d1673bd0f390d3b3c03a9d0d151ac66d732eb978cff06f", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f", + "initial_public_parameter_elements": 9541824, + "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe", + "initial_public_parameter_tensors": 115, + "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_group": 4, + "transformer_blocks_per_attnres_group": 2, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 54.54310071476275, + "measured_steps": 7980, + "median_ms": 54.49852900346741, + "p95_ms": 54.8562506082817, + "peak_allocated_bytes": 6536921088, + "peak_reserved_bytes": 6889144320, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.030921126746609, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.566610336303711, + "step": 1, + "unclipped_grad_norm": 17.857467651367188 + }, + { + "bits_per_byte": 7.460191884344154, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.171010971069336, + "step": 10, + "unclipped_grad_norm": 12.879206657409668 + }, + { + "bits_per_byte": 6.7765918040416455, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6971755027771, + "step": 20, + "unclipped_grad_norm": 4.5564117431640625 + }, + { + "bits_per_byte": 6.553465640395858, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.542516231536865, + "step": 30, + "unclipped_grad_norm": 3.0218148231506348 + }, + { + "bits_per_byte": 6.42131485940348, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.450916290283203, + "step": 40, + "unclipped_grad_norm": 2.865441083908081 + }, + { + "bits_per_byte": 6.256338093806218, + "learning_rate": 3.75e-05, + "loss_nats": 4.3365631103515625, + "step": 50, + "unclipped_grad_norm": 2.8344333171844482 + }, + { + "bits_per_byte": 5.946641367160921, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.1218976974487305, + "step": 60, + "unclipped_grad_norm": 2.6930737495422363 + }, + { + "bits_per_byte": 5.708490111205155, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9568238258361816, + "step": 70, + "unclipped_grad_norm": 2.6865241527557373 + }, + { + "bits_per_byte": 5.509431199721109, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8188467025756836, + "step": 80, + "unclipped_grad_norm": 2.448140859603882 + }, + { + "bits_per_byte": 5.311447867735559, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.681615114212036, + "step": 90, + "unclipped_grad_norm": 2.2759759426116943 + }, + { + "bits_per_byte": 5.0864742664542035, + "learning_rate": 7.5e-05, + "loss_nats": 3.5256752967834473, + "step": 100, + "unclipped_grad_norm": 2.1487066745758057 + }, + { + "bits_per_byte": 4.9338616775927715, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4198923110961914, + "step": 110, + "unclipped_grad_norm": 1.9959912300109863 + }, + { + "bits_per_byte": 4.731707078393195, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2797694206237793, + "step": 120, + "unclipped_grad_norm": 1.7048262357711792 + }, + { + "bits_per_byte": 4.582112785054405, + "learning_rate": 9.75e-05, + "loss_nats": 3.1760785579681396, + "step": 130, + "unclipped_grad_norm": 1.5641754865646362 + }, + { + "bits_per_byte": 4.436114304979855, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0748801231384277, + "step": 140, + "unclipped_grad_norm": 1.380750060081482 + }, + { + "bits_per_byte": 4.229324085408923, + "learning_rate": 0.0001125, + "loss_nats": 2.931544065475464, + "step": 150, + "unclipped_grad_norm": 1.1272099018096924 + }, + { + "bits_per_byte": 4.108468093029308, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8477730751037598, + "step": 160, + "unclipped_grad_norm": 0.8926234245300293 + }, + { + "bits_per_byte": 4.025982461953091, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7905983924865723, + "step": 170, + "unclipped_grad_norm": 0.915916919708252 + }, + { + "bits_per_byte": 3.9299425341970746, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7240285873413086, + "step": 180, + "unclipped_grad_norm": 0.7168256640434265 + }, + { + "bits_per_byte": 3.8518424607178594, + "learning_rate": 0.0001425, + "loss_nats": 2.669893741607666, + "step": 190, + "unclipped_grad_norm": 0.8937639594078064 + }, + { + "bits_per_byte": 3.7890044716415114, + "learning_rate": 0.00015, + "loss_nats": 2.626337766647339, + "step": 200, + "unclipped_grad_norm": 0.7755346894264221 + }, + { + "bits_per_byte": 3.7186869570791403, + "learning_rate": 0.0001575, + "loss_nats": 2.5775973796844482, + "step": 210, + "unclipped_grad_norm": 0.731957197189331 + }, + { + "bits_per_byte": 3.706538446587493, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.56917667388916, + "step": 220, + "unclipped_grad_norm": 0.8748186230659485 + }, + { + "bits_per_byte": 3.655573795761412, + "learning_rate": 0.0001725, + "loss_nats": 2.53385066986084, + "step": 230, + "unclipped_grad_norm": 0.8331225514411926 + }, + { + "bits_per_byte": 3.652975825837478, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5320498943328857, + "step": 240, + "unclipped_grad_norm": 1.103366494178772 + }, + { + "bits_per_byte": 3.571490789823981, + "learning_rate": 0.0001875, + "loss_nats": 2.4755687713623047, + "step": 250, + "unclipped_grad_norm": 0.6647337675094604 + }, + { + "bits_per_byte": 3.5855342051628347, + "learning_rate": 0.000195, + "loss_nats": 2.4853029251098633, + "step": 260, + "unclipped_grad_norm": 0.8029841780662537 + }, + { + "bits_per_byte": 3.576924065730039, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.479334831237793, + "step": 270, + "unclipped_grad_norm": 0.7433680891990662 + }, + { + "bits_per_byte": 3.5153632198189984, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.43666410446167, + "step": 280, + "unclipped_grad_norm": 0.824142336845398 + }, + { + "bits_per_byte": 3.511655617831698, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.434094190597534, + "step": 290, + "unclipped_grad_norm": 0.8560227751731873 + }, + { + "bits_per_byte": 3.4998641433232422, + "learning_rate": 0.000225, + "loss_nats": 2.4259209632873535, + "step": 300, + "unclipped_grad_norm": 0.8836220502853394 + }, + { + "bits_per_byte": 3.4156380478202935, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.367539882659912, + "step": 310, + "unclipped_grad_norm": 0.9033657908439636 + }, + { + "bits_per_byte": 3.481727196928552, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.4133493900299072, + "step": 320, + "unclipped_grad_norm": 1.119826316833496 + }, + { + "bits_per_byte": 3.4809216301919106, + "learning_rate": 0.0002475, + "loss_nats": 2.4127910137176514, + "step": 330, + "unclipped_grad_norm": 0.9842678308486938 + }, + { + "bits_per_byte": 3.422977579428515, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3726272583007812, + "step": 340, + "unclipped_grad_norm": 1.075486421585083 + }, + { + "bits_per_byte": 3.385370477129646, + "learning_rate": 0.0002625, + "loss_nats": 2.346560001373291, + "step": 350, + "unclipped_grad_norm": 0.7245537042617798 + }, + { + "bits_per_byte": 3.385430671057448, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.346601724624634, + "step": 360, + "unclipped_grad_norm": 1.202561616897583 + }, + { + "bits_per_byte": 3.3953991294667585, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.353511333465576, + "step": 370, + "unclipped_grad_norm": 0.7641725540161133 + }, + { + "bits_per_byte": 3.295281837024086, + "learning_rate": 0.000285, + "loss_nats": 2.2841153144836426, + "step": 380, + "unclipped_grad_norm": 1.061362862586975 + }, + { + "bits_per_byte": 3.380484450018634, + "learning_rate": 0.0002925, + "loss_nats": 2.3431732654571533, + "step": 390, + "unclipped_grad_norm": 1.0704491138458252 + }, + { + "bits_per_byte": 3.354825670405808, + "learning_rate": 0.0003, + "loss_nats": 2.325387954711914, + "step": 400, + "unclipped_grad_norm": 1.066178560256958 + }, + { + "bits_per_byte": 3.38879568360423, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.3489341735839844, + "step": 410, + "unclipped_grad_norm": 0.9359472990036011 + }, + { + "bits_per_byte": 3.3128674070401103, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.296304702758789, + "step": 420, + "unclipped_grad_norm": 0.9575046896934509 + }, + { + "bits_per_byte": 3.3331142365955855, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.3103387355804443, + "step": 430, + "unclipped_grad_norm": 0.9350728988647461 + }, + { + "bits_per_byte": 3.2769842588334903, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.271432399749756, + "step": 440, + "unclipped_grad_norm": 1.028943419456482 + }, + { + "bits_per_byte": 3.3396626480098375, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.31487774848938, + "step": 450, + "unclipped_grad_norm": 1.1271060705184937 + }, + { + "bits_per_byte": 3.2591895579140173, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2590980529785156, + "step": 460, + "unclipped_grad_norm": 1.1319539546966553 + }, + { + "bits_per_byte": 3.2327510089621794, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.240772247314453, + "step": 470, + "unclipped_grad_norm": 1.1909105777740479 + }, + { + "bits_per_byte": 3.2186394884935843, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.2309908866882324, + "step": 480, + "unclipped_grad_norm": 1.3586266040802002 + }, + { + "bits_per_byte": 3.169725902761693, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1970865726470947, + "step": 490, + "unclipped_grad_norm": 1.5543664693832397 + }, + { + "bits_per_byte": 3.101864644418944, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1500487327575684, + "step": 500, + "unclipped_grad_norm": 1.9330527782440186 + }, + { + "bits_per_byte": 3.1523112835006266, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1850156784057617, + "step": 510, + "unclipped_grad_norm": 1.367343544960022 + }, + { + "bits_per_byte": 3.137257298105303, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1745810508728027, + "step": 520, + "unclipped_grad_norm": 1.4051828384399414 + }, + { + "bits_per_byte": 2.96818390576449, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.0573883056640625, + "step": 530, + "unclipped_grad_norm": 0.88100665807724 + }, + { + "bits_per_byte": 2.9700385666713944, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.058673858642578, + "step": 540, + "unclipped_grad_norm": 1.1694613695144653 + }, + { + "bits_per_byte": 2.981698990399893, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.066756248474121, + "step": 550, + "unclipped_grad_norm": 2.2637624740600586 + }, + { + "bits_per_byte": 2.9860140351100406, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.06974720954895, + "step": 560, + "unclipped_grad_norm": 2.0154106616973877 + }, + { + "bits_per_byte": 2.908624249978889, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0161046981811523, + "step": 570, + "unclipped_grad_norm": 1.2818057537078857 + }, + { + "bits_per_byte": 2.9258029090429716, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0280120372772217, + "step": 580, + "unclipped_grad_norm": 1.5401909351348877 + }, + { + "bits_per_byte": 2.865440266260492, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.986171841621399, + "step": 590, + "unclipped_grad_norm": 1.2538588047027588 + }, + { + "bits_per_byte": 2.8203701488100825, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.954931616783142, + "step": 600, + "unclipped_grad_norm": 1.5663292407989502 + }, + { + "bits_per_byte": 2.8417604910111445, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9697582721710205, + "step": 610, + "unclipped_grad_norm": 1.3150807619094849 + }, + { + "bits_per_byte": 2.806068931477586, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9450187683105469, + "step": 620, + "unclipped_grad_norm": 1.0626381635665894 + }, + { + "bits_per_byte": 2.766462358879786, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9175655841827393, + "step": 630, + "unclipped_grad_norm": 2.55611515045166 + }, + { + "bits_per_byte": 2.7411267346679313, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.900004267692566, + "step": 640, + "unclipped_grad_norm": 0.9799621105194092 + }, + { + "bits_per_byte": 2.7289859633955733, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8915889263153076, + "step": 650, + "unclipped_grad_norm": 1.4919346570968628 + }, + { + "bits_per_byte": 2.7446909031244124, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.9024747610092163, + "step": 660, + "unclipped_grad_norm": 1.184576153755188 + }, + { + "bits_per_byte": 2.706914225931793, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.876289963722229, + "step": 670, + "unclipped_grad_norm": 1.2165472507476807 + }, + { + "bits_per_byte": 2.7108362902847194, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8790085315704346, + "step": 680, + "unclipped_grad_norm": 2.3380022048950195 + }, + { + "bits_per_byte": 2.6699751042488753, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.850685715675354, + "step": 690, + "unclipped_grad_norm": 1.790702223777771 + }, + { + "bits_per_byte": 2.6284332108809165, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8218910694122314, + "step": 700, + "unclipped_grad_norm": 1.6820380687713623 + }, + { + "bits_per_byte": 2.6224609413470548, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.817751407623291, + "step": 710, + "unclipped_grad_norm": 1.3093626499176025 + }, + { + "bits_per_byte": 2.6378994799497075, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8284525871276855, + "step": 720, + "unclipped_grad_norm": 1.689491868019104 + }, + { + "bits_per_byte": 2.604935393276182, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8056036233901978, + "step": 730, + "unclipped_grad_norm": 1.4546315670013428 + }, + { + "bits_per_byte": 2.542083817570416, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.762038230895996, + "step": 740, + "unclipped_grad_norm": 1.6848912239074707 + }, + { + "bits_per_byte": 2.5572903515770546, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7725785970687866, + "step": 750, + "unclipped_grad_norm": 1.477650761604309 + }, + { + "bits_per_byte": 2.448532478772384, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.6971933841705322, + "step": 760, + "unclipped_grad_norm": 1.4033355712890625 + }, + { + "bits_per_byte": 2.5017822630804796, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7341033220291138, + "step": 770, + "unclipped_grad_norm": 1.3305320739746094 + }, + { + "bits_per_byte": 2.4567315796696176, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7028765678405762, + "step": 780, + "unclipped_grad_norm": 1.7414066791534424 + }, + { + "bits_per_byte": 2.463544844366199, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.70759916305542, + "step": 790, + "unclipped_grad_norm": 1.3770233392715454 + }, + { + "bits_per_byte": 2.390281954925151, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6568171977996826, + "step": 800, + "unclipped_grad_norm": 1.4013909101486206 + }, + { + "bits_per_byte": 2.4192710065892844, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.6769108772277832, + "step": 810, + "unclipped_grad_norm": 1.360757827758789 + }, + { + "bits_per_byte": 2.3909581907083437, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6572859287261963, + "step": 820, + "unclipped_grad_norm": 1.4178366661071777 + }, + { + "bits_per_byte": 2.4097108349931227, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6702842712402344, + "step": 830, + "unclipped_grad_norm": 1.1894793510437012 + }, + { + "bits_per_byte": 2.395687713607071, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6605641841888428, + "step": 840, + "unclipped_grad_norm": 1.7937202453613281 + }, + { + "bits_per_byte": 2.326945216161301, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6129155158996582, + "step": 850, + "unclipped_grad_norm": 1.4669246673583984 + }, + { + "bits_per_byte": 2.3940583499727963, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6594347953796387, + "step": 860, + "unclipped_grad_norm": 1.3730710744857788 + }, + { + "bits_per_byte": 2.3742401011832714, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.645697832107544, + "step": 870, + "unclipped_grad_norm": 1.3468742370605469 + }, + { + "bits_per_byte": 2.3406023584663167, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6223819255828857, + "step": 880, + "unclipped_grad_norm": 1.3746341466903687 + }, + { + "bits_per_byte": 2.2996515694693733, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5939970016479492, + "step": 890, + "unclipped_grad_norm": 1.5287024974822998 + }, + { + "bits_per_byte": 2.338895602639154, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6211988925933838, + "step": 900, + "unclipped_grad_norm": 1.2028048038482666 + }, + { + "bits_per_byte": 2.322601450348459, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6099046468734741, + "step": 910, + "unclipped_grad_norm": 1.5355594158172607 + }, + { + "bits_per_byte": 2.307329906899794, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5993192195892334, + "step": 920, + "unclipped_grad_norm": 1.5230286121368408 + }, + { + "bits_per_byte": 2.2540851100885737, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5624127388000488, + "step": 930, + "unclipped_grad_norm": 1.3632712364196777 + }, + { + "bits_per_byte": 2.306159220995371, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5985077619552612, + "step": 940, + "unclipped_grad_norm": 1.4536393880844116 + }, + { + "bits_per_byte": 2.330239371855927, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6151988506317139, + "step": 950, + "unclipped_grad_norm": 1.4820353984832764 + }, + { + "bits_per_byte": 2.256347369878023, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5639808177947998, + "step": 960, + "unclipped_grad_norm": 1.5091183185577393 + }, + { + "bits_per_byte": 2.2135099312011146, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5342881679534912, + "step": 970, + "unclipped_grad_norm": 1.3972843885421753 + }, + { + "bits_per_byte": 2.3381242604500345, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6206642389297485, + "step": 980, + "unclipped_grad_norm": 1.3591992855072021 + }, + { + "bits_per_byte": 2.202303885636196, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5265207290649414, + "step": 990, + "unclipped_grad_norm": 1.2794140577316284 + }, + { + "bits_per_byte": 2.194666824041982, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5212271213531494, + "step": 1000, + "unclipped_grad_norm": 1.2117714881896973 + }, + { + "bits_per_byte": 2.205168084703665, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5285060405731201, + "step": 1010, + "unclipped_grad_norm": 1.4173126220703125 + }, + { + "bits_per_byte": 2.1627664500640433, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.4991154670715332, + "step": 1020, + "unclipped_grad_norm": 1.5233873128890991 + }, + { + "bits_per_byte": 2.171931921476475, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.505468487739563, + "step": 1030, + "unclipped_grad_norm": 1.5871013402938843 + }, + { + "bits_per_byte": 2.2533218510840447, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5618836879730225, + "step": 1040, + "unclipped_grad_norm": 1.408625602722168 + }, + { + "bits_per_byte": 2.203781560572409, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5275449752807617, + "step": 1050, + "unclipped_grad_norm": 1.9470032453536987 + }, + { + "bits_per_byte": 2.2074819392883733, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5301098823547363, + "step": 1060, + "unclipped_grad_norm": 1.358320713043213 + }, + { + "bits_per_byte": 2.2549842353872847, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5630359649658203, + "step": 1070, + "unclipped_grad_norm": 1.3524690866470337 + }, + { + "bits_per_byte": 2.232926944466177, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.547747015953064, + "step": 1080, + "unclipped_grad_norm": 1.3045321702957153 + }, + { + "bits_per_byte": 2.1562297334700493, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.494584560394287, + "step": 1090, + "unclipped_grad_norm": 1.3637882471084595 + }, + { + "bits_per_byte": 2.1766043461351154, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5087071657180786, + "step": 1100, + "unclipped_grad_norm": 1.3327077627182007 + }, + { + "bits_per_byte": 2.113153238908442, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.464726209640503, + "step": 1110, + "unclipped_grad_norm": 1.3556562662124634 + }, + { + "bits_per_byte": 2.158234707213808, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4959743022918701, + "step": 1120, + "unclipped_grad_norm": 1.5276546478271484 + }, + { + "bits_per_byte": 2.1778080527085044, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5095415115356445, + "step": 1130, + "unclipped_grad_norm": 1.3574931621551514 + }, + { + "bits_per_byte": 2.120840347454057, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4700545072555542, + "step": 1140, + "unclipped_grad_norm": 1.49063241481781 + }, + { + "bits_per_byte": 2.172341240185528, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5057522058486938, + "step": 1150, + "unclipped_grad_norm": 1.270938515663147 + }, + { + "bits_per_byte": 2.081790654679751, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.442987322807312, + "step": 1160, + "unclipped_grad_norm": 1.387803077697754 + }, + { + "bits_per_byte": 2.1352082940550883, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4800136089324951, + "step": 1170, + "unclipped_grad_norm": 1.2486525774002075 + }, + { + "bits_per_byte": 2.1410305947173978, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4840493202209473, + "step": 1180, + "unclipped_grad_norm": 1.3077996969223022 + }, + { + "bits_per_byte": 2.1241448221077346, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.472344994544983, + "step": 1190, + "unclipped_grad_norm": 1.846550464630127 + }, + { + "bits_per_byte": 2.128028362346868, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.475036859512329, + "step": 1200, + "unclipped_grad_norm": 1.1341090202331543 + }, + { + "bits_per_byte": 2.1484036629425374, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4891599416732788, + "step": 1210, + "unclipped_grad_norm": 1.2377034425735474 + }, + { + "bits_per_byte": 2.096246140450072, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4530071020126343, + "step": 1220, + "unclipped_grad_norm": 1.182817816734314 + }, + { + "bits_per_byte": 2.11190705262029, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.463862419128418, + "step": 1230, + "unclipped_grad_norm": 1.3176664113998413 + }, + { + "bits_per_byte": 2.0884407078406633, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.447596788406372, + "step": 1240, + "unclipped_grad_norm": 1.3174140453338623 + }, + { + "bits_per_byte": 2.01052448381522, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.3935893774032593, + "step": 1250, + "unclipped_grad_norm": 1.3046211004257202 + }, + { + "bits_per_byte": 2.0807407005962335, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4422595500946045, + "step": 1260, + "unclipped_grad_norm": 1.3089001178741455 + }, + { + "bits_per_byte": 2.0962698740558907, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4530235528945923, + "step": 1270, + "unclipped_grad_norm": 1.318946361541748 + }, + { + "bits_per_byte": 2.1189218809836823, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4687247276306152, + "step": 1280, + "unclipped_grad_norm": 1.1660429239273071 + }, + { + "bits_per_byte": 2.0943455601753866, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4516897201538086, + "step": 1290, + "unclipped_grad_norm": 1.5281683206558228 + }, + { + "bits_per_byte": 2.0472271293532014, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.419029712677002, + "step": 1300, + "unclipped_grad_norm": 1.416712999343872 + }, + { + "bits_per_byte": 2.0644360573638356, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4309580326080322, + "step": 1310, + "unclipped_grad_norm": 1.2950114011764526 + }, + { + "bits_per_byte": 2.044286569988749, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4169914722442627, + "step": 1320, + "unclipped_grad_norm": 1.1484317779541016 + }, + { + "bits_per_byte": 2.0443660259734475, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4170465469360352, + "step": 1330, + "unclipped_grad_norm": 1.206559181213379 + }, + { + "bits_per_byte": 2.040463051694767, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4143412113189697, + "step": 1340, + "unclipped_grad_norm": 1.1644346714019775 + }, + { + "bits_per_byte": 2.073596541279392, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.437307596206665, + "step": 1350, + "unclipped_grad_norm": 1.295727014541626 + }, + { + "bits_per_byte": 2.028423750186418, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4059962034225464, + "step": 1360, + "unclipped_grad_norm": 1.2224998474121094 + }, + { + "bits_per_byte": 2.049744783379182, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4207748174667358, + "step": 1370, + "unclipped_grad_norm": 1.2044527530670166 + }, + { + "bits_per_byte": 2.01407248590252, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.396048665046692, + "step": 1380, + "unclipped_grad_norm": 1.241443157196045 + }, + { + "bits_per_byte": 2.0615071928196422, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4289278984069824, + "step": 1390, + "unclipped_grad_norm": 1.1473987102508545 + }, + { + "bits_per_byte": 2.1662895146669676, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.501557469367981, + "step": 1400, + "unclipped_grad_norm": 1.1963618993759155 + }, + { + "bits_per_byte": 2.064517577140345, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4310145378112793, + "step": 1410, + "unclipped_grad_norm": 1.5827157497406006 + }, + { + "bits_per_byte": 2.0563607839752236, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4253606796264648, + "step": 1420, + "unclipped_grad_norm": 1.2223409414291382 + }, + { + "bits_per_byte": 1.996439276692207, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.3838262557983398, + "step": 1430, + "unclipped_grad_norm": 1.1598049402236938 + }, + { + "bits_per_byte": 2.0215600945731342, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4012386798858643, + "step": 1440, + "unclipped_grad_norm": 1.1466177701950073 + }, + { + "bits_per_byte": 2.0019069491284376, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3876161575317383, + "step": 1450, + "unclipped_grad_norm": 1.1313462257385254 + }, + { + "bits_per_byte": 2.1132414660083345, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4647873640060425, + "step": 1460, + "unclipped_grad_norm": 1.1495121717453003 + }, + { + "bits_per_byte": 2.102186421210873, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4571245908737183, + "step": 1470, + "unclipped_grad_norm": 1.337440013885498 + }, + { + "bits_per_byte": 2.0400563127254765, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4140592813491821, + "step": 1480, + "unclipped_grad_norm": 1.06326162815094 + }, + { + "bits_per_byte": 2.057447370363375, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4261138439178467, + "step": 1490, + "unclipped_grad_norm": 1.1691441535949707 + }, + { + "bits_per_byte": 1.9426222936015662, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3465231657028198, + "step": 1500, + "unclipped_grad_norm": 1.0966376066207886 + }, + { + "bits_per_byte": 1.953619208263036, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3541456460952759, + "step": 1510, + "unclipped_grad_norm": 1.2253862619400024 + }, + { + "bits_per_byte": 1.9856435817322469, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3763432502746582, + "step": 1520, + "unclipped_grad_norm": 1.1588174104690552 + }, + { + "bits_per_byte": 2.043245215037775, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4162696599960327, + "step": 1530, + "unclipped_grad_norm": 1.1396243572235107 + }, + { + "bits_per_byte": 2.030725050037613, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4075913429260254, + "step": 1540, + "unclipped_grad_norm": 1.1333942413330078 + }, + { + "bits_per_byte": 2.081739919797746, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4429521560668945, + "step": 1550, + "unclipped_grad_norm": 1.1568241119384766 + }, + { + "bits_per_byte": 1.9638389333252566, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.361229419708252, + "step": 1560, + "unclipped_grad_norm": 1.0923500061035156 + }, + { + "bits_per_byte": 1.9637097743544585, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3611398935317993, + "step": 1570, + "unclipped_grad_norm": 1.181723952293396 + }, + { + "bits_per_byte": 1.984943956308537, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3758583068847656, + "step": 1580, + "unclipped_grad_norm": 1.15809965133667 + }, + { + "bits_per_byte": 1.9967120411764756, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3840153217315674, + "step": 1590, + "unclipped_grad_norm": 1.2961064577102661 + }, + { + "bits_per_byte": 1.9713934432470563, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3664658069610596, + "step": 1600, + "unclipped_grad_norm": 1.1767901182174683 + }, + { + "bits_per_byte": 1.9459826626167747, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3488523960113525, + "step": 1610, + "unclipped_grad_norm": 1.1538722515106201 + }, + { + "bits_per_byte": 1.9305324291938633, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3381431102752686, + "step": 1620, + "unclipped_grad_norm": 1.0682140588760376 + }, + { + "bits_per_byte": 1.9835519287324535, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3748934268951416, + "step": 1630, + "unclipped_grad_norm": 1.1328599452972412 + }, + { + "bits_per_byte": 1.952673819631243, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3534903526306152, + "step": 1640, + "unclipped_grad_norm": 1.1611915826797485 + }, + { + "bits_per_byte": 1.9528083100642177, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.353583574295044, + "step": 1650, + "unclipped_grad_norm": 1.2886978387832642 + }, + { + "bits_per_byte": 1.9660846827802232, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.362786054611206, + "step": 1660, + "unclipped_grad_norm": 1.1841517686843872 + }, + { + "bits_per_byte": 1.9119278619714772, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3252474069595337, + "step": 1670, + "unclipped_grad_norm": 1.1872814893722534 + }, + { + "bits_per_byte": 1.93973676868539, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3445230722427368, + "step": 1680, + "unclipped_grad_norm": 1.1895164251327515 + }, + { + "bits_per_byte": 1.9710771671521197, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3662465810775757, + "step": 1690, + "unclipped_grad_norm": 1.0209845304489136 + }, + { + "bits_per_byte": 1.9189868898761413, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3301403522491455, + "step": 1700, + "unclipped_grad_norm": 1.2754039764404297 + }, + { + "bits_per_byte": 1.9587719804655366, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3577172756195068, + "step": 1710, + "unclipped_grad_norm": 0.9888415336608887 + }, + { + "bits_per_byte": 1.8892168649944399, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3095053434371948, + "step": 1720, + "unclipped_grad_norm": 1.1411211490631104 + }, + { + "bits_per_byte": 1.970308060737461, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3657134771347046, + "step": 1730, + "unclipped_grad_norm": 1.1278765201568604 + }, + { + "bits_per_byte": 1.9812284431192968, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3732829093933105, + "step": 1740, + "unclipped_grad_norm": 1.0859284400939941 + }, + { + "bits_per_byte": 1.9480999410315467, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3503199815750122, + "step": 1750, + "unclipped_grad_norm": 1.063671588897705 + }, + { + "bits_per_byte": 1.9687923776354073, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3646628856658936, + "step": 1760, + "unclipped_grad_norm": 1.02903151512146 + }, + { + "bits_per_byte": 1.9746323925107556, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3687108755111694, + "step": 1770, + "unclipped_grad_norm": 1.1287299394607544 + }, + { + "bits_per_byte": 2.0187137817013547, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3992657661437988, + "step": 1780, + "unclipped_grad_norm": 1.1014562845230103 + }, + { + "bits_per_byte": 1.8992608237874788, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.31646728515625, + "step": 1790, + "unclipped_grad_norm": 1.2110038995742798 + }, + { + "bits_per_byte": 2.0498698147663594, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.4208614826202393, + "step": 1800, + "unclipped_grad_norm": 1.5255318880081177 + }, + { + "bits_per_byte": 1.8942502812372417, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3129942417144775, + "step": 1810, + "unclipped_grad_norm": 1.0947346687316895 + }, + { + "bits_per_byte": 1.9627021280030534, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3604414463043213, + "step": 1820, + "unclipped_grad_norm": 1.1853052377700806 + }, + { + "bits_per_byte": 1.9780426364947141, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3710746765136719, + "step": 1830, + "unclipped_grad_norm": 1.0857312679290771 + }, + { + "bits_per_byte": 1.975353859731125, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.369210958480835, + "step": 1840, + "unclipped_grad_norm": 1.1014374494552612 + }, + { + "bits_per_byte": 1.8969483450639775, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3148643970489502, + "step": 1850, + "unclipped_grad_norm": 1.0865846872329712 + }, + { + "bits_per_byte": 1.943441103002324, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.347090721130371, + "step": 1860, + "unclipped_grad_norm": 0.9979931116104126 + }, + { + "bits_per_byte": 2.018713093770751, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3992652893066406, + "step": 1870, + "unclipped_grad_norm": 1.1247007846832275 + }, + { + "bits_per_byte": 1.895716605318498, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3140106201171875, + "step": 1880, + "unclipped_grad_norm": 1.2656805515289307 + }, + { + "bits_per_byte": 1.939341724536358, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3442492485046387, + "step": 1890, + "unclipped_grad_norm": 1.1298930644989014 + }, + { + "bits_per_byte": 1.9376002282137212, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3430421352386475, + "step": 1900, + "unclipped_grad_norm": 1.1912769079208374 + }, + { + "bits_per_byte": 1.9616757355427041, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3597300052642822, + "step": 1910, + "unclipped_grad_norm": 1.0634745359420776 + }, + { + "bits_per_byte": 1.8880833273426032, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3087196350097656, + "step": 1920, + "unclipped_grad_norm": 1.0458215475082397 + }, + { + "bits_per_byte": 1.9484389188363973, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3505549430847168, + "step": 1930, + "unclipped_grad_norm": 1.0484954118728638 + }, + { + "bits_per_byte": 1.9728711181832694, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3674900531768799, + "step": 1940, + "unclipped_grad_norm": 1.0497392416000366 + }, + { + "bits_per_byte": 1.9061248233660644, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3212250471115112, + "step": 1950, + "unclipped_grad_norm": 1.2195848226547241 + }, + { + "bits_per_byte": 1.8858399856447488, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3071646690368652, + "step": 1960, + "unclipped_grad_norm": 1.0152307748794556 + }, + { + "bits_per_byte": 1.8880905506139394, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3087246417999268, + "step": 1970, + "unclipped_grad_norm": 1.1332975625991821 + }, + { + "bits_per_byte": 1.9225495104887642, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.33260977268219, + "step": 1980, + "unclipped_grad_norm": 1.1374293565750122 + }, + { + "bits_per_byte": 1.8947043154355196, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3133089542388916, + "step": 1990, + "unclipped_grad_norm": 1.0456688404083252 + }, + { + "bits_per_byte": 1.977381019236845, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.37061607837677, + "step": 2000, + "unclipped_grad_norm": 1.1185020208358765 + }, + { + "bits_per_byte": 1.9636655748131868, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3611092567443848, + "step": 2010, + "unclipped_grad_norm": 1.1275893449783325 + }, + { + "bits_per_byte": 1.8762688071589317, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3005304336547852, + "step": 2020, + "unclipped_grad_norm": 1.0362930297851562 + }, + { + "bits_per_byte": 1.8977705941177525, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3154343366622925, + "step": 2030, + "unclipped_grad_norm": 1.0457504987716675 + }, + { + "bits_per_byte": 1.8942691993288368, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3130073547363281, + "step": 2040, + "unclipped_grad_norm": 0.9790334105491638 + }, + { + "bits_per_byte": 1.854481013051726, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2854282855987549, + "step": 2050, + "unclipped_grad_norm": 1.067064881324768 + }, + { + "bits_per_byte": 1.8548413167052835, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2856780290603638, + "step": 2060, + "unclipped_grad_norm": 1.0005096197128296 + }, + { + "bits_per_byte": 1.9705749778116002, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3658984899520874, + "step": 2070, + "unclipped_grad_norm": 1.1014602184295654 + }, + { + "bits_per_byte": 1.9435265783798028, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3471499681472778, + "step": 2080, + "unclipped_grad_norm": 1.1029977798461914 + }, + { + "bits_per_byte": 1.8586646630166146, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2883281707763672, + "step": 2090, + "unclipped_grad_norm": 1.0793615579605103 + }, + { + "bits_per_byte": 1.8191048696998495, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2609074115753174, + "step": 2100, + "unclipped_grad_norm": 1.0433158874511719 + }, + { + "bits_per_byte": 1.851254274556238, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2831916809082031, + "step": 2110, + "unclipped_grad_norm": 1.1101045608520508 + }, + { + "bits_per_byte": 1.8581610978148881, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2879791259765625, + "step": 2120, + "unclipped_grad_norm": 1.1086524724960327 + }, + { + "bits_per_byte": 1.9049420986760808, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3204052448272705, + "step": 2130, + "unclipped_grad_norm": 1.0729707479476929 + }, + { + "bits_per_byte": 1.941282376768694, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3455944061279297, + "step": 2140, + "unclipped_grad_norm": 1.0528757572174072 + }, + { + "bits_per_byte": 1.843757550805128, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2779953479766846, + "step": 2150, + "unclipped_grad_norm": 1.1246051788330078 + }, + { + "bits_per_byte": 1.9154739722496175, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3277053833007812, + "step": 2160, + "unclipped_grad_norm": 1.0607515573501587 + }, + { + "bits_per_byte": 1.850249895875199, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2824954986572266, + "step": 2170, + "unclipped_grad_norm": 1.0301073789596558 + }, + { + "bits_per_byte": 1.9307078514977434, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3382647037506104, + "step": 2180, + "unclipped_grad_norm": 1.0744229555130005 + }, + { + "bits_per_byte": 1.895962540509232, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3141810894012451, + "step": 2190, + "unclipped_grad_norm": 1.1302496194839478 + }, + { + "bits_per_byte": 1.8514286649642127, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2833125591278076, + "step": 2200, + "unclipped_grad_norm": 1.043583869934082 + }, + { + "bits_per_byte": 1.8687825743495246, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2953413724899292, + "step": 2210, + "unclipped_grad_norm": 1.1252996921539307 + }, + { + "bits_per_byte": 1.862911258631719, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.291271686553955, + "step": 2220, + "unclipped_grad_norm": 1.0556633472442627 + }, + { + "bits_per_byte": 1.8208848901362797, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.262141227722168, + "step": 2230, + "unclipped_grad_norm": 1.0693137645721436 + }, + { + "bits_per_byte": 1.90406292336487, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.319795846939087, + "step": 2240, + "unclipped_grad_norm": 1.0752713680267334 + }, + { + "bits_per_byte": 1.8964874315596651, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.314544916152954, + "step": 2250, + "unclipped_grad_norm": 1.0330802202224731 + }, + { + "bits_per_byte": 1.935825367256817, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3418118953704834, + "step": 2260, + "unclipped_grad_norm": 1.078423023223877 + }, + { + "bits_per_byte": 1.832309869615747, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.270060420036316, + "step": 2270, + "unclipped_grad_norm": 1.056177020072937 + }, + { + "bits_per_byte": 1.887026837918353, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.3079873323440552, + "step": 2280, + "unclipped_grad_norm": 1.0121575593948364 + }, + { + "bits_per_byte": 1.834445722156812, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.271540880203247, + "step": 2290, + "unclipped_grad_norm": 1.0369646549224854 + }, + { + "bits_per_byte": 1.8213642057842343, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.262473464012146, + "step": 2300, + "unclipped_grad_norm": 0.9950895309448242 + }, + { + "bits_per_byte": 1.8373876573824712, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2735800743103027, + "step": 2310, + "unclipped_grad_norm": 1.0178841352462769 + }, + { + "bits_per_byte": 1.8979250345382275, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.315541386604309, + "step": 2320, + "unclipped_grad_norm": 0.9894719123840332 + }, + { + "bits_per_byte": 1.8294843666447218, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2681019306182861, + "step": 2330, + "unclipped_grad_norm": 0.9816614389419556 + }, + { + "bits_per_byte": 1.848164090285535, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2810497283935547, + "step": 2340, + "unclipped_grad_norm": 1.0238531827926636 + }, + { + "bits_per_byte": 1.905264050198496, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3206284046173096, + "step": 2350, + "unclipped_grad_norm": 1.0559433698654175 + }, + { + "bits_per_byte": 1.8307938425483912, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2690095901489258, + "step": 2360, + "unclipped_grad_norm": 1.0712460279464722 + }, + { + "bits_per_byte": 1.7703326537069675, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2271010875701904, + "step": 2370, + "unclipped_grad_norm": 1.1017504930496216 + }, + { + "bits_per_byte": 1.886526368404342, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.3076404333114624, + "step": 2380, + "unclipped_grad_norm": 1.1038753986358643 + }, + { + "bits_per_byte": 1.8630619154338748, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2913761138916016, + "step": 2390, + "unclipped_grad_norm": 1.0329313278198242 + }, + { + "bits_per_byte": 1.9135943738583379, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3264025449752808, + "step": 2400, + "unclipped_grad_norm": 1.1860445737838745 + }, + { + "bits_per_byte": 1.835037686441082, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2719511985778809, + "step": 2410, + "unclipped_grad_norm": 1.153725266456604 + }, + { + "bits_per_byte": 1.871177432762789, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2970013618469238, + "step": 2420, + "unclipped_grad_norm": 1.0672882795333862 + }, + { + "bits_per_byte": 1.8586395535495885, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2883107662200928, + "step": 2430, + "unclipped_grad_norm": 1.0162642002105713 + }, + { + "bits_per_byte": 1.8218378460047104, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2628017663955688, + "step": 2440, + "unclipped_grad_norm": 1.065905213356018 + }, + { + "bits_per_byte": 1.9384147380482075, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.34360671043396, + "step": 2450, + "unclipped_grad_norm": 1.218255639076233 + }, + { + "bits_per_byte": 1.8158471743272062, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2586493492126465, + "step": 2460, + "unclipped_grad_norm": 1.0468804836273193 + }, + { + "bits_per_byte": 1.8206396428761493, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2619712352752686, + "step": 2470, + "unclipped_grad_norm": 1.1381754875183105 + }, + { + "bits_per_byte": 1.8851444878046595, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.3066825866699219, + "step": 2480, + "unclipped_grad_norm": 1.035730242729187 + }, + { + "bits_per_byte": 1.8941151028736636, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.3129005432128906, + "step": 2490, + "unclipped_grad_norm": 1.0220366716384888 + }, + { + "bits_per_byte": 1.8318142156159603, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2697168588638306, + "step": 2500, + "unclipped_grad_norm": 0.982593834400177 + }, + { + "bits_per_byte": 1.8129355080480989, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2566311359405518, + "step": 2510, + "unclipped_grad_norm": 1.091998815536499 + }, + { + "bits_per_byte": 1.841321932503609, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2763071060180664, + "step": 2520, + "unclipped_grad_norm": 1.000153660774231 + }, + { + "bits_per_byte": 1.848013777448681, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2809455394744873, + "step": 2530, + "unclipped_grad_norm": 1.0528815984725952 + }, + { + "bits_per_byte": 1.8343394368785788, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2714672088623047, + "step": 2540, + "unclipped_grad_norm": 0.9793087840080261 + }, + { + "bits_per_byte": 1.7887870800751502, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2398927211761475, + "step": 2550, + "unclipped_grad_norm": 1.0650514364242554 + }, + { + "bits_per_byte": 1.7584782335482958, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2188842296600342, + "step": 2560, + "unclipped_grad_norm": 0.9875280857086182 + }, + { + "bits_per_byte": 1.81654955147333, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2591361999511719, + "step": 2570, + "unclipped_grad_norm": 1.0060385465621948 + }, + { + "bits_per_byte": 1.7945154782100887, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2438633441925049, + "step": 2580, + "unclipped_grad_norm": 1.0824097394943237 + }, + { + "bits_per_byte": 1.7956424805211926, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2446445226669312, + "step": 2590, + "unclipped_grad_norm": 0.9988609552383423 + }, + { + "bits_per_byte": 1.815006351147138, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2580665349960327, + "step": 2600, + "unclipped_grad_norm": 0.9940101504325867 + }, + { + "bits_per_byte": 1.840550934279791, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2757726907730103, + "step": 2610, + "unclipped_grad_norm": 1.0804052352905273 + }, + { + "bits_per_byte": 1.7607050649116673, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2204277515411377, + "step": 2620, + "unclipped_grad_norm": 1.0194178819656372 + }, + { + "bits_per_byte": 1.7479795526263755, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2116070985794067, + "step": 2630, + "unclipped_grad_norm": 0.9564489126205444 + }, + { + "bits_per_byte": 1.7532458333784458, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2152574062347412, + "step": 2640, + "unclipped_grad_norm": 0.9848498106002808 + }, + { + "bits_per_byte": 1.813315245741204, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2568943500518799, + "step": 2650, + "unclipped_grad_norm": 0.9513642191886902 + }, + { + "bits_per_byte": 1.7231679595517, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1944090127944946, + "step": 2660, + "unclipped_grad_norm": 1.0000529289245605 + }, + { + "bits_per_byte": 1.802969457395901, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.249723196029663, + "step": 2670, + "unclipped_grad_norm": 0.9922699928283691 + }, + { + "bits_per_byte": 1.9163316497294702, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.328299880027771, + "step": 2680, + "unclipped_grad_norm": 1.192367672920227 + }, + { + "bits_per_byte": 1.8243539521868333, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2645457983016968, + "step": 2690, + "unclipped_grad_norm": 1.0074772834777832 + }, + { + "bits_per_byte": 1.751833683832211, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2142785787582397, + "step": 2700, + "unclipped_grad_norm": 0.9746342301368713 + }, + { + "bits_per_byte": 1.826312318632208, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2659032344818115, + "step": 2710, + "unclipped_grad_norm": 1.0221014022827148 + }, + { + "bits_per_byte": 1.8721807795479226, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.297696828842163, + "step": 2720, + "unclipped_grad_norm": 1.0561023950576782 + }, + { + "bits_per_byte": 1.742347808741222, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2077034711837769, + "step": 2730, + "unclipped_grad_norm": 0.946500301361084 + }, + { + "bits_per_byte": 1.826287381147833, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2658859491348267, + "step": 2740, + "unclipped_grad_norm": 1.0504826307296753 + }, + { + "bits_per_byte": 1.7872662374935704, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.23883855342865, + "step": 2750, + "unclipped_grad_norm": 1.0072195529937744 + }, + { + "bits_per_byte": 1.9127833036768689, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.3258403539657593, + "step": 2760, + "unclipped_grad_norm": 1.1406866312026978 + }, + { + "bits_per_byte": 1.8309943743192973, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.269148588180542, + "step": 2770, + "unclipped_grad_norm": 1.0432147979736328 + }, + { + "bits_per_byte": 1.835324725485372, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2721501588821411, + "step": 2780, + "unclipped_grad_norm": 1.0946613550186157 + }, + { + "bits_per_byte": 1.8014186898330709, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2486482858657837, + "step": 2790, + "unclipped_grad_norm": 1.0307797193527222 + }, + { + "bits_per_byte": 1.8122052697125355, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2561249732971191, + "step": 2800, + "unclipped_grad_norm": 1.0519623756408691 + }, + { + "bits_per_byte": 1.7642310532196568, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2228717803955078, + "step": 2810, + "unclipped_grad_norm": 1.0939170122146606 + }, + { + "bits_per_byte": 1.8221561858914574, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2630224227905273, + "step": 2820, + "unclipped_grad_norm": 1.006143569946289 + }, + { + "bits_per_byte": 1.7646739085456284, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.223178744316101, + "step": 2830, + "unclipped_grad_norm": 0.9335127472877502 + }, + { + "bits_per_byte": 1.8147093371090979, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2578606605529785, + "step": 2840, + "unclipped_grad_norm": 0.9985573887825012 + }, + { + "bits_per_byte": 1.758499215431701, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2188987731933594, + "step": 2850, + "unclipped_grad_norm": 0.9980565905570984 + }, + { + "bits_per_byte": 1.783597331602714, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.236295461654663, + "step": 2860, + "unclipped_grad_norm": 1.0390338897705078 + }, + { + "bits_per_byte": 1.7758223399225082, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2309062480926514, + "step": 2870, + "unclipped_grad_norm": 1.0219858884811401 + }, + { + "bits_per_byte": 1.7873183482367818, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2388746738433838, + "step": 2880, + "unclipped_grad_norm": 1.002677083015442 + }, + { + "bits_per_byte": 1.7586128959639213, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2189775705337524, + "step": 2890, + "unclipped_grad_norm": 1.0140609741210938 + }, + { + "bits_per_byte": 1.7806794739481757, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.2342729568481445, + "step": 2900, + "unclipped_grad_norm": 0.9781115651130676 + }, + { + "bits_per_byte": 1.7540770255300655, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2158335447311401, + "step": 2910, + "unclipped_grad_norm": 0.9601871967315674 + }, + { + "bits_per_byte": 1.798447689539416, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.246588945388794, + "step": 2920, + "unclipped_grad_norm": 1.0120164155960083 + }, + { + "bits_per_byte": 1.749404256906123, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2125946283340454, + "step": 2930, + "unclipped_grad_norm": 0.9387297630310059 + }, + { + "bits_per_byte": 1.7688945347804526, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2261042594909668, + "step": 2940, + "unclipped_grad_norm": 1.073227047920227 + }, + { + "bits_per_byte": 1.8185380148826058, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.260514497756958, + "step": 2950, + "unclipped_grad_norm": 1.033491849899292 + }, + { + "bits_per_byte": 1.8193900169349801, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2611050605773926, + "step": 2960, + "unclipped_grad_norm": 1.0577077865600586 + }, + { + "bits_per_byte": 1.799499191466791, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2473177909851074, + "step": 2970, + "unclipped_grad_norm": 1.0431829690933228 + }, + { + "bits_per_byte": 1.8318936716006589, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.269771933555603, + "step": 2980, + "unclipped_grad_norm": 1.3554320335388184 + }, + { + "bits_per_byte": 1.7557352822496848, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2169829607009888, + "step": 2990, + "unclipped_grad_norm": 1.0419063568115234 + }, + { + "bits_per_byte": 1.8299745171996809, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2684416770935059, + "step": 3000, + "unclipped_grad_norm": 1.0811892747879028 + }, + { + "bits_per_byte": 1.7247794369902845, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1955260038375854, + "step": 3010, + "unclipped_grad_norm": 1.0030642747879028 + }, + { + "bits_per_byte": 1.7923558920632043, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2423664331436157, + "step": 3020, + "unclipped_grad_norm": 1.002471685409546 + }, + { + "bits_per_byte": 1.7403096423458468, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2062907218933105, + "step": 3030, + "unclipped_grad_norm": 1.0230085849761963 + }, + { + "bits_per_byte": 1.7392983843587735, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.205589771270752, + "step": 3040, + "unclipped_grad_norm": 1.0195822715759277 + }, + { + "bits_per_byte": 1.8747281865725025, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2994625568389893, + "step": 3050, + "unclipped_grad_norm": 1.0621304512023926 + }, + { + "bits_per_byte": 1.80563828417199, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.251573085784912, + "step": 3060, + "unclipped_grad_norm": 1.0812959671020508 + }, + { + "bits_per_byte": 1.7309672007856773, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.199815034866333, + "step": 3070, + "unclipped_grad_norm": 1.048651099205017 + }, + { + "bits_per_byte": 1.80954315025983, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2542797327041626, + "step": 3080, + "unclipped_grad_norm": 1.0254682302474976 + }, + { + "bits_per_byte": 1.787755528135275, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2391777038574219, + "step": 3090, + "unclipped_grad_norm": 1.0376405715942383 + }, + { + "bits_per_byte": 1.748834134418513, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2121994495391846, + "step": 3100, + "unclipped_grad_norm": 1.0169665813446045 + }, + { + "bits_per_byte": 1.8895020122295705, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.30970299243927, + "step": 3110, + "unclipped_grad_norm": 1.009265661239624 + }, + { + "bits_per_byte": 1.772648744066137, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2287064790725708, + "step": 3120, + "unclipped_grad_norm": 1.0459308624267578 + }, + { + "bits_per_byte": 1.7184131552032957, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1911132335662842, + "step": 3130, + "unclipped_grad_norm": 0.9758033752441406 + }, + { + "bits_per_byte": 1.7503415623533254, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2132443189620972, + "step": 3140, + "unclipped_grad_norm": 1.0032713413238525 + }, + { + "bits_per_byte": 1.7368366246943234, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.203883409500122, + "step": 3150, + "unclipped_grad_norm": 0.9544663429260254 + }, + { + "bits_per_byte": 1.7448823170669872, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2094602584838867, + "step": 3160, + "unclipped_grad_norm": 0.9832491278648376 + }, + { + "bits_per_byte": 1.8183364512157945, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2603747844696045, + "step": 3170, + "unclipped_grad_norm": 1.080575942993164 + }, + { + "bits_per_byte": 1.796097030667423, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2449595928192139, + "step": 3180, + "unclipped_grad_norm": 1.0869723558425903 + }, + { + "bits_per_byte": 1.7798367589589479, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2336888313293457, + "step": 3190, + "unclipped_grad_norm": 1.0730317831039429 + }, + { + "bits_per_byte": 1.7660776309419706, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.2241517305374146, + "step": 3200, + "unclipped_grad_norm": 1.0053741931915283 + }, + { + "bits_per_byte": 1.736418362887425, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.2035934925079346, + "step": 3210, + "unclipped_grad_norm": 0.9884828925132751 + }, + { + "bits_per_byte": 1.7094793444215763, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1849207878112793, + "step": 3220, + "unclipped_grad_norm": 0.9791789054870605 + }, + { + "bits_per_byte": 1.7767474346014993, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.231547474861145, + "step": 3230, + "unclipped_grad_norm": 1.0538209676742554 + }, + { + "bits_per_byte": 1.783878351254224, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.236490249633789, + "step": 3240, + "unclipped_grad_norm": 1.0962731838226318 + }, + { + "bits_per_byte": 1.7619694813608107, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.221304178237915, + "step": 3250, + "unclipped_grad_norm": 0.9704980850219727 + }, + { + "bits_per_byte": 1.7647327266122235, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2232195138931274, + "step": 3260, + "unclipped_grad_norm": 0.9566147923469543 + }, + { + "bits_per_byte": 1.793783004100064, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.243355631828308, + "step": 3270, + "unclipped_grad_norm": 1.0367501974105835 + }, + { + "bits_per_byte": 1.7882781834112471, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2395399808883667, + "step": 3280, + "unclipped_grad_norm": 1.1419988870620728 + }, + { + "bits_per_byte": 1.7570841421804018, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2179179191589355, + "step": 3290, + "unclipped_grad_norm": 1.0371057987213135 + }, + { + "bits_per_byte": 1.7704131415875712, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2271568775177002, + "step": 3300, + "unclipped_grad_norm": 1.0890001058578491 + }, + { + "bits_per_byte": 1.73421681297373, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.202067494392395, + "step": 3310, + "unclipped_grad_norm": 1.0024704933166504 + }, + { + "bits_per_byte": 1.7556792159055035, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2169440984725952, + "step": 3320, + "unclipped_grad_norm": 1.0351438522338867 + }, + { + "bits_per_byte": 1.751791376100099, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.2142492532730103, + "step": 3330, + "unclipped_grad_norm": 0.967126727104187 + }, + { + "bits_per_byte": 1.7023927992927743, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1800087690353394, + "step": 3340, + "unclipped_grad_norm": 0.9899824857711792 + }, + { + "bits_per_byte": 1.7427576433982281, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.2079875469207764, + "step": 3350, + "unclipped_grad_norm": 1.044270396232605 + }, + { + "bits_per_byte": 1.7489617455454531, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2122879028320312, + "step": 3360, + "unclipped_grad_norm": 1.0146241188049316 + }, + { + "bits_per_byte": 1.6929069242017853, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1734336614608765, + "step": 3370, + "unclipped_grad_norm": 0.9934349656105042 + }, + { + "bits_per_byte": 1.7374774320514381, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2043275833129883, + "step": 3380, + "unclipped_grad_norm": 1.0039739608764648 + }, + { + "bits_per_byte": 1.7582206035373031, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.218705654144287, + "step": 3390, + "unclipped_grad_norm": 1.007129430770874 + }, + { + "bits_per_byte": 1.7465608677394082, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2106237411499023, + "step": 3400, + "unclipped_grad_norm": 0.9504732489585876 + }, + { + "bits_per_byte": 1.744059208099958, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2088897228240967, + "step": 3410, + "unclipped_grad_norm": 1.002007007598877 + }, + { + "bits_per_byte": 1.7186439559207538, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1912732124328613, + "step": 3420, + "unclipped_grad_norm": 0.9931132197380066 + }, + { + "bits_per_byte": 1.762099672227514, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.221394419670105, + "step": 3430, + "unclipped_grad_norm": 0.943431556224823 + }, + { + "bits_per_byte": 1.763225298677411, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.2221746444702148, + "step": 3440, + "unclipped_grad_norm": 0.9422610402107239 + }, + { + "bits_per_byte": 1.8310394337738234, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2691798210144043, + "step": 3450, + "unclipped_grad_norm": 1.1188790798187256 + }, + { + "bits_per_byte": 1.822499291279929, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2632602453231812, + "step": 3460, + "unclipped_grad_norm": 1.14117431640625 + }, + { + "bits_per_byte": 1.7955301758501792, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2445666790008545, + "step": 3470, + "unclipped_grad_norm": 1.0871022939682007 + }, + { + "bits_per_byte": 1.7607453088519693, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2204556465148926, + "step": 3480, + "unclipped_grad_norm": 1.0999677181243896 + }, + { + "bits_per_byte": 1.6814413568167144, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1654863357543945, + "step": 3490, + "unclipped_grad_norm": 1.0303022861480713 + }, + { + "bits_per_byte": 1.6926489502254911, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1732548475265503, + "step": 3500, + "unclipped_grad_norm": 1.0132246017456055 + }, + { + "bits_per_byte": 1.7370531508517597, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2040334939956665, + "step": 3510, + "unclipped_grad_norm": 1.0184191465377808 + }, + { + "bits_per_byte": 1.7040419409318979, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1811518669128418, + "step": 3520, + "unclipped_grad_norm": 0.9375280141830444 + }, + { + "bits_per_byte": 1.7216300906876847, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1933430433273315, + "step": 3530, + "unclipped_grad_norm": 1.0306724309921265 + }, + { + "bits_per_byte": 1.7320362449434406, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.2005560398101807, + "step": 3540, + "unclipped_grad_norm": 0.9943498373031616 + }, + { + "bits_per_byte": 1.7834315403272825, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2361805438995361, + "step": 3550, + "unclipped_grad_norm": 1.066209316253662 + }, + { + "bits_per_byte": 1.7059953198803977, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1825058460235596, + "step": 3560, + "unclipped_grad_norm": 1.0326995849609375 + }, + { + "bits_per_byte": 1.768657198722262, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.2259397506713867, + "step": 3570, + "unclipped_grad_norm": 1.0416048765182495 + }, + { + "bits_per_byte": 1.8330426876910733, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2705683708190918, + "step": 3580, + "unclipped_grad_norm": 1.1256591081619263 + }, + { + "bits_per_byte": 1.759539710469421, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2196199893951416, + "step": 3590, + "unclipped_grad_norm": 1.0512219667434692 + }, + { + "bits_per_byte": 1.727328735824024, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1972930431365967, + "step": 3600, + "unclipped_grad_norm": 0.9710429906845093 + }, + { + "bits_per_byte": 1.7248793589104356, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1955952644348145, + "step": 3610, + "unclipped_grad_norm": 1.0369946956634521 + }, + { + "bits_per_byte": 1.7248757472747676, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1955927610397339, + "step": 3620, + "unclipped_grad_norm": 1.0838762521743774 + }, + { + "bits_per_byte": 1.7412607064051182, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.2069499492645264, + "step": 3630, + "unclipped_grad_norm": 0.9884288311004639 + }, + { + "bits_per_byte": 1.6889442719432552, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.170686960220337, + "step": 3640, + "unclipped_grad_norm": 1.0365123748779297 + }, + { + "bits_per_byte": 1.7701531038194667, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2269766330718994, + "step": 3650, + "unclipped_grad_norm": 1.1143370866775513 + }, + { + "bits_per_byte": 1.6641216719789245, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1534812450408936, + "step": 3660, + "unclipped_grad_norm": 1.0202206373214722 + }, + { + "bits_per_byte": 1.6856676584790171, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1684157848358154, + "step": 3670, + "unclipped_grad_norm": 1.0151705741882324 + }, + { + "bits_per_byte": 1.7531821997976265, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.2152132987976074, + "step": 3680, + "unclipped_grad_norm": 1.013437271118164 + }, + { + "bits_per_byte": 1.7126154480600597, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1870945692062378, + "step": 3690, + "unclipped_grad_norm": 1.0570411682128906 + }, + { + "bits_per_byte": 1.6629353356532728, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1526589393615723, + "step": 3700, + "unclipped_grad_norm": 1.0999462604522705 + }, + { + "bits_per_byte": 1.7333302424085324, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.2014529705047607, + "step": 3710, + "unclipped_grad_norm": 1.0060557126998901 + }, + { + "bits_per_byte": 1.714764887230543, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1885844469070435, + "step": 3720, + "unclipped_grad_norm": 1.061936378479004 + }, + { + "bits_per_byte": 1.761044214699169, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.2206628322601318, + "step": 3730, + "unclipped_grad_norm": 1.0892959833145142 + }, + { + "bits_per_byte": 1.704870553343755, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1817262172698975, + "step": 3740, + "unclipped_grad_norm": 1.0097936391830444 + }, + { + "bits_per_byte": 1.6862241943372092, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1688015460968018, + "step": 3750, + "unclipped_grad_norm": 1.0313822031021118 + }, + { + "bits_per_byte": 1.6824576023006628, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.16619074344635, + "step": 3760, + "unclipped_grad_norm": 1.0350043773651123 + }, + { + "bits_per_byte": 1.7407710717981117, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2066105604171753, + "step": 3770, + "unclipped_grad_norm": 1.0363857746124268 + }, + { + "bits_per_byte": 1.6889002443846344, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.170656442642212, + "step": 3780, + "unclipped_grad_norm": 0.9539183974266052 + }, + { + "bits_per_byte": 1.7337478162848272, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.20174241065979, + "step": 3790, + "unclipped_grad_norm": 0.9895081520080566 + }, + { + "bits_per_byte": 1.6773657119565675, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1626613140106201, + "step": 3800, + "unclipped_grad_norm": 0.9700848460197449 + }, + { + "bits_per_byte": 1.6665356204664348, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1551544666290283, + "step": 3810, + "unclipped_grad_norm": 1.059192180633545 + }, + { + "bits_per_byte": 1.7701960994821824, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.227006435394287, + "step": 3820, + "unclipped_grad_norm": 0.9839220643043518 + }, + { + "bits_per_byte": 1.6973205149708774, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1764929294586182, + "step": 3830, + "unclipped_grad_norm": 1.0503867864608765 + }, + { + "bits_per_byte": 1.6894750104038179, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1710548400878906, + "step": 3840, + "unclipped_grad_norm": 1.0203979015350342 + }, + { + "bits_per_byte": 1.7223587811793903, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1938481330871582, + "step": 3850, + "unclipped_grad_norm": 1.0148154497146606 + }, + { + "bits_per_byte": 1.6589467140144625, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1498942375183105, + "step": 3860, + "unclipped_grad_norm": 0.9799250364303589 + }, + { + "bits_per_byte": 1.719725382829379, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1920228004455566, + "step": 3870, + "unclipped_grad_norm": 0.9761002659797668 + }, + { + "bits_per_byte": 1.70201718918329, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1797484159469604, + "step": 3880, + "unclipped_grad_norm": 1.047680139541626 + }, + { + "bits_per_byte": 1.7301960305792086, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1992805004119873, + "step": 3890, + "unclipped_grad_norm": 1.0218281745910645 + }, + { + "bits_per_byte": 1.728929550338255, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1984026432037354, + "step": 3900, + "unclipped_grad_norm": 1.1127806901931763 + }, + { + "bits_per_byte": 1.7357486624449652, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.2031292915344238, + "step": 3910, + "unclipped_grad_norm": 0.9941173791885376 + }, + { + "bits_per_byte": 1.7061793413168207, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.182633399963379, + "step": 3920, + "unclipped_grad_norm": 0.9868062734603882 + }, + { + "bits_per_byte": 1.6839027725158628, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1671924591064453, + "step": 3930, + "unclipped_grad_norm": 1.0294983386993408 + }, + { + "bits_per_byte": 1.6929858642385314, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1734883785247803, + "step": 3940, + "unclipped_grad_norm": 1.0079408884048462 + }, + { + "bits_per_byte": 1.7648030675164263, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2232682704925537, + "step": 3950, + "unclipped_grad_norm": 1.091759204864502 + }, + { + "bits_per_byte": 1.7722222270919972, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.2284108400344849, + "step": 3960, + "unclipped_grad_norm": 1.075370192527771 + }, + { + "bits_per_byte": 1.7441799399208635, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.2089734077453613, + "step": 3970, + "unclipped_grad_norm": 1.0237821340560913 + }, + { + "bits_per_byte": 1.7199797451700052, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1921991109848022, + "step": 3980, + "unclipped_grad_norm": 1.0290383100509644 + }, + { + "bits_per_byte": 1.7549034021674614, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.2164063453674316, + "step": 3990, + "unclipped_grad_norm": 1.0765279531478882 + }, + { + "bits_per_byte": 1.7501554771250918, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.2131153345108032, + "step": 4000, + "unclipped_grad_norm": 1.037840485572815 + }, + { + "bits_per_byte": 1.783230320625773, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.2360410690307617, + "step": 4010, + "unclipped_grad_norm": 1.1067007780075073 + }, + { + "bits_per_byte": 1.634854696350997, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.133194923400879, + "step": 4020, + "unclipped_grad_norm": 0.9555695652961731 + }, + { + "bits_per_byte": 1.6858748975733067, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1685594320297241, + "step": 4030, + "unclipped_grad_norm": 0.9827634692192078 + }, + { + "bits_per_byte": 1.6858767893824662, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1685607433319092, + "step": 4040, + "unclipped_grad_norm": 1.0647450685501099 + }, + { + "bits_per_byte": 1.6753074235910415, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1612346172332764, + "step": 4050, + "unclipped_grad_norm": 1.0498672723770142 + }, + { + "bits_per_byte": 1.7132395731000407, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1875271797180176, + "step": 4060, + "unclipped_grad_norm": 1.0244723558425903 + }, + { + "bits_per_byte": 1.659084988065756, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1499900817871094, + "step": 4070, + "unclipped_grad_norm": 0.954911470413208 + }, + { + "bits_per_byte": 1.7001991605810192, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1784882545471191, + "step": 4080, + "unclipped_grad_norm": 1.0178123712539673 + }, + { + "bits_per_byte": 1.6512532421107655, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.144561529159546, + "step": 4090, + "unclipped_grad_norm": 0.988084614276886 + }, + { + "bits_per_byte": 1.6312373852553994, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1306875944137573, + "step": 4100, + "unclipped_grad_norm": 0.9640962481498718 + }, + { + "bits_per_byte": 1.7389465078551083, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.205345869064331, + "step": 4110, + "unclipped_grad_norm": 1.0591799020767212 + }, + { + "bits_per_byte": 1.670625711869254, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.157989501953125, + "step": 4120, + "unclipped_grad_norm": 1.0143104791641235 + }, + { + "bits_per_byte": 1.6531302607622822, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1458625793457031, + "step": 4130, + "unclipped_grad_norm": 1.0070092678070068 + }, + { + "bits_per_byte": 1.6853878427060633, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1682218313217163, + "step": 4140, + "unclipped_grad_norm": 1.0397000312805176 + }, + { + "bits_per_byte": 1.716280570332597, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1896350383758545, + "step": 4150, + "unclipped_grad_norm": 1.0394762754440308 + }, + { + "bits_per_byte": 1.7032425655706875, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1805977821350098, + "step": 4160, + "unclipped_grad_norm": 1.0539857149124146 + }, + { + "bits_per_byte": 1.6362312454885026, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1341490745544434, + "step": 4170, + "unclipped_grad_norm": 1.0312221050262451 + }, + { + "bits_per_byte": 1.6176134356020002, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.121244192123413, + "step": 4180, + "unclipped_grad_norm": 0.9745424389839172 + }, + { + "bits_per_byte": 1.7779335989445002, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.2323696613311768, + "step": 4190, + "unclipped_grad_norm": 1.067311406135559 + }, + { + "bits_per_byte": 1.675829562919061, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1615965366363525, + "step": 4200, + "unclipped_grad_norm": 1.045929193496704 + }, + { + "bits_per_byte": 1.681460618873611, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1654996871948242, + "step": 4210, + "unclipped_grad_norm": 1.0254682302474976 + }, + { + "bits_per_byte": 1.7232076875440492, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1944365501403809, + "step": 4220, + "unclipped_grad_norm": 1.0498327016830444 + }, + { + "bits_per_byte": 1.6892706950145928, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1709132194519043, + "step": 4230, + "unclipped_grad_norm": 1.0495660305023193 + }, + { + "bits_per_byte": 1.7317139494557237, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.2003326416015625, + "step": 4240, + "unclipped_grad_norm": 1.1651132106781006 + }, + { + "bits_per_byte": 1.744415040204593, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.2091363668441772, + "step": 4250, + "unclipped_grad_norm": 1.0255383253097534 + }, + { + "bits_per_byte": 1.73589312787169, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.2032294273376465, + "step": 4260, + "unclipped_grad_norm": 1.1120972633361816 + }, + { + "bits_per_byte": 1.7234627378152787, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1946133375167847, + "step": 4270, + "unclipped_grad_norm": 1.016249656677246 + }, + { + "bits_per_byte": 1.7430716837187037, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.208205223083496, + "step": 4280, + "unclipped_grad_norm": 1.0722230672836304 + }, + { + "bits_per_byte": 1.7030208799337254, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1804441213607788, + "step": 4290, + "unclipped_grad_norm": 1.03997004032135 + }, + { + "bits_per_byte": 1.6244537015747667, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1259855031967163, + "step": 4300, + "unclipped_grad_norm": 1.037449836730957 + }, + { + "bits_per_byte": 1.7698973656176336, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.2267993688583374, + "step": 4310, + "unclipped_grad_norm": 1.235188603401184 + }, + { + "bits_per_byte": 1.6511122163370582, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1444637775421143, + "step": 4320, + "unclipped_grad_norm": 1.1049147844314575 + }, + { + "bits_per_byte": 1.6449793150072904, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1402127742767334, + "step": 4330, + "unclipped_grad_norm": 1.0132696628570557 + }, + { + "bits_per_byte": 1.6583323919855806, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1494684219360352, + "step": 4340, + "unclipped_grad_norm": 1.0222140550613403 + }, + { + "bits_per_byte": 1.5761074866080718, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0924744606018066, + "step": 4350, + "unclipped_grad_norm": 1.0156723260879517 + }, + { + "bits_per_byte": 1.7219245249859618, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1935471296310425, + "step": 4360, + "unclipped_grad_norm": 1.0623725652694702 + }, + { + "bits_per_byte": 1.6842240361076746, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1674151420593262, + "step": 4370, + "unclipped_grad_norm": 1.070603847503662 + }, + { + "bits_per_byte": 1.6398215553079147, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1366376876831055, + "step": 4380, + "unclipped_grad_norm": 1.0265616178512573 + }, + { + "bits_per_byte": 1.6765849107216504, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1621201038360596, + "step": 4390, + "unclipped_grad_norm": 1.020019769668579 + }, + { + "bits_per_byte": 1.6365060737645816, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1343395709991455, + "step": 4400, + "unclipped_grad_norm": 1.0334678888320923 + }, + { + "bits_per_byte": 1.6461109608499676, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1409971714019775, + "step": 4410, + "unclipped_grad_norm": 1.064432144165039 + }, + { + "bits_per_byte": 1.6319834459948421, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1312047243118286, + "step": 4420, + "unclipped_grad_norm": 1.0356578826904297 + }, + { + "bits_per_byte": 1.6589714795161867, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1499114036560059, + "step": 4430, + "unclipped_grad_norm": 1.0542991161346436 + }, + { + "bits_per_byte": 1.6678407968038327, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1560591459274292, + "step": 4440, + "unclipped_grad_norm": 1.1220062971115112 + }, + { + "bits_per_byte": 1.6463864770566499, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.141188144683838, + "step": 4450, + "unclipped_grad_norm": 1.069725751876831 + }, + { + "bits_per_byte": 1.6193611233000682, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.1224555969238281, + "step": 4460, + "unclipped_grad_norm": 1.0096163749694824 + }, + { + "bits_per_byte": 1.7261988098078551, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.196509838104248, + "step": 4470, + "unclipped_grad_norm": 1.0757471323013306 + }, + { + "bits_per_byte": 1.6253762165139949, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1266249418258667, + "step": 4480, + "unclipped_grad_norm": 1.0172395706176758 + }, + { + "bits_per_byte": 1.5785597872267245, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.0941742658615112, + "step": 4490, + "unclipped_grad_norm": 1.081101894378662 + }, + { + "bits_per_byte": 1.6683827141367014, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1564347743988037, + "step": 4500, + "unclipped_grad_norm": 1.027239441871643 + }, + { + "bits_per_byte": 1.6472935135573004, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1418168544769287, + "step": 4510, + "unclipped_grad_norm": 1.0604325532913208 + }, + { + "bits_per_byte": 1.6464067710094517, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1412022113800049, + "step": 4520, + "unclipped_grad_norm": 1.0223703384399414 + }, + { + "bits_per_byte": 1.6663331968863693, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.155014157295227, + "step": 4530, + "unclipped_grad_norm": 1.096794843673706 + }, + { + "bits_per_byte": 1.5983706407622622, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1079061031341553, + "step": 4540, + "unclipped_grad_norm": 0.991118311882019 + }, + { + "bits_per_byte": 1.6821887934173643, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1660044193267822, + "step": 4550, + "unclipped_grad_norm": 1.0131337642669678 + }, + { + "bits_per_byte": 1.6779839895864195, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1630898714065552, + "step": 4560, + "unclipped_grad_norm": 1.0416655540466309 + }, + { + "bits_per_byte": 1.6489940780090317, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1429955959320068, + "step": 4570, + "unclipped_grad_norm": 1.0762525796890259 + }, + { + "bits_per_byte": 1.5938887728807776, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.104799509048462, + "step": 4580, + "unclipped_grad_norm": 1.039039969444275 + }, + { + "bits_per_byte": 1.7129355077733153, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1873164176940918, + "step": 4590, + "unclipped_grad_norm": 1.1662163734436035 + }, + { + "bits_per_byte": 1.6753789683738005, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1612842082977295, + "step": 4600, + "unclipped_grad_norm": 1.0834790468215942 + }, + { + "bits_per_byte": 1.6489213293477167, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1429451704025269, + "step": 4610, + "unclipped_grad_norm": 1.0408029556274414 + }, + { + "bits_per_byte": 1.5772676815707922, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.0932786464691162, + "step": 4620, + "unclipped_grad_norm": 0.9947658777236938 + }, + { + "bits_per_byte": 1.6629637127906651, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1526786088943481, + "step": 4630, + "unclipped_grad_norm": 1.0645713806152344 + }, + { + "bits_per_byte": 1.6351708004632826, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1334140300750732, + "step": 4640, + "unclipped_grad_norm": 1.0213215351104736 + }, + { + "bits_per_byte": 1.6143973600308656, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1190149784088135, + "step": 4650, + "unclipped_grad_norm": 1.0158205032348633 + }, + { + "bits_per_byte": 1.6442494206370286, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1397068500518799, + "step": 4660, + "unclipped_grad_norm": 1.032319188117981 + }, + { + "bits_per_byte": 1.6788175894951514, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1636676788330078, + "step": 4670, + "unclipped_grad_norm": 1.1467437744140625 + }, + { + "bits_per_byte": 1.6228953667752988, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1249053478240967, + "step": 4680, + "unclipped_grad_norm": 1.0684930086135864 + }, + { + "bits_per_byte": 1.6116814100084402, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1171324253082275, + "step": 4690, + "unclipped_grad_norm": 1.0818438529968262 + }, + { + "bits_per_byte": 1.7118176205427071, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1865415573120117, + "step": 4700, + "unclipped_grad_norm": 1.0913734436035156 + }, + { + "bits_per_byte": 1.6314229545356804, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1308162212371826, + "step": 4710, + "unclipped_grad_norm": 1.0516728162765503 + }, + { + "bits_per_byte": 1.6067035441618671, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1136820316314697, + "step": 4720, + "unclipped_grad_norm": 1.027355670928955 + }, + { + "bits_per_byte": 1.723685455348146, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.194767713546753, + "step": 4730, + "unclipped_grad_norm": 1.0737906694412231 + }, + { + "bits_per_byte": 1.6626813172779484, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1524828672409058, + "step": 4740, + "unclipped_grad_norm": 1.093795657157898 + }, + { + "bits_per_byte": 1.6942468410346574, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1743624210357666, + "step": 4750, + "unclipped_grad_norm": 1.098819375038147 + }, + { + "bits_per_byte": 1.567997816689287, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.086853265762329, + "step": 4760, + "unclipped_grad_norm": 0.9886804819107056 + }, + { + "bits_per_byte": 1.674730077832095, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1608344316482544, + "step": 4770, + "unclipped_grad_norm": 1.0352309942245483 + }, + { + "bits_per_byte": 1.6338455021557339, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.132495403289795, + "step": 4780, + "unclipped_grad_norm": 1.037882685661316 + }, + { + "bits_per_byte": 1.6704422063807836, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1578623056411743, + "step": 4790, + "unclipped_grad_norm": 1.117195725440979 + }, + { + "bits_per_byte": 1.6306944360266256, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1303112506866455, + "step": 4800, + "unclipped_grad_norm": 1.0563780069351196 + }, + { + "bits_per_byte": 1.6131430905581232, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1181455850601196, + "step": 4810, + "unclipped_grad_norm": 1.0533764362335205 + }, + { + "bits_per_byte": 1.675093477173368, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1610863208770752, + "step": 4820, + "unclipped_grad_norm": 1.0993541479110718 + }, + { + "bits_per_byte": 1.6710443176414542, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1582796573638916, + "step": 4830, + "unclipped_grad_norm": 1.2536643743515015 + }, + { + "bits_per_byte": 1.6771435103716528, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1625072956085205, + "step": 4840, + "unclipped_grad_norm": 1.074759840965271 + }, + { + "bits_per_byte": 1.6143529885069432, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1189842224121094, + "step": 4850, + "unclipped_grad_norm": 1.0702130794525146 + }, + { + "bits_per_byte": 1.6332604171774985, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1320898532867432, + "step": 4860, + "unclipped_grad_norm": 1.0851078033447266 + }, + { + "bits_per_byte": 1.6331224870915066, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1319942474365234, + "step": 4870, + "unclipped_grad_norm": 1.08431875705719 + }, + { + "bits_per_byte": 1.7854224114936703, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.237560510635376, + "step": 4880, + "unclipped_grad_norm": 1.1104563474655151 + }, + { + "bits_per_byte": 1.5755340964500952, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0920770168304443, + "step": 4890, + "unclipped_grad_norm": 1.042919635772705 + }, + { + "bits_per_byte": 1.6730817961062259, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1596919298171997, + "step": 4900, + "unclipped_grad_norm": 1.0552027225494385 + }, + { + "bits_per_byte": 1.6497614645971816, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1435275077819824, + "step": 4910, + "unclipped_grad_norm": 1.0571706295013428 + }, + { + "bits_per_byte": 1.5965801293841295, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.106665015220642, + "step": 4920, + "unclipped_grad_norm": 1.0640807151794434 + }, + { + "bits_per_byte": 1.6007018655947076, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.1095219850540161, + "step": 4930, + "unclipped_grad_norm": 1.0152149200439453 + }, + { + "bits_per_byte": 1.6331477685411835, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.1320117712020874, + "step": 4940, + "unclipped_grad_norm": 1.100314736366272 + }, + { + "bits_per_byte": 1.6360496318091915, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1340231895446777, + "step": 4950, + "unclipped_grad_norm": 1.079222559928894 + }, + { + "bits_per_byte": 1.6569551549174713, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1485137939453125, + "step": 4960, + "unclipped_grad_norm": 1.0604397058486938 + }, + { + "bits_per_byte": 1.638398398872025, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1356512308120728, + "step": 4970, + "unclipped_grad_norm": 1.0775542259216309 + }, + { + "bits_per_byte": 1.680847672705936, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1650748252868652, + "step": 4980, + "unclipped_grad_norm": 1.1589142084121704 + }, + { + "bits_per_byte": 1.633999426628256, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1326020956039429, + "step": 4990, + "unclipped_grad_norm": 1.075969934463501 + }, + { + "bits_per_byte": 1.6444650868812105, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1398563385009766, + "step": 5000, + "unclipped_grad_norm": 1.0691334009170532 + }, + { + "bits_per_byte": 1.6738108305632333, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1601972579956055, + "step": 5010, + "unclipped_grad_norm": 1.0487086772918701 + }, + { + "bits_per_byte": 1.6477535671483583, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.142135739326477, + "step": 5020, + "unclipped_grad_norm": 1.1224150657653809 + }, + { + "bits_per_byte": 1.5805776596692978, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0955729484558105, + "step": 5030, + "unclipped_grad_norm": 0.9987351894378662 + }, + { + "bits_per_byte": 1.6501410303076358, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.143790602684021, + "step": 5040, + "unclipped_grad_norm": 1.069442868232727 + }, + { + "bits_per_byte": 1.6202205206064295, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1230512857437134, + "step": 5050, + "unclipped_grad_norm": 1.1084743738174438 + }, + { + "bits_per_byte": 1.6928149134835737, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1733698844909668, + "step": 5060, + "unclipped_grad_norm": 1.0974880456924438 + }, + { + "bits_per_byte": 1.6259845191500966, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1270465850830078, + "step": 5070, + "unclipped_grad_norm": 1.0709881782531738 + }, + { + "bits_per_byte": 1.6998562271751987, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1782505512237549, + "step": 5080, + "unclipped_grad_norm": 1.1272931098937988 + }, + { + "bits_per_byte": 1.6260173678364114, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.127069354057312, + "step": 5090, + "unclipped_grad_norm": 1.0618640184402466 + }, + { + "bits_per_byte": 1.6234274810970684, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.1252741813659668, + "step": 5100, + "unclipped_grad_norm": 1.1292778253555298 + }, + { + "bits_per_byte": 1.6568031222541084, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1484084129333496, + "step": 5110, + "unclipped_grad_norm": 1.1496297121047974 + }, + { + "bits_per_byte": 1.5598939941806313, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0812361240386963, + "step": 5120, + "unclipped_grad_norm": 1.035429835319519 + }, + { + "bits_per_byte": 1.6914442117561972, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.172419786453247, + "step": 5130, + "unclipped_grad_norm": 1.0999172925949097 + }, + { + "bits_per_byte": 1.6623124144918477, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1522271633148193, + "step": 5140, + "unclipped_grad_norm": 1.0919309854507446 + }, + { + "bits_per_byte": 1.6100444791375281, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1159977912902832, + "step": 5150, + "unclipped_grad_norm": 1.0898085832595825 + }, + { + "bits_per_byte": 1.6321538808018472, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.1313228607177734, + "step": 5160, + "unclipped_grad_norm": 1.0807921886444092 + }, + { + "bits_per_byte": 1.6844840738757791, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.167595386505127, + "step": 5170, + "unclipped_grad_norm": 1.176076889038086 + }, + { + "bits_per_byte": 1.65267932225172, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.145550012588501, + "step": 5180, + "unclipped_grad_norm": 1.110230803489685 + }, + { + "bits_per_byte": 1.5426296877564192, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0692694187164307, + "step": 5190, + "unclipped_grad_norm": 1.0403128862380981 + }, + { + "bits_per_byte": 1.6380544335702993, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.1354128122329712, + "step": 5200, + "unclipped_grad_norm": 1.1156731843948364 + }, + { + "bits_per_byte": 1.6025732088187459, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1108191013336182, + "step": 5210, + "unclipped_grad_norm": 1.112536907196045 + }, + { + "bits_per_byte": 1.595609287320009, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.105992078781128, + "step": 5220, + "unclipped_grad_norm": 1.0583246946334839 + }, + { + "bits_per_byte": 1.6907153492818405, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1719145774841309, + "step": 5230, + "unclipped_grad_norm": 1.1788045167922974 + }, + { + "bits_per_byte": 1.6472453584150588, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.1417834758758545, + "step": 5240, + "unclipped_grad_norm": 1.1392273902893066 + }, + { + "bits_per_byte": 1.6431225903085755, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.1389257907867432, + "step": 5250, + "unclipped_grad_norm": 1.087467074394226 + }, + { + "bits_per_byte": 1.5745493237912547, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0913944244384766, + "step": 5260, + "unclipped_grad_norm": 1.0672780275344849 + }, + { + "bits_per_byte": 1.625834722261195, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.126942753791809, + "step": 5270, + "unclipped_grad_norm": 1.0955538749694824 + }, + { + "bits_per_byte": 1.5722745092682924, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0898176431655884, + "step": 5280, + "unclipped_grad_norm": 1.1191643476486206 + }, + { + "bits_per_byte": 1.581137635180507, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.095961093902588, + "step": 5290, + "unclipped_grad_norm": 1.0905182361602783 + }, + { + "bits_per_byte": 1.5625956696430352, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0831087827682495, + "step": 5300, + "unclipped_grad_norm": 1.0970019102096558 + }, + { + "bits_per_byte": 1.575699715742876, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0921918153762817, + "step": 5310, + "unclipped_grad_norm": 1.085243582725525 + }, + { + "bits_per_byte": 1.5934223559316376, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.1044762134552002, + "step": 5320, + "unclipped_grad_norm": 1.1693148612976074 + }, + { + "bits_per_byte": 1.6214211314921028, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1238834857940674, + "step": 5330, + "unclipped_grad_norm": 1.1232852935791016 + }, + { + "bits_per_byte": 1.6839337293930183, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1672139167785645, + "step": 5340, + "unclipped_grad_norm": 1.1369119882583618 + }, + { + "bits_per_byte": 1.5416975417887426, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0686233043670654, + "step": 5350, + "unclipped_grad_norm": 1.0887452363967896 + }, + { + "bits_per_byte": 1.5866414239734188, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.099776029586792, + "step": 5360, + "unclipped_grad_norm": 1.0925503969192505 + }, + { + "bits_per_byte": 1.6646420914804352, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1538419723510742, + "step": 5370, + "unclipped_grad_norm": 1.1924686431884766 + }, + { + "bits_per_byte": 1.586491799067168, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0996723175048828, + "step": 5380, + "unclipped_grad_norm": 1.0599968433380127 + }, + { + "bits_per_byte": 1.5783984675002152, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0940624475479126, + "step": 5390, + "unclipped_grad_norm": 1.0973831415176392 + }, + { + "bits_per_byte": 1.642996527025493, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.1388384103775024, + "step": 5400, + "unclipped_grad_norm": 1.1477714776992798 + }, + { + "bits_per_byte": 1.5308346016122951, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0610936880111694, + "step": 5410, + "unclipped_grad_norm": 1.0651572942733765 + }, + { + "bits_per_byte": 1.6877225071915258, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1698400974273682, + "step": 5420, + "unclipped_grad_norm": 1.230097770690918 + }, + { + "bits_per_byte": 1.606558218821888, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.1135812997817993, + "step": 5430, + "unclipped_grad_norm": 1.1141976118087769 + }, + { + "bits_per_byte": 1.6079992614534675, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.1145801544189453, + "step": 5440, + "unclipped_grad_norm": 1.0613634586334229 + }, + { + "bits_per_byte": 1.6121270170568258, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.1174412965774536, + "step": 5450, + "unclipped_grad_norm": 1.1033883094787598 + }, + { + "bits_per_byte": 1.6549515570349194, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.147125005722046, + "step": 5460, + "unclipped_grad_norm": 1.0616251230239868 + }, + { + "bits_per_byte": 1.6138821000088808, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.1186578273773193, + "step": 5470, + "unclipped_grad_norm": 1.1187942028045654 + }, + { + "bits_per_byte": 1.566549894751673, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.085849642753601, + "step": 5480, + "unclipped_grad_norm": 1.095421314239502 + }, + { + "bits_per_byte": 1.6100902265226575, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1160295009613037, + "step": 5490, + "unclipped_grad_norm": 1.1169681549072266 + }, + { + "bits_per_byte": 1.6345392801693144, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.1329762935638428, + "step": 5500, + "unclipped_grad_norm": 1.0606133937835693 + }, + { + "bits_per_byte": 1.6112232482265416, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1168148517608643, + "step": 5510, + "unclipped_grad_norm": 1.082831859588623 + }, + { + "bits_per_byte": 1.6200284159854157, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.1229181289672852, + "step": 5520, + "unclipped_grad_norm": 1.1288034915924072 + }, + { + "bits_per_byte": 1.6500037881522474, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.1436954736709595, + "step": 5530, + "unclipped_grad_norm": 1.1526131629943848 + }, + { + "bits_per_byte": 1.6647621353707376, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1539251804351807, + "step": 5540, + "unclipped_grad_norm": 1.1722888946533203 + }, + { + "bits_per_byte": 1.5974680758105342, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.1072804927825928, + "step": 5550, + "unclipped_grad_norm": 1.073315143585205 + }, + { + "bits_per_byte": 1.594942510582614, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.1055299043655396, + "step": 5560, + "unclipped_grad_norm": 1.0879392623901367 + }, + { + "bits_per_byte": 1.6466642290377933, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.1413806676864624, + "step": 5570, + "unclipped_grad_norm": 1.21271812915802 + }, + { + "bits_per_byte": 1.5387917229197647, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0666091442108154, + "step": 5580, + "unclipped_grad_norm": 1.0752229690551758 + }, + { + "bits_per_byte": 1.5950678859350929, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.105616807937622, + "step": 5590, + "unclipped_grad_norm": 1.115165114402771 + }, + { + "bits_per_byte": 1.60302053569364, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.1111291646957397, + "step": 5600, + "unclipped_grad_norm": 1.0947556495666504 + }, + { + "bits_per_byte": 1.6150159816260192, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.1194437742233276, + "step": 5610, + "unclipped_grad_norm": 1.0436956882476807 + }, + { + "bits_per_byte": 1.5445598490470525, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.070607304573059, + "step": 5620, + "unclipped_grad_norm": 1.1328603029251099 + }, + { + "bits_per_byte": 1.5627000631121089, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0831811428070068, + "step": 5630, + "unclipped_grad_norm": 1.0480417013168335 + }, + { + "bits_per_byte": 1.5462631652211978, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.07178795337677, + "step": 5640, + "unclipped_grad_norm": 1.1326069831848145 + }, + { + "bits_per_byte": 1.6566736193180087, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1483186483383179, + "step": 5650, + "unclipped_grad_norm": 1.174943447113037 + }, + { + "bits_per_byte": 1.6152889180929386, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1196329593658447, + "step": 5660, + "unclipped_grad_norm": 1.1866281032562256 + }, + { + "bits_per_byte": 1.6043573568387968, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.112055778503418, + "step": 5670, + "unclipped_grad_norm": 1.1592957973480225 + }, + { + "bits_per_byte": 1.6221047625292826, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.1243573427200317, + "step": 5680, + "unclipped_grad_norm": 1.1609989404678345 + }, + { + "bits_per_byte": 1.6421648189259206, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1382619142532349, + "step": 5690, + "unclipped_grad_norm": 1.1130645275115967 + }, + { + "bits_per_byte": 1.598431178655366, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1079480648040771, + "step": 5700, + "unclipped_grad_norm": 1.1522045135498047 + }, + { + "bits_per_byte": 1.6208697551134366, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.1235013008117676, + "step": 5710, + "unclipped_grad_norm": 1.1070868968963623 + }, + { + "bits_per_byte": 1.648464543427025, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.14262855052948, + "step": 5720, + "unclipped_grad_norm": 1.2136235237121582 + }, + { + "bits_per_byte": 1.5559942875723172, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0785330533981323, + "step": 5730, + "unclipped_grad_norm": 1.1237554550170898 + }, + { + "bits_per_byte": 1.6165538504900345, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1205097436904907, + "step": 5740, + "unclipped_grad_norm": 1.1293457746505737 + }, + { + "bits_per_byte": 1.5847578699811693, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0984704494476318, + "step": 5750, + "unclipped_grad_norm": 1.1838749647140503 + }, + { + "bits_per_byte": 1.5751051717188433, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0917797088623047, + "step": 5760, + "unclipped_grad_norm": 1.1379411220550537 + }, + { + "bits_per_byte": 1.5366526027083331, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0651264190673828, + "step": 5770, + "unclipped_grad_norm": 1.1725109815597534 + }, + { + "bits_per_byte": 1.6123841311198657, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.117619514465332, + "step": 5780, + "unclipped_grad_norm": 1.1070845127105713 + }, + { + "bits_per_byte": 1.5625800192218067, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0830979347229004, + "step": 5790, + "unclipped_grad_norm": 1.0731046199798584 + }, + { + "bits_per_byte": 1.581076581339451, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0959187746047974, + "step": 5800, + "unclipped_grad_norm": 1.1906232833862305 + }, + { + "bits_per_byte": 1.6024834338749956, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.1107568740844727, + "step": 5810, + "unclipped_grad_norm": 1.198943853378296 + }, + { + "bits_per_byte": 1.6068543729466738, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.1137865781784058, + "step": 5820, + "unclipped_grad_norm": 1.1112759113311768 + }, + { + "bits_per_byte": 1.567694095327863, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0866427421569824, + "step": 5830, + "unclipped_grad_norm": 1.2037192583084106 + }, + { + "bits_per_byte": 1.5486041930647423, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0734106302261353, + "step": 5840, + "unclipped_grad_norm": 1.1501859426498413 + }, + { + "bits_per_byte": 1.540087784176667, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0675075054168701, + "step": 5850, + "unclipped_grad_norm": 1.1106252670288086 + }, + { + "bits_per_byte": 1.591153904766757, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.1029038429260254, + "step": 5860, + "unclipped_grad_norm": 1.267377257347107 + }, + { + "bits_per_byte": 1.6187461133405827, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.1220293045043945, + "step": 5870, + "unclipped_grad_norm": 1.1557857990264893 + }, + { + "bits_per_byte": 1.6194158137830426, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.1224935054779053, + "step": 5880, + "unclipped_grad_norm": 1.1605706214904785 + }, + { + "bits_per_byte": 1.6297550667876128, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.1296601295471191, + "step": 5890, + "unclipped_grad_norm": 1.1718353033065796 + }, + { + "bits_per_byte": 1.5804839291245776, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0955079793930054, + "step": 5900, + "unclipped_grad_norm": 1.1900224685668945 + }, + { + "bits_per_byte": 1.5770434161940672, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.093123197555542, + "step": 5910, + "unclipped_grad_norm": 1.1988046169281006 + }, + { + "bits_per_byte": 1.5761890063845807, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0925309658050537, + "step": 5920, + "unclipped_grad_norm": 1.1799134016036987 + }, + { + "bits_per_byte": 1.6240858306845711, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1257305145263672, + "step": 5930, + "unclipped_grad_norm": 1.1704758405685425 + }, + { + "bits_per_byte": 1.6230097352381225, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.124984622001648, + "step": 5940, + "unclipped_grad_norm": 1.1741182804107666 + }, + { + "bits_per_byte": 1.5942451209333652, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.1050465106964111, + "step": 5950, + "unclipped_grad_norm": 1.223347544670105 + }, + { + "bits_per_byte": 1.587844614598855, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.1006100177764893, + "step": 5960, + "unclipped_grad_norm": 1.1676011085510254 + }, + { + "bits_per_byte": 1.6162547726601841, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.120302438735962, + "step": 5970, + "unclipped_grad_norm": 1.2063930034637451 + }, + { + "bits_per_byte": 1.5550433954956966, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.077873945236206, + "step": 5980, + "unclipped_grad_norm": 1.1823816299438477 + }, + { + "bits_per_byte": 1.5403349232459567, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0676788091659546, + "step": 5990, + "unclipped_grad_norm": 1.1445503234863281 + }, + { + "bits_per_byte": 1.6087774829486217, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.1151195764541626, + "step": 6000, + "unclipped_grad_norm": 1.182093858718872 + }, + { + "bits_per_byte": 1.585380103211991, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0989017486572266, + "step": 6010, + "unclipped_grad_norm": 1.1548501253128052 + }, + { + "bits_per_byte": 1.57159036228316, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0893434286117554, + "step": 6020, + "unclipped_grad_norm": 1.132990837097168 + }, + { + "bits_per_byte": 1.6225496816470646, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.1246657371520996, + "step": 6030, + "unclipped_grad_norm": 1.1933883428573608 + }, + { + "bits_per_byte": 1.5568839538252304, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0791497230529785, + "step": 6040, + "unclipped_grad_norm": 1.1697132587432861 + }, + { + "bits_per_byte": 1.5810753774608948, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0959179401397705, + "step": 6050, + "unclipped_grad_norm": 1.1444475650787354 + }, + { + "bits_per_byte": 1.6146938581209531, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.119220495223999, + "step": 6060, + "unclipped_grad_norm": 1.1966716051101685 + }, + { + "bits_per_byte": 1.5827436091742642, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.097074270248413, + "step": 6070, + "unclipped_grad_norm": 1.1267356872558594 + }, + { + "bits_per_byte": 1.5673551175230125, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0864077806472778, + "step": 6080, + "unclipped_grad_norm": 1.097261905670166 + }, + { + "bits_per_byte": 1.5774123189801679, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0933789014816284, + "step": 6090, + "unclipped_grad_norm": 1.1282033920288086 + }, + { + "bits_per_byte": 1.5779040173789847, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.093719720840454, + "step": 6100, + "unclipped_grad_norm": 1.1281672716140747 + }, + { + "bits_per_byte": 1.5504769121499873, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0747087001800537, + "step": 6110, + "unclipped_grad_norm": 1.1312857866287231 + }, + { + "bits_per_byte": 1.5543487575688617, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0773924589157104, + "step": 6120, + "unclipped_grad_norm": 1.157880187034607 + }, + { + "bits_per_byte": 1.5931889754744168, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.1043144464492798, + "step": 6130, + "unclipped_grad_norm": 1.1723217964172363 + }, + { + "bits_per_byte": 1.6084328297162926, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.1148806810379028, + "step": 6140, + "unclipped_grad_norm": 1.1961045265197754 + }, + { + "bits_per_byte": 1.542341444833573, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0690696239471436, + "step": 6150, + "unclipped_grad_norm": 1.1969319581985474 + }, + { + "bits_per_byte": 1.6357159854665178, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.1337919235229492, + "step": 6160, + "unclipped_grad_norm": 1.2412800788879395 + }, + { + "bits_per_byte": 1.4931134748159045, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.034947395324707, + "step": 6170, + "unclipped_grad_norm": 1.0997314453125 + }, + { + "bits_per_byte": 1.5659319610871227, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0854213237762451, + "step": 6180, + "unclipped_grad_norm": 1.128227710723877 + }, + { + "bits_per_byte": 1.581150533879322, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0959700345993042, + "step": 6190, + "unclipped_grad_norm": 1.1780462265014648 + }, + { + "bits_per_byte": 1.5351257407339731, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.064068078994751, + "step": 6200, + "unclipped_grad_norm": 1.13433837890625 + }, + { + "bits_per_byte": 1.5793001725386888, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0946874618530273, + "step": 6210, + "unclipped_grad_norm": 1.1643598079681396 + }, + { + "bits_per_byte": 1.5609047362197521, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0819367170333862, + "step": 6220, + "unclipped_grad_norm": 1.131072998046875 + }, + { + "bits_per_byte": 1.5253499028936293, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0572919845581055, + "step": 6230, + "unclipped_grad_norm": 1.1126173734664917 + }, + { + "bits_per_byte": 1.5400140036194467, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0674563646316528, + "step": 6240, + "unclipped_grad_norm": 1.1826318502426147 + }, + { + "bits_per_byte": 1.5344780540708238, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0636191368103027, + "step": 6250, + "unclipped_grad_norm": 1.1302508115768433 + }, + { + "bits_per_byte": 1.600364263651064, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.109287977218628, + "step": 6260, + "unclipped_grad_norm": 1.1724376678466797 + }, + { + "bits_per_byte": 1.5544997583363194, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.077497124671936, + "step": 6270, + "unclipped_grad_norm": 1.081700086593628 + }, + { + "bits_per_byte": 1.6754886933050508, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.161360263824463, + "step": 6280, + "unclipped_grad_norm": 1.2664587497711182 + }, + { + "bits_per_byte": 1.5413214157313058, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0683625936508179, + "step": 6290, + "unclipped_grad_norm": 1.2083820104599 + }, + { + "bits_per_byte": 1.5888871734283854, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.101332664489746, + "step": 6300, + "unclipped_grad_norm": 1.2475577592849731 + }, + { + "bits_per_byte": 1.6228484155116132, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.1248728036880493, + "step": 6310, + "unclipped_grad_norm": 1.2506906986236572 + }, + { + "bits_per_byte": 1.5728649257087044, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0902268886566162, + "step": 6320, + "unclipped_grad_norm": 1.1333374977111816 + }, + { + "bits_per_byte": 1.5253509347895344, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0572926998138428, + "step": 6330, + "unclipped_grad_norm": 1.180550217628479 + }, + { + "bits_per_byte": 1.5807446548232857, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0956887006759644, + "step": 6340, + "unclipped_grad_norm": 1.1644010543823242 + }, + { + "bits_per_byte": 1.5717518539923203, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0894553661346436, + "step": 6350, + "unclipped_grad_norm": 1.142453908920288 + }, + { + "bits_per_byte": 1.5231024336121541, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0557341575622559, + "step": 6360, + "unclipped_grad_norm": 1.1504549980163574 + }, + { + "bits_per_byte": 1.6077399116159663, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.1144003868103027, + "step": 6370, + "unclipped_grad_norm": 1.2130403518676758 + }, + { + "bits_per_byte": 1.5316929670227515, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0616886615753174, + "step": 6380, + "unclipped_grad_norm": 1.1643673181533813 + }, + { + "bits_per_byte": 1.5801346323606753, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0952658653259277, + "step": 6390, + "unclipped_grad_norm": 1.1411337852478027 + }, + { + "bits_per_byte": 1.5548263533903077, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.077723503112793, + "step": 6400, + "unclipped_grad_norm": 1.1154334545135498 + }, + { + "bits_per_byte": 1.5450890396637573, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0709741115570068, + "step": 6410, + "unclipped_grad_norm": 1.137816071510315 + }, + { + "bits_per_byte": 1.5323659351355776, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0621551275253296, + "step": 6420, + "unclipped_grad_norm": 1.1425901651382446 + }, + { + "bits_per_byte": 1.5643414655319436, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0843188762664795, + "step": 6430, + "unclipped_grad_norm": 1.2278902530670166 + }, + { + "bits_per_byte": 1.5636070996127593, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0838098526000977, + "step": 6440, + "unclipped_grad_norm": 1.204923152923584 + }, + { + "bits_per_byte": 1.5427707135301267, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0693671703338623, + "step": 6450, + "unclipped_grad_norm": 1.1532135009765625 + }, + { + "bits_per_byte": 1.5522318231193915, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0759251117706299, + "step": 6460, + "unclipped_grad_norm": 1.5724140405654907 + }, + { + "bits_per_byte": 1.5677630603708592, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0866905450820923, + "step": 6470, + "unclipped_grad_norm": 1.182875394821167 + }, + { + "bits_per_byte": 1.5839062118940967, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0978801250457764, + "step": 6480, + "unclipped_grad_norm": 1.211335301399231 + }, + { + "bits_per_byte": 1.561875234318571, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0826094150543213, + "step": 6490, + "unclipped_grad_norm": 1.2880244255065918 + }, + { + "bits_per_byte": 1.577221590220361, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0932466983795166, + "step": 6500, + "unclipped_grad_norm": 1.2181490659713745 + }, + { + "bits_per_byte": 1.5067461955844976, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0443968772888184, + "step": 6510, + "unclipped_grad_norm": 1.1337521076202393 + }, + { + "bits_per_byte": 1.4936307986296997, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0353059768676758, + "step": 6520, + "unclipped_grad_norm": 1.2117457389831543 + }, + { + "bits_per_byte": 1.5784743118492457, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0941150188446045, + "step": 6530, + "unclipped_grad_norm": 1.2337595224380493 + }, + { + "bits_per_byte": 1.5448394928373554, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0708011388778687, + "step": 6540, + "unclipped_grad_norm": 1.1313438415527344 + }, + { + "bits_per_byte": 1.5964084906985685, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.1065460443496704, + "step": 6550, + "unclipped_grad_norm": 1.2166260480880737 + }, + { + "bits_per_byte": 1.5631484218829081, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0834919214248657, + "step": 6560, + "unclipped_grad_norm": 1.1645575761795044 + }, + { + "bits_per_byte": 1.4810062401604644, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0265552997589111, + "step": 6570, + "unclipped_grad_norm": 1.2005696296691895 + }, + { + "bits_per_byte": 1.5898447728283895, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.1019964218139648, + "step": 6580, + "unclipped_grad_norm": 1.1795482635498047 + }, + { + "bits_per_byte": 1.5634753609021985, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0837185382843018, + "step": 6590, + "unclipped_grad_norm": 1.1887481212615967 + }, + { + "bits_per_byte": 1.47844799822888, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.0247820615768433, + "step": 6600, + "unclipped_grad_norm": 1.1429756879806519 + }, + { + "bits_per_byte": 1.6001293353499855, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.1091251373291016, + "step": 6610, + "unclipped_grad_norm": 1.1761175394058228 + }, + { + "bits_per_byte": 1.6007649832275743, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.1095657348632812, + "step": 6620, + "unclipped_grad_norm": 1.2725090980529785 + }, + { + "bits_per_byte": 1.5866947385951862, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0998129844665527, + "step": 6630, + "unclipped_grad_norm": 1.242309808731079 + }, + { + "bits_per_byte": 1.557243053600232, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0793986320495605, + "step": 6640, + "unclipped_grad_norm": 1.2104532718658447 + }, + { + "bits_per_byte": 1.5462134622350983, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0717535018920898, + "step": 6650, + "unclipped_grad_norm": 1.1630686521530151 + }, + { + "bits_per_byte": 1.588834890702523, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.1012964248657227, + "step": 6660, + "unclipped_grad_norm": 1.204805612564087 + }, + { + "bits_per_byte": 1.587033544417386, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.1000478267669678, + "step": 6670, + "unclipped_grad_norm": 1.2499597072601318 + }, + { + "bits_per_byte": 1.5912992301067361, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.1030045747756958, + "step": 6680, + "unclipped_grad_norm": 1.2160732746124268 + }, + { + "bits_per_byte": 1.56477726956923, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0846209526062012, + "step": 6690, + "unclipped_grad_norm": 1.2467725276947021 + }, + { + "bits_per_byte": 1.6040180350686444, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.1118205785751343, + "step": 6700, + "unclipped_grad_norm": 1.267776370048523 + }, + { + "bits_per_byte": 1.537844786444114, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0659527778625488, + "step": 6710, + "unclipped_grad_norm": 1.2301596403121948 + }, + { + "bits_per_byte": 1.5327245189626266, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.062403678894043, + "step": 6720, + "unclipped_grad_norm": 1.180388331413269 + }, + { + "bits_per_byte": 1.5337890915714674, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0631415843963623, + "step": 6730, + "unclipped_grad_norm": 1.2269387245178223 + }, + { + "bits_per_byte": 1.5501341507268178, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.074471116065979, + "step": 6740, + "unclipped_grad_norm": 1.1682806015014648 + }, + { + "bits_per_byte": 1.5184903748639662, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0525373220443726, + "step": 6750, + "unclipped_grad_norm": 1.23702871799469 + }, + { + "bits_per_byte": 1.500455070215936, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0400362014770508, + "step": 6760, + "unclipped_grad_norm": 1.1704752445220947 + }, + { + "bits_per_byte": 1.5177950490065277, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0520553588867188, + "step": 6770, + "unclipped_grad_norm": 1.209053874015808 + }, + { + "bits_per_byte": 1.5265771710901863, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0581426620483398, + "step": 6780, + "unclipped_grad_norm": 1.2882649898529053 + }, + { + "bits_per_byte": 1.6213150181965206, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.1238099336624146, + "step": 6790, + "unclipped_grad_norm": 1.1679272651672363 + }, + { + "bits_per_byte": 1.5863846538756805, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0995980501174927, + "step": 6800, + "unclipped_grad_norm": 1.280809998512268 + }, + { + "bits_per_byte": 1.578056394007649, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.093825340270996, + "step": 6810, + "unclipped_grad_norm": 1.24360191822052 + }, + { + "bits_per_byte": 1.5169633409069554, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0514788627624512, + "step": 6820, + "unclipped_grad_norm": 1.2292150259017944 + }, + { + "bits_per_byte": 1.5347601056182387, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.063814640045166, + "step": 6830, + "unclipped_grad_norm": 1.242634892463684 + }, + { + "bits_per_byte": 1.5555476486280264, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.078223466873169, + "step": 6840, + "unclipped_grad_norm": 1.2094796895980835 + }, + { + "bits_per_byte": 1.5217322478327302, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0547844171524048, + "step": 6850, + "unclipped_grad_norm": 1.1779916286468506 + }, + { + "bits_per_byte": 1.549958384457636, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.074349284172058, + "step": 6860, + "unclipped_grad_norm": 1.2639429569244385 + }, + { + "bits_per_byte": 1.5571720247654257, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.079349398612976, + "step": 6870, + "unclipped_grad_norm": 1.2982949018478394 + }, + { + "bits_per_byte": 1.597770421310751, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.107490062713623, + "step": 6880, + "unclipped_grad_norm": 1.2857638597488403 + }, + { + "bits_per_byte": 1.4860104193526196, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0300239324569702, + "step": 6890, + "unclipped_grad_norm": 1.2555214166641235 + }, + { + "bits_per_byte": 1.5064882216082034, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0442180633544922, + "step": 6900, + "unclipped_grad_norm": 1.202001690864563 + }, + { + "bits_per_byte": 1.685866814388716, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1685538291931152, + "step": 6910, + "unclipped_grad_norm": 1.3379162549972534 + }, + { + "bits_per_byte": 1.5053493524941899, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.043428659439087, + "step": 6920, + "unclipped_grad_norm": 1.1770343780517578 + }, + { + "bits_per_byte": 1.516240325842728, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 1.0509777069091797, + "step": 6930, + "unclipped_grad_norm": 1.2326538562774658 + }, + { + "bits_per_byte": 1.5463919802266939, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0718772411346436, + "step": 6940, + "unclipped_grad_norm": 1.2218632698059082 + }, + { + "bits_per_byte": 1.5410003241221448, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0681400299072266, + "step": 6950, + "unclipped_grad_norm": 1.2248505353927612 + }, + { + "bits_per_byte": 1.574234251574874, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0911760330200195, + "step": 6960, + "unclipped_grad_norm": 1.2728900909423828 + }, + { + "bits_per_byte": 1.556295085228676, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0787415504455566, + "step": 6970, + "unclipped_grad_norm": 1.2423036098480225 + }, + { + "bits_per_byte": 1.6109532354646872, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.1166276931762695, + "step": 6980, + "unclipped_grad_norm": 1.2731002569198608 + }, + { + "bits_per_byte": 1.5507145921734797, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.074873447418213, + "step": 6990, + "unclipped_grad_norm": 1.2125478982925415 + }, + { + "bits_per_byte": 1.506560110356264, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0442678928375244, + "step": 7000, + "unclipped_grad_norm": 1.2366911172866821 + }, + { + "bits_per_byte": 1.5357397187975532, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0644936561584473, + "step": 7010, + "unclipped_grad_norm": 1.2639423608779907 + }, + { + "bits_per_byte": 1.5037485379799589, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0423190593719482, + "step": 7020, + "unclipped_grad_norm": 1.2011712789535522 + }, + { + "bits_per_byte": 1.528263632964547, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0593116283416748, + "step": 7030, + "unclipped_grad_norm": 1.1849545240402222 + }, + { + "bits_per_byte": 1.5983039114937276, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.1078598499298096, + "step": 7040, + "unclipped_grad_norm": 1.3057087659835815 + }, + { + "bits_per_byte": 1.5441888824691414, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.070350170135498, + "step": 7050, + "unclipped_grad_norm": 1.2155181169509888 + }, + { + "bits_per_byte": 1.5741694141154987, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0911310911178589, + "step": 7060, + "unclipped_grad_norm": 1.3050142526626587 + }, + { + "bits_per_byte": 1.5094996378248111, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0463054180145264, + "step": 7070, + "unclipped_grad_norm": 1.1835323572158813 + }, + { + "bits_per_byte": 1.4442604269771167, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0010850429534912, + "step": 7080, + "unclipped_grad_norm": 1.167786717414856 + }, + { + "bits_per_byte": 1.5378045425038123, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.065924882888794, + "step": 7090, + "unclipped_grad_norm": 1.240479588508606 + }, + { + "bits_per_byte": 1.5019334330827527, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0410609245300293, + "step": 7100, + "unclipped_grad_norm": 1.1977860927581787 + }, + { + "bits_per_byte": 1.5780336922977352, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0938096046447754, + "step": 7110, + "unclipped_grad_norm": 1.2797318696975708 + }, + { + "bits_per_byte": 1.56051364767169, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0816656351089478, + "step": 7120, + "unclipped_grad_norm": 1.297847032546997 + }, + { + "bits_per_byte": 1.4993894657111901, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0392975807189941, + "step": 7130, + "unclipped_grad_norm": 1.224601149559021 + }, + { + "bits_per_byte": 1.6077283887783587, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.1143923997879028, + "step": 7140, + "unclipped_grad_norm": 1.3082928657531738 + }, + { + "bits_per_byte": 1.624090990164097, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.1257340908050537, + "step": 7150, + "unclipped_grad_norm": 1.2189027070999146 + }, + { + "bits_per_byte": 1.5477993142587043, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0728527307510376, + "step": 7160, + "unclipped_grad_norm": 1.2660733461380005 + }, + { + "bits_per_byte": 1.5516317756505311, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0755091905593872, + "step": 7170, + "unclipped_grad_norm": 1.2526509761810303 + }, + { + "bits_per_byte": 1.4692106380683878, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0183792114257812, + "step": 7180, + "unclipped_grad_norm": 1.1470739841461182 + }, + { + "bits_per_byte": 1.4673153892558797, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0170655250549316, + "step": 7190, + "unclipped_grad_norm": 1.2286535501480103 + }, + { + "bits_per_byte": 1.553365876719181, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0767111778259277, + "step": 7200, + "unclipped_grad_norm": 1.2306180000305176 + }, + { + "bits_per_byte": 1.561453876823957, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0823173522949219, + "step": 7210, + "unclipped_grad_norm": 1.2458006143569946 + }, + { + "bits_per_byte": 1.5086355969868763, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0457065105438232, + "step": 7220, + "unclipped_grad_norm": 1.1648746728897095 + }, + { + "bits_per_byte": 1.5304512522835219, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0608279705047607, + "step": 7230, + "unclipped_grad_norm": 1.2855236530303955 + }, + { + "bits_per_byte": 1.493701999447157, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0353553295135498, + "step": 7240, + "unclipped_grad_norm": 1.2490638494491577 + }, + { + "bits_per_byte": 1.5430964486708607, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0695929527282715, + "step": 7250, + "unclipped_grad_norm": 1.286012887954712 + }, + { + "bits_per_byte": 1.5196966611771179, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0533734560012817, + "step": 7260, + "unclipped_grad_norm": 1.2369425296783447 + }, + { + "bits_per_byte": 1.5993731276341416, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.1086009740829468, + "step": 7270, + "unclipped_grad_norm": 1.2697807550430298 + }, + { + "bits_per_byte": 1.5305549578219921, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0608998537063599, + "step": 7280, + "unclipped_grad_norm": 1.2743209600448608 + }, + { + "bits_per_byte": 1.5476247518680786, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0727317333221436, + "step": 7290, + "unclipped_grad_norm": 1.2424356937408447 + }, + { + "bits_per_byte": 1.5365045256459402, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0650237798690796, + "step": 7300, + "unclipped_grad_norm": 1.298574686050415 + }, + { + "bits_per_byte": 1.5764911799021466, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0927404165267944, + "step": 7310, + "unclipped_grad_norm": 1.3189024925231934 + }, + { + "bits_per_byte": 1.5490052566065542, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.0736886262893677, + "step": 7320, + "unclipped_grad_norm": 1.2497724294662476 + }, + { + "bits_per_byte": 1.5554974296939745, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0781886577606201, + "step": 7330, + "unclipped_grad_norm": 1.279703140258789 + }, + { + "bits_per_byte": 1.5241250424541843, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0564429759979248, + "step": 7340, + "unclipped_grad_norm": 1.2488007545471191 + }, + { + "bits_per_byte": 1.57090466745417, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0888681411743164, + "step": 7350, + "unclipped_grad_norm": 1.238014578819275 + }, + { + "bits_per_byte": 1.5500565865512788, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0744173526763916, + "step": 7360, + "unclipped_grad_norm": 1.2180218696594238 + }, + { + "bits_per_byte": 1.5597980278614498, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.081169605255127, + "step": 7370, + "unclipped_grad_norm": 1.2368392944335938 + }, + { + "bits_per_byte": 1.6215841710451209, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.1239964962005615, + "step": 7380, + "unclipped_grad_norm": 1.3172329664230347 + }, + { + "bits_per_byte": 1.5928735592927343, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.1040958166122437, + "step": 7390, + "unclipped_grad_norm": 1.3043875694274902 + }, + { + "bits_per_byte": 1.5057738056765193, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0437228679656982, + "step": 7400, + "unclipped_grad_norm": 1.2024799585342407 + }, + { + "bits_per_byte": 1.5152624324899222, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.050299882888794, + "step": 7410, + "unclipped_grad_norm": 1.2088910341262817 + }, + { + "bits_per_byte": 1.597047062281222, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.1069886684417725, + "step": 7420, + "unclipped_grad_norm": 1.3177212476730347 + }, + { + "bits_per_byte": 1.5299019396966662, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0604472160339355, + "step": 7430, + "unclipped_grad_norm": 1.290786862373352 + }, + { + "bits_per_byte": 1.5250764504787575, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0571024417877197, + "step": 7440, + "unclipped_grad_norm": 1.2367006540298462 + }, + { + "bits_per_byte": 1.4892672548120085, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0322813987731934, + "step": 7450, + "unclipped_grad_norm": 1.3495734930038452 + }, + { + "bits_per_byte": 1.5046927227331957, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.042973518371582, + "step": 7460, + "unclipped_grad_norm": 1.2188435792922974 + }, + { + "bits_per_byte": 1.5481360562890936, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.073086142539978, + "step": 7470, + "unclipped_grad_norm": 1.1978274583816528 + }, + { + "bits_per_byte": 1.4779339420854511, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.024425745010376, + "step": 7480, + "unclipped_grad_norm": 1.25648832321167 + }, + { + "bits_per_byte": 1.5058897219832008, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0438032150268555, + "step": 7490, + "unclipped_grad_norm": 1.2401517629623413 + }, + { + "bits_per_byte": 1.5728836718176484, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0902398824691772, + "step": 7500, + "unclipped_grad_norm": 1.2475262880325317 + }, + { + "bits_per_byte": 1.5378353273983165, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0659462213516235, + "step": 7510, + "unclipped_grad_norm": 1.2484221458435059 + }, + { + "bits_per_byte": 1.5312337733449477, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0613703727722168, + "step": 7520, + "unclipped_grad_norm": 1.1939175128936768 + }, + { + "bits_per_byte": 1.5599504044901142, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.081275224685669, + "step": 7530, + "unclipped_grad_norm": 1.2631168365478516 + }, + { + "bits_per_byte": 1.5976060058965262, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.1073760986328125, + "step": 7540, + "unclipped_grad_norm": 1.3131186962127686 + }, + { + "bits_per_byte": 1.5182279293387495, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.052355408668518, + "step": 7550, + "unclipped_grad_norm": 1.2190253734588623 + }, + { + "bits_per_byte": 1.5653272700866891, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0850021839141846, + "step": 7560, + "unclipped_grad_norm": 1.2437095642089844 + }, + { + "bits_per_byte": 1.566524613301996, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.085832118988037, + "step": 7570, + "unclipped_grad_norm": 1.2195208072662354 + }, + { + "bits_per_byte": 1.514368810636039, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.049680471420288, + "step": 7580, + "unclipped_grad_norm": 1.2027956247329712 + }, + { + "bits_per_byte": 1.5218865162705542, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0548913478851318, + "step": 7590, + "unclipped_grad_norm": 1.22147798538208 + }, + { + "bits_per_byte": 1.5124059726417418, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.048319935798645, + "step": 7600, + "unclipped_grad_norm": 1.2925957441329956 + }, + { + "bits_per_byte": 1.451400458710337, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0060341358184814, + "step": 7610, + "unclipped_grad_norm": 1.22130286693573 + }, + { + "bits_per_byte": 1.554240580481469, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.077317476272583, + "step": 7620, + "unclipped_grad_norm": 1.2743861675262451 + }, + { + "bits_per_byte": 1.5089792183233002, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0459446907043457, + "step": 7630, + "unclipped_grad_norm": 1.1950352191925049 + }, + { + "bits_per_byte": 1.523324463214418, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.055888056755066, + "step": 7640, + "unclipped_grad_norm": 1.2816646099090576 + }, + { + "bits_per_byte": 1.6003928127711071, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.1093077659606934, + "step": 7650, + "unclipped_grad_norm": 1.3542457818984985 + }, + { + "bits_per_byte": 1.5396506042781735, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.067204475402832, + "step": 7660, + "unclipped_grad_norm": 1.277113437652588 + }, + { + "bits_per_byte": 1.515347907867401, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0503591299057007, + "step": 7670, + "unclipped_grad_norm": 1.3265182971954346 + }, + { + "bits_per_byte": 1.4941618810555641, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0356740951538086, + "step": 7680, + "unclipped_grad_norm": 1.2858244180679321 + }, + { + "bits_per_byte": 1.5394590156051124, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0670716762542725, + "step": 7690, + "unclipped_grad_norm": 1.2726463079452515 + }, + { + "bits_per_byte": 1.5472200766905984, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0724512338638306, + "step": 7700, + "unclipped_grad_norm": 1.3285934925079346 + }, + { + "bits_per_byte": 1.4615679010466955, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.013081669807434, + "step": 7710, + "unclipped_grad_norm": 1.2261079549789429 + }, + { + "bits_per_byte": 1.4667946257890672, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0167045593261719, + "step": 7720, + "unclipped_grad_norm": 1.24894380569458 + }, + { + "bits_per_byte": 1.5217016349208765, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0547631978988647, + "step": 7730, + "unclipped_grad_norm": 1.2570104598999023 + }, + { + "bits_per_byte": 1.5186754281962946, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0526655912399292, + "step": 7740, + "unclipped_grad_norm": 1.2608413696289062 + }, + { + "bits_per_byte": 1.530832365837834, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0610921382904053, + "step": 7750, + "unclipped_grad_norm": 1.268287181854248 + }, + { + "bits_per_byte": 1.5045843736631521, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.042898416519165, + "step": 7760, + "unclipped_grad_norm": 1.2538435459136963 + }, + { + "bits_per_byte": 1.5028014295216574, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.041662573814392, + "step": 7770, + "unclipped_grad_norm": 1.3365142345428467 + }, + { + "bits_per_byte": 1.6460007199707645, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.1409207582473755, + "step": 7780, + "unclipped_grad_norm": 1.3043659925460815 + }, + { + "bits_per_byte": 1.5196737874845532, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0533576011657715, + "step": 7790, + "unclipped_grad_norm": 1.2850605249404907 + }, + { + "bits_per_byte": 1.559599903847656, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0810322761535645, + "step": 7800, + "unclipped_grad_norm": 1.3824384212493896 + }, + { + "bits_per_byte": 1.4742115495901766, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 1.0218455791473389, + "step": 7810, + "unclipped_grad_norm": 1.2828388214111328 + }, + { + "bits_per_byte": 1.5144011433744011, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0497028827667236, + "step": 7820, + "unclipped_grad_norm": 1.3298958539962769 + }, + { + "bits_per_byte": 1.549842984098907, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0742692947387695, + "step": 7830, + "unclipped_grad_norm": 1.2448348999023438 + }, + { + "bits_per_byte": 1.5177059619933808, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.0519936084747314, + "step": 7840, + "unclipped_grad_norm": 1.234107255935669 + }, + { + "bits_per_byte": 1.486972662284197, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 1.0306909084320068, + "step": 7850, + "unclipped_grad_norm": 1.2269891500473022 + }, + { + "bits_per_byte": 1.4944172752920954, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0358511209487915, + "step": 7860, + "unclipped_grad_norm": 1.266727328300476 + }, + { + "bits_per_byte": 1.549148518154723, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0737879276275635, + "step": 7870, + "unclipped_grad_norm": 1.3428150415420532 + }, + { + "bits_per_byte": 1.522268833703422, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.0551563501358032, + "step": 7880, + "unclipped_grad_norm": 1.305385947227478 + }, + { + "bits_per_byte": 1.5751703531435204, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0918248891830444, + "step": 7890, + "unclipped_grad_norm": 1.2760140895843506 + }, + { + "bits_per_byte": 1.5609104116472305, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0819406509399414, + "step": 7900, + "unclipped_grad_norm": 1.2964600324630737 + }, + { + "bits_per_byte": 1.5527064952357728, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.07625412940979, + "step": 7910, + "unclipped_grad_norm": 1.3213040828704834 + }, + { + "bits_per_byte": 1.4860267577044515, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0300352573394775, + "step": 7920, + "unclipped_grad_norm": 1.2681679725646973 + }, + { + "bits_per_byte": 1.482107273091288, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 1.0273184776306152, + "step": 7930, + "unclipped_grad_norm": 1.2925180196762085 + }, + { + "bits_per_byte": 1.5584990428994832, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.08026921749115, + "step": 7940, + "unclipped_grad_norm": 1.4040197134017944 + }, + { + "bits_per_byte": 1.488459280318255, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0317213535308838, + "step": 7950, + "unclipped_grad_norm": 1.3141639232635498 + }, + { + "bits_per_byte": 1.530331896323823, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0607452392578125, + "step": 7960, + "unclipped_grad_norm": 1.263742446899414 + }, + { + "bits_per_byte": 1.5034190192209056, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.042090654373169, + "step": 7970, + "unclipped_grad_norm": 1.2354859113693237 + }, + { + "bits_per_byte": 1.5362255697862408, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0648304224014282, + "step": 7980, + "unclipped_grad_norm": 1.2929013967514038 + }, + { + "bits_per_byte": 1.5535658925421343, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0768498182296753, + "step": 7990, + "unclipped_grad_norm": 1.2544541358947754 + }, + { + "bits_per_byte": 1.4716885641020192, + "learning_rate": 3e-05, + "loss_nats": 1.020096778869629, + "step": 8000, + "unclipped_grad_norm": 1.2711973190307617 + } + ] + }, + "depth-32-baseline-seed-2026073001": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "6574e479d5e2794b43a6244a723ba3a2b034f8ef3bfeae24909948fb4a990d99", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002768727717921138, + 0.0002554995589889586, + 0.00023783760843798518, + 0.00022115583124104887, + 0.00020655356638599187, + 0.00019376208365429193, + 0.00018488570640329272, + 0.00017590694187674671, + 0.0001684112212387845, + 0.0001647331955609843, + 0.00015843863366171718, + 0.0001538397918920964, + 0.00015132295084185898, + 0.00014730033581145108, + 0.00014253577683120966, + 0.0001394751889165491, + 0.0001364792260574177, + 0.0001337067224085331, + 0.00013106675760354847, + 0.00012915431580040604, + 0.00012709009752143174, + 0.00012574320135172457, + 0.0001242366706719622, + 0.0001228937617270276, + 0.00012132258416386321, + 0.00011988351616309956, + 0.00011879197700181976, + 0.00011764531518565491, + 0.00011639683361863717, + 0.00011549647024367005, + 0.00011481995898066089, + 0.00011415599146857858 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0002190592585975537, + "first_to_last_ratio": 1.8672887091156776, + "imbalance_abs_log_ratio": 0.6244874905881238, + "last_quartile_mean": 0.00011731408085324802, + "mean": 0.0001546067051094724, + "normalized": [ + 1.7908199492128654, + 1.6525774791465027, + 1.538339545297078, + 1.4304413969915148, + 1.3359935860462033, + 1.2532579587482624, + 1.1958453307208163, + 1.1377704592578457, + 1.0892879524179597, + 1.065498391187767, + 1.0247850088360109, + 0.9950395863049215, + 0.978760596021445, + 0.9527422223192203, + 0.9219249367631522, + 0.9021289782858439, + 0.8827510162691898, + 0.8648183939620171, + 0.8477430361816715, + 0.835373315206192, + 0.8220219002238166, + 0.8133101424203405, + 0.80356586464988, + 0.7948798963150414, + 0.7847174809007043, + 0.775409553409819, + 0.7683494510649245, + 0.7609328140222236, + 0.7528576043077825, + 0.7470340316863389, + 0.7426583400723811, + 0.7383637817502684 + ], + "population_cv": 0.27888268733798954 + }, + "activation_output_rms_by_block": [ + 0.028270306065678596, + 0.0284538883715868, + 0.02867252007126808, + 0.02881420962512493, + 0.029061447829008102, + 0.02929086796939373, + 0.029533835127949715, + 0.029811719432473183, + 0.030115574598312378, + 0.030315518379211426, + 0.030812423676252365, + 0.03120259940624237, + 0.03179442882537842, + 0.03198441118001938, + 0.03240228444337845, + 0.0329107940196991, + 0.033366311341524124, + 0.033811621367931366, + 0.03420914337038994, + 0.03470747172832489, + 0.03500960022211075, + 0.03567802160978317, + 0.035978782922029495, + 0.036836784332990646, + 0.03769209608435631, + 0.038124579936265945, + 0.038542140275239944, + 0.03904740884900093, + 0.0400773324072361, + 0.04093807935714722, + 0.04163828864693642, + 0.042313966900110245 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.028988599311560392, + "first_to_last_ratio": 0.728416493900548, + "imbalance_abs_log_ratio": 0.31688228737002583, + "last_quartile_mean": 0.03979673655703664, + "mean": 0.03379432682413608, + "normalized": [ + 0.8365399971656723, + 0.8419723381282118, + 0.8484418174825138, + 0.8526345198433033, + 0.8599504875549783, + 0.8667392051281828, + 0.873928789339177, + 0.8821515986281313, + 0.8911429055838948, + 0.8970593951159853, + 0.9117632032322649, + 0.9233088017589188, + 0.9408214873116122, + 0.9464432106153376, + 0.9588084003566117, + 0.9738555879797565, + 0.9873346942272493, + 1.0005117587897308, + 1.0122747391422395, + 1.0270206567197142, + 1.0359608701276664, + 1.055739970659861, + 1.0646397281194921, + 1.0900286466627183, + 1.1153379761196018, + 1.1281355043603702, + 1.1404914344295491, + 1.1554427183059932, + 1.1859189447920195, + 1.2113891059344621, + 1.2321088348236646, + 1.2521026715611152 + ], + "population_cv": 0.12455242949918699 + }, + "bits_per_byte": 7.95915895791699, + "branch_output_rms_by_sublayer": [ + 0.0022265268489718437, + 0.002717922907322645, + 0.0021652730647474527, + 0.002733128145337105, + 0.0025650423485785723, + 0.002695153234526515, + 0.002537536434829235, + 0.002696225419640541, + 0.0027307753916829824, + 0.0027451792266219854, + 0.0029185698367655277, + 0.0026901187375187874, + 0.002936159959062934, + 0.0027115964330732822, + 0.0030124466866254807, + 0.0027617362793534994, + 0.0031113612931221724, + 0.002723218873143196, + 0.0034121612552553415, + 0.0027921765577048063, + 0.0034701526165008545, + 0.002744668396189809, + 0.0039346772246062756, + 0.0027058592531830072, + 0.004295050632208586, + 0.0026920263189822435, + 0.004176756367087364, + 0.0027182905469089746, + 0.004023758694529533, + 0.0027150253299623728, + 0.004447397775948048, + 0.0027387896552681923, + 0.004260339308530092, + 0.00275799329392612, + 0.005327676888555288, + 0.002721488941460848, + 0.00474865036085248, + 0.0027454239316284657, + 0.00498866755515337, + 0.002719792304560542, + 0.0052231852896511555, + 0.0027601351030170918, + 0.005184618756175041, + 0.002755985129624605, + 0.0052438946440815926, + 0.0027470553759485483, + 0.0059704589657485485, + 0.0027777766808867455, + 0.0060942270793020725, + 0.0026787330862134695, + 0.005698176100850105, + 0.002644060645252466, + 0.006003713235259056, + 0.002661966485902667, + 0.006405353546142578, + 0.002804658142849803, + 0.006667613051831722, + 0.0026854604948312044, + 0.006290186662226915, + 0.0027102255262434483, + 0.0072579155676066875, + 0.0027289679273962975, + 0.006407012697309256, + 0.0026884053368121386 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.005888019831616423, + 0.005779358340731243, + 0.0060348395662513914, + 0.005586875351280001, + 0.005559071516712478, + 0.0054082748429522395, + 0.005481135467826832, + 0.0053805923812947415, + 0.004971746645687392, + 0.005439263395635174, + 0.005139424262259116, + 0.005696496994849179, + 0.005782818048228185, + 0.0054294510739720785, + 0.005067071208887471, + 0.005218073376917496, + 0.005181036474299639, + 0.005605812513830226, + 0.005170579144406511, + 0.004834742615880465, + 0.005150211089068095, + 0.00476651480071292, + 0.004776402486441839, + 0.00519066249960807, + 0.0053994045182471796, + 0.004795508114414784, + 0.004895792813412353, + 0.00496559115560922, + 0.00502742614163479, + 0.004657478744424888, + 0.005287606581362069, + 0.004870555415551657 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005639770912333169, + "first_to_last_ratio": 1.130799174678946, + "imbalance_abs_log_ratio": 0.12292461698112994, + "last_quartile_mean": 0.0049874204355821175, + "mean": 0.005263682419187692, + "normalized": [ + 1.1186122875029152, + 1.0979686615711766, + 1.146505257280075, + 1.061400537941684, + 1.0561183357962487, + 1.0274698228824493, + 1.0413119621059315, + 1.0222106792919112, + 0.9445377303850805, + 1.0333570611721248, + 0.976393302818647, + 1.0822265747043835, + 1.0986259404914112, + 1.0314929058371969, + 0.962647592570647, + 0.9913351455049914, + 0.9842988352437859, + 1.0649982402804865, + 0.9823121405573801, + 0.9185095586801336, + 0.9784425956805525, + 0.9055475655859389, + 0.9074260386664719, + 0.9861275977985596, + 1.0257846291342996, + 0.9110557462459602, + 0.9301079403205877, + 0.9433683038148654, + 0.9551157804103686, + 0.884832779319472, + 1.0045451378463044, + 0.9253133125579595 + ], + "population_cv": 0.06750436074287791 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028027428314089775, + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346 + ], + "loss_nats": 5.516868591308594, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028122643008828163, + 0.028270306065678596, + 0.028331812471151352, + 0.0284538883715868, + 0.02856677956879139, + 0.02867252007126808, + 0.02870391495525837, + 0.02881420962512493, + 0.028933145105838776, + 0.029061447829008102, + 0.029177924618124962, + 0.02929086796939373, + 0.029417123645544052, + 0.029533835127949715, + 0.02969702146947384, + 0.029811719432473183, + 0.02999095991253853, + 0.030115574598312378, + 0.0302139800041914, + 0.030315518379211426, + 0.03069150820374489, + 0.030812423676252365, + 0.03109365701675415, + 0.03120259940624237, + 0.03165328875184059, + 0.03179442882537842, + 0.03191109746694565, + 0.03198441118001938, + 0.03227011114358902, + 0.03240228444337845, + 0.03283532336354256, + 0.0329107940196991, + 0.03322411701083183, + 0.033366311341524124, + 0.0337243415415287, + 0.033811621367931366, + 0.03413747623562813, + 0.03420914337038994, + 0.03456577658653259, + 0.03470747172832489, + 0.03486170619726181, + 0.03500960022211075, + 0.03554223105311394, + 0.03567802160978317, + 0.03587620332837105, + 0.035978782922029495, + 0.036742135882377625, + 0.036836784332990646, + 0.03761643171310425, + 0.03769209608435631, + 0.03803960233926773, + 0.038124579936265945, + 0.03846931457519531, + 0.038542140275239944, + 0.03895273432135582, + 0.03904740884900093, + 0.03994588926434517, + 0.0400773324072361, + 0.04087882116436958, + 0.04093807935714722, + 0.04153488203883171, + 0.04163828864693642, + 0.042190924286842346, + 0.042313966900110245 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.172436016844586e-05, + 6.834617670392618e-05, + 6.473411485785618e-05, + 6.156626477604732e-05, + 5.8372206694912165e-05, + 5.494635479408316e-05, + 5.167145354789682e-05, + 4.809222446056083e-05, + 4.4674157834379e-05, + 4.1754625272005796e-05, + 3.8718964788131416e-05, + 3.608527549658902e-05, + 3.353153806529008e-05, + 3.1093299185158685e-05, + 2.904756365751382e-05, + 2.7216165108256973e-05, + 2.5600058506825007e-05, + 2.4208331524278037e-05, + 2.304886766069103e-05, + 2.1986410501995124e-05, + 2.1054642274975777e-05, + 2.025439971475862e-05, + 1.956623418664094e-05, + 1.9002633052878082e-05, + 1.8555338101577945e-05, + 1.8124846974387765e-05, + 1.7765629308996722e-05, + 1.744573273754213e-05, + 1.7174377717310563e-05, + 1.6944877643254586e-05, + 1.6726755347917788e-05, + 1.656309177633375e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.9931644500466064e-05, + "first_to_last_ratio": 3.4418587232383575, + "imbalance_abs_log_ratio": 1.2360116519842717, + "last_quartile_mean": 1.7412581200915156e-05, + "mean": 3.3612405388794286e-05, + "normalized": [ + 2.1338657361414946, + 2.033361668508004, + 1.9258995037420696, + 1.831653047852752, + 1.7366268798594315, + 1.634704632367703, + 1.537273305799265, + 1.4307879458276982, + 1.3290973174229443, + 1.242238536309156, + 1.1519248426368067, + 1.0735701619444094, + 0.9975941226886675, + 0.9250542716447356, + 0.8641917566303572, + 0.8097059640167944, + 0.761625305023828, + 0.7202201462305526, + 0.6857250290208349, + 0.654115950574753, + 0.626394988143127, + 0.602587035366146, + 0.582113477459246, + 0.5653458249439413, + 0.5520383884148895, + 0.5392308811207612, + 0.5285438249212427, + 0.5190266074607738, + 0.510953545830914, + 0.5041257073765888, + 0.49763636831222313, + 0.4927672264078892 + ], + "population_cv": 0.513984504821457 + }, + "activation_output_rms_by_block": [ + 0.029439672827720642, + 0.030752120539546013, + 0.03267423436045647, + 0.034982286393642426, + 0.03797784447669983, + 0.04108294099569321, + 0.044510453939437866, + 0.04866119846701622, + 0.05275992304086685, + 0.05702463164925575, + 0.061875153332948685, + 0.06669966131448746, + 0.071797676384449, + 0.07734881341457367, + 0.08311311900615692, + 0.08836977183818817, + 0.09436330199241638, + 0.10090246796607971, + 0.10806354880332947, + 0.11583591252565384, + 0.1238454207777977, + 0.13231202960014343, + 0.1417466551065445, + 0.15185882151126862, + 0.16391249001026154, + 0.1746179163455963, + 0.18763093650341034, + 0.20137503743171692, + 0.21650369465351105, + 0.23270079493522644, + 0.2497490495443344, + 0.2673082947731018 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.037510094000026584, + "first_to_last_ratio": 0.17716440452291277, + "imbalance_abs_log_ratio": 1.7306771383999247, + "last_quartile_mean": 0.21172477677464485, + "mean": 0.11005612107692286, + "normalized": [ + 0.2674969146617844, + 0.27942217332966024, + 0.29688702491721536, + 0.3178586165979067, + 0.3450770761778485, + 0.37329083192909013, + 0.4044341514483109, + 0.4421489508339555, + 0.4793910827003499, + 0.5181413908763769, + 0.5622145567869099, + 0.6060513550888119, + 0.6523733135594211, + 0.702812462021184, + 0.7551885183020911, + 0.8029519028425762, + 0.857410756158323, + 0.9168274056792781, + 0.9818949436515149, + 1.0525167650120182, + 1.1252933463940358, + 1.2022232685055743, + 1.2879488548162785, + 1.3798307629352853, + 1.4893536897933755, + 1.586626119809806, + 1.7048659780791944, + 1.8297486360705681, + 1.9672117510137168, + 2.114383031658748, + 2.269288133185925, + 2.428836235162865 + ], + "population_cv": 0.6268498746310547 + }, + "bits_per_byte": 5.1517691996807775, + "branch_output_rms_by_sublayer": [ + 0.0028861628379672766, + 0.004992613103240728, + 0.002704753540456295, + 0.005112498998641968, + 0.002895323559641838, + 0.005543495528399944, + 0.0029729304369539022, + 0.005716898012906313, + 0.00318311620503664, + 0.005760234780609608, + 0.003378538880497217, + 0.0057550352066755295, + 0.0039320578798651695, + 0.005740197841078043, + 0.0034711055923253298, + 0.006492512300610542, + 0.0037959336768835783, + 0.006124566774815321, + 0.004324590787291527, + 0.0062123509123921394, + 0.004316650331020355, + 0.006645897403359413, + 0.0046909768134355545, + 0.006303280591964722, + 0.0053819348104298115, + 0.006548910401761532, + 0.005383738782256842, + 0.006836008746176958, + 0.00522694643586874, + 0.006793942768126726, + 0.005513017997145653, + 0.006699956953525543, + 0.005531978793442249, + 0.0068945568054914474, + 0.006940840277820826, + 0.0070524453185498714, + 0.006634449120610952, + 0.007447477430105209, + 0.0070588430389761925, + 0.007686031982302666, + 0.007738462649285793, + 0.007927682250738144, + 0.008234621025621891, + 0.008245103061199188, + 0.008838997222483158, + 0.008133068680763245, + 0.009296931326389313, + 0.008390788920223713, + 0.013193055056035519, + 0.008053022436797619, + 0.00934307835996151, + 0.008179600350558758, + 0.012704792432487011, + 0.008694498799741268, + 0.012802425771951675, + 0.009316418319940567, + 0.014256707392632961, + 0.008604929782450199, + 0.015807906165719032, + 0.009046261198818684, + 0.0164310485124588, + 0.009124966338276863, + 0.01581309363245964, + 0.009515758603811264 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004018231655400003, + 0.0003405186139403606, + 0.0003221632759320079, + 0.0003173401597627506, + 0.0002956526561473367, + 0.0002913580859417252, + 0.00030034368266542127, + 0.00026684124508166604, + 0.0002497190139849492, + 0.00026578754174316435, + 0.0002404550275593826, + 0.00024493210822064133, + 0.0002532169075836586, + 0.00023865218150919594, + 0.0002121496513808785, + 0.0002091708005241373, + 0.0001980500112256327, + 0.00020258502349527826, + 0.00019483821933596074, + 0.0001851504169444552, + 0.00018439346409065922, + 0.00017525536186022104, + 0.000171573793780652, + 0.0001670148315601783, + 0.00018406810665917416, + 0.00014791927954418835, + 0.00017020274695916728, + 0.00015649953794173187, + 0.00016066786897129027, + 0.00016871534344057424, + 0.00016946426752177343, + 0.0001617385113992835 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00031700511062640856, + "first_to_last_ratio": 1.9222979375866573, + "imbalance_abs_log_ratio": 0.653521312887717, + "last_quartile_mean": 0.0001649094578046479, + "mean": 0.00022650815319523427, + "normalized": [ + 1.773989853661721, + 1.5033393241560593, + 1.4223032212634112, + 1.401009876625613, + 1.305262755343313, + 1.286302862972861, + 1.3259729437047942, + 1.1780646361619815, + 1.1024725179305521, + 1.1734126917456873, + 1.061573387833537, + 1.0813390368757583, + 1.117915200895233, + 1.0536140891294732, + 0.9366093378458668, + 0.9234581518301755, + 0.8743615116358631, + 0.8943829201621014, + 0.8601819254074433, + 0.8174117104953322, + 0.814069875585118, + 0.7737265055936556, + 0.7574729269580301, + 0.7373457829406393, + 0.8126334706394442, + 0.6530417446682029, + 0.7514199579935843, + 0.6909223166321972, + 0.7093248817088087, + 0.7448532914184036, + 0.7481596804849096, + 0.7140516097002307 + ], + "population_cv": 0.27857383691000986 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0285640936344862, + 0.02861291915178299, + 0.029439672827720642, + 0.02950597181916237, + 0.030752120539546013, + 0.030888807028532028, + 0.03267423436045647, + 0.03270668536424637, + 0.034982286393642426, + 0.03526782616972923, + 0.03797784447669983, + 0.03822308033704758, + 0.04108294099569321, + 0.041396427899599075, + 0.044510453939437866, + 0.04485016688704491, + 0.04866119846701622, + 0.04912792518734932, + 0.05275992304086685, + 0.05317053198814392, + 0.05702463164925575, + 0.05767740681767464, + 0.061875153332948685, + 0.06268688291311264, + 0.06669966131448746, + 0.06753430515527725, + 0.071797676384449, + 0.07279384881258011, + 0.07734881341457367, + 0.078476682305336, + 0.08311311900615692, + 0.08398299664258957, + 0.08836977183818817, + 0.08972389250993729, + 0.09436330199241638, + 0.09631941467523575, + 0.10090246796607971, + 0.10291995853185654, + 0.10806354880332947, + 0.11053454130887985, + 0.11583591252565384, + 0.11846786737442017, + 0.1238454207777977, + 0.12666596472263336, + 0.13231202960014343, + 0.13596534729003906, + 0.1417466551065445, + 0.1458624303340912, + 0.15185882151126862, + 0.1581486165523529, + 0.16391249001026154, + 0.16865970194339752, + 0.1746179163455963, + 0.18122267723083496, + 0.18763093650341034, + 0.19437597692012787, + 0.20137503743171692, + 0.2100502997636795, + 0.21650369465351105, + 0.22599515318870544, + 0.23270079493522644, + 0.2427441030740738, + 0.2497490495443344, + 0.2599930763244629 + ], + "loss_nats": 3.570934295654297, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.02861291915178299, + 0.029439672827720642, + 0.02950597181916237, + 0.030752120539546013, + 0.030888807028532028, + 0.03267423436045647, + 0.03270668536424637, + 0.034982286393642426, + 0.03526782616972923, + 0.03797784447669983, + 0.03822308033704758, + 0.04108294099569321, + 0.041396427899599075, + 0.044510453939437866, + 0.04485016688704491, + 0.04866119846701622, + 0.04912792518734932, + 0.05275992304086685, + 0.05317053198814392, + 0.05702463164925575, + 0.05767740681767464, + 0.061875153332948685, + 0.06268688291311264, + 0.06669966131448746, + 0.06753430515527725, + 0.071797676384449, + 0.07279384881258011, + 0.07734881341457367, + 0.078476682305336, + 0.08311311900615692, + 0.08398299664258957, + 0.08836977183818817, + 0.08972389250993729, + 0.09436330199241638, + 0.09631941467523575, + 0.10090246796607971, + 0.10291995853185654, + 0.10806354880332947, + 0.11053454130887985, + 0.11583591252565384, + 0.11846786737442017, + 0.1238454207777977, + 0.12666596472263336, + 0.13231202960014343, + 0.13596534729003906, + 0.1417466551065445, + 0.1458624303340912, + 0.15185882151126862, + 0.1581486165523529, + 0.16391249001026154, + 0.16865970194339752, + 0.1746179163455963, + 0.18122267723083496, + 0.18763093650341034, + 0.19437597692012787, + 0.20137503743171692, + 0.2100502997636795, + 0.21650369465351105, + 0.22599515318870544, + 0.23270079493522644, + 0.2427441030740738, + 0.2497490495443344, + 0.2599930763244629, + 0.2673082947731018 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014451092283707112, + 0.00014379274216480553, + 0.00014628663484472781, + 0.0001445338421035558, + 0.00013843078340869397, + 0.0001292457600357011, + 0.00012031046207994223, + 0.00011329296830808744, + 0.0001077206979971379, + 0.0001003394863801077, + 9.358100214740261e-05, + 8.73711978783831e-05, + 8.221827738452703e-05, + 7.71504346630536e-05, + 7.201205153251067e-05, + 6.728569132974371e-05, + 6.256035703700036e-05, + 5.796365803689696e-05, + 5.370515646063723e-05, + 4.9705678975442424e-05, + 4.572505713440478e-05, + 4.197436646791175e-05, + 3.853226371575147e-05, + 3.558541357051581e-05, + 3.2570587791269645e-05, + 2.9896507840021513e-05, + 2.8078287868993357e-05, + 2.6684616386773996e-05, + 2.568897070887033e-05, + 2.5203555196640082e-05, + 2.4927356207626872e-05, + 2.5036461011040956e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013505051447282312, + "first_to_last_ratio": 4.954020049421058, + "imbalance_abs_log_ratio": 1.6001993781766728, + "last_quartile_mean": 2.7260792876404594e-05, + "mean": 7.412253904703903e-05, + "normalized": [ + 1.9496218652920512, + 1.9399327655728709, + 1.9735783032458267, + 1.9499310730820074, + 1.8675936521932173, + 1.743676912547211, + 1.6231292617160862, + 1.5284550389752622, + 1.4532785760182485, + 1.3536973728926245, + 1.2625174926619152, + 1.1787399487615544, + 1.1092210067487078, + 1.040849863684419, + 0.9715270477554875, + 0.90776290443914, + 0.8440126018524382, + 0.7819977402570161, + 0.7245455586263081, + 0.670587915828121, + 0.616884657787925, + 0.5662834410094119, + 0.5198454371793504, + 0.4800889719648283, + 0.4394154357097774, + 0.4033389603808477, + 0.3788090401379066, + 0.36000677701879086, + 0.34657434889767885, + 0.34002552422881266, + 0.3362992758762309, + 0.3377712276579264 + ], + "population_cv": 0.5690574657123064 + }, + "activation_output_rms_by_block": [ + 0.049516160041093826, + 0.057937320321798325, + 0.06586314737796783, + 0.0719481110572815, + 0.076947420835495, + 0.08064281940460205, + 0.08439230173826218, + 0.08712083846330643, + 0.0897950753569603, + 0.09305135160684586, + 0.09555979818105698, + 0.09745962917804718, + 0.10008277744054794, + 0.10209457576274872, + 0.1030229851603508, + 0.10483334213495255, + 0.10759729146957397, + 0.11058001965284348, + 0.11383581161499023, + 0.1180567741394043, + 0.1237289309501648, + 0.13110940158367157, + 0.14003770053386688, + 0.15068796277046204, + 0.16247524321079254, + 0.1767880618572235, + 0.1922861486673355, + 0.20968382060527802, + 0.22965215146541595, + 0.2505997121334076, + 0.27601101994514465, + 0.307341605424881 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07179601490497589, + "first_to_last_ratio": 0.31823808816289667, + "imbalance_abs_log_ratio": 1.1449554714268964, + "last_quartile_mean": 0.22560472041368484, + "mean": 0.13002310344018042, + "normalized": [ + 0.3808258588741859, + 0.44559250463094413, + 0.506549571847971, + 0.5533486676879896, + 0.5917980635718029, + 0.6202191554495797, + 0.6490562023624398, + 0.6700412169702441, + 0.6906086147856962, + 0.7156524428726306, + 0.7349447571448028, + 0.7495562450013764, + 0.7697307231756155, + 0.785203345109504, + 0.7923436868875267, + 0.8062670353287107, + 0.8275244062227459, + 0.8504643923048483, + 0.8755044957634206, + 0.9079676689436854, + 0.9515918915678598, + 1.008354654786339, + 1.0770216740619014, + 1.1589322111496045, + 1.2495874880077935, + 1.359666529868349, + 1.4788613990882042, + 1.6126658651994643, + 1.7662411170724883, + 1.9273475674936549, + 2.1227844332459633, + 2.363746113522658 + ], + "population_cv": 0.4886061625493153 + }, + "bits_per_byte": 3.0864900833624125, + "branch_output_rms_by_sublayer": [ + 0.01111105177551508, + 0.01691170036792755, + 0.01229762565344572, + 0.013536608777940273, + 0.012528475373983383, + 0.011229105293750763, + 0.017575321719050407, + 0.009298570454120636, + 0.015520633198320866, + 0.008989781141281128, + 0.012922349385917187, + 0.008074384182691574, + 0.013718407601118088, + 0.0077475146390497684, + 0.012880085967481136, + 0.00855444185435772, + 0.019964123144745827, + 0.007989896461367607, + 0.02377571351826191, + 0.008533375337719917, + 0.02730517089366913, + 0.0086820675060153, + 0.02310827188193798, + 0.008458662778139114, + 0.019315505400300026, + 0.008968821726739407, + 0.031517889350652695, + 0.009528025984764099, + 0.018633879721164703, + 0.008953212760388851, + 0.016267014667391777, + 0.008700341917574406, + 0.016277894377708435, + 0.010046470910310745, + 0.022248439490795135, + 0.010587109252810478, + 0.016605904325842857, + 0.012151478789746761, + 0.01819506660103798, + 0.012493276968598366, + 0.017910068854689598, + 0.013204623945057392, + 0.017647475004196167, + 0.015671590343117714, + 0.02343275025486946, + 0.016670797020196915, + 0.03074510768055916, + 0.019418122246861458, + 0.017775433138012886, + 0.019835535436868668, + 0.017164910212159157, + 0.021999742835760117, + 0.01125776395201683, + 0.023901214823126793, + 0.007435314357280731, + 0.02528192289173603, + 0.006950905080884695, + 0.027673469856381416, + 0.007996064610779285, + 0.026783082634210587, + 0.010837847366929054, + 0.030296670272946358, + 0.021106943488121033, + 0.028989506885409355 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006755996472372422, + 0.0003902215048764158, + 0.00033650809592979206, + 0.00029675556364760884, + 0.00026186200490798707, + 0.0002455230107742763, + 0.00021966924842671773, + 0.00022418062520577193, + 0.00022548545903395049, + 0.00022091596273961857, + 0.00024014950731391624, + 0.00021478395214610848, + 0.0001956916368008953, + 0.00020741213919383669, + 0.0001699546796737774, + 0.00016808908215382032, + 0.00015850476160987511, + 0.00015061268581609982, + 0.00013300713992511992, + 0.0001284985461118983, + 0.0001312583398227583, + 0.00012527717278626568, + 0.0001234119340024874, + 0.00012326054887191204, + 0.00010914072106764207, + 0.00010077507233480893, + 9.59134879196041e-05, + 8.362219426315277e-05, + 8.733675859901561e-05, + 7.698995894839292e-05, + 8.92787325176121e-05, + 9.631544502550806e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003312899626257265, + "first_to_last_ratio": 3.584553340265607, + "imbalance_abs_log_ratio": 1.276633875010675, + "last_quartile_mean": 9.242154633446708e-05, + "mean": 0.00019081267561512152, + "normalized": [ + 3.540643435030279, + 2.045050223306504, + 1.763552105985586, + 1.5552193411206041, + 1.3723512028947988, + 1.2867227503769625, + 1.1512298526215383, + 1.17487281430902, + 1.1817111117332983, + 1.1577635606620644, + 1.2585616051960276, + 1.1256272753039647, + 1.0255693767201026, + 1.0869935056735807, + 0.8906886249872848, + 0.8809115097409158, + 0.8306825586869656, + 0.7893222257408777, + 0.697056102254977, + 0.6734277253733716, + 0.6878910921385158, + 0.6565453389425546, + 0.6467701037399387, + 0.6459767333305183, + 0.5719783589628198, + 0.5281361524329609, + 0.5026578428839092, + 0.43824234419217944, + 0.45770941745598764, + 0.4034845101364514, + 0.4678868017012227, + 0.5047643963642175 + ], + "population_cv": 0.6065148734399883 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.043906208127737045, + 0.04462560638785362, + 0.049516160041093826, + 0.052572574466466904, + 0.057937320321798325, + 0.060727521777153015, + 0.06586314737796783, + 0.06803189963102341, + 0.0719481110572815, + 0.07307156920433044, + 0.076947420835495, + 0.07774755358695984, + 0.08064281940460205, + 0.08214583247900009, + 0.08439230173826218, + 0.08468613773584366, + 0.08712083846330643, + 0.08735909312963486, + 0.0897950753569603, + 0.09098032861948013, + 0.09305135160684586, + 0.09323208779096603, + 0.09555979818105698, + 0.09555087983608246, + 0.09745962917804718, + 0.09809984266757965, + 0.10008277744054794, + 0.09976384043693542, + 0.10209457576274872, + 0.10044334083795547, + 0.1030229851603508, + 0.1023804098367691, + 0.10483334213495255, + 0.10451660305261612, + 0.10759729146957397, + 0.10710933804512024, + 0.11058001965284348, + 0.10916344821453094, + 0.11383581161499023, + 0.11319266259670258, + 0.1180567741394043, + 0.11788330227136612, + 0.1237289309501648, + 0.12335778772830963, + 0.13110940158367157, + 0.13146334886550903, + 0.14003770053386688, + 0.13964787125587463, + 0.15068796277046204, + 0.15031130611896515, + 0.16247524321079254, + 0.16256581246852875, + 0.1767880618572235, + 0.1762392520904541, + 0.1922861486673355, + 0.19244804978370667, + 0.20968382060527802, + 0.21054796874523163, + 0.22965215146541595, + 0.23175573348999023, + 0.2505997121334076, + 0.2547222673892975, + 0.27601101994514465, + 0.28623583912849426 + ], + "loss_nats": 2.1393918991088867, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04462560638785362, + 0.049516160041093826, + 0.052572574466466904, + 0.057937320321798325, + 0.060727521777153015, + 0.06586314737796783, + 0.06803189963102341, + 0.0719481110572815, + 0.07307156920433044, + 0.076947420835495, + 0.07774755358695984, + 0.08064281940460205, + 0.08214583247900009, + 0.08439230173826218, + 0.08468613773584366, + 0.08712083846330643, + 0.08735909312963486, + 0.0897950753569603, + 0.09098032861948013, + 0.09305135160684586, + 0.09323208779096603, + 0.09555979818105698, + 0.09555087983608246, + 0.09745962917804718, + 0.09809984266757965, + 0.10008277744054794, + 0.09976384043693542, + 0.10209457576274872, + 0.10044334083795547, + 0.1030229851603508, + 0.1023804098367691, + 0.10483334213495255, + 0.10451660305261612, + 0.10759729146957397, + 0.10710933804512024, + 0.11058001965284348, + 0.10916344821453094, + 0.11383581161499023, + 0.11319266259670258, + 0.1180567741394043, + 0.11788330227136612, + 0.1237289309501648, + 0.12335778772830963, + 0.13110940158367157, + 0.13146334886550903, + 0.14003770053386688, + 0.13964787125587463, + 0.15068796277046204, + 0.15031130611896515, + 0.16247524321079254, + 0.16256581246852875, + 0.1767880618572235, + 0.1762392520904541, + 0.1922861486673355, + 0.19244804978370667, + 0.20968382060527802, + 0.21054796874523163, + 0.22965215146541595, + 0.23175573348999023, + 0.2505997121334076, + 0.2547222673892975, + 0.27601101994514465, + 0.28623583912849426, + 0.307341605424881 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010485891834832728, + 0.00010495351307326928, + 0.00011622957390500233, + 0.00011138651461806148, + 0.00010787966311909258, + 0.000104747356090229, + 0.00010223284334642813, + 9.973083797376603e-05, + 9.735774074215442e-05, + 9.589978435542434e-05, + 9.408760524820536e-05, + 9.229916759068146e-05, + 9.143509669229388e-05, + 8.827183046378195e-05, + 8.612661622464657e-05, + 8.33196536405012e-05, + 8.087792957667261e-05, + 7.82960414653644e-05, + 8.221158350352198e-05, + 7.83153300289996e-05, + 7.789687515469268e-05, + 7.347232894971967e-05, + 6.715180643368512e-05, + 6.132270209491253e-05, + 5.5257889471249655e-05, + 5.0392085540806875e-05, + 4.541657472145744e-05, + 4.046381582156755e-05, + 3.4924181818496436e-05, + 2.9514063498936594e-05, + 2.4251618015114218e-05, + 2.1234091036603786e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010650240255927201, + "first_to_last_ratio": 2.8263626166920495, + "imbalance_abs_log_ratio": 1.038990590500678, + "last_quartile_mean": 3.768178999052907e-05, + "mean": 7.755673851761458e-05, + "normalized": [ + 1.3520284678359944, + 1.353248151989171, + 1.4986392688316106, + 1.4361938981325721, + 1.3909773048874547, + 1.3505900079390127, + 1.3181684183793927, + 1.2859080961883835, + 1.255309887999515, + 1.236511310150616, + 1.213145460298544, + 1.1900857276214447, + 1.1789445822496427, + 1.1381581040019337, + 1.1104981703825207, + 1.0743057951254327, + 1.0428227272386361, + 1.0095324140994133, + 1.060018575753417, + 1.009781116713833, + 1.0043856490561534, + 0.947336496531926, + 0.8658410309303259, + 0.7906818062106236, + 0.7124834092746121, + 0.6497447740064765, + 0.585591601574923, + 0.5217317875271078, + 0.4503049314092096, + 0.380548022816011, + 0.3126951761851902, + 0.2737878286589002 + ], + "population_cv": 0.34367475536074504 + }, + "activation_output_rms_by_block": [ + 0.09794864058494568, + 0.13700027763843536, + 0.20758503675460815, + 0.22207145392894745, + 0.22574232518672943, + 0.2265917956829071, + 0.22746813297271729, + 0.22716933488845825, + 0.22722382843494415, + 0.22613705694675446, + 0.22211141884326935, + 0.21741008758544922, + 0.21311214566230774, + 0.20640577375888824, + 0.19932369887828827, + 0.19335229694843292, + 0.19345322251319885, + 0.19645750522613525, + 0.20112650096416473, + 0.2038118690252304, + 0.20720544457435608, + 0.21031972765922546, + 0.2129739224910736, + 0.21647654473781586, + 0.22099705040454865, + 0.23361735045909882, + 0.2512694001197815, + 0.2773953676223755, + 0.3133004307746887, + 0.35808902978897095, + 0.4303363263607025, + 0.54518723487854 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.1964471247047186, + "first_to_last_ratio": 0.5975141297159507, + "imbalance_abs_log_ratio": 0.5149773473959137, + "last_quartile_mean": 0.32877402380108833, + "mean": 0.2358959447592497, + "normalized": [ + 0.4152196880065486, + 0.5807657176059338, + 0.879985609614718, + 0.9413958097312304, + 0.9569572101678865, + 0.9605582491643161, + 0.964273180723248, + 0.9630065286637394, + 0.9632375353753703, + 0.958630540162719, + 0.9415652272867661, + 0.9216355448896473, + 0.9034158933075572, + 0.8749865283591098, + 0.8449644994182233, + 0.8196507877478102, + 0.8200786270854848, + 0.832814253872128, + 0.8526068609167068, + 0.8639905583507864, + 0.8783764586789548, + 0.8915783943376951, + 0.9028299435517223, + 0.9176781099766134, + 0.9368412442617167, + 0.9903406804959011, + 1.065170494457722, + 1.1759225785143497, + 1.32812978660755, + 1.5179956999872501, + 1.824263349672647, + 2.311134409007948 + ], + "population_cv": 0.335075401661643 + }, + "bits_per_byte": 2.08044695422856, + "branch_output_rms_by_sublayer": [ + 0.018880587071180344, + 0.05959044396877289, + 0.060257185250520706, + 0.016382697969675064, + 0.09490419179201126, + 0.0035255588591098785, + 0.0323294997215271, + 0.004731281660497189, + 0.02052305079996586, + 0.00493080448359251, + 0.01820369064807892, + 0.005357450805604458, + 0.019246362149715424, + 0.006479416508227587, + 0.024384424090385437, + 0.009427462704479694, + 0.017510300502181053, + 0.009214801713824272, + 0.01695968210697174, + 0.0109269879758358, + 0.019068937748670578, + 0.012604814022779465, + 0.019973911345005035, + 0.013778526335954666, + 0.022227400913834572, + 0.013526024296879768, + 0.027813225984573364, + 0.017681727185845375, + 0.027307361364364624, + 0.018625596538186073, + 0.03342962637543678, + 0.019818320870399475, + 0.030618051066994667, + 0.021072659641504288, + 0.025097575038671494, + 0.022579019889235497, + 0.025198329240083694, + 0.01933942921459675, + 0.03349718078970909, + 0.022255338728427887, + 0.03462665155529976, + 0.024717653170228004, + 0.03921883553266525, + 0.025107676163315773, + 0.043981269001960754, + 0.0301958154886961, + 0.038896940648555756, + 0.03620966896414757, + 0.04658674821257591, + 0.046204980462789536, + 0.033179834485054016, + 0.04984369874000549, + 0.02075173333287239, + 0.05490582063794136, + 0.011197025887668133, + 0.06379986554384232, + 0.012507086619734764, + 0.07197272032499313, + 0.011961712501943111, + 0.07922235131263733, + 0.010508585721254349, + 0.11548902839422226, + 0.021868467330932617, + 0.1684120148420334 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00040070945427075937, + 0.00021125547826331077, + 0.00020379424936248455, + 0.00011927406183950472, + 9.446000613801463e-05, + 0.00010295610627477164, + 9.818733926828671e-05, + 0.00012673421109679544, + 0.00011573807851122185, + 0.00011146965874368272, + 0.0001232427596852727, + 0.00012517422740451753, + 0.000137325623762558, + 0.0001763842480315643, + 0.00017420922346804142, + 0.00019637547238758496, + 0.00020814888113781, + 0.0002138469295217413, + 0.0002110578549773679, + 0.0002602958562879386, + 0.0002719585176535882, + 0.00027346390406242154, + 0.00028937535779988523, + 0.00028561131716422746, + 0.00028061956919165, + 0.00023874030969169658, + 0.00021716224392712217, + 0.0001981302869037355, + 0.00018664421118786767, + 0.0001673158872138231, + 0.0001540361578610053, + 0.00014575183058398037 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016967136331424098, + "first_to_last_ratio": 0.8545520537501935, + "imbalance_abs_log_ratio": 0.1571778610785998, + "last_quartile_mean": 0.0001985500620701101, + "mean": 0.00019123279105231976, + "normalized": [ + 2.0954013799920355, + 1.104703210682696, + 1.0656867383519393, + 0.6237113477409442, + 0.4939529754192163, + 0.5383810261212141, + 0.5134440528111281, + 0.6627221743687356, + 0.605220882225982, + 0.582900339059453, + 0.6444645764311125, + 0.6545646628682571, + 0.71810709348998, + 0.9223535726323577, + 0.9109798717542076, + 1.0268922568507521, + 1.0884581038241612, + 1.1182545019867147, + 1.1036697933233854, + 1.3611465630741315, + 1.4221332866452938, + 1.4300052964640568, + 1.5132099270606496, + 1.493526897727945, + 1.4674239059496585, + 1.2484276800958218, + 1.1355910392360917, + 1.0360685832877303, + 0.976005266465014, + 0.8749330399515364, + 0.8054902980465429, + 0.7621696560612549 + ], + "population_cv": 0.3661316865844204 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0565960556268692, + 0.06250853836536407, + 0.09794864058494568, + 0.12857383489608765, + 0.13700027763843536, + 0.2065405696630478, + 0.20758503675460815, + 0.22102168202400208, + 0.22207145392894745, + 0.2248813658952713, + 0.22574232518672943, + 0.2264578640460968, + 0.2265917956829071, + 0.22759772837162018, + 0.22746813297271729, + 0.22716368734836578, + 0.22716933488845825, + 0.22757723927497864, + 0.22722382843494415, + 0.22673146426677704, + 0.22613705694675446, + 0.2244347184896469, + 0.22211141884326935, + 0.2183273881673813, + 0.21741008758544922, + 0.21451692283153534, + 0.21311214566230774, + 0.20762479305267334, + 0.20640577375888824, + 0.20071867108345032, + 0.19932369887828827, + 0.19245882332324982, + 0.19335229694843292, + 0.19078245759010315, + 0.19345322251319885, + 0.1916658580303192, + 0.19645750522613525, + 0.19844752550125122, + 0.20112650096416473, + 0.2010338455438614, + 0.2038118690252304, + 0.20468150079250336, + 0.20720544457435608, + 0.20727655291557312, + 0.21031972765922546, + 0.20860347151756287, + 0.2129739224910736, + 0.21107539534568787, + 0.21647654473781586, + 0.2111099362373352, + 0.22099705040454865, + 0.2196389138698578, + 0.23361735045909882, + 0.23286758363246918, + 0.2512694001197815, + 0.25084227323532104, + 0.2773953676223755, + 0.27732276916503906, + 0.3133004307746887, + 0.31342777609825134, + 0.35808902978897095, + 0.35860511660575867, + 0.4303363263607025, + 0.4337502121925354 + ], + "loss_nats": 1.4420559406280518, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.06250853836536407, + 0.09794864058494568, + 0.12857383489608765, + 0.13700027763843536, + 0.2065405696630478, + 0.20758503675460815, + 0.22102168202400208, + 0.22207145392894745, + 0.2248813658952713, + 0.22574232518672943, + 0.2264578640460968, + 0.2265917956829071, + 0.22759772837162018, + 0.22746813297271729, + 0.22716368734836578, + 0.22716933488845825, + 0.22757723927497864, + 0.22722382843494415, + 0.22673146426677704, + 0.22613705694675446, + 0.2244347184896469, + 0.22211141884326935, + 0.2183273881673813, + 0.21741008758544922, + 0.21451692283153534, + 0.21311214566230774, + 0.20762479305267334, + 0.20640577375888824, + 0.20071867108345032, + 0.19932369887828827, + 0.19245882332324982, + 0.19335229694843292, + 0.19078245759010315, + 0.19345322251319885, + 0.1916658580303192, + 0.19645750522613525, + 0.19844752550125122, + 0.20112650096416473, + 0.2010338455438614, + 0.2038118690252304, + 0.20468150079250336, + 0.20720544457435608, + 0.20727655291557312, + 0.21031972765922546, + 0.20860347151756287, + 0.2129739224910736, + 0.21107539534568787, + 0.21647654473781586, + 0.2111099362373352, + 0.22099705040454865, + 0.2196389138698578, + 0.23361735045909882, + 0.23286758363246918, + 0.2512694001197815, + 0.25084227323532104, + 0.2773953676223755, + 0.27732276916503906, + 0.3133004307746887, + 0.31342777609825134, + 0.35808902978897095, + 0.35860511660575867, + 0.4303363263607025, + 0.4337502121925354, + 0.54518723487854 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010598741209832951, + 0.00010775360715342686, + 0.00011871542665176094, + 0.00011485091090435162, + 0.00011244894267292693, + 0.00011016445205314085, + 0.00010846564691746607, + 0.00010656306403689086, + 0.0001046601973939687, + 0.00010342278255848214, + 0.00010179441596847028, + 0.00010021102934842929, + 9.965883509721607e-05, + 9.537018195260316e-05, + 9.32025650399737e-05, + 9.038933058036491e-05, + 8.758632611716166e-05, + 8.443772094324231e-05, + 8.744140359340236e-05, + 8.20954010123387e-05, + 8.16298124846071e-05, + 7.812595140421763e-05, + 7.176945655373856e-05, + 6.528423546114936e-05, + 5.917211092310026e-05, + 5.3287756600184366e-05, + 4.791216633748263e-05, + 4.251671634847298e-05, + 3.64475381502416e-05, + 3.0126682759146206e-05, + 2.3703445549472235e-05, + 1.9818620785372332e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.0001106186828110367, + "first_to_last_ratio": 2.827449739094469, + "imbalance_abs_log_ratio": 1.0393751531099071, + "last_quartile_mean": 3.912312968168408e-05, + "mean": 8.203169204534788e-05, + "normalized": [ + 1.292030061256553, + 1.3135607040003474, + 1.4471897835062812, + 1.4000797501636437, + 1.3707987714158585, + 1.342949892978447, + 1.3222407610159408, + 1.2990474946924393, + 1.275850769189444, + 1.2607661743867122, + 1.2409157172108236, + 1.2216135843331344, + 1.2148821097353903, + 1.1626016674126585, + 1.1361775274420824, + 1.1018830445481589, + 1.067713269509827, + 1.0293304799389529, + 1.06594660445458, + 1.000776638460116, + 0.9951009231832177, + 0.9523874182801066, + 0.8748991367149143, + 0.7958416294163435, + 0.7213323222735595, + 0.6495996275528049, + 0.5840689755734447, + 0.5182962253779837, + 0.444310451747028, + 0.36725662982170226, + 0.28895473150022005, + 0.24159712290728336 + ], + "population_cv": 0.3450366052372569 + }, + "activation_output_rms_by_block": [ + 0.11079900711774826, + 0.18173734843730927, + 0.25566670298576355, + 0.2647266387939453, + 0.2669948637485504, + 0.2675777077674866, + 0.2675747275352478, + 0.26725223660469055, + 0.2656837999820709, + 0.265005499124527, + 0.25976043939590454, + 0.2535160183906555, + 0.24569463729858398, + 0.23344992101192474, + 0.22194796800613403, + 0.21299737691879272, + 0.21236176788806915, + 0.21684354543685913, + 0.22108736634254456, + 0.22472712397575378, + 0.22933396697044373, + 0.23535853624343872, + 0.2425893247127533, + 0.2497948557138443, + 0.26040923595428467, + 0.27719539403915405, + 0.2986113131046295, + 0.3278995454311371, + 0.36809566617012024, + 0.4197922646999359, + 0.5092319846153259, + 0.6714946031570435 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23529115412384272, + "first_to_last_ratio": 0.6008590681870453, + "imbalance_abs_log_ratio": 0.5093948674743478, + "last_quartile_mean": 0.39159125089645386, + "mean": 0.2751628558617085, + "normalized": [ + 0.40266701975732394, + 0.6604719516672225, + 0.9291468580855864, + 0.962072580490267, + 0.9703157895800326, + 0.9724339679842758, + 0.972423137190165, + 0.9712511369594387, + 0.9655511066348229, + 0.9630860178952118, + 0.9440243617999629, + 0.9213308155155495, + 0.8929062628353637, + 0.8484063747661208, + 0.806605845513105, + 0.7740775049443485, + 0.7717675673303741, + 0.7880552945919432, + 0.803478237097738, + 0.8167058859452209, + 0.8334481274816485, + 0.8553426860845104, + 0.881620900295765, + 0.9078073235269309, + 0.9463822256778773, + 1.007386673506787, + 1.0852166516788362, + 1.191656280802424, + 1.3377374828349577, + 1.5256138528774221, + 1.850656706627787, + 2.4403533720209807 + ], + "population_cv": 0.3576152457433685 + }, + "bits_per_byte": 1.9501283044158226, + "branch_output_rms_by_sublayer": [ + 0.02630079723894596, + 0.06842244416475296, + 0.10497251898050308, + 0.019157659262418747, + 0.09477359801530838, + 0.0031024799682199955, + 0.0338861346244812, + 0.0047894855961203575, + 0.022547677159309387, + 0.004550074227154255, + 0.021251287311315536, + 0.004262411035597324, + 0.01840241253376007, + 0.006118505261838436, + 0.02267448604106903, + 0.009554358199238777, + 0.022321712225675583, + 0.00963375624269247, + 0.01962294802069664, + 0.011137282475829124, + 0.019759519025683403, + 0.013955196365714073, + 0.021449215710163116, + 0.01607326976954937, + 0.026240134611725807, + 0.015075869858264923, + 0.039021123200654984, + 0.02189606800675392, + 0.031211141496896744, + 0.022710295394062996, + 0.03785466402769089, + 0.026124922558665276, + 0.0325845405459404, + 0.026591502130031586, + 0.03150996193289757, + 0.029410794377326965, + 0.02998937852680683, + 0.024189060553908348, + 0.04107306897640228, + 0.029402343556284904, + 0.040071673691272736, + 0.032556869089603424, + 0.04498095437884331, + 0.03419904410839081, + 0.05116673558950424, + 0.041954249143600464, + 0.04688612371683121, + 0.050006624311208725, + 0.052477724850177765, + 0.06379573792219162, + 0.04486159235239029, + 0.06914625316858292, + 0.032891806215047836, + 0.0747767761349678, + 0.015299716033041477, + 0.08387137204408646, + 0.017307085916399956, + 0.09213023632764816, + 0.017144370824098587, + 0.10062393546104431, + 0.017122838646173477, + 0.15051087737083435, + 0.027784937992691994, + 0.24076412618160248 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00032837002715831827, + 0.0002079177996867109, + 0.00018809406473136247, + 0.00010899326506499365, + 9.474430680287842e-05, + 8.318366756728252e-05, + 9.647881676005136e-05, + 0.00013404889675397192, + 0.00012865721885494213, + 0.0001321695767247016, + 0.00014056789256563647, + 0.00015176892928157556, + 0.00016698296163968515, + 0.00022933649843535653, + 0.000217708018553423, + 0.0002507549770035681, + 0.00025689149388127776, + 0.0002828493196393162, + 0.00027498157120012506, + 0.0003306027661240732, + 0.00032889440594708014, + 0.00033897343531885953, + 0.000372930739966371, + 0.0003793341532875068, + 0.0003649148263870391, + 0.0003190518951646816, + 0.00028800578855572547, + 0.0002568521570555004, + 0.00024004963937954607, + 0.00021095610212654392, + 0.00019368945551758278, + 0.00017448279084866967 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015522885556569618, + "first_to_last_ratio": 0.6063619309634984, + "imbalance_abs_log_ratio": 0.5002782253787689, + "last_quartile_mean": 0.00025600033187941115, + "mean": 0.00022728867056201116, + "normalized": [ + 1.4447267712304608, + 0.9147741467825811, + 0.8275558313851142, + 0.4795367265578555, + 0.41684570800914306, + 0.36598246345317736, + 0.42447701648080627, + 0.5897737728084399, + 0.56605205414249, + 0.581505344713791, + 0.6184553423540983, + 0.6677364468114501, + 0.7346734935216545, + 1.0090098106002459, + 0.9578480881387607, + 1.1032445056919573, + 1.1302432859766764, + 1.244449707677561, + 1.2098340428503753, + 1.4545501335662696, + 1.4470338760565185, + 1.491378494496396, + 1.6407801543483633, + 1.668953196609125, + 1.605512608634311, + 1.4037298664107178, + 1.2671365794149818, + 1.1300702160841907, + 1.05614432424626, + 0.928141739774877, + 0.8521738238806694, + 0.7676704272906798 + ], + "population_cv": 0.3864254083322711 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06033600866794586, + 0.06945198029279709, + 0.11079900711774826, + 0.17341101169586182, + 0.18173734843730927, + 0.25461602210998535, + 0.25566670298576355, + 0.26339152455329895, + 0.2647266387939453, + 0.2662918269634247, + 0.2669948637485504, + 0.2675604224205017, + 0.2675777077674866, + 0.2677975296974182, + 0.2675747275352478, + 0.2675320506095886, + 0.26725223660469055, + 0.26636025309562683, + 0.2656837999820709, + 0.2656107544898987, + 0.265005499124527, + 0.26352381706237793, + 0.25976043939590454, + 0.2553655505180359, + 0.2535160183906555, + 0.24811074137687683, + 0.24569463729858398, + 0.23600345849990845, + 0.23344992101192474, + 0.22498825192451477, + 0.22194796800613403, + 0.21307076513767242, + 0.21299737691879272, + 0.20943206548690796, + 0.21236176788806915, + 0.21104833483695984, + 0.21684354543685913, + 0.21818861365318298, + 0.22108736634254456, + 0.22058506309986115, + 0.22472712397575378, + 0.22517690062522888, + 0.22933396697044373, + 0.2308174967765808, + 0.23535853624343872, + 0.23589646816253662, + 0.2425893247127533, + 0.241608664393425, + 0.2497948557138443, + 0.24810902774333954, + 0.26040923595428467, + 0.26088401675224304, + 0.27719539403915405, + 0.2782979905605316, + 0.2986113131046295, + 0.2985531687736511, + 0.3278995454311371, + 0.3279455602169037, + 0.36809566617012024, + 0.3681766986846924, + 0.4197922646999359, + 0.4200114607810974, + 0.5092319846153259, + 0.5118712186813354 + ], + "loss_nats": 1.3517259359359741, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06945198029279709, + 0.11079900711774826, + 0.17341101169586182, + 0.18173734843730927, + 0.25461602210998535, + 0.25566670298576355, + 0.26339152455329895, + 0.2647266387939453, + 0.2662918269634247, + 0.2669948637485504, + 0.2675604224205017, + 0.2675777077674866, + 0.2677975296974182, + 0.2675747275352478, + 0.2675320506095886, + 0.26725223660469055, + 0.26636025309562683, + 0.2656837999820709, + 0.2656107544898987, + 0.265005499124527, + 0.26352381706237793, + 0.25976043939590454, + 0.2553655505180359, + 0.2535160183906555, + 0.24811074137687683, + 0.24569463729858398, + 0.23600345849990845, + 0.23344992101192474, + 0.22498825192451477, + 0.22194796800613403, + 0.21307076513767242, + 0.21299737691879272, + 0.20943206548690796, + 0.21236176788806915, + 0.21104833483695984, + 0.21684354543685913, + 0.21818861365318298, + 0.22108736634254456, + 0.22058506309986115, + 0.22472712397575378, + 0.22517690062522888, + 0.22933396697044373, + 0.2308174967765808, + 0.23535853624343872, + 0.23589646816253662, + 0.2425893247127533, + 0.241608664393425, + 0.2497948557138443, + 0.24810902774333954, + 0.26040923595428467, + 0.26088401675224304, + 0.27719539403915405, + 0.2782979905605316, + 0.2986113131046295, + 0.2985531687736511, + 0.3278995454311371, + 0.3279455602169037, + 0.36809566617012024, + 0.3681766986846924, + 0.4197922646999359, + 0.4200114607810974, + 0.5092319846153259, + 0.5118712186813354, + 0.6714946031570435 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018891454965341836, + 0.0001931450387928635, + 0.0002085021696984768, + 0.00020282011246308684, + 0.00019955210154876113, + 0.00019673256610985845, + 0.00019503913063090295, + 0.0001924382959259674, + 0.00019001582404598594, + 0.00018850783817470074, + 0.00018595026631373912, + 0.0001838041644077748, + 0.00018427071336191148, + 0.00017771514831110835, + 0.0001728432544041425, + 0.0001671368081588298, + 0.0001615467481315136, + 0.00015400326810777187, + 0.000160209892783314, + 0.00015067463391460478, + 0.00014840754738543183, + 0.00014097621897235513, + 0.00012854386295657605, + 0.00011642893514363095, + 0.00010425484651932493, + 9.334496280644089e-05, + 8.331386925419793e-05, + 7.313644891837612e-05, + 6.15216777077876e-05, + 4.904071465716697e-05, + 3.525121428538114e-05, + 2.4962779207271524e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019714299560291693, + "first_to_last_ratio": 3.0050767723956184, + "imbalance_abs_log_ratio": 1.1003031158789967, + "last_quartile_mean": 6.560331416949339e-05, + "mean": 0.00014728142508602104, + "normalized": [ + 1.282677360998382, + 1.311401208129643, + 1.4156718647765612, + 1.3770922731405397, + 1.3549033860326305, + 1.3357595229334251, + 1.324261566025645, + 1.3066026202121013, + 1.2901547084773621, + 1.2799159029361715, + 1.262550699826086, + 1.247979263511487, + 1.2511470014245618, + 1.206636534154339, + 1.1735577266664268, + 1.1348125404219307, + 1.0968575842959203, + 1.045639448544342, + 1.0877807075110928, + 1.023038946198422, + 1.0076460578701834, + 0.9571894004285788, + 0.8727771535446431, + 0.7905201560592559, + 0.707861473084158, + 0.6337863905914947, + 0.5656780493910737, + 0.4965761899415362, + 0.41771511697320496, + 0.33297284181304126, + 0.23934596141225786, + 0.1694903426735027 + ], + "population_cv": 0.361085575415688 + }, + "activation_output_rms_by_block": [ + 0.1016591340303421, + 0.18086770176887512, + 0.2325568050146103, + 0.2376081645488739, + 0.23870401084423065, + 0.23941391706466675, + 0.23885947465896606, + 0.2381371706724167, + 0.23675362765789032, + 0.23536528646945953, + 0.22870509326457977, + 0.22131286561489105, + 0.21254637837409973, + 0.19929367303848267, + 0.18746964633464813, + 0.17704539000988007, + 0.17596758902072906, + 0.1784123033285141, + 0.18117663264274597, + 0.18366514146327972, + 0.1872515082359314, + 0.19141052663326263, + 0.1969655156135559, + 0.2033752053976059, + 0.21314463019371033, + 0.22718842327594757, + 0.2457481175661087, + 0.269707053899765, + 0.30305343866348267, + 0.347743958234787, + 0.4299577474594116, + 0.6035616993904114 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2134757973253727, + "first_to_last_ratio": 0.6468706108937196, + "imbalance_abs_log_ratio": 0.4356089876432565, + "last_quartile_mean": 0.33001313358545303, + "mean": 0.23576961969956756, + "normalized": [ + 0.4311799550759871, + 0.7671374369579427, + 0.986373076017804, + 1.0077980566438083, + 1.0124460104249322, + 1.015457026947505, + 1.0131053990897376, + 1.010041798327817, + 1.0041735994636487, + 0.9982850494876174, + 0.9700363157730422, + 0.9386827102529232, + 0.9015002808459363, + 0.8452898778580343, + 0.7951391132306771, + 0.7509253746750002, + 0.7463539587711004, + 0.7567230398719659, + 0.7684477451913126, + 0.7790025775895867, + 0.7942138960674365, + 0.8118540755045961, + 0.8354151644496934, + 0.8626014057992686, + 0.9040377232033228, + 0.9636034683579899, + 1.0423230858973958, + 1.143943202875089, + 1.2853795117863462, + 1.4749311581276, + 1.8236350722679655, + 2.5599638331669174 + ], + "population_cv": 0.3675345504070441 + }, + "bits_per_byte": 1.8720084529317582, + "branch_output_rms_by_sublayer": [ + 0.03573180362582207, + 0.052011821419000626, + 0.10664675384759903, + 0.01823282428085804, + 0.06781794875860214, + 0.0026424091774970293, + 0.02450474537909031, + 0.003992235753685236, + 0.01725216954946518, + 0.0036646369844675064, + 0.018646538257598877, + 0.003319106763228774, + 0.015910238027572632, + 0.004927239380776882, + 0.018212314695119858, + 0.008043181151151657, + 0.016784843057394028, + 0.007957165129482746, + 0.01594853773713112, + 0.009037249721586704, + 0.0176989808678627, + 0.012198803015053272, + 0.019605962559580803, + 0.013432997278869152, + 0.022475004196166992, + 0.012972000986337662, + 0.03453320264816284, + 0.018862294033169746, + 0.026713354513049126, + 0.01990535296499729, + 0.0319337360560894, + 0.023401649668812752, + 0.02801610343158245, + 0.023707188665866852, + 0.028282491490244865, + 0.026515409350395203, + 0.02569115161895752, + 0.022723721340298653, + 0.03440641239285469, + 0.027487577870488167, + 0.034427426755428314, + 0.03072489984333515, + 0.039456624537706375, + 0.03320617973804474, + 0.043320003896951675, + 0.04054662957787514, + 0.04175209999084473, + 0.04858461394906044, + 0.043093886226415634, + 0.060716040432453156, + 0.03683279827237129, + 0.0668661892414093, + 0.03015786223113537, + 0.0722465068101883, + 0.015251493081450462, + 0.07915013283491135, + 0.017382971942424774, + 0.08651599287986755, + 0.015752796083688736, + 0.09494592994451523, + 0.018999194726347923, + 0.14218401908874512, + 0.03338775411248207, + 0.24767111241817474 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004570302696620844, + 0.00031493934518203267, + 0.0002143615651481701, + 0.00013786409436507824, + 0.0001291171736265367, + 0.00013682107280354788, + 0.00015837516984797755, + 0.00019563631023119442, + 0.00019674727625245423, + 0.000191343414336242, + 0.00021955065404037036, + 0.0002332172506517699, + 0.00027919484030541734, + 0.0003858941801364767, + 0.0003704235701666674, + 0.00042305543536477747, + 0.0004472346343569547, + 0.0004940425030518999, + 0.0004673224693273912, + 0.0005594278228367747, + 0.0005764247107952261, + 0.0005972083894683241, + 0.0006512089060709392, + 0.0006542639335938355, + 0.0006197665059356166, + 0.0005424206044372668, + 0.0004994517842756415, + 0.0004394000632885711, + 0.0004055610719669202, + 0.0003512581615295656, + 0.00030757820471853463, + 0.00025263519960351045 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00021801812510832775, + "first_to_last_ratio": 0.5102716406035511, + "imbalance_abs_log_ratio": 0.6728120664256351, + "last_quartile_mean": 0.0004272589494694534, + "mean": 0.0003721492683555553, + "normalized": [ + 1.2280832142478726, + 0.8462715688618158, + 0.5760096374645209, + 0.3704537562958782, + 0.34694995961452973, + 0.36765105950128485, + 0.42556894051626687, + 0.5256931206546978, + 0.5286783905872947, + 0.5141577066152674, + 0.5899532061705127, + 0.6266766334979106, + 0.7502227306239702, + 1.0369338675356186, + 0.9953628870573538, + 1.1367896468913272, + 1.2017614231332092, + 1.3275385579419885, + 1.2557393203871798, + 1.5032350468100115, + 1.5489072794427852, + 1.6047549740115168, + 1.7498594285795221, + 1.7580685741634856, + 1.6653707494152192, + 1.4575350553128925, + 1.3420738041017997, + 1.1807091956143891, + 1.0897806510785424, + 0.9438636359052839, + 0.8264914937967075, + 0.6788544841693472 + ], + "population_cv": 0.43653500885793567 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06193641945719719, + 0.07578708976507187, + 0.1016591340303421, + 0.17375877499580383, + 0.18086770176887512, + 0.23162810504436493, + 0.2325568050146103, + 0.2364615947008133, + 0.2376081645488739, + 0.23820650577545166, + 0.23870401084423065, + 0.23957808315753937, + 0.23941391706466675, + 0.2393639236688614, + 0.23885947465896606, + 0.23879873752593994, + 0.2381371706724167, + 0.23766019940376282, + 0.23675362765789032, + 0.23612910509109497, + 0.23536528646945953, + 0.2328052520751953, + 0.22870509326457977, + 0.22353777289390564, + 0.22131286561489105, + 0.21547429263591766, + 0.21254637837409973, + 0.20247745513916016, + 0.19929367303848267, + 0.19119344651699066, + 0.18746964633464813, + 0.1784159243106842, + 0.17704539000988007, + 0.17436346411705017, + 0.17596758902072906, + 0.17439287900924683, + 0.1784123033285141, + 0.1789977103471756, + 0.18117663264274597, + 0.18044494092464447, + 0.18366514146327972, + 0.1840001940727234, + 0.1872515082359314, + 0.18770679831504822, + 0.19141052663326263, + 0.19118990004062653, + 0.1969655156135559, + 0.19665203988552094, + 0.2033752053976059, + 0.20245759189128876, + 0.21314463019371033, + 0.21310319006443024, + 0.22718842327594757, + 0.2287222445011139, + 0.2457481175661087, + 0.2456687092781067, + 0.269707053899765, + 0.26972076296806335, + 0.30305343866348267, + 0.30293965339660645, + 0.347743958234787, + 0.3480367064476013, + 0.4299577474594116, + 0.43200185894966125 + ], + "loss_nats": 1.2975773811340332, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07578708976507187, + 0.1016591340303421, + 0.17375877499580383, + 0.18086770176887512, + 0.23162810504436493, + 0.2325568050146103, + 0.2364615947008133, + 0.2376081645488739, + 0.23820650577545166, + 0.23870401084423065, + 0.23957808315753937, + 0.23941391706466675, + 0.2393639236688614, + 0.23885947465896606, + 0.23879873752593994, + 0.2381371706724167, + 0.23766019940376282, + 0.23675362765789032, + 0.23612910509109497, + 0.23536528646945953, + 0.2328052520751953, + 0.22870509326457977, + 0.22353777289390564, + 0.22131286561489105, + 0.21547429263591766, + 0.21254637837409973, + 0.20247745513916016, + 0.19929367303848267, + 0.19119344651699066, + 0.18746964633464813, + 0.1784159243106842, + 0.17704539000988007, + 0.17436346411705017, + 0.17596758902072906, + 0.17439287900924683, + 0.1784123033285141, + 0.1789977103471756, + 0.18117663264274597, + 0.18044494092464447, + 0.18366514146327972, + 0.1840001940727234, + 0.1872515082359314, + 0.18770679831504822, + 0.19141052663326263, + 0.19118990004062653, + 0.1969655156135559, + 0.19665203988552094, + 0.2033752053976059, + 0.20245759189128876, + 0.21314463019371033, + 0.21310319006443024, + 0.22718842327594757, + 0.2287222445011139, + 0.2457481175661087, + 0.2456687092781067, + 0.269707053899765, + 0.26972076296806335, + 0.30305343866348267, + 0.30293965339660645, + 0.347743958234787, + 0.3480367064476013, + 0.4299577474594116, + 0.43200185894966125, + 0.6035616993904114 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.95552135286859, + "cross_entropy_nats": 5.514347195625305, + "step": 0 + }, + { + "bits_per_byte": 5.166060485015187, + "cross_entropy_nats": 3.5808402597904205, + "step": 100 + }, + { + "bits_per_byte": 3.1277970484292927, + "cross_entropy_nats": 2.168023705482483, + "step": 500 + }, + { + "bits_per_byte": 1.9613801618593651, + "cross_entropy_nats": 1.359525129199028, + "step": 2000 + }, + { + "bits_per_byte": 1.7971937855298066, + "cross_entropy_nats": 1.2457198053598404, + "step": 4000 + }, + { + "bits_per_byte": 1.7178972072507075, + "cross_entropy_nats": 1.1907556056976318, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "final_optimizer_state": "06d1b38e173978179caca1f5bf426eaad5609e9e9a1b567ef88c8b18eeab7e4d", + "final_public_parameters": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 42.00224438344842, + "measured_steps": 7980, + "median_ms": 41.91726399585605, + "p95_ms": 42.462232409161516, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958522622108798, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516427516937256, + "step": 1, + "unclipped_grad_norm": 22.09486961364746 + }, + { + "bits_per_byte": 7.238571600789322, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.017395496368408, + "step": 10, + "unclipped_grad_norm": 11.172341346740723 + }, + { + "bits_per_byte": 6.729949421197048, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.6648454666137695, + "step": 20, + "unclipped_grad_norm": 3.462282180786133 + }, + { + "bits_per_byte": 6.60904492970989, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.581040859222412, + "step": 30, + "unclipped_grad_norm": 3.06864070892334 + }, + { + "bits_per_byte": 6.370936325451538, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.415996551513672, + "step": 40, + "unclipped_grad_norm": 3.234333038330078 + }, + { + "bits_per_byte": 6.086515545038246, + "learning_rate": 3.75e-05, + "loss_nats": 4.218851089477539, + "step": 50, + "unclipped_grad_norm": 2.8305625915527344 + }, + { + "bits_per_byte": 5.8833778930063385, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.078046798706055, + "step": 60, + "unclipped_grad_norm": 2.670325517654419 + }, + { + "bits_per_byte": 5.715139992383416, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.961433172225952, + "step": 70, + "unclipped_grad_norm": 3.2375030517578125 + }, + { + "bits_per_byte": 5.566718964688812, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.858555555343628, + "step": 80, + "unclipped_grad_norm": 2.6123173236846924 + }, + { + "bits_per_byte": 5.355997221719156, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.712494373321533, + "step": 90, + "unclipped_grad_norm": 2.440842628479004 + }, + { + "bits_per_byte": 5.182009597112589, + "learning_rate": 7.5e-05, + "loss_nats": 3.591895341873169, + "step": 100, + "unclipped_grad_norm": 2.292849063873291 + }, + { + "bits_per_byte": 4.992510513259403, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4605445861816406, + "step": 110, + "unclipped_grad_norm": 2.057412624359131 + }, + { + "bits_per_byte": 4.778040580396842, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.311885356903076, + "step": 120, + "unclipped_grad_norm": 2.002248525619507 + }, + { + "bits_per_byte": 4.588237775182233, + "learning_rate": 9.75e-05, + "loss_nats": 3.180324077606201, + "step": 130, + "unclipped_grad_norm": 1.7531825304031372 + }, + { + "bits_per_byte": 4.430652823919056, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.071094512939453, + "step": 140, + "unclipped_grad_norm": 1.4980645179748535 + }, + { + "bits_per_byte": 4.274817968111058, + "learning_rate": 0.0001125, + "loss_nats": 2.963078022003174, + "step": 150, + "unclipped_grad_norm": 1.3038520812988281 + }, + { + "bits_per_byte": 4.169977000179788, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8904078006744385, + "step": 160, + "unclipped_grad_norm": 1.0998529195785522 + }, + { + "bits_per_byte": 4.028519206053318, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7923567295074463, + "step": 170, + "unclipped_grad_norm": 0.9818326234817505 + }, + { + "bits_per_byte": 3.950109907767851, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7380075454711914, + "step": 180, + "unclipped_grad_norm": 0.8779997229576111 + }, + { + "bits_per_byte": 3.846655463967837, + "learning_rate": 0.0001425, + "loss_nats": 2.6662983894348145, + "step": 190, + "unclipped_grad_norm": 1.2310962677001953 + }, + { + "bits_per_byte": 3.7855644746589534, + "learning_rate": 0.00015, + "loss_nats": 2.623953342437744, + "step": 200, + "unclipped_grad_norm": 1.0928086042404175 + }, + { + "bits_per_byte": 3.7365611139833117, + "learning_rate": 0.0001575, + "loss_nats": 2.589986801147461, + "step": 210, + "unclipped_grad_norm": 0.7771895527839661 + }, + { + "bits_per_byte": 3.6369384436445213, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5209336280822754, + "step": 220, + "unclipped_grad_norm": 0.7917730212211609 + }, + { + "bits_per_byte": 3.630690657903977, + "learning_rate": 0.0001725, + "loss_nats": 2.5166029930114746, + "step": 230, + "unclipped_grad_norm": 1.0061120986938477 + }, + { + "bits_per_byte": 3.503533565162051, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.428464412689209, + "step": 240, + "unclipped_grad_norm": 0.7126089930534363 + }, + { + "bits_per_byte": 3.5514149110234636, + "learning_rate": 0.0001875, + "loss_nats": 2.461653232574463, + "step": 250, + "unclipped_grad_norm": 1.2347668409347534 + }, + { + "bits_per_byte": 3.5407041754930297, + "learning_rate": 0.000195, + "loss_nats": 2.4542291164398193, + "step": 260, + "unclipped_grad_norm": 0.9977514147758484 + }, + { + "bits_per_byte": 3.5337584841552845, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4494147300720215, + "step": 270, + "unclipped_grad_norm": 0.7908692955970764 + }, + { + "bits_per_byte": 3.478037825102243, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.410792112350464, + "step": 280, + "unclipped_grad_norm": 1.0802956819534302 + }, + { + "bits_per_byte": 3.3705159916093237, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.336263656616211, + "step": 290, + "unclipped_grad_norm": 0.9241761565208435 + }, + { + "bits_per_byte": 3.410749956917471, + "learning_rate": 0.000225, + "loss_nats": 2.3641517162323, + "step": 300, + "unclipped_grad_norm": 0.9868955612182617 + }, + { + "bits_per_byte": 3.3596274260178975, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.328716278076172, + "step": 310, + "unclipped_grad_norm": 0.8471657037734985 + }, + { + "bits_per_byte": 3.3434163413475693, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174796104431152, + "step": 320, + "unclipped_grad_norm": 0.9364105463027954 + }, + { + "bits_per_byte": 3.3338296844231747, + "learning_rate": 0.0002475, + "loss_nats": 2.3108346462249756, + "step": 330, + "unclipped_grad_norm": 0.9398566484451294 + }, + { + "bits_per_byte": 3.4261592584694767, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3748326301574707, + "step": 340, + "unclipped_grad_norm": 0.949837863445282 + }, + { + "bits_per_byte": 3.364650007353695, + "learning_rate": 0.0002625, + "loss_nats": 2.332197666168213, + "step": 350, + "unclipped_grad_norm": 0.8382212519645691 + }, + { + "bits_per_byte": 3.2974426270495267, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2856130599975586, + "step": 360, + "unclipped_grad_norm": 1.0729857683181763 + }, + { + "bits_per_byte": 3.304221839181237, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2903120517730713, + "step": 370, + "unclipped_grad_norm": 0.9106518030166626 + }, + { + "bits_per_byte": 3.3613300542614395, + "learning_rate": 0.000285, + "loss_nats": 2.3298964500427246, + "step": 380, + "unclipped_grad_norm": 1.290928602218628 + }, + { + "bits_per_byte": 3.2658243046190027, + "learning_rate": 0.0002925, + "loss_nats": 2.2636969089508057, + "step": 390, + "unclipped_grad_norm": 1.1257888078689575 + }, + { + "bits_per_byte": 3.2284232375358677, + "learning_rate": 0.0003, + "loss_nats": 2.2377724647521973, + "step": 400, + "unclipped_grad_norm": 1.3445736169815063 + }, + { + "bits_per_byte": 3.1961283353568497, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2153873443603516, + "step": 410, + "unclipped_grad_norm": 0.7953492403030396 + }, + { + "bits_per_byte": 3.2387222466001364, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2449111938476562, + "step": 420, + "unclipped_grad_norm": 0.8595701456069946 + }, + { + "bits_per_byte": 3.2223075344711862, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2335333824157715, + "step": 430, + "unclipped_grad_norm": 1.2960833311080933 + }, + { + "bits_per_byte": 3.2629112624786885, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2616777420043945, + "step": 440, + "unclipped_grad_norm": 0.9954231977462769 + }, + { + "bits_per_byte": 3.169579029577856, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1969847679138184, + "step": 450, + "unclipped_grad_norm": 1.0229040384292603 + }, + { + "bits_per_byte": 3.210129442963591, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.2250921726226807, + "step": 460, + "unclipped_grad_norm": 1.1146044731140137 + }, + { + "bits_per_byte": 3.160413042217472, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.19063138961792, + "step": 470, + "unclipped_grad_norm": 0.9123480319976807 + }, + { + "bits_per_byte": 3.1596232978847096, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1900839805603027, + "step": 480, + "unclipped_grad_norm": 0.7869184613227844 + }, + { + "bits_per_byte": 3.0856473683731847, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.138807773590088, + "step": 490, + "unclipped_grad_norm": 0.8606863021850586 + }, + { + "bits_per_byte": 3.05385947104891, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.116774082183838, + "step": 500, + "unclipped_grad_norm": 0.9675024151802063 + }, + { + "bits_per_byte": 3.0816491157059263, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1360363960266113, + "step": 510, + "unclipped_grad_norm": 0.8225864768028259 + }, + { + "bits_per_byte": 3.0308351216820006, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.1008148193359375, + "step": 520, + "unclipped_grad_norm": 1.098574161529541 + }, + { + "bits_per_byte": 3.0891052515514326, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.141204595565796, + "step": 530, + "unclipped_grad_norm": 0.8098863363265991 + }, + { + "bits_per_byte": 3.0354875963531414, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.1040396690368652, + "step": 540, + "unclipped_grad_norm": 1.0088073015213013 + }, + { + "bits_per_byte": 3.0042414443790846, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0823814868927, + "step": 550, + "unclipped_grad_norm": 1.054882287979126 + }, + { + "bits_per_byte": 2.9880702596837563, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0711724758148193, + "step": 560, + "unclipped_grad_norm": 0.723493754863739 + }, + { + "bits_per_byte": 2.93916561704971, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0372743606567383, + "step": 570, + "unclipped_grad_norm": 1.0234330892562866 + }, + { + "bits_per_byte": 3.0140372322069284, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0891714096069336, + "step": 580, + "unclipped_grad_norm": 1.0687707662582397 + }, + { + "bits_per_byte": 2.915940047981291, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0211756229400635, + "step": 590, + "unclipped_grad_norm": 1.149570107460022 + }, + { + "bits_per_byte": 2.9613039117993742, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.052619457244873, + "step": 600, + "unclipped_grad_norm": 0.7275651693344116 + }, + { + "bits_per_byte": 2.880448160305384, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9965745210647583, + "step": 610, + "unclipped_grad_norm": 0.9942755699157715 + }, + { + "bits_per_byte": 2.9149380770573643, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0204811096191406, + "step": 620, + "unclipped_grad_norm": 0.8311108350753784 + }, + { + "bits_per_byte": 2.871522948656208, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.990388035774231, + "step": 630, + "unclipped_grad_norm": 1.2078016996383667 + }, + { + "bits_per_byte": 2.795491826466872, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9376872777938843, + "step": 640, + "unclipped_grad_norm": 0.7877771258354187 + }, + { + "bits_per_byte": 2.7606237198656447, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.9135185480117798, + "step": 650, + "unclipped_grad_norm": 1.1925899982452393 + }, + { + "bits_per_byte": 2.6852617821708162, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8612816333770752, + "step": 660, + "unclipped_grad_norm": 1.126668930053711 + }, + { + "bits_per_byte": 2.7363994475436653, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8967275619506836, + "step": 670, + "unclipped_grad_norm": 1.136749267578125 + }, + { + "bits_per_byte": 2.6585009377312616, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8427324295043945, + "step": 680, + "unclipped_grad_norm": 1.2957242727279663 + }, + { + "bits_per_byte": 2.734127384743117, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8951526880264282, + "step": 690, + "unclipped_grad_norm": 0.7534775137901306 + }, + { + "bits_per_byte": 2.6542058430086137, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8397552967071533, + "step": 700, + "unclipped_grad_norm": 0.8543418049812317 + }, + { + "bits_per_byte": 2.6774995172067744, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8559012413024902, + "step": 710, + "unclipped_grad_norm": 1.3297739028930664 + }, + { + "bits_per_byte": 2.6677150802338874, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8491191864013672, + "step": 720, + "unclipped_grad_norm": 0.7874022126197815 + }, + { + "bits_per_byte": 2.6473420154126788, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8349976539611816, + "step": 730, + "unclipped_grad_norm": 1.0262641906738281 + }, + { + "bits_per_byte": 2.582519862493327, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7900663614273071, + "step": 740, + "unclipped_grad_norm": 1.1010198593139648 + }, + { + "bits_per_byte": 2.5681288702170795, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7800912857055664, + "step": 750, + "unclipped_grad_norm": 0.9110153913497925 + }, + { + "bits_per_byte": 2.5014427693276766, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7338680028915405, + "step": 760, + "unclipped_grad_norm": 1.0652238130569458 + }, + { + "bits_per_byte": 2.634176399523829, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.8258719444274902, + "step": 770, + "unclipped_grad_norm": 0.9231922626495361 + }, + { + "bits_per_byte": 2.5180344516043642, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.745368480682373, + "step": 780, + "unclipped_grad_norm": 0.8394486308097839 + }, + { + "bits_per_byte": 2.482792454720161, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7209405899047852, + "step": 790, + "unclipped_grad_norm": 0.8228755593299866 + }, + { + "bits_per_byte": 2.430754632152696, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.684870719909668, + "step": 800, + "unclipped_grad_norm": 0.9281390905380249 + }, + { + "bits_per_byte": 2.489219790348206, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.725395679473877, + "step": 810, + "unclipped_grad_norm": 0.8680029511451721 + }, + { + "bits_per_byte": 2.4286650429447127, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.683422327041626, + "step": 820, + "unclipped_grad_norm": 1.0944814682006836 + }, + { + "bits_per_byte": 2.483867002322752, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7216854095458984, + "step": 830, + "unclipped_grad_norm": 0.8509150743484497 + }, + { + "bits_per_byte": 2.566762812021276, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7791444063186646, + "step": 840, + "unclipped_grad_norm": 1.2704709768295288 + }, + { + "bits_per_byte": 2.458091274507339, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.7038190364837646, + "step": 850, + "unclipped_grad_norm": 0.905615508556366 + }, + { + "bits_per_byte": 2.442831941826886, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.693242073059082, + "step": 860, + "unclipped_grad_norm": 0.7913721203804016 + }, + { + "bits_per_byte": 2.3506050414231483, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6293152570724487, + "step": 870, + "unclipped_grad_norm": 1.0574872493743896 + }, + { + "bits_per_byte": 2.3881724157296675, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6553549766540527, + "step": 880, + "unclipped_grad_norm": 0.923473060131073 + }, + { + "bits_per_byte": 2.321517099734769, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6091530323028564, + "step": 890, + "unclipped_grad_norm": 1.0251853466033936 + }, + { + "bits_per_byte": 2.330866592583624, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6156336069107056, + "step": 900, + "unclipped_grad_norm": 0.9670726656913757 + }, + { + "bits_per_byte": 2.356522276508734, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.633416771888733, + "step": 910, + "unclipped_grad_norm": 0.8682661652565002 + }, + { + "bits_per_byte": 2.329593920967239, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6147514581680298, + "step": 920, + "unclipped_grad_norm": 0.9293603301048279 + }, + { + "bits_per_byte": 2.356717820782765, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6335523128509521, + "step": 930, + "unclipped_grad_norm": 1.1330991983413696 + }, + { + "bits_per_byte": 2.3506738344834934, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.629362940788269, + "step": 940, + "unclipped_grad_norm": 0.8093944787979126 + }, + { + "bits_per_byte": 2.337553106066519, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6202683448791504, + "step": 950, + "unclipped_grad_norm": 1.0507299900054932 + }, + { + "bits_per_byte": 2.3753796582278888, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6464877128601074, + "step": 960, + "unclipped_grad_norm": 0.8050268888473511 + }, + { + "bits_per_byte": 2.2641018956227765, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.569355845451355, + "step": 970, + "unclipped_grad_norm": 0.8510206341743469 + }, + { + "bits_per_byte": 2.35890939570271, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6350713968276978, + "step": 980, + "unclipped_grad_norm": 0.8616457581520081 + }, + { + "bits_per_byte": 2.3101203254100433, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.6012533903121948, + "step": 990, + "unclipped_grad_norm": 0.8957601189613342 + }, + { + "bits_per_byte": 2.2887248237294555, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5864231586456299, + "step": 1000, + "unclipped_grad_norm": 1.0717068910598755 + }, + { + "bits_per_byte": 2.2687179099719343, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.572555422782898, + "step": 1010, + "unclipped_grad_norm": 0.9444129467010498 + }, + { + "bits_per_byte": 2.2310683279583023, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5464587211608887, + "step": 1020, + "unclipped_grad_norm": 0.9090887308120728 + }, + { + "bits_per_byte": 2.2341806979909666, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5486160516738892, + "step": 1030, + "unclipped_grad_norm": 0.8268627524375916 + }, + { + "bits_per_byte": 2.26161898209227, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5676348209381104, + "step": 1040, + "unclipped_grad_norm": 0.7536599636077881 + }, + { + "bits_per_byte": 2.2636605881406626, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5690499544143677, + "step": 1050, + "unclipped_grad_norm": 0.8793148398399353 + }, + { + "bits_per_byte": 2.1893296864377567, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.51752769947052, + "step": 1060, + "unclipped_grad_norm": 0.9039056301116943 + }, + { + "bits_per_byte": 2.2431745307178375, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5548501014709473, + "step": 1070, + "unclipped_grad_norm": 0.862832248210907 + }, + { + "bits_per_byte": 2.24520426996332, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5562570095062256, + "step": 1080, + "unclipped_grad_norm": 0.9674477577209473 + }, + { + "bits_per_byte": 2.1985577595351025, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5239241123199463, + "step": 1090, + "unclipped_grad_norm": 0.7859865427017212 + }, + { + "bits_per_byte": 2.2599985615558404, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.566511631011963, + "step": 1100, + "unclipped_grad_norm": 0.788394570350647 + }, + { + "bits_per_byte": 2.119760812354591, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.469306230545044, + "step": 1110, + "unclipped_grad_norm": 0.8257600665092468 + }, + { + "bits_per_byte": 2.1691607650231224, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5035476684570312, + "step": 1120, + "unclipped_grad_norm": 0.9345623254776001 + }, + { + "bits_per_byte": 2.1036112974732717, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4581122398376465, + "step": 1130, + "unclipped_grad_norm": 0.6939264535903931 + }, + { + "bits_per_byte": 2.2552164119659492, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5631968975067139, + "step": 1140, + "unclipped_grad_norm": 0.8951012492179871 + }, + { + "bits_per_byte": 2.1146625586524137, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4657723903656006, + "step": 1150, + "unclipped_grad_norm": 0.9783032536506653 + }, + { + "bits_per_byte": 2.1242843000375844, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4724416732788086, + "step": 1160, + "unclipped_grad_norm": 0.8440186977386475 + }, + { + "bits_per_byte": 2.174440976369912, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5072076320648193, + "step": 1170, + "unclipped_grad_norm": 0.9151178002357483 + }, + { + "bits_per_byte": 2.1585999983642408, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.496227502822876, + "step": 1180, + "unclipped_grad_norm": 0.7418354153633118 + }, + { + "bits_per_byte": 2.091859378974514, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4499664306640625, + "step": 1190, + "unclipped_grad_norm": 0.9675256609916687 + }, + { + "bits_per_byte": 2.0676216920057677, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4331661462783813, + "step": 1200, + "unclipped_grad_norm": 0.937224805355072 + }, + { + "bits_per_byte": 2.116782072841647, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.4672415256500244, + "step": 1210, + "unclipped_grad_norm": 0.8679532408714294 + }, + { + "bits_per_byte": 2.1441637746508166, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4862210750579834, + "step": 1220, + "unclipped_grad_norm": 0.7687044143676758 + }, + { + "bits_per_byte": 2.083470065265426, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4441514015197754, + "step": 1230, + "unclipped_grad_norm": 0.7017022967338562 + }, + { + "bits_per_byte": 2.025742368676816, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4041376113891602, + "step": 1240, + "unclipped_grad_norm": 0.8267619013786316 + }, + { + "bits_per_byte": 2.1229982137744323, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4715502262115479, + "step": 1250, + "unclipped_grad_norm": 1.0167953968048096 + }, + { + "bits_per_byte": 2.098591123894586, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4546325206756592, + "step": 1260, + "unclipped_grad_norm": 0.839733898639679 + }, + { + "bits_per_byte": 2.0788139789586175, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.440924048423767, + "step": 1270, + "unclipped_grad_norm": 0.7740461230278015 + }, + { + "bits_per_byte": 1.9768289549275753, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.370233416557312, + "step": 1280, + "unclipped_grad_norm": 0.6849040389060974 + }, + { + "bits_per_byte": 2.098678835046526, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.45469331741333, + "step": 1290, + "unclipped_grad_norm": 0.7928483486175537 + }, + { + "bits_per_byte": 2.041355641652745, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4149599075317383, + "step": 1300, + "unclipped_grad_norm": 0.7687125205993652 + }, + { + "bits_per_byte": 2.0466036922438238, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4185975790023804, + "step": 1310, + "unclipped_grad_norm": 0.7732207179069519 + }, + { + "bits_per_byte": 2.038361079735922, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.41288423538208, + "step": 1320, + "unclipped_grad_norm": 0.8016889691352844 + }, + { + "bits_per_byte": 2.1092168999954937, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4619977474212646, + "step": 1330, + "unclipped_grad_norm": 0.7918544411659241 + }, + { + "bits_per_byte": 2.0672703314500547, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.432922601699829, + "step": 1340, + "unclipped_grad_norm": 0.9473646879196167 + }, + { + "bits_per_byte": 2.0426272813732247, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4158413410186768, + "step": 1350, + "unclipped_grad_norm": 0.7250363826751709 + }, + { + "bits_per_byte": 2.10636370781768, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4600200653076172, + "step": 1360, + "unclipped_grad_norm": 0.7928993105888367 + }, + { + "bits_per_byte": 2.104627027009267, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4588162899017334, + "step": 1370, + "unclipped_grad_norm": 0.794548749923706 + }, + { + "bits_per_byte": 2.0561698832327657, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4252283573150635, + "step": 1380, + "unclipped_grad_norm": 0.7989111542701721 + }, + { + "bits_per_byte": 2.0527231789268243, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4228392839431763, + "step": 1390, + "unclipped_grad_norm": 0.8791808485984802 + }, + { + "bits_per_byte": 1.9977298345042815, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.384720802307129, + "step": 1400, + "unclipped_grad_norm": 0.7230417132377625 + }, + { + "bits_per_byte": 2.02384763581226, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4028242826461792, + "step": 1410, + "unclipped_grad_norm": 0.7468129396438599 + }, + { + "bits_per_byte": 2.0636549121636167, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4304165840148926, + "step": 1420, + "unclipped_grad_norm": 0.7919746041297913 + }, + { + "bits_per_byte": 2.0795395737626077, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4414269924163818, + "step": 1430, + "unclipped_grad_norm": 0.6952440738677979 + }, + { + "bits_per_byte": 2.0428340045195617, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4159846305847168, + "step": 1440, + "unclipped_grad_norm": 0.9689956307411194 + }, + { + "bits_per_byte": 1.9802144334098097, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3725800514221191, + "step": 1450, + "unclipped_grad_norm": 0.7872179746627808 + }, + { + "bits_per_byte": 2.013908586436248, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.39593505859375, + "step": 1460, + "unclipped_grad_norm": 0.7631422281265259 + }, + { + "bits_per_byte": 1.9493597139491168, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3511931896209717, + "step": 1470, + "unclipped_grad_norm": 0.7200304865837097 + }, + { + "bits_per_byte": 2.0209578113298123, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4008212089538574, + "step": 1480, + "unclipped_grad_norm": 0.7176085114479065 + }, + { + "bits_per_byte": 2.0333618880406417, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.409419059753418, + "step": 1490, + "unclipped_grad_norm": 0.7242412567138672 + }, + { + "bits_per_byte": 2.033254742849154, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.4093447923660278, + "step": 1500, + "unclipped_grad_norm": 0.7850398421287537 + }, + { + "bits_per_byte": 2.0298035669942904, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.4069526195526123, + "step": 1510, + "unclipped_grad_norm": 0.7650813460350037 + }, + { + "bits_per_byte": 1.981937699571455, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.373774528503418, + "step": 1520, + "unclipped_grad_norm": 0.7211347222328186 + }, + { + "bits_per_byte": 2.0555194448472025, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4247775077819824, + "step": 1530, + "unclipped_grad_norm": 0.7807993292808533 + }, + { + "bits_per_byte": 1.9955152139091212, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3831857442855835, + "step": 1540, + "unclipped_grad_norm": 0.9850755929946899 + }, + { + "bits_per_byte": 2.0666652964843193, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4325032234191895, + "step": 1550, + "unclipped_grad_norm": 0.8105318546295166 + }, + { + "bits_per_byte": 1.9693205363562067, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.365028977394104, + "step": 1560, + "unclipped_grad_norm": 0.6913962364196777 + }, + { + "bits_per_byte": 2.059840508950131, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4277726411819458, + "step": 1570, + "unclipped_grad_norm": 0.725780189037323 + }, + { + "bits_per_byte": 1.9988298355392002, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3854832649230957, + "step": 1580, + "unclipped_grad_norm": 0.7167323231697083 + }, + { + "bits_per_byte": 2.0030889858878176, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3884354829788208, + "step": 1590, + "unclipped_grad_norm": 0.7256693840026855 + }, + { + "bits_per_byte": 2.027964040560662, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.4056775569915771, + "step": 1600, + "unclipped_grad_norm": 0.709171712398529 + }, + { + "bits_per_byte": 1.961726126459407, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3597649335861206, + "step": 1610, + "unclipped_grad_norm": 0.6563887000083923 + }, + { + "bits_per_byte": 1.9260072216843611, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3350064754486084, + "step": 1620, + "unclipped_grad_norm": 0.743120014667511 + }, + { + "bits_per_byte": 1.954758421342351, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3549352884292603, + "step": 1630, + "unclipped_grad_norm": 0.6699017286300659 + }, + { + "bits_per_byte": 1.9950130245686017, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3828376531600952, + "step": 1640, + "unclipped_grad_norm": 0.7793066501617432 + }, + { + "bits_per_byte": 2.0499475509245495, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4209153652191162, + "step": 1650, + "unclipped_grad_norm": 0.6984328031539917 + }, + { + "bits_per_byte": 1.996117325169792, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3836030960083008, + "step": 1660, + "unclipped_grad_norm": 0.7241947054862976 + }, + { + "bits_per_byte": 2.017874850330446, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3986842632293701, + "step": 1670, + "unclipped_grad_norm": 0.7746724486351013 + }, + { + "bits_per_byte": 2.0244201660569825, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4032211303710938, + "step": 1680, + "unclipped_grad_norm": 0.7699829936027527 + }, + { + "bits_per_byte": 2.06043436504356, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4281842708587646, + "step": 1690, + "unclipped_grad_norm": 0.7417407035827637 + }, + { + "bits_per_byte": 1.95363795437198, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.354158639907837, + "step": 1700, + "unclipped_grad_norm": 0.6731021404266357 + }, + { + "bits_per_byte": 1.8890682719840945, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.309402346611023, + "step": 1710, + "unclipped_grad_norm": 0.6633850932121277 + }, + { + "bits_per_byte": 1.8893066399381904, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3095675706863403, + "step": 1720, + "unclipped_grad_norm": 0.7114297151565552 + }, + { + "bits_per_byte": 1.8703618910323976, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.296436071395874, + "step": 1730, + "unclipped_grad_norm": 0.6913781762123108 + }, + { + "bits_per_byte": 1.988379825707474, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3782398700714111, + "step": 1740, + "unclipped_grad_norm": 0.7100496888160706 + }, + { + "bits_per_byte": 2.0086424776668284, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.392284870147705, + "step": 1750, + "unclipped_grad_norm": 0.7558912038803101 + }, + { + "bits_per_byte": 1.996569639541561, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3839166164398193, + "step": 1760, + "unclipped_grad_norm": 0.6800652742385864 + }, + { + "bits_per_byte": 2.0072186333003352, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3912979364395142, + "step": 1770, + "unclipped_grad_norm": 0.7516586780548096 + }, + { + "bits_per_byte": 1.8758278436421194, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.300224781036377, + "step": 1780, + "unclipped_grad_norm": 0.6571854948997498 + }, + { + "bits_per_byte": 1.963898439322455, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3612706661224365, + "step": 1790, + "unclipped_grad_norm": 0.6896912455558777 + }, + { + "bits_per_byte": 1.880059304783948, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3031578063964844, + "step": 1800, + "unclipped_grad_norm": 0.6719921827316284 + }, + { + "bits_per_byte": 1.9405041552735398, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3450549840927124, + "step": 1810, + "unclipped_grad_norm": 0.7259154915809631 + }, + { + "bits_per_byte": 1.9116848504858082, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3250789642333984, + "step": 1820, + "unclipped_grad_norm": 0.6911426782608032 + }, + { + "bits_per_byte": 1.887859061965878, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3085641860961914, + "step": 1830, + "unclipped_grad_norm": 0.6866450309753418 + }, + { + "bits_per_byte": 1.904207044826293, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3198957443237305, + "step": 1840, + "unclipped_grad_norm": 0.6746156215667725 + }, + { + "bits_per_byte": 1.913379911492712, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.326253890991211, + "step": 1850, + "unclipped_grad_norm": 0.6992620825767517 + }, + { + "bits_per_byte": 1.8924212457453158, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.311726450920105, + "step": 1860, + "unclipped_grad_norm": 0.7686212658882141 + }, + { + "bits_per_byte": 1.9034685513234884, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3193838596343994, + "step": 1870, + "unclipped_grad_norm": 0.6957508325576782 + }, + { + "bits_per_byte": 1.8975736739825146, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3152978420257568, + "step": 1880, + "unclipped_grad_norm": 0.686790406703949 + }, + { + "bits_per_byte": 1.9108474669587572, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3244985342025757, + "step": 1890, + "unclipped_grad_norm": 0.6941424608230591 + }, + { + "bits_per_byte": 1.9178445811091105, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3293485641479492, + "step": 1900, + "unclipped_grad_norm": 0.6622544527053833 + }, + { + "bits_per_byte": 1.857984643615103, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2878568172454834, + "step": 1910, + "unclipped_grad_norm": 0.6576761603355408 + }, + { + "bits_per_byte": 1.8615044405476613, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2902965545654297, + "step": 1920, + "unclipped_grad_norm": 0.7134246826171875 + }, + { + "bits_per_byte": 1.9495869030309065, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3513506650924683, + "step": 1930, + "unclipped_grad_norm": 0.68708336353302 + }, + { + "bits_per_byte": 1.8269496863363057, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2663450241088867, + "step": 1940, + "unclipped_grad_norm": 0.671497106552124 + }, + { + "bits_per_byte": 1.904890503880822, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3203694820404053, + "step": 1950, + "unclipped_grad_norm": 0.6914620995521545 + }, + { + "bits_per_byte": 1.81755221032786, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259831190109253, + "step": 1960, + "unclipped_grad_norm": 0.654005765914917 + }, + { + "bits_per_byte": 1.9009173606805894, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3176155090332031, + "step": 1970, + "unclipped_grad_norm": 0.6446248292922974 + }, + { + "bits_per_byte": 1.9534350148439619, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.354017972946167, + "step": 1980, + "unclipped_grad_norm": 0.6792307496070862 + }, + { + "bits_per_byte": 1.9578730271494766, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.357094168663025, + "step": 1990, + "unclipped_grad_norm": 0.8862876892089844 + }, + { + "bits_per_byte": 1.9120136813142579, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3253068923950195, + "step": 2000, + "unclipped_grad_norm": 0.6821991801261902 + }, + { + "bits_per_byte": 1.879799782963796, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3029779195785522, + "step": 2010, + "unclipped_grad_norm": 0.6441471576690674 + }, + { + "bits_per_byte": 1.8960643542385427, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3142516613006592, + "step": 2020, + "unclipped_grad_norm": 0.6215534806251526 + }, + { + "bits_per_byte": 1.9880719767624297, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3780264854431152, + "step": 2030, + "unclipped_grad_norm": 0.6645570993423462 + }, + { + "bits_per_byte": 1.9132447331291338, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.326160192489624, + "step": 2040, + "unclipped_grad_norm": 0.6700881719589233 + }, + { + "bits_per_byte": 1.943815681215903, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.3473503589630127, + "step": 2050, + "unclipped_grad_norm": 0.6627214550971985 + }, + { + "bits_per_byte": 1.9209480080439298, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.331499695777893, + "step": 2060, + "unclipped_grad_norm": 0.6492555737495422 + }, + { + "bits_per_byte": 1.8667327131338907, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2939205169677734, + "step": 2070, + "unclipped_grad_norm": 0.733633279800415 + }, + { + "bits_per_byte": 1.8747223391623733, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2994585037231445, + "step": 2080, + "unclipped_grad_norm": 0.7179054021835327 + }, + { + "bits_per_byte": 1.870721334772701, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2966852188110352, + "step": 2090, + "unclipped_grad_norm": 0.6603640913963318 + }, + { + "bits_per_byte": 1.8546041526297437, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2855136394500732, + "step": 2100, + "unclipped_grad_norm": 0.6752005815505981 + }, + { + "bits_per_byte": 1.9160201891487578, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3280839920043945, + "step": 2110, + "unclipped_grad_norm": 0.6652648448944092 + }, + { + "bits_per_byte": 1.871099696604599, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969474792480469, + "step": 2120, + "unclipped_grad_norm": 0.7032200694084167 + }, + { + "bits_per_byte": 1.8333405616423677, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.2707748413085938, + "step": 2130, + "unclipped_grad_norm": 0.6664465069770813 + }, + { + "bits_per_byte": 1.966146252569232, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3628287315368652, + "step": 2140, + "unclipped_grad_norm": 0.7021564245223999 + }, + { + "bits_per_byte": 1.835432558607463, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2722249031066895, + "step": 2150, + "unclipped_grad_norm": 0.6662127375602722 + }, + { + "bits_per_byte": 1.8787831935145458, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3022732734680176, + "step": 2160, + "unclipped_grad_norm": 0.6593465209007263 + }, + { + "bits_per_byte": 1.9513302911627028, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3525590896606445, + "step": 2170, + "unclipped_grad_norm": 0.721863329410553 + }, + { + "bits_per_byte": 1.8313610413309367, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2694027423858643, + "step": 2180, + "unclipped_grad_norm": 0.661525547504425 + }, + { + "bits_per_byte": 1.8437549710653651, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.2779935598373413, + "step": 2190, + "unclipped_grad_norm": 0.6355888247489929 + }, + { + "bits_per_byte": 1.8767926663134598, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3008935451507568, + "step": 2200, + "unclipped_grad_norm": 0.650397777557373 + }, + { + "bits_per_byte": 1.8339084483555168, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2711684703826904, + "step": 2210, + "unclipped_grad_norm": 0.6309477686882019 + }, + { + "bits_per_byte": 1.8366491638796665, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2730681896209717, + "step": 2220, + "unclipped_grad_norm": 0.6397845149040222 + }, + { + "bits_per_byte": 1.9122785345965865, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3254904747009277, + "step": 2230, + "unclipped_grad_norm": 0.7579862475395203 + }, + { + "bits_per_byte": 1.8301518313127203, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2685645818710327, + "step": 2240, + "unclipped_grad_norm": 0.6486738324165344 + }, + { + "bits_per_byte": 1.854603636681791, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2855132818222046, + "step": 2250, + "unclipped_grad_norm": 0.6761392951011658 + }, + { + "bits_per_byte": 1.8292511581701518, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2679402828216553, + "step": 2260, + "unclipped_grad_norm": 0.6474469900131226 + }, + { + "bits_per_byte": 1.8217264012469514, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.26272451877594, + "step": 2270, + "unclipped_grad_norm": 0.6697631478309631 + }, + { + "bits_per_byte": 1.839495476751446, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.275041103363037, + "step": 2280, + "unclipped_grad_norm": 0.672606885433197 + }, + { + "bits_per_byte": 1.8894086256501519, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.309638261795044, + "step": 2290, + "unclipped_grad_norm": 0.7677630186080933 + }, + { + "bits_per_byte": 1.8035906587308173, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2501537799835205, + "step": 2300, + "unclipped_grad_norm": 0.6455734372138977 + }, + { + "bits_per_byte": 1.834394643309506, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2715054750442505, + "step": 2310, + "unclipped_grad_norm": 0.6606747508049011 + }, + { + "bits_per_byte": 1.8063805612931139, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2520875930786133, + "step": 2320, + "unclipped_grad_norm": 0.634936511516571 + }, + { + "bits_per_byte": 1.8747587994843562, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2994837760925293, + "step": 2330, + "unclipped_grad_norm": 0.67148357629776 + }, + { + "bits_per_byte": 1.7920827836136342, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.242177128791809, + "step": 2340, + "unclipped_grad_norm": 0.6197777390480042 + }, + { + "bits_per_byte": 1.8459198886744261, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2794941663742065, + "step": 2350, + "unclipped_grad_norm": 0.6368024349212646 + }, + { + "bits_per_byte": 1.8583048753110096, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.288078784942627, + "step": 2360, + "unclipped_grad_norm": 0.662507951259613 + }, + { + "bits_per_byte": 1.8678564477746282, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2946994304656982, + "step": 2370, + "unclipped_grad_norm": 0.6344826221466064 + }, + { + "bits_per_byte": 1.8307873072076584, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2690050601959229, + "step": 2380, + "unclipped_grad_norm": 0.680084764957428 + }, + { + "bits_per_byte": 1.888488346485385, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3090003728866577, + "step": 2390, + "unclipped_grad_norm": 0.6502870917320251 + }, + { + "bits_per_byte": 1.8376442554975587, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2737579345703125, + "step": 2400, + "unclipped_grad_norm": 0.6415471434593201 + }, + { + "bits_per_byte": 1.8272317378837208, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.26654052734375, + "step": 2410, + "unclipped_grad_norm": 0.6551245450973511 + }, + { + "bits_per_byte": 1.8084292186301916, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2535076141357422, + "step": 2420, + "unclipped_grad_norm": 0.6829442381858826 + }, + { + "bits_per_byte": 1.8282901191171306, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2672741413116455, + "step": 2430, + "unclipped_grad_norm": 0.6490575075149536 + }, + { + "bits_per_byte": 1.8846385148458211, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3063318729400635, + "step": 2440, + "unclipped_grad_norm": 0.6281237006187439 + }, + { + "bits_per_byte": 1.8146233457836665, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2578010559082031, + "step": 2450, + "unclipped_grad_norm": 0.634181559085846 + }, + { + "bits_per_byte": 1.8319019267679002, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2697776556015015, + "step": 2460, + "unclipped_grad_norm": 0.6390114426612854 + }, + { + "bits_per_byte": 1.8154657168075925, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2583849430084229, + "step": 2470, + "unclipped_grad_norm": 0.6522298455238342 + }, + { + "bits_per_byte": 1.8417133650169726, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.276578426361084, + "step": 2480, + "unclipped_grad_norm": 0.6301186680793762 + }, + { + "bits_per_byte": 1.7981788806561174, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.246402621269226, + "step": 2490, + "unclipped_grad_norm": 0.613478422164917 + }, + { + "bits_per_byte": 1.8422764362158974, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2769687175750732, + "step": 2500, + "unclipped_grad_norm": 0.6680112481117249 + }, + { + "bits_per_byte": 1.7934540012889635, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2431275844573975, + "step": 2510, + "unclipped_grad_norm": 0.6324169635772705 + }, + { + "bits_per_byte": 1.8373082013977726, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2735249996185303, + "step": 2520, + "unclipped_grad_norm": 0.6562070250511169 + }, + { + "bits_per_byte": 1.8465858054985669, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2799557447433472, + "step": 2530, + "unclipped_grad_norm": 0.67587810754776 + }, + { + "bits_per_byte": 1.8180633427662245, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2601854801177979, + "step": 2540, + "unclipped_grad_norm": 0.6210066080093384 + }, + { + "bits_per_byte": 1.8116361791208304, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2557305097579956, + "step": 2550, + "unclipped_grad_norm": 0.656908392906189 + }, + { + "bits_per_byte": 1.8229865181298233, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2635979652404785, + "step": 2560, + "unclipped_grad_norm": 0.6474040746688843 + }, + { + "bits_per_byte": 1.8231736352539618, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2637276649475098, + "step": 2570, + "unclipped_grad_norm": 0.6998929977416992 + }, + { + "bits_per_byte": 1.8511411099719701, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2831132411956787, + "step": 2580, + "unclipped_grad_norm": 0.6996825337409973 + }, + { + "bits_per_byte": 1.805761251767357, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.251658320426941, + "step": 2590, + "unclipped_grad_norm": 0.6730060577392578 + }, + { + "bits_per_byte": 1.7987187341971758, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.246776819229126, + "step": 2600, + "unclipped_grad_norm": 0.6812606453895569 + }, + { + "bits_per_byte": 1.7378126262379698, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2045599222183228, + "step": 2610, + "unclipped_grad_norm": 0.6152759194374084 + }, + { + "bits_per_byte": 1.8067871282797534, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2523694038391113, + "step": 2620, + "unclipped_grad_norm": 0.6381004452705383 + }, + { + "bits_per_byte": 1.815846142431301, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2586486339569092, + "step": 2630, + "unclipped_grad_norm": 0.6937149167060852 + }, + { + "bits_per_byte": 1.6830253170311609, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1665842533111572, + "step": 2640, + "unclipped_grad_norm": 0.5872177481651306 + }, + { + "bits_per_byte": 1.7705318096166665, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2272391319274902, + "step": 2650, + "unclipped_grad_norm": 0.6221135854721069 + }, + { + "bits_per_byte": 1.813369076310924, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2569316625595093, + "step": 2660, + "unclipped_grad_norm": 0.6537755727767944 + }, + { + "bits_per_byte": 1.8137383230623265, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2571876049041748, + "step": 2670, + "unclipped_grad_norm": 0.6585802435874939 + }, + { + "bits_per_byte": 1.7472491423081613, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2111008167266846, + "step": 2680, + "unclipped_grad_norm": 0.6200863122940063 + }, + { + "bits_per_byte": 1.9002486921340347, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3171520233154297, + "step": 2690, + "unclipped_grad_norm": 0.6396089792251587 + }, + { + "bits_per_byte": 1.8072413344606821, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.252684235572815, + "step": 2700, + "unclipped_grad_norm": 0.6738898158073425 + }, + { + "bits_per_byte": 1.7328827435509875, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2011427879333496, + "step": 2710, + "unclipped_grad_norm": 0.7000491619110107 + }, + { + "bits_per_byte": 1.7288494064229527, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1983470916748047, + "step": 2720, + "unclipped_grad_norm": 0.6326305270195007 + }, + { + "bits_per_byte": 1.776559801529408, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2314174175262451, + "step": 2730, + "unclipped_grad_norm": 0.647907555103302 + }, + { + "bits_per_byte": 1.783826928441616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2364546060562134, + "step": 2740, + "unclipped_grad_norm": 0.6690548658370972 + }, + { + "bits_per_byte": 1.7821005665922551, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2352579832077026, + "step": 2750, + "unclipped_grad_norm": 0.664342999458313 + }, + { + "bits_per_byte": 1.8057218677403093, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2516310214996338, + "step": 2760, + "unclipped_grad_norm": 0.6517228484153748 + }, + { + "bits_per_byte": 1.7048344369870736, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.1817011833190918, + "step": 2770, + "unclipped_grad_norm": 0.684334397315979 + }, + { + "bits_per_byte": 1.7173274287283988, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.19036066532135, + "step": 2780, + "unclipped_grad_norm": 0.6013396382331848 + }, + { + "bits_per_byte": 1.804812079517245, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2510004043579102, + "step": 2790, + "unclipped_grad_norm": 0.6293013095855713 + }, + { + "bits_per_byte": 1.8557677872454814, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2863202095031738, + "step": 2800, + "unclipped_grad_norm": 0.7250765562057495 + }, + { + "bits_per_byte": 1.7793906359626097, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.233379602432251, + "step": 2810, + "unclipped_grad_norm": 0.7088303565979004 + }, + { + "bits_per_byte": 1.7744306563117265, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2299416065216064, + "step": 2820, + "unclipped_grad_norm": 0.6359960436820984 + }, + { + "bits_per_byte": 1.8690950668261421, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.295557975769043, + "step": 2830, + "unclipped_grad_norm": 0.6622759103775024 + }, + { + "bits_per_byte": 1.7494551637707785, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2126299142837524, + "step": 2840, + "unclipped_grad_norm": 0.6139791011810303 + }, + { + "bits_per_byte": 1.7661483158114752, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.22420072555542, + "step": 2850, + "unclipped_grad_norm": 0.6314926743507385 + }, + { + "bits_per_byte": 1.7415515290677273, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2071515321731567, + "step": 2860, + "unclipped_grad_norm": 0.664522111415863 + }, + { + "bits_per_byte": 1.8281711071227333, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2671916484832764, + "step": 2870, + "unclipped_grad_norm": 0.6572005152702332 + }, + { + "bits_per_byte": 1.8333617155084239, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2707895040512085, + "step": 2880, + "unclipped_grad_norm": 0.6237366795539856 + }, + { + "bits_per_byte": 1.7312996432497951, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2000454664230347, + "step": 2890, + "unclipped_grad_norm": 0.6335425972938538 + }, + { + "bits_per_byte": 1.7034657990515074, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1807525157928467, + "step": 2900, + "unclipped_grad_norm": 0.6068143844604492 + }, + { + "bits_per_byte": 1.7597350827608012, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2197554111480713, + "step": 2910, + "unclipped_grad_norm": 0.6452642679214478 + }, + { + "bits_per_byte": 1.7579383800072375, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2185100317001343, + "step": 2920, + "unclipped_grad_norm": 0.6799261569976807 + }, + { + "bits_per_byte": 1.7601423376780443, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2200376987457275, + "step": 2930, + "unclipped_grad_norm": 0.6305575370788574 + }, + { + "bits_per_byte": 1.77748111259008, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2320560216903687, + "step": 2940, + "unclipped_grad_norm": 0.6869999766349792 + }, + { + "bits_per_byte": 1.7661192507434795, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2241805791854858, + "step": 2950, + "unclipped_grad_norm": 0.6630687713623047 + }, + { + "bits_per_byte": 1.7912226983766693, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2415809631347656, + "step": 2960, + "unclipped_grad_norm": 0.6753889918327332 + }, + { + "bits_per_byte": 1.7573228540997994, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218083381652832, + "step": 2970, + "unclipped_grad_norm": 0.6314569115638733 + }, + { + "bits_per_byte": 1.7556446473926801, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2169201374053955, + "step": 2980, + "unclipped_grad_norm": 0.685643196105957 + }, + { + "bits_per_byte": 1.778294762511312, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2326200008392334, + "step": 2990, + "unclipped_grad_norm": 0.7002031803131104 + }, + { + "bits_per_byte": 1.7854260231293384, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2375630140304565, + "step": 3000, + "unclipped_grad_norm": 0.6241077184677124 + }, + { + "bits_per_byte": 1.7589776711664014, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2192304134368896, + "step": 3010, + "unclipped_grad_norm": 0.5821192860603333 + }, + { + "bits_per_byte": 1.7213342805282006, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1931380033493042, + "step": 3020, + "unclipped_grad_norm": 0.6891434192657471 + }, + { + "bits_per_byte": 1.8083322204151049, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.2534403800964355, + "step": 3030, + "unclipped_grad_norm": 0.6679258346557617 + }, + { + "bits_per_byte": 1.7077268412092843, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1837060451507568, + "step": 3040, + "unclipped_grad_norm": 0.6403138637542725 + }, + { + "bits_per_byte": 1.7462220619172086, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2103888988494873, + "step": 3050, + "unclipped_grad_norm": 0.6389932632446289 + }, + { + "bits_per_byte": 1.7619969985849488, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2213232517242432, + "step": 3060, + "unclipped_grad_norm": 0.676324188709259 + }, + { + "bits_per_byte": 1.6793940753408436, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.164067268371582, + "step": 3070, + "unclipped_grad_norm": 0.5887739062309265 + }, + { + "bits_per_byte": 1.723725699288448, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.1947956085205078, + "step": 3080, + "unclipped_grad_norm": 0.6651143431663513 + }, + { + "bits_per_byte": 1.792202311555984, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2422599792480469, + "step": 3090, + "unclipped_grad_norm": 0.7115829586982727 + }, + { + "bits_per_byte": 1.6949688242029795, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1748628616333008, + "step": 3100, + "unclipped_grad_norm": 0.648972749710083 + }, + { + "bits_per_byte": 1.7872602181007902, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2388343811035156, + "step": 3110, + "unclipped_grad_norm": 0.6126834154129028 + }, + { + "bits_per_byte": 1.7418012478767801, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2073246240615845, + "step": 3120, + "unclipped_grad_norm": 0.6380596160888672 + }, + { + "bits_per_byte": 1.7994572276999805, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.247288703918457, + "step": 3130, + "unclipped_grad_norm": 0.6736929416656494 + }, + { + "bits_per_byte": 1.7652089465724625, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.2235496044158936, + "step": 3140, + "unclipped_grad_norm": 0.6578004956245422 + }, + { + "bits_per_byte": 1.7230821402089194, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1943495273590088, + "step": 3150, + "unclipped_grad_norm": 0.6959549784660339 + }, + { + "bits_per_byte": 1.745404112429705, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2098219394683838, + "step": 3160, + "unclipped_grad_norm": 0.6303315758705139 + }, + { + "bits_per_byte": 1.6651441088383039, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.154189944267273, + "step": 3170, + "unclipped_grad_norm": 0.6514579057693481 + }, + { + "bits_per_byte": 1.748495672561615, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2119648456573486, + "step": 3180, + "unclipped_grad_norm": 0.6430810689926147 + }, + { + "bits_per_byte": 1.7413002624148166, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.206977367401123, + "step": 3190, + "unclipped_grad_norm": 0.6930623650550842 + }, + { + "bits_per_byte": 1.7004863716079601, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178687334060669, + "step": 3200, + "unclipped_grad_norm": 0.6456758975982666 + }, + { + "bits_per_byte": 1.6959322710131128, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755306720733643, + "step": 3210, + "unclipped_grad_norm": 0.6182776689529419 + }, + { + "bits_per_byte": 1.7081593775762043, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.184005856513977, + "step": 3220, + "unclipped_grad_norm": 0.6259344816207886 + }, + { + "bits_per_byte": 1.8040880325571125, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2504985332489014, + "step": 3230, + "unclipped_grad_norm": 0.6837271451950073 + }, + { + "bits_per_byte": 1.680291136847744, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.164689064025879, + "step": 3240, + "unclipped_grad_norm": 0.6078165173530579 + }, + { + "bits_per_byte": 1.833327662943553, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2707659006118774, + "step": 3250, + "unclipped_grad_norm": 0.6908280849456787 + }, + { + "bits_per_byte": 1.6877314502893708, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1698462963104248, + "step": 3260, + "unclipped_grad_norm": 0.6267281174659729 + }, + { + "bits_per_byte": 1.7156755353668616, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1892156600952148, + "step": 3270, + "unclipped_grad_norm": 0.6226681470870972 + }, + { + "bits_per_byte": 1.77760270432424, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.232140302658081, + "step": 3280, + "unclipped_grad_norm": 0.62666916847229 + }, + { + "bits_per_byte": 1.7647146684338828, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2232069969177246, + "step": 3290, + "unclipped_grad_norm": 0.6992575526237488 + }, + { + "bits_per_byte": 1.6858417049216903, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1685364246368408, + "step": 3300, + "unclipped_grad_norm": 0.6361157894134521 + }, + { + "bits_per_byte": 1.7274259060217614, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.1973603963851929, + "step": 3310, + "unclipped_grad_norm": 0.6828696727752686 + }, + { + "bits_per_byte": 1.7324201102201664, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.200822114944458, + "step": 3320, + "unclipped_grad_norm": 0.6315762996673584 + }, + { + "bits_per_byte": 1.7279007501207935, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1976895332336426, + "step": 3330, + "unclipped_grad_norm": 0.6427719593048096 + }, + { + "bits_per_byte": 1.6987109947031034, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1774567365646362, + "step": 3340, + "unclipped_grad_norm": 0.6303005814552307 + }, + { + "bits_per_byte": 1.7007044456092542, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1788384914398193, + "step": 3350, + "unclipped_grad_norm": 0.6034587025642395 + }, + { + "bits_per_byte": 1.7301495952634756, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1992483139038086, + "step": 3360, + "unclipped_grad_norm": 0.6297745108604431 + }, + { + "bits_per_byte": 1.6889530430584494, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.170693039894104, + "step": 3370, + "unclipped_grad_norm": 0.6365876197814941 + }, + { + "bits_per_byte": 1.7185201284121325, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1911873817443848, + "step": 3380, + "unclipped_grad_norm": 0.612015962600708 + }, + { + "bits_per_byte": 1.7554757604295328, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2168030738830566, + "step": 3390, + "unclipped_grad_norm": 0.7148704528808594 + }, + { + "bits_per_byte": 1.805198180568432, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2512680292129517, + "step": 3400, + "unclipped_grad_norm": 0.7259010672569275 + }, + { + "bits_per_byte": 1.8248889902136676, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2649166584014893, + "step": 3410, + "unclipped_grad_norm": 0.6750312447547913 + }, + { + "bits_per_byte": 1.6908183668897072, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1719859838485718, + "step": 3420, + "unclipped_grad_norm": 0.6068294644355774 + }, + { + "bits_per_byte": 1.6181469257849768, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1216139793395996, + "step": 3430, + "unclipped_grad_norm": 0.6401124596595764 + }, + { + "bits_per_byte": 1.7073711810873, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1834595203399658, + "step": 3440, + "unclipped_grad_norm": 0.6253624558448792 + }, + { + "bits_per_byte": 1.7759955264519272, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.231026291847229, + "step": 3450, + "unclipped_grad_norm": 0.6825960874557495 + }, + { + "bits_per_byte": 1.7035373438342662, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.1808021068572998, + "step": 3460, + "unclipped_grad_norm": 0.6307380795478821 + }, + { + "bits_per_byte": 1.694909318205781, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1748216152191162, + "step": 3470, + "unclipped_grad_norm": 0.6624554395675659 + }, + { + "bits_per_byte": 1.640782594360936, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1373038291931152, + "step": 3480, + "unclipped_grad_norm": 0.6500871777534485 + }, + { + "bits_per_byte": 1.8527976468650806, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2842614650726318, + "step": 3490, + "unclipped_grad_norm": 0.6960679888725281 + }, + { + "bits_per_byte": 1.7002731131208901, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.178539514541626, + "step": 3500, + "unclipped_grad_norm": 0.7090469002723694 + }, + { + "bits_per_byte": 1.7481523951904927, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2117269039154053, + "step": 3510, + "unclipped_grad_norm": 0.6682069301605225 + }, + { + "bits_per_byte": 1.7343439081527179, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.202155590057373, + "step": 3520, + "unclipped_grad_norm": 0.6754779815673828 + }, + { + "bits_per_byte": 1.703965236669613, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1810986995697021, + "step": 3530, + "unclipped_grad_norm": 0.625368595123291 + }, + { + "bits_per_byte": 1.7204380789345546, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.192516803741455, + "step": 3540, + "unclipped_grad_norm": 0.6671978235244751 + }, + { + "bits_per_byte": 1.7171148581719324, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1902133226394653, + "step": 3550, + "unclipped_grad_norm": 0.6819398999214172 + }, + { + "bits_per_byte": 1.6516749435706812, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1448538303375244, + "step": 3560, + "unclipped_grad_norm": 0.6344290375709534 + }, + { + "bits_per_byte": 1.6805387918649863, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.164860725402832, + "step": 3570, + "unclipped_grad_norm": 0.6333562135696411 + }, + { + "bits_per_byte": 1.6952945593437136, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.17508864402771, + "step": 3580, + "unclipped_grad_norm": 0.655963659286499 + }, + { + "bits_per_byte": 1.7344106374212527, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.2022018432617188, + "step": 3590, + "unclipped_grad_norm": 0.7093035578727722 + }, + { + "bits_per_byte": 1.7542852965202604, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2159779071807861, + "step": 3600, + "unclipped_grad_norm": 0.668968915939331 + }, + { + "bits_per_byte": 1.6751485116216442, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1611244678497314, + "step": 3610, + "unclipped_grad_norm": 0.6502929925918579 + }, + { + "bits_per_byte": 1.6413291552253781, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1376826763153076, + "step": 3620, + "unclipped_grad_norm": 0.6599135398864746 + }, + { + "bits_per_byte": 1.6914022479893867, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1723906993865967, + "step": 3630, + "unclipped_grad_norm": 0.6377425789833069 + }, + { + "bits_per_byte": 1.6712699588793862, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1584360599517822, + "step": 3640, + "unclipped_grad_norm": 0.6370565891265869 + }, + { + "bits_per_byte": 1.7179845744373456, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1908161640167236, + "step": 3650, + "unclipped_grad_norm": 0.6391654014587402 + }, + { + "bits_per_byte": 1.6386034021918534, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1357933282852173, + "step": 3660, + "unclipped_grad_norm": 0.651786208152771 + }, + { + "bits_per_byte": 1.7137386667528447, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.187873125076294, + "step": 3670, + "unclipped_grad_norm": 0.64645916223526 + }, + { + "bits_per_byte": 1.651732729741371, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1448938846588135, + "step": 3680, + "unclipped_grad_norm": 0.6200698018074036 + }, + { + "bits_per_byte": 1.6382305438047828, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1355348825454712, + "step": 3690, + "unclipped_grad_norm": 0.6166419386863708 + }, + { + "bits_per_byte": 1.6489116983192684, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.142938494682312, + "step": 3700, + "unclipped_grad_norm": 0.6672006845474243 + }, + { + "bits_per_byte": 1.7161945790071655, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.189575433731079, + "step": 3710, + "unclipped_grad_norm": 0.6522265672683716 + }, + { + "bits_per_byte": 1.6845286173823526, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1676262617111206, + "step": 3720, + "unclipped_grad_norm": 0.6373200416564941 + }, + { + "bits_per_byte": 1.6158851819434799, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1200462579727173, + "step": 3730, + "unclipped_grad_norm": 0.6535916328430176 + }, + { + "bits_per_byte": 1.6679503497524322, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.156135082244873, + "step": 3740, + "unclipped_grad_norm": 0.6719768643379211 + }, + { + "bits_per_byte": 1.6882193650698685, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1701844930648804, + "step": 3750, + "unclipped_grad_norm": 0.6530092358589172 + }, + { + "bits_per_byte": 1.7695160800806709, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2265350818634033, + "step": 3760, + "unclipped_grad_norm": 0.6655569672584534 + }, + { + "bits_per_byte": 1.7330591977507726, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.2012650966644287, + "step": 3770, + "unclipped_grad_norm": 0.6482370495796204 + }, + { + "bits_per_byte": 1.555590988256044, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0782535076141357, + "step": 3780, + "unclipped_grad_norm": 0.6347804665565491 + }, + { + "bits_per_byte": 1.661505987841952, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1516681909561157, + "step": 3790, + "unclipped_grad_norm": 0.6471394896507263 + }, + { + "bits_per_byte": 1.671339095905033, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1584839820861816, + "step": 3800, + "unclipped_grad_norm": 0.6512831449508667 + }, + { + "bits_per_byte": 1.6910716973344282, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17216157913208, + "step": 3810, + "unclipped_grad_norm": 0.66470867395401 + }, + { + "bits_per_byte": 1.662058912064476, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1520514488220215, + "step": 3820, + "unclipped_grad_norm": 0.6363425254821777 + }, + { + "bits_per_byte": 1.7124487968713735, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.186979055404663, + "step": 3830, + "unclipped_grad_norm": 0.6397905945777893 + }, + { + "bits_per_byte": 1.6528970522877124, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1457009315490723, + "step": 3840, + "unclipped_grad_norm": 0.6409087777137756 + }, + { + "bits_per_byte": 1.5958772362900533, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106177806854248, + "step": 3850, + "unclipped_grad_norm": 0.6576514840126038 + }, + { + "bits_per_byte": 1.6597844415068153, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1504749059677124, + "step": 3860, + "unclipped_grad_norm": 0.6520916819572449 + }, + { + "bits_per_byte": 1.7387071080051073, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2051799297332764, + "step": 3870, + "unclipped_grad_norm": 0.6458169221878052 + }, + { + "bits_per_byte": 1.631997548572213, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1312144994735718, + "step": 3880, + "unclipped_grad_norm": 0.6073523163795471 + }, + { + "bits_per_byte": 1.6031751480967658, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.111236333847046, + "step": 3890, + "unclipped_grad_norm": 0.632803201675415 + }, + { + "bits_per_byte": 1.6778582702686387, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1630027294158936, + "step": 3900, + "unclipped_grad_norm": 0.7183825969696045 + }, + { + "bits_per_byte": 1.69528785202033, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1750839948654175, + "step": 3910, + "unclipped_grad_norm": 0.6837812662124634 + }, + { + "bits_per_byte": 1.6365686754494955, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.134382963180542, + "step": 3920, + "unclipped_grad_norm": 0.6382866501808167 + }, + { + "bits_per_byte": 1.6653448125918606, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1543290615081787, + "step": 3930, + "unclipped_grad_norm": 0.6285881400108337 + }, + { + "bits_per_byte": 1.6987961261152804, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1775157451629639, + "step": 3940, + "unclipped_grad_norm": 0.666064977645874 + }, + { + "bits_per_byte": 1.7229675997634448, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.194270133972168, + "step": 3950, + "unclipped_grad_norm": 0.6532164812088013 + }, + { + "bits_per_byte": 1.7031080751377128, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.180504560470581, + "step": 3960, + "unclipped_grad_norm": 0.651078999042511 + }, + { + "bits_per_byte": 1.631284336519085, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1307201385498047, + "step": 3970, + "unclipped_grad_norm": 0.6460844278335571 + }, + { + "bits_per_byte": 1.6744433827531067, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1606357097625732, + "step": 3980, + "unclipped_grad_norm": 0.6715673208236694 + }, + { + "bits_per_byte": 1.6984889651008395, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1773028373718262, + "step": 3990, + "unclipped_grad_norm": 0.7048564553260803 + }, + { + "bits_per_byte": 1.5840455178412955, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0979766845703125, + "step": 4000, + "unclipped_grad_norm": 0.6564667224884033 + }, + { + "bits_per_byte": 1.67799809216379, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1630996465682983, + "step": 4010, + "unclipped_grad_norm": 0.6846939921379089 + }, + { + "bits_per_byte": 1.645402220345762, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1405059099197388, + "step": 4020, + "unclipped_grad_norm": 0.6446395516395569 + }, + { + "bits_per_byte": 1.678588508604202, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1635088920593262, + "step": 4030, + "unclipped_grad_norm": 0.744084894657135 + }, + { + "bits_per_byte": 1.6238756678852169, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1255848407745361, + "step": 4040, + "unclipped_grad_norm": 0.6811469793319702 + }, + { + "bits_per_byte": 1.6502275375810198, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.143850564956665, + "step": 4050, + "unclipped_grad_norm": 0.6219719052314758 + }, + { + "bits_per_byte": 1.655425885185999, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.147453784942627, + "step": 4060, + "unclipped_grad_norm": 0.6728122234344482 + }, + { + "bits_per_byte": 1.6906052803852882, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1718382835388184, + "step": 4070, + "unclipped_grad_norm": 0.6714834570884705 + }, + { + "bits_per_byte": 1.7592499197027172, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2194191217422485, + "step": 4080, + "unclipped_grad_norm": 0.7062271237373352 + }, + { + "bits_per_byte": 1.656125338627058, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.14793860912323, + "step": 4090, + "unclipped_grad_norm": 0.7272074222564697 + }, + { + "bits_per_byte": 1.6359631245358075, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1339632272720337, + "step": 4100, + "unclipped_grad_norm": 0.6811342239379883 + }, + { + "bits_per_byte": 1.6284930580955814, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1287853717803955, + "step": 4110, + "unclipped_grad_norm": 0.6341036558151245 + }, + { + "bits_per_byte": 1.6773249520683131, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1626330614089966, + "step": 4120, + "unclipped_grad_norm": 0.6589056849479675 + }, + { + "bits_per_byte": 1.6258371300183072, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1269444227218628, + "step": 4130, + "unclipped_grad_norm": 0.6739165186882019 + }, + { + "bits_per_byte": 1.688376729195408, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1702935695648193, + "step": 4140, + "unclipped_grad_norm": 0.6829613447189331 + }, + { + "bits_per_byte": 1.6211284170203344, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.123680591583252, + "step": 4150, + "unclipped_grad_norm": 0.6710932850837708 + }, + { + "bits_per_byte": 1.6548134549662765, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.1470292806625366, + "step": 4160, + "unclipped_grad_norm": 0.7125453948974609 + }, + { + "bits_per_byte": 1.6949535177470527, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1748522520065308, + "step": 4170, + "unclipped_grad_norm": 0.6789602041244507 + }, + { + "bits_per_byte": 1.7346840898361244, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.2023913860321045, + "step": 4180, + "unclipped_grad_norm": 0.7571533918380737 + }, + { + "bits_per_byte": 1.6432809863300202, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1390355825424194, + "step": 4190, + "unclipped_grad_norm": 0.6785401701927185 + }, + { + "bits_per_byte": 1.6844624040617706, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1675803661346436, + "step": 4200, + "unclipped_grad_norm": 0.7054091095924377 + }, + { + "bits_per_byte": 1.672555185229284, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1593269109725952, + "step": 4210, + "unclipped_grad_norm": 0.6699897050857544 + }, + { + "bits_per_byte": 1.6983758005165717, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1772243976593018, + "step": 4220, + "unclipped_grad_norm": 0.6806776523590088 + }, + { + "bits_per_byte": 1.6692897506373519, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1570634841918945, + "step": 4230, + "unclipped_grad_norm": 0.6995632648468018 + }, + { + "bits_per_byte": 1.611002250520183, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1166616678237915, + "step": 4240, + "unclipped_grad_norm": 0.6468433737754822 + }, + { + "bits_per_byte": 1.646616933808806, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.141347885131836, + "step": 4250, + "unclipped_grad_norm": 0.6655133962631226 + }, + { + "bits_per_byte": 1.61702336312689, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1208351850509644, + "step": 4260, + "unclipped_grad_norm": 0.6660183072090149 + }, + { + "bits_per_byte": 1.6213850151354217, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1238584518432617, + "step": 4270, + "unclipped_grad_norm": 0.6792552471160889 + }, + { + "bits_per_byte": 1.6175876382043708, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1212263107299805, + "step": 4280, + "unclipped_grad_norm": 0.6596150398254395 + }, + { + "bits_per_byte": 1.673857609844268, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1602296829223633, + "step": 4290, + "unclipped_grad_norm": 0.6736516952514648 + }, + { + "bits_per_byte": 1.5981075073064421, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.1077237129211426, + "step": 4300, + "unclipped_grad_norm": 0.65042644739151 + }, + { + "bits_per_byte": 1.6804373221009774, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.164790391921997, + "step": 4310, + "unclipped_grad_norm": 0.7301046252250671 + }, + { + "bits_per_byte": 1.6671354959526443, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1555702686309814, + "step": 4320, + "unclipped_grad_norm": 0.6921650767326355 + }, + { + "bits_per_byte": 1.6208601240849883, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1234946250915527, + "step": 4330, + "unclipped_grad_norm": 0.6875792145729065 + }, + { + "bits_per_byte": 1.651699193124453, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.144870638847351, + "step": 4340, + "unclipped_grad_norm": 0.6566780209541321 + }, + { + "bits_per_byte": 1.638596522885819, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.1357885599136353, + "step": 4350, + "unclipped_grad_norm": 0.6942516565322876 + }, + { + "bits_per_byte": 1.6271663839268258, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1278657913208008, + "step": 4360, + "unclipped_grad_norm": 0.6855849623680115 + }, + { + "bits_per_byte": 1.6394584999319433, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1363860368728638, + "step": 4370, + "unclipped_grad_norm": 0.7280991673469543 + }, + { + "bits_per_byte": 1.69825386481711, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1771398782730103, + "step": 4380, + "unclipped_grad_norm": 0.6875903010368347 + }, + { + "bits_per_byte": 1.6491371675745496, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.143094778060913, + "step": 4390, + "unclipped_grad_norm": 0.7626568078994751 + }, + { + "bits_per_byte": 1.5979716410122606, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1076295375823975, + "step": 4400, + "unclipped_grad_norm": 0.6716449856758118 + }, + { + "bits_per_byte": 1.6724951632841327, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.159285306930542, + "step": 4410, + "unclipped_grad_norm": 0.7023071646690369 + }, + { + "bits_per_byte": 1.6178187828871304, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1213865280151367, + "step": 4420, + "unclipped_grad_norm": 0.6506969332695007 + }, + { + "bits_per_byte": 1.6647528483075908, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.153918743133545, + "step": 4430, + "unclipped_grad_norm": 0.6848753094673157 + }, + { + "bits_per_byte": 1.604941065955825, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1124603748321533, + "step": 4440, + "unclipped_grad_norm": 0.6678289771080017 + }, + { + "bits_per_byte": 1.6073562183218915, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.114134430885315, + "step": 4450, + "unclipped_grad_norm": 0.6393097639083862 + }, + { + "bits_per_byte": 1.581368607880616, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0961211919784546, + "step": 4460, + "unclipped_grad_norm": 0.6358886957168579 + }, + { + "bits_per_byte": 1.6384745871863573, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1357040405273438, + "step": 4470, + "unclipped_grad_norm": 0.7018935680389404 + }, + { + "bits_per_byte": 1.6524677835911588, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1454033851623535, + "step": 4480, + "unclipped_grad_norm": 0.7296347618103027 + }, + { + "bits_per_byte": 1.6287001252072204, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1289288997650146, + "step": 4490, + "unclipped_grad_norm": 0.6741621494293213 + }, + { + "bits_per_byte": 1.644477125666771, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1398646831512451, + "step": 4500, + "unclipped_grad_norm": 0.656035304069519 + }, + { + "bits_per_byte": 1.6005383100937371, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1094086170196533, + "step": 4510, + "unclipped_grad_norm": 0.6776733994483948 + }, + { + "bits_per_byte": 1.6433757487706455, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.139101266860962, + "step": 4520, + "unclipped_grad_norm": 0.6692283153533936 + }, + { + "bits_per_byte": 1.6564529655769518, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1481657028198242, + "step": 4530, + "unclipped_grad_norm": 0.6900064945220947 + }, + { + "bits_per_byte": 1.7115691056122104, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1863692998886108, + "step": 4540, + "unclipped_grad_norm": 0.7291412353515625 + }, + { + "bits_per_byte": 1.568905197155239, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.087482213973999, + "step": 4550, + "unclipped_grad_norm": 0.6766284108161926 + }, + { + "bits_per_byte": 1.6280190739098037, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1284568309783936, + "step": 4560, + "unclipped_grad_norm": 0.7198469042778015 + }, + { + "bits_per_byte": 1.6109731854521872, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1166415214538574, + "step": 4570, + "unclipped_grad_norm": 0.6274889707565308 + }, + { + "bits_per_byte": 1.5632356170868957, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.083552360534668, + "step": 4580, + "unclipped_grad_norm": 0.7059792280197144 + }, + { + "bits_per_byte": 1.6166451732776426, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1205730438232422, + "step": 4590, + "unclipped_grad_norm": 0.7005748748779297 + }, + { + "bits_per_byte": 1.6071720249028174, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.114006757736206, + "step": 4600, + "unclipped_grad_norm": 0.6841927170753479 + }, + { + "bits_per_byte": 1.66910400937442, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1569347381591797, + "step": 4610, + "unclipped_grad_norm": 0.7007519006729126 + }, + { + "bits_per_byte": 1.6744798430750896, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.160660982131958, + "step": 4620, + "unclipped_grad_norm": 0.7218911051750183 + }, + { + "bits_per_byte": 1.648005693714523, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1423105001449585, + "step": 4630, + "unclipped_grad_norm": 0.7027076482772827 + }, + { + "bits_per_byte": 1.692054922149411, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.172843098640442, + "step": 4640, + "unclipped_grad_norm": 0.6847496628761292 + }, + { + "bits_per_byte": 1.6069941948418252, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1138834953308105, + "step": 4650, + "unclipped_grad_norm": 0.695062518119812 + }, + { + "bits_per_byte": 1.5888770264519845, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1013256311416626, + "step": 4660, + "unclipped_grad_norm": 0.7018181681632996 + }, + { + "bits_per_byte": 1.721095740591454, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1929726600646973, + "step": 4670, + "unclipped_grad_norm": 0.7332665920257568 + }, + { + "bits_per_byte": 1.7135371030860336, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1877334117889404, + "step": 4680, + "unclipped_grad_norm": 0.7111047506332397 + }, + { + "bits_per_byte": 1.6797499074454787, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1643139123916626, + "step": 4690, + "unclipped_grad_norm": 0.7494509816169739 + }, + { + "bits_per_byte": 1.6073170062774949, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1141072511672974, + "step": 4700, + "unclipped_grad_norm": 0.6868770718574524 + }, + { + "bits_per_byte": 1.6766973873753146, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1621980667114258, + "step": 4710, + "unclipped_grad_norm": 0.7650524377822876 + }, + { + "bits_per_byte": 1.6150646527162134, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1194775104522705, + "step": 4720, + "unclipped_grad_norm": 0.7059918642044067 + }, + { + "bits_per_byte": 1.5217877982289587, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0548229217529297, + "step": 4730, + "unclipped_grad_norm": 0.6677312850952148 + }, + { + "bits_per_byte": 1.5772625220912664, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.0932750701904297, + "step": 4740, + "unclipped_grad_norm": 0.6796184182167053 + }, + { + "bits_per_byte": 1.6299009080755444, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1297612190246582, + "step": 4750, + "unclipped_grad_norm": 0.7213169932365417 + }, + { + "bits_per_byte": 1.6096504668844014, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1157246828079224, + "step": 4760, + "unclipped_grad_norm": 0.7036519050598145 + }, + { + "bits_per_byte": 1.5962356481344513, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1064262390136719, + "step": 4770, + "unclipped_grad_norm": 0.7376033663749695 + }, + { + "bits_per_byte": 1.626534347684905, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1274276971817017, + "step": 4780, + "unclipped_grad_norm": 0.7321033477783203 + }, + { + "bits_per_byte": 1.5732460392630163, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0904910564422607, + "step": 4790, + "unclipped_grad_norm": 0.6640080809593201 + }, + { + "bits_per_byte": 1.5751923669228307, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.091840147972107, + "step": 4800, + "unclipped_grad_norm": 0.7126999497413635 + }, + { + "bits_per_byte": 1.637988564215019, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1353671550750732, + "step": 4810, + "unclipped_grad_norm": 0.7521406412124634 + }, + { + "bits_per_byte": 1.57020504203046, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0883831977844238, + "step": 4820, + "unclipped_grad_norm": 0.7331681847572327 + }, + { + "bits_per_byte": 1.6098945102659756, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.115893840789795, + "step": 4830, + "unclipped_grad_norm": 0.7234310507774353 + }, + { + "bits_per_byte": 1.6131886659606018, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1181771755218506, + "step": 4840, + "unclipped_grad_norm": 0.7273361682891846 + }, + { + "bits_per_byte": 1.6277132887565695, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1282448768615723, + "step": 4850, + "unclipped_grad_norm": 0.6802273988723755 + }, + { + "bits_per_byte": 1.5401256203598566, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0675337314605713, + "step": 4860, + "unclipped_grad_norm": 0.6796969771385193 + }, + { + "bits_per_byte": 1.619305056955887, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1224167346954346, + "step": 4870, + "unclipped_grad_norm": 0.7333647608757019 + }, + { + "bits_per_byte": 1.4835698135542255, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.028332233428955, + "step": 4880, + "unclipped_grad_norm": 0.7043541073799133 + }, + { + "bits_per_byte": 1.5411697270332447, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.068257451057434, + "step": 4890, + "unclipped_grad_norm": 0.741724967956543 + }, + { + "bits_per_byte": 1.588994318619873, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1014069318771362, + "step": 4900, + "unclipped_grad_norm": 0.6961487531661987 + }, + { + "bits_per_byte": 1.693890149016768, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1741151809692383, + "step": 4910, + "unclipped_grad_norm": 0.7309879064559937 + }, + { + "bits_per_byte": 1.6231887691776707, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1251087188720703, + "step": 4920, + "unclipped_grad_norm": 0.7229281067848206 + }, + { + "bits_per_byte": 1.5846615596966862, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0984036922454834, + "step": 4930, + "unclipped_grad_norm": 0.6784905791282654 + }, + { + "bits_per_byte": 1.5777433855830787, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0936083793640137, + "step": 4940, + "unclipped_grad_norm": 0.6933380365371704 + }, + { + "bits_per_byte": 1.5189858568811019, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0528807640075684, + "step": 4950, + "unclipped_grad_norm": 0.6876996159553528 + }, + { + "bits_per_byte": 1.5293708572708018, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0600790977478027, + "step": 4960, + "unclipped_grad_norm": 0.7548285126686096 + }, + { + "bits_per_byte": 1.613208615948102, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1181910037994385, + "step": 4970, + "unclipped_grad_norm": 0.7192531824111938 + }, + { + "bits_per_byte": 1.6464662770066503, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1412434577941895, + "step": 4980, + "unclipped_grad_norm": 0.7251392602920532 + }, + { + "bits_per_byte": 1.5932895853251714, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.104384183883667, + "step": 4990, + "unclipped_grad_norm": 0.770077109336853 + }, + { + "bits_per_byte": 1.5394951319617936, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670967102050781, + "step": 5000, + "unclipped_grad_norm": 0.689707338809967 + }, + { + "bits_per_byte": 1.5463551759394092, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0718517303466797, + "step": 5010, + "unclipped_grad_norm": 0.6723923087120056 + }, + { + "bits_per_byte": 1.5950135394174203, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.105579137802124, + "step": 5020, + "unclipped_grad_norm": 0.6902953386306763 + }, + { + "bits_per_byte": 1.575558174021216, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0920937061309814, + "step": 5030, + "unclipped_grad_norm": 0.7085601687431335 + }, + { + "bits_per_byte": 1.5476398863413545, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.072742223739624, + "step": 5040, + "unclipped_grad_norm": 0.7076513767242432 + }, + { + "bits_per_byte": 1.517178147237883, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0516277551651, + "step": 5050, + "unclipped_grad_norm": 0.6860785484313965 + }, + { + "bits_per_byte": 1.5405944450661087, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0678586959838867, + "step": 5060, + "unclipped_grad_norm": 0.6969667673110962 + }, + { + "bits_per_byte": 1.603204729112714, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1112568378448486, + "step": 5070, + "unclipped_grad_norm": 0.6925143003463745 + }, + { + "bits_per_byte": 1.568753508457178, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.0873770713806152, + "step": 5080, + "unclipped_grad_norm": 0.7227545380592346 + }, + { + "bits_per_byte": 1.5571398640097143, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.07932710647583, + "step": 5090, + "unclipped_grad_norm": 0.7346438765525818 + }, + { + "bits_per_byte": 1.5756090808858714, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0921289920806885, + "step": 5100, + "unclipped_grad_norm": 0.687656581401825 + }, + { + "bits_per_byte": 1.4874313400140482, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0310088396072388, + "step": 5110, + "unclipped_grad_norm": 0.7022747993469238 + }, + { + "bits_per_byte": 1.5848280389027214, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0985190868377686, + "step": 5120, + "unclipped_grad_norm": 0.6745842695236206 + }, + { + "bits_per_byte": 1.5742394110543998, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.091179609298706, + "step": 5130, + "unclipped_grad_norm": 0.6994429230690002 + }, + { + "bits_per_byte": 1.5480249554966363, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0730091333389282, + "step": 5140, + "unclipped_grad_norm": 0.7030988335609436 + }, + { + "bits_per_byte": 1.639304919424723, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.136279582977295, + "step": 5150, + "unclipped_grad_norm": 0.7419465184211731 + }, + { + "bits_per_byte": 1.5116879450743894, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0478222370147705, + "step": 5160, + "unclipped_grad_norm": 0.7094154953956604 + }, + { + "bits_per_byte": 1.6830882626813766, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1666278839111328, + "step": 5170, + "unclipped_grad_norm": 0.7866556644439697 + }, + { + "bits_per_byte": 1.5735232752962072, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0906832218170166, + "step": 5180, + "unclipped_grad_norm": 0.7095000743865967 + }, + { + "bits_per_byte": 1.51639820591622, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0510871410369873, + "step": 5190, + "unclipped_grad_norm": 0.701707124710083 + }, + { + "bits_per_byte": 1.549430569702138, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0739834308624268, + "step": 5200, + "unclipped_grad_norm": 0.7002435922622681 + }, + { + "bits_per_byte": 1.5581800150821326, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0800480842590332, + "step": 5210, + "unclipped_grad_norm": 0.6994377374649048 + }, + { + "bits_per_byte": 1.6434304392536199, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.139139175415039, + "step": 5220, + "unclipped_grad_norm": 0.764930009841919 + }, + { + "bits_per_byte": 1.5617875231666307, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.0825486183166504, + "step": 5230, + "unclipped_grad_norm": 0.7441672086715698 + }, + { + "bits_per_byte": 1.5466518460121477, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0720573663711548, + "step": 5240, + "unclipped_grad_norm": 0.688214123249054 + }, + { + "bits_per_byte": 1.5248055778036484, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0569146871566772, + "step": 5250, + "unclipped_grad_norm": 0.6841322779655457 + }, + { + "bits_per_byte": 1.5447307998020101, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0707257986068726, + "step": 5260, + "unclipped_grad_norm": 0.694692850112915 + }, + { + "bits_per_byte": 1.539800573149726, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0673084259033203, + "step": 5270, + "unclipped_grad_norm": 0.6789148449897766 + }, + { + "bits_per_byte": 1.5548948024853513, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0777709484100342, + "step": 5280, + "unclipped_grad_norm": 0.7349528074264526 + }, + { + "bits_per_byte": 1.5569278094012005, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.079180121421814, + "step": 5290, + "unclipped_grad_norm": 0.7407549023628235 + }, + { + "bits_per_byte": 1.5483455311578447, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0732313394546509, + "step": 5300, + "unclipped_grad_norm": 0.7291215658187866 + }, + { + "bits_per_byte": 1.5444855525418797, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0705558061599731, + "step": 5310, + "unclipped_grad_norm": 0.7253612279891968 + }, + { + "bits_per_byte": 1.5505493168460005, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0747588872909546, + "step": 5320, + "unclipped_grad_norm": 0.7087648510932922 + }, + { + "bits_per_byte": 1.5517671259967603, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0756030082702637, + "step": 5330, + "unclipped_grad_norm": 0.7454737424850464 + }, + { + "bits_per_byte": 1.5001372462771416, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.039815902709961, + "step": 5340, + "unclipped_grad_norm": 0.7009119391441345 + }, + { + "bits_per_byte": 1.5857225206698589, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0991390943527222, + "step": 5350, + "unclipped_grad_norm": 0.7474352121353149 + }, + { + "bits_per_byte": 1.4946740453898335, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0360291004180908, + "step": 5360, + "unclipped_grad_norm": 0.7168655395507812 + }, + { + "bits_per_byte": 1.4766953230339372, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0235671997070312, + "step": 5370, + "unclipped_grad_norm": 0.7242471575737 + }, + { + "bits_per_byte": 1.5813711876203789, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.0961229801177979, + "step": 5380, + "unclipped_grad_norm": 0.7494375109672546 + }, + { + "bits_per_byte": 1.5460944502407012, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0716710090637207, + "step": 5390, + "unclipped_grad_norm": 0.7461474537849426 + }, + { + "bits_per_byte": 1.5453937929210861, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0711853504180908, + "step": 5400, + "unclipped_grad_norm": 0.7291334271430969 + }, + { + "bits_per_byte": 1.5672311180317404, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0863218307495117, + "step": 5410, + "unclipped_grad_norm": 0.720815896987915 + }, + { + "bits_per_byte": 1.541436472124733, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0684423446655273, + "step": 5420, + "unclipped_grad_norm": 0.743694543838501 + }, + { + "bits_per_byte": 1.5782952779096975, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0939909219741821, + "step": 5430, + "unclipped_grad_norm": 0.7555011510848999 + }, + { + "bits_per_byte": 1.508865365808429, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.045865774154663, + "step": 5440, + "unclipped_grad_norm": 0.7259441018104553 + }, + { + "bits_per_byte": 1.5083157092562716, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0454847812652588, + "step": 5450, + "unclipped_grad_norm": 0.7505731582641602 + }, + { + "bits_per_byte": 1.5045496331676778, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0428743362426758, + "step": 5460, + "unclipped_grad_norm": 0.7536665201187134 + }, + { + "bits_per_byte": 1.5449808625763646, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0708991289138794, + "step": 5470, + "unclipped_grad_norm": 0.7403784394264221 + }, + { + "bits_per_byte": 1.550094938682421, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0744439363479614, + "step": 5480, + "unclipped_grad_norm": 0.725517213344574 + }, + { + "bits_per_byte": 1.5697909078071826, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0880961418151855, + "step": 5490, + "unclipped_grad_norm": 0.776546061038971 + }, + { + "bits_per_byte": 1.546138821764624, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0717017650604248, + "step": 5500, + "unclipped_grad_norm": 0.7528209686279297 + }, + { + "bits_per_byte": 1.6319261757721049, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1311650276184082, + "step": 5510, + "unclipped_grad_norm": 0.8170714974403381 + }, + { + "bits_per_byte": 1.5554193495704827, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.078134536743164, + "step": 5520, + "unclipped_grad_norm": 0.7434027194976807 + }, + { + "bits_per_byte": 1.5251368163892103, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.057144284248352, + "step": 5530, + "unclipped_grad_norm": 0.7136013507843018 + }, + { + "bits_per_byte": 1.586380354309409, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.099595069885254, + "step": 5540, + "unclipped_grad_norm": 0.7905043959617615 + }, + { + "bits_per_byte": 1.5182255215816374, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0523537397384644, + "step": 5550, + "unclipped_grad_norm": 0.7156078815460205 + }, + { + "bits_per_byte": 1.5143791295950908, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0496876239776611, + "step": 5560, + "unclipped_grad_norm": 0.7718254923820496 + }, + { + "bits_per_byte": 1.4761380992451418, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0231809616088867, + "step": 5570, + "unclipped_grad_norm": 0.7351405620574951 + }, + { + "bits_per_byte": 1.5698151573609542, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0881129503250122, + "step": 5580, + "unclipped_grad_norm": 0.7265479564666748 + }, + { + "bits_per_byte": 1.5069491351125157, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0445375442504883, + "step": 5590, + "unclipped_grad_norm": 0.7370763421058655 + }, + { + "bits_per_byte": 1.5604274843636077, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0816059112548828, + "step": 5600, + "unclipped_grad_norm": 0.7772327661514282 + }, + { + "bits_per_byte": 1.5487797873512732, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0735323429107666, + "step": 5610, + "unclipped_grad_norm": 0.6945621371269226 + }, + { + "bits_per_byte": 1.5643851491252627, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0843491554260254, + "step": 5620, + "unclipped_grad_norm": 0.772422194480896 + }, + { + "bits_per_byte": 1.5390314667350675, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0667753219604492, + "step": 5630, + "unclipped_grad_norm": 0.7249912023544312 + }, + { + "bits_per_byte": 1.5236104703628028, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0560863018035889, + "step": 5640, + "unclipped_grad_norm": 0.7168114185333252 + }, + { + "bits_per_byte": 1.5290047062071148, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0598253011703491, + "step": 5650, + "unclipped_grad_norm": 0.7599905729293823 + }, + { + "bits_per_byte": 1.6052637054088437, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1126840114593506, + "step": 5660, + "unclipped_grad_norm": 0.8106476068496704 + }, + { + "bits_per_byte": 1.5110031101586539, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0473475456237793, + "step": 5670, + "unclipped_grad_norm": 0.7452697157859802 + }, + { + "bits_per_byte": 1.532641279359609, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0623459815979004, + "step": 5680, + "unclipped_grad_norm": 0.7387374043464661 + }, + { + "bits_per_byte": 1.6103865526300942, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1162348985671997, + "step": 5690, + "unclipped_grad_norm": 0.7787995338439941 + }, + { + "bits_per_byte": 1.5250291552497701, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0570696592330933, + "step": 5700, + "unclipped_grad_norm": 0.7521193027496338 + }, + { + "bits_per_byte": 1.5451155249919901, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0709924697875977, + "step": 5710, + "unclipped_grad_norm": 0.7714623808860779 + }, + { + "bits_per_byte": 1.5521905472831843, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0758965015411377, + "step": 5720, + "unclipped_grad_norm": 0.7254540324211121 + }, + { + "bits_per_byte": 1.554917504195265, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0777866840362549, + "step": 5730, + "unclipped_grad_norm": 0.7474860548973083 + }, + { + "bits_per_byte": 1.5468128217733552, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0721689462661743, + "step": 5740, + "unclipped_grad_norm": 0.7887082099914551 + }, + { + "bits_per_byte": 1.541234736475271, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0683025121688843, + "step": 5750, + "unclipped_grad_norm": 0.7614914774894714 + }, + { + "bits_per_byte": 1.5622339901282707, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0828580856323242, + "step": 5760, + "unclipped_grad_norm": 0.7596094608306885 + }, + { + "bits_per_byte": 1.5113494832174914, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0475876331329346, + "step": 5770, + "unclipped_grad_norm": 0.7298161387443542 + }, + { + "bits_per_byte": 1.4968841944360716, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.037561058998108, + "step": 5780, + "unclipped_grad_norm": 0.7402821779251099 + }, + { + "bits_per_byte": 1.524952966935438, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0570168495178223, + "step": 5790, + "unclipped_grad_norm": 0.7289884686470032 + }, + { + "bits_per_byte": 1.4886130328081264, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0318279266357422, + "step": 5800, + "unclipped_grad_norm": 0.7239735722541809 + }, + { + "bits_per_byte": 1.5780909625204724, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0938493013381958, + "step": 5810, + "unclipped_grad_norm": 0.7893304824829102 + }, + { + "bits_per_byte": 1.5071924905634866, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0447062253952026, + "step": 5820, + "unclipped_grad_norm": 0.7217836976051331 + }, + { + "bits_per_byte": 1.516913465938205, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0514442920684814, + "step": 5830, + "unclipped_grad_norm": 0.7600290775299072 + }, + { + "bits_per_byte": 1.5193552756151552, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0531368255615234, + "step": 5840, + "unclipped_grad_norm": 0.7521220445632935 + }, + { + "bits_per_byte": 1.4707125625583728, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0194202661514282, + "step": 5850, + "unclipped_grad_norm": 0.7758342027664185 + }, + { + "bits_per_byte": 1.5340661556220072, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0633336305618286, + "step": 5860, + "unclipped_grad_norm": 0.7845423817634583 + }, + { + "bits_per_byte": 1.5391652692374387, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0668680667877197, + "step": 5870, + "unclipped_grad_norm": 0.7568844556808472 + }, + { + "bits_per_byte": 1.550266749350633, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0745630264282227, + "step": 5880, + "unclipped_grad_norm": 0.7594192028045654 + }, + { + "bits_per_byte": 1.5444004211297027, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0704967975616455, + "step": 5890, + "unclipped_grad_norm": 0.722252607345581 + }, + { + "bits_per_byte": 1.5452520792167752, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071087121963501, + "step": 5900, + "unclipped_grad_norm": 0.7907101511955261 + }, + { + "bits_per_byte": 1.554658154357764, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0776069164276123, + "step": 5910, + "unclipped_grad_norm": 0.7964528799057007 + }, + { + "bits_per_byte": 1.4817756905404247, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0270886421203613, + "step": 5920, + "unclipped_grad_norm": 0.7341460585594177 + }, + { + "bits_per_byte": 1.5817024262059407, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.0963525772094727, + "step": 5930, + "unclipped_grad_norm": 0.8211336731910706 + }, + { + "bits_per_byte": 1.4638614616786019, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0146714448928833, + "step": 5940, + "unclipped_grad_norm": 0.7324587106704712 + }, + { + "bits_per_byte": 1.5159049596735457, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0507452487945557, + "step": 5950, + "unclipped_grad_norm": 0.7514658570289612 + }, + { + "bits_per_byte": 1.553970911684916, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0771305561065674, + "step": 5960, + "unclipped_grad_norm": 0.8067389130592346 + }, + { + "bits_per_byte": 1.5228774803048255, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0555782318115234, + "step": 5970, + "unclipped_grad_norm": 0.8027852177619934 + }, + { + "bits_per_byte": 1.564259601790133, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0842621326446533, + "step": 5980, + "unclipped_grad_norm": 0.7799409031867981 + }, + { + "bits_per_byte": 1.5324249251848234, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0621960163116455, + "step": 5990, + "unclipped_grad_norm": 0.7881770133972168 + }, + { + "bits_per_byte": 1.480203081180935, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.025998592376709, + "step": 6000, + "unclipped_grad_norm": 0.7544389367103577 + }, + { + "bits_per_byte": 1.5204526969103107, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.053897500038147, + "step": 6010, + "unclipped_grad_norm": 0.7601525783538818 + }, + { + "bits_per_byte": 1.4842316027947455, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0287909507751465, + "step": 6020, + "unclipped_grad_norm": 0.7522627711296082 + }, + { + "bits_per_byte": 1.524829999340071, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0569316148757935, + "step": 6030, + "unclipped_grad_norm": 0.7871755361557007 + }, + { + "bits_per_byte": 1.5740971814021363, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0910810232162476, + "step": 6040, + "unclipped_grad_norm": 0.8342390656471252 + }, + { + "bits_per_byte": 1.475741851217554, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0229063034057617, + "step": 6050, + "unclipped_grad_norm": 0.7723444104194641 + }, + { + "bits_per_byte": 1.4753325325085003, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0226225852966309, + "step": 6060, + "unclipped_grad_norm": 0.769443154335022 + }, + { + "bits_per_byte": 1.5574803896584226, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0795631408691406, + "step": 6070, + "unclipped_grad_norm": 0.8035310506820679 + }, + { + "bits_per_byte": 1.4788967009649812, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0250930786132812, + "step": 6080, + "unclipped_grad_norm": 0.7756995558738708 + }, + { + "bits_per_byte": 1.4841172343319218, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0287116765975952, + "step": 6090, + "unclipped_grad_norm": 0.7475231289863586 + }, + { + "bits_per_byte": 1.4853364193438883, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0295567512512207, + "step": 6100, + "unclipped_grad_norm": 0.7613393664360046 + }, + { + "bits_per_byte": 1.5560739155396666, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0785882472991943, + "step": 6110, + "unclipped_grad_norm": 0.8045461177825928 + }, + { + "bits_per_byte": 1.548306147130797, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0732040405273438, + "step": 6120, + "unclipped_grad_norm": 0.8037658333778381 + }, + { + "bits_per_byte": 1.482889794152714, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0278608798980713, + "step": 6130, + "unclipped_grad_norm": 0.7671685814857483 + }, + { + "bits_per_byte": 1.4915240111566304, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0338456630706787, + "step": 6140, + "unclipped_grad_norm": 0.7935377359390259 + }, + { + "bits_per_byte": 1.5420002312542613, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0688331127166748, + "step": 6150, + "unclipped_grad_norm": 0.7919352650642395 + }, + { + "bits_per_byte": 1.4144471484587233, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.9804200530052185, + "step": 6160, + "unclipped_grad_norm": 0.7595162987709045 + }, + { + "bits_per_byte": 1.5291687776560379, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0599390268325806, + "step": 6170, + "unclipped_grad_norm": 0.7883366942405701 + }, + { + "bits_per_byte": 1.4831051164315943, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0280101299285889, + "step": 6180, + "unclipped_grad_norm": 0.7718276977539062 + }, + { + "bits_per_byte": 1.4948415564917739, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0361452102661133, + "step": 6190, + "unclipped_grad_norm": 0.7770188450813293 + }, + { + "bits_per_byte": 1.5074203675758797, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0448641777038574, + "step": 6200, + "unclipped_grad_norm": 0.7592265009880066 + }, + { + "bits_per_byte": 1.5232448352470684, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.055832862854004, + "step": 6210, + "unclipped_grad_norm": 0.8102302551269531 + }, + { + "bits_per_byte": 1.5164056011702072, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.051092267036438, + "step": 6220, + "unclipped_grad_norm": 0.787604033946991 + }, + { + "bits_per_byte": 1.4586058438508853, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.011028528213501, + "step": 6230, + "unclipped_grad_norm": 0.7935925126075745 + }, + { + "bits_per_byte": 1.4934343944424144, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0351698398590088, + "step": 6240, + "unclipped_grad_norm": 0.8212590217590332 + }, + { + "bits_per_byte": 1.5503186881111937, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.074599027633667, + "step": 6250, + "unclipped_grad_norm": 0.8379690647125244 + }, + { + "bits_per_byte": 1.403218659157867, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9726370573043823, + "step": 6260, + "unclipped_grad_norm": 0.7395487427711487 + }, + { + "bits_per_byte": 1.5229197880369378, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.055607557296753, + "step": 6270, + "unclipped_grad_norm": 0.8249105215072632 + }, + { + "bits_per_byte": 1.509112332895068, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.046036958694458, + "step": 6280, + "unclipped_grad_norm": 0.7966778874397278 + }, + { + "bits_per_byte": 1.4545608119025921, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0082247257232666, + "step": 6290, + "unclipped_grad_norm": 0.7541455030441284 + }, + { + "bits_per_byte": 1.5209793077872527, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0542625188827515, + "step": 6300, + "unclipped_grad_norm": 0.8022943139076233 + }, + { + "bits_per_byte": 1.4739374092447013, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.021655559539795, + "step": 6310, + "unclipped_grad_norm": 0.7839627861976624 + }, + { + "bits_per_byte": 1.4204840834693098, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.98460453748703, + "step": 6320, + "unclipped_grad_norm": 0.7670372128486633 + }, + { + "bits_per_byte": 1.5195884840897251, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0532984733581543, + "step": 6330, + "unclipped_grad_norm": 0.7843371033668518 + }, + { + "bits_per_byte": 1.4664117924082465, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0164391994476318, + "step": 6340, + "unclipped_grad_norm": 0.7673349380493164 + }, + { + "bits_per_byte": 1.577301734135663, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0933022499084473, + "step": 6350, + "unclipped_grad_norm": 0.8766481280326843 + }, + { + "bits_per_byte": 1.4611757806027283, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0128098726272583, + "step": 6360, + "unclipped_grad_norm": 0.8388718962669373 + }, + { + "bits_per_byte": 1.4358727471032122, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9952711462974548, + "step": 6370, + "unclipped_grad_norm": 0.775425136089325 + }, + { + "bits_per_byte": 1.4795478272811478, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0255444049835205, + "step": 6380, + "unclipped_grad_norm": 0.7803093194961548 + }, + { + "bits_per_byte": 1.5296866174177859, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.060297966003418, + "step": 6390, + "unclipped_grad_norm": 0.8152122497558594 + }, + { + "bits_per_byte": 1.5162291469704219, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0509699583053589, + "step": 6400, + "unclipped_grad_norm": 0.792400598526001 + }, + { + "bits_per_byte": 1.5084371290077807, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0455689430236816, + "step": 6410, + "unclipped_grad_norm": 0.8240766525268555 + }, + { + "bits_per_byte": 1.4750114408993396, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0224000215530396, + "step": 6420, + "unclipped_grad_norm": 0.853355348110199 + }, + { + "bits_per_byte": 1.5300785658791023, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0605696439743042, + "step": 6430, + "unclipped_grad_norm": 0.8168259859085083 + }, + { + "bits_per_byte": 1.4651398087224652, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0155575275421143, + "step": 6440, + "unclipped_grad_norm": 0.8302746415138245 + }, + { + "bits_per_byte": 1.53347470728569, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0629236698150635, + "step": 6450, + "unclipped_grad_norm": 0.8165877461433411 + }, + { + "bits_per_byte": 1.585776179256928, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.099176287651062, + "step": 6460, + "unclipped_grad_norm": 0.8728512525558472 + }, + { + "bits_per_byte": 1.5696127337808887, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.087972640991211, + "step": 6470, + "unclipped_grad_norm": 0.8505071401596069 + }, + { + "bits_per_byte": 1.4561444281517368, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0093224048614502, + "step": 6480, + "unclipped_grad_norm": 0.7639529705047607 + }, + { + "bits_per_byte": 1.5541174409034513, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0772321224212646, + "step": 6490, + "unclipped_grad_norm": 0.8572817444801331 + }, + { + "bits_per_byte": 1.5173276001614828, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0517313480377197, + "step": 6500, + "unclipped_grad_norm": 0.825986921787262 + }, + { + "bits_per_byte": 1.4589312350263177, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.011254072189331, + "step": 6510, + "unclipped_grad_norm": 0.7764277458190918 + }, + { + "bits_per_byte": 1.4310413244838487, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9919222593307495, + "step": 6520, + "unclipped_grad_norm": 0.7579286098480225 + }, + { + "bits_per_byte": 1.4675702675444584, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.017242193222046, + "step": 6530, + "unclipped_grad_norm": 0.8082018494606018 + }, + { + "bits_per_byte": 1.5128018767040279, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.048594355583191, + "step": 6540, + "unclipped_grad_norm": 0.8025459051132202 + }, + { + "bits_per_byte": 1.4392037930764485, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9975800514221191, + "step": 6550, + "unclipped_grad_norm": 0.790289044380188 + }, + { + "bits_per_byte": 1.527554032547087, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0588197708129883, + "step": 6560, + "unclipped_grad_norm": 0.818975567817688 + }, + { + "bits_per_byte": 1.47142164702788, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.019911766052246, + "step": 6570, + "unclipped_grad_norm": 0.8489425778388977 + }, + { + "bits_per_byte": 1.514791200026558, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0499732494354248, + "step": 6580, + "unclipped_grad_norm": 0.8219936490058899 + }, + { + "bits_per_byte": 1.5010776474120593, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0404677391052246, + "step": 6590, + "unclipped_grad_norm": 0.7910152077674866 + }, + { + "bits_per_byte": 1.4169470882716653, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9821528792381287, + "step": 6600, + "unclipped_grad_norm": 0.7658081650733948 + }, + { + "bits_per_byte": 1.448498423459678, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0040225982666016, + "step": 6610, + "unclipped_grad_norm": 0.7563595771789551 + }, + { + "bits_per_byte": 1.4411885588587308, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9989557862281799, + "step": 6620, + "unclipped_grad_norm": 0.8031777143478394 + }, + { + "bits_per_byte": 1.471761484745985, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0201473236083984, + "step": 6630, + "unclipped_grad_norm": 0.802291989326477 + }, + { + "bits_per_byte": 1.511551906797557, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0477279424667358, + "step": 6640, + "unclipped_grad_norm": 0.8319684863090515 + }, + { + "bits_per_byte": 1.5027432993856658, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.041622281074524, + "step": 6650, + "unclipped_grad_norm": 0.7944393754005432 + }, + { + "bits_per_byte": 1.5208112807373597, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0541460514068604, + "step": 6660, + "unclipped_grad_norm": 0.7983800768852234 + }, + { + "bits_per_byte": 1.499563684136514, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.039418339729309, + "step": 6670, + "unclipped_grad_norm": 0.8299691677093506 + }, + { + "bits_per_byte": 1.4686320884308854, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0179781913757324, + "step": 6680, + "unclipped_grad_norm": 0.8146572113037109 + }, + { + "bits_per_byte": 1.4999098852127009, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0396583080291748, + "step": 6690, + "unclipped_grad_norm": 0.8203337788581848 + }, + { + "bits_per_byte": 1.5587066259590745, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0804131031036377, + "step": 6700, + "unclipped_grad_norm": 0.8495987057685852 + }, + { + "bits_per_byte": 1.5200210204566451, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0535982847213745, + "step": 6710, + "unclipped_grad_norm": 0.7851044535636902 + }, + { + "bits_per_byte": 1.5404384568017762, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0677505731582642, + "step": 6720, + "unclipped_grad_norm": 0.8347281217575073 + }, + { + "bits_per_byte": 1.4888245714686876, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0319745540618896, + "step": 6730, + "unclipped_grad_norm": 0.8212841153144836 + }, + { + "bits_per_byte": 1.4304995791336308, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9915467500686646, + "step": 6740, + "unclipped_grad_norm": 0.7957568168640137 + }, + { + "bits_per_byte": 1.5212219753076202, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0544307231903076, + "step": 6750, + "unclipped_grad_norm": 0.7967842817306519 + }, + { + "bits_per_byte": 1.4749558905031108, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0223615169525146, + "step": 6760, + "unclipped_grad_norm": 0.8139022588729858 + }, + { + "bits_per_byte": 1.3835263876600992, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9589874148368835, + "step": 6770, + "unclipped_grad_norm": 0.785581648349762 + }, + { + "bits_per_byte": 1.4903192726873364, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0330106019973755, + "step": 6780, + "unclipped_grad_norm": 0.8513718843460083 + }, + { + "bits_per_byte": 1.4442869123053497, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.001103401184082, + "step": 6790, + "unclipped_grad_norm": 0.8081458210945129 + }, + { + "bits_per_byte": 1.5181372944817448, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0522925853729248, + "step": 6800, + "unclipped_grad_norm": 0.8491948843002319 + }, + { + "bits_per_byte": 1.5056836867674672, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0436604022979736, + "step": 6810, + "unclipped_grad_norm": 0.8421627879142761 + }, + { + "bits_per_byte": 1.5049580919634769, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0431574583053589, + "step": 6820, + "unclipped_grad_norm": 0.8799064755439758 + }, + { + "bits_per_byte": 1.4726698971078425, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0207769870758057, + "step": 6830, + "unclipped_grad_norm": 0.8110744953155518 + }, + { + "bits_per_byte": 1.4402100635666468, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9982775449752808, + "step": 6840, + "unclipped_grad_norm": 0.8030053973197937 + }, + { + "bits_per_byte": 1.4826992373755579, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.027728796005249, + "step": 6850, + "unclipped_grad_norm": 0.8305994868278503 + }, + { + "bits_per_byte": 1.4689192994578262, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0181772708892822, + "step": 6860, + "unclipped_grad_norm": 0.8366835117340088 + }, + { + "bits_per_byte": 1.419515391188325, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9839330911636353, + "step": 6870, + "unclipped_grad_norm": 0.8165588974952698 + }, + { + "bits_per_byte": 1.4317985640955977, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9924471378326416, + "step": 6880, + "unclipped_grad_norm": 0.7981967926025391 + }, + { + "bits_per_byte": 1.426708565560662, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9889190196990967, + "step": 6890, + "unclipped_grad_norm": 0.8219130635261536 + }, + { + "bits_per_byte": 1.544462506866664, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0705398321151733, + "step": 6900, + "unclipped_grad_norm": 0.8717687129974365 + }, + { + "bits_per_byte": 1.4952656657088017, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0364391803741455, + "step": 6910, + "unclipped_grad_norm": 0.8866252899169922 + }, + { + "bits_per_byte": 1.4499126367977229, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.0050028562545776, + "step": 6920, + "unclipped_grad_norm": 0.8052312135696411 + }, + { + "bits_per_byte": 1.437854245215128, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9966446161270142, + "step": 6930, + "unclipped_grad_norm": 0.8604745268821716 + }, + { + "bits_per_byte": 1.4202836376897292, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9844655990600586, + "step": 6940, + "unclipped_grad_norm": 0.7888393998146057 + }, + { + "bits_per_byte": 1.4300773617257627, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9912540912628174, + "step": 6950, + "unclipped_grad_norm": 0.8240157961845398 + }, + { + "bits_per_byte": 1.4910837355704216, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0335404872894287, + "step": 6960, + "unclipped_grad_norm": 0.833236575126648 + }, + { + "bits_per_byte": 1.496710663941351, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0374407768249512, + "step": 6970, + "unclipped_grad_norm": 0.828235387802124 + }, + { + "bits_per_byte": 1.4445393828368163, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0012784004211426, + "step": 6980, + "unclipped_grad_norm": 0.8125858902931213 + }, + { + "bits_per_byte": 1.4867339503647994, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0305254459381104, + "step": 6990, + "unclipped_grad_norm": 0.8594081401824951 + }, + { + "bits_per_byte": 1.533073471761227, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0626455545425415, + "step": 7000, + "unclipped_grad_norm": 0.8734340071678162 + }, + { + "bits_per_byte": 1.532040715942796, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.061929702758789, + "step": 7010, + "unclipped_grad_norm": 0.8863697648048401 + }, + { + "bits_per_byte": 1.459764662952399, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0118317604064941, + "step": 7020, + "unclipped_grad_norm": 0.8206458687782288 + }, + { + "bits_per_byte": 1.5177566968753855, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.052028775215149, + "step": 7030, + "unclipped_grad_norm": 0.8729474544525146 + }, + { + "bits_per_byte": 1.50918250181662, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0460855960845947, + "step": 7040, + "unclipped_grad_norm": 0.8456775546073914 + }, + { + "bits_per_byte": 1.4815705152379453, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0269464254379272, + "step": 7050, + "unclipped_grad_norm": 0.8360646367073059 + }, + { + "bits_per_byte": 1.4998292253494463, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0396023988723755, + "step": 7060, + "unclipped_grad_norm": 0.8408570289611816 + }, + { + "bits_per_byte": 1.453465282416596, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0074653625488281, + "step": 7070, + "unclipped_grad_norm": 0.8465698957443237 + }, + { + "bits_per_byte": 1.431578598285144, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9922946691513062, + "step": 7080, + "unclipped_grad_norm": 0.8291451930999756 + }, + { + "bits_per_byte": 1.4576069686146742, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.01033616065979, + "step": 7090, + "unclipped_grad_norm": 0.8671156764030457 + }, + { + "bits_per_byte": 1.4654204844086733, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0157520771026611, + "step": 7100, + "unclipped_grad_norm": 0.831190288066864 + }, + { + "bits_per_byte": 1.478559958934592, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0248596668243408, + "step": 7110, + "unclipped_grad_norm": 0.8307176828384399 + }, + { + "bits_per_byte": 1.5234764958777807, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0559934377670288, + "step": 7120, + "unclipped_grad_norm": 0.8790873289108276 + }, + { + "bits_per_byte": 1.486035012871693, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.030040979385376, + "step": 7130, + "unclipped_grad_norm": 0.8940773606300354 + }, + { + "bits_per_byte": 1.4639995637472447, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0147671699523926, + "step": 7140, + "unclipped_grad_norm": 0.8168482184410095 + }, + { + "bits_per_byte": 1.430851627619947, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.991790771484375, + "step": 7150, + "unclipped_grad_norm": 0.7878822088241577 + }, + { + "bits_per_byte": 1.5069802639723218, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.044559121131897, + "step": 7160, + "unclipped_grad_norm": 0.8418039679527283 + }, + { + "bits_per_byte": 1.4531985373251077, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0072804689407349, + "step": 7170, + "unclipped_grad_norm": 0.815416157245636 + }, + { + "bits_per_byte": 1.4725130489302556, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0206682682037354, + "step": 7180, + "unclipped_grad_norm": 0.8564274907112122 + }, + { + "bits_per_byte": 1.437259873173746, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9962326288223267, + "step": 7190, + "unclipped_grad_norm": 0.8627841472625732 + }, + { + "bits_per_byte": 1.4854186270510006, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.029613733291626, + "step": 7200, + "unclipped_grad_norm": 0.8707078695297241 + }, + { + "bits_per_byte": 1.5339277095880628, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0632376670837402, + "step": 7210, + "unclipped_grad_norm": 0.8451290726661682 + }, + { + "bits_per_byte": 1.5149174352922914, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.050060749053955, + "step": 7220, + "unclipped_grad_norm": 0.8569709062576294 + }, + { + "bits_per_byte": 1.4518221601702528, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.00632643699646, + "step": 7230, + "unclipped_grad_norm": 0.8588427901268005 + }, + { + "bits_per_byte": 1.4720953030713098, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0203787088394165, + "step": 7240, + "unclipped_grad_norm": 0.8185886144638062 + }, + { + "bits_per_byte": 1.4930615360553439, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0349113941192627, + "step": 7250, + "unclipped_grad_norm": 0.9033716917037964 + }, + { + "bits_per_byte": 1.392136183127593, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9649552702903748, + "step": 7260, + "unclipped_grad_norm": 0.8070151209831238 + }, + { + "bits_per_byte": 1.419755822934231, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9840997457504272, + "step": 7270, + "unclipped_grad_norm": 0.8482468724250793 + }, + { + "bits_per_byte": 1.499443640246212, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0393351316452026, + "step": 7280, + "unclipped_grad_norm": 0.9195974469184875 + }, + { + "bits_per_byte": 1.424472533125469, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.9873691201210022, + "step": 7290, + "unclipped_grad_norm": 0.84366774559021 + }, + { + "bits_per_byte": 1.449468749575846, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.004695177078247, + "step": 7300, + "unclipped_grad_norm": 0.8698141574859619 + }, + { + "bits_per_byte": 1.4526158601039845, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0068765878677368, + "step": 7310, + "unclipped_grad_norm": 0.8154390454292297 + }, + { + "bits_per_byte": 1.40692677709312, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9752073287963867, + "step": 7320, + "unclipped_grad_norm": 0.835978627204895 + }, + { + "bits_per_byte": 1.4261922736427717, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9885611534118652, + "step": 7330, + "unclipped_grad_norm": 0.8695172071456909 + }, + { + "bits_per_byte": 1.392467249730504, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.96518474817276, + "step": 7340, + "unclipped_grad_norm": 0.8504277467727661 + }, + { + "bits_per_byte": 1.4759263886019296, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0230342149734497, + "step": 7350, + "unclipped_grad_norm": 0.8514381051063538 + }, + { + "bits_per_byte": 1.4321242992363319, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9926729202270508, + "step": 7360, + "unclipped_grad_norm": 0.8621233105659485 + }, + { + "bits_per_byte": 1.4814723131443026, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0268783569335938, + "step": 7370, + "unclipped_grad_norm": 0.8794771432876587 + }, + { + "bits_per_byte": 1.401024074541532, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9711158871650696, + "step": 7380, + "unclipped_grad_norm": 0.8035550117492676 + }, + { + "bits_per_byte": 1.474961909895891, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.022365689277649, + "step": 7390, + "unclipped_grad_norm": 0.8444681763648987 + }, + { + "bits_per_byte": 1.4183169300857874, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9831023812294006, + "step": 7400, + "unclipped_grad_norm": 0.880998969078064 + }, + { + "bits_per_byte": 1.4093773578852642, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9769059419631958, + "step": 7410, + "unclipped_grad_norm": 0.8176257014274597 + }, + { + "bits_per_byte": 1.476101638923159, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.023155689239502, + "step": 7420, + "unclipped_grad_norm": 0.8866280913352966 + }, + { + "bits_per_byte": 1.489828262219123, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.032670259475708, + "step": 7430, + "unclipped_grad_norm": 0.8741231560707092 + }, + { + "bits_per_byte": 1.5070485410847143, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0446064472198486, + "step": 7440, + "unclipped_grad_norm": 0.8906005620956421 + }, + { + "bits_per_byte": 1.4470038942236803, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0029866695404053, + "step": 7450, + "unclipped_grad_norm": 0.833862841129303 + }, + { + "bits_per_byte": 1.4545604679372903, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 1.0082244873046875, + "step": 7460, + "unclipped_grad_norm": 0.851222813129425 + }, + { + "bits_per_byte": 1.4397255884391662, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9979417324066162, + "step": 7470, + "unclipped_grad_norm": 0.8808716535568237 + }, + { + "bits_per_byte": 1.469680494670545, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.018704891204834, + "step": 7480, + "unclipped_grad_norm": 0.9037331938743591 + }, + { + "bits_per_byte": 1.5297206699826567, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.060321569442749, + "step": 7490, + "unclipped_grad_norm": 0.8992816209793091 + }, + { + "bits_per_byte": 1.48344512613235, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0282458066940308, + "step": 7500, + "unclipped_grad_norm": 0.8851720690727234 + }, + { + "bits_per_byte": 1.4271278592634655, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9892096519470215, + "step": 7510, + "unclipped_grad_norm": 0.8391884565353394 + }, + { + "bits_per_byte": 1.4636686691269847, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0145378112792969, + "step": 7520, + "unclipped_grad_norm": 0.8814444541931152 + }, + { + "bits_per_byte": 1.5040419403823309, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0425224304199219, + "step": 7530, + "unclipped_grad_norm": 0.8750982880592346 + }, + { + "bits_per_byte": 1.448819515068839, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0042451620101929, + "step": 7540, + "unclipped_grad_norm": 0.8588963150978088 + }, + { + "bits_per_byte": 1.4846842611318165, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0291047096252441, + "step": 7550, + "unclipped_grad_norm": 0.8553798198699951 + }, + { + "bits_per_byte": 1.4456509067093426, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0020488500595093, + "step": 7560, + "unclipped_grad_norm": 0.8477172255516052 + }, + { + "bits_per_byte": 1.4068580700241002, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9751597046852112, + "step": 7570, + "unclipped_grad_norm": 0.8558363914489746 + }, + { + "bits_per_byte": 1.5845748804406512, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.0983436107635498, + "step": 7580, + "unclipped_grad_norm": 0.9179992079734802 + }, + { + "bits_per_byte": 1.435446746077025, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9949758648872375, + "step": 7590, + "unclipped_grad_norm": 0.8465545177459717 + }, + { + "bits_per_byte": 1.4142041369730542, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9802516102790833, + "step": 7600, + "unclipped_grad_norm": 0.8474127054214478 + }, + { + "bits_per_byte": 1.441303443269507, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9990354180335999, + "step": 7610, + "unclipped_grad_norm": 0.8852339386940002 + }, + { + "bits_per_byte": 1.4014744111128163, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9714280366897583, + "step": 7620, + "unclipped_grad_norm": 0.8255653977394104 + }, + { + "bits_per_byte": 1.3991135192730972, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.96979159116745, + "step": 7630, + "unclipped_grad_norm": 0.8985438942909241 + }, + { + "bits_per_byte": 1.4200022740729177, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9842705726623535, + "step": 7640, + "unclipped_grad_norm": 0.8505365252494812 + }, + { + "bits_per_byte": 1.5509151239443857, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.075012445449829, + "step": 7650, + "unclipped_grad_norm": 0.8840228915214539 + }, + { + "bits_per_byte": 1.4633312391659918, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0143039226531982, + "step": 7660, + "unclipped_grad_norm": 0.9004685878753662 + }, + { + "bits_per_byte": 1.5004719245157205, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0400478839874268, + "step": 7670, + "unclipped_grad_norm": 0.9166978597640991 + }, + { + "bits_per_byte": 1.4376227565670667, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9964841604232788, + "step": 7680, + "unclipped_grad_norm": 0.8822170495986938 + }, + { + "bits_per_byte": 1.4453855374790612, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0018649101257324, + "step": 7690, + "unclipped_grad_norm": 0.8656983375549316 + }, + { + "bits_per_byte": 1.396036663657836, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9676588773727417, + "step": 7700, + "unclipped_grad_norm": 0.840412974357605 + }, + { + "bits_per_byte": 1.4832948132954957, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0281416177749634, + "step": 7710, + "unclipped_grad_norm": 0.8699836730957031 + }, + { + "bits_per_byte": 1.4846416094344024, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0290751457214355, + "step": 7720, + "unclipped_grad_norm": 0.8882943391799927 + }, + { + "bits_per_byte": 1.4868813394965887, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.0306276082992554, + "step": 7730, + "unclipped_grad_norm": 0.8993004560470581 + }, + { + "bits_per_byte": 1.4329027787054622, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9932125210762024, + "step": 7740, + "unclipped_grad_norm": 0.8985061049461365 + }, + { + "bits_per_byte": 1.457792709877606, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 1.0104649066925049, + "step": 7750, + "unclipped_grad_norm": 0.8967045545578003 + }, + { + "bits_per_byte": 1.475737723633933, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0229034423828125, + "step": 7760, + "unclipped_grad_norm": 0.8843175768852234 + }, + { + "bits_per_byte": 1.4006856986759595, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9708813428878784, + "step": 7770, + "unclipped_grad_norm": 0.8820574879646301 + }, + { + "bits_per_byte": 1.4787585988963383, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.024997353553772, + "step": 7780, + "unclipped_grad_norm": 0.8838117718696594 + }, + { + "bits_per_byte": 1.4172842602586817, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.982386589050293, + "step": 7790, + "unclipped_grad_norm": 0.8568381071090698 + }, + { + "bits_per_byte": 1.4471380406913532, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0030796527862549, + "step": 7800, + "unclipped_grad_norm": 0.8466761708259583 + }, + { + "bits_per_byte": 1.4200249757828314, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9842863082885742, + "step": 7810, + "unclipped_grad_norm": 0.9039835929870605 + }, + { + "bits_per_byte": 1.410308987904988, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.9775516986846924, + "step": 7820, + "unclipped_grad_norm": 0.8540316224098206 + }, + { + "bits_per_byte": 1.4409317887609925, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 0.9987778067588806, + "step": 7830, + "unclipped_grad_norm": 0.9217174649238586 + }, + { + "bits_per_byte": 1.4393295123942291, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9976671934127808, + "step": 7840, + "unclipped_grad_norm": 0.894000232219696 + }, + { + "bits_per_byte": 1.3879004224194929, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9620192646980286, + "step": 7850, + "unclipped_grad_norm": 0.856924295425415 + }, + { + "bits_per_byte": 1.4481823193473922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0038034915924072, + "step": 7860, + "unclipped_grad_norm": 0.8965994119644165 + }, + { + "bits_per_byte": 1.4733294505739012, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.021234154701233, + "step": 7870, + "unclipped_grad_norm": 0.9309923648834229 + }, + { + "bits_per_byte": 1.3911279348369099, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9642564058303833, + "step": 7880, + "unclipped_grad_norm": 0.8823378086090088 + }, + { + "bits_per_byte": 1.46931073197119, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0184485912322998, + "step": 7890, + "unclipped_grad_norm": 0.9113782048225403 + }, + { + "bits_per_byte": 1.449501254296859, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0047177076339722, + "step": 7900, + "unclipped_grad_norm": 0.8579812049865723 + }, + { + "bits_per_byte": 1.457684704772864, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.010390043258667, + "step": 7910, + "unclipped_grad_norm": 0.8884916305541992 + }, + { + "bits_per_byte": 1.4556867823177908, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0090051889419556, + "step": 7920, + "unclipped_grad_norm": 0.8975357413291931 + }, + { + "bits_per_byte": 1.4187827450956494, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9834252595901489, + "step": 7930, + "unclipped_grad_norm": 0.844153106212616 + }, + { + "bits_per_byte": 1.4233246349222852, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9865734577178955, + "step": 7940, + "unclipped_grad_norm": 0.9031311273574829 + }, + { + "bits_per_byte": 1.4939792354803478, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.0355474948883057, + "step": 7950, + "unclipped_grad_norm": 0.910707950592041 + }, + { + "bits_per_byte": 1.4252896226997183, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.987935483455658, + "step": 7960, + "unclipped_grad_norm": 0.8988999128341675 + }, + { + "bits_per_byte": 1.446669559950403, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 1.0027549266815186, + "step": 7970, + "unclipped_grad_norm": 0.9068711400032043 + }, + { + "bits_per_byte": 1.484100552014788, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0287001132965088, + "step": 7980, + "unclipped_grad_norm": 0.9153758883476257 + }, + { + "bits_per_byte": 1.5191428770413398, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0529896020889282, + "step": 7990, + "unclipped_grad_norm": 0.9229548573493958 + }, + { + "bits_per_byte": 1.419257675186007, + "learning_rate": 3e-05, + "loss_nats": 0.9837544560432434, + "step": 8000, + "unclipped_grad_norm": 0.8808227777481079 + } + ] + }, + "depth-32-baseline-seed-2026073002": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "890c06c98d3b195e1aa79852477608280a7bcd106a01b463c2d793e8ec58d94f", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00042041894630528986, + 0.000374794501112774, + 0.0003388339828234166, + 0.000304876797599718, + 0.0002806047268677503, + 0.0002645689237397164, + 0.00024906310136429965, + 0.00023202023294288665, + 0.00021741738601122051, + 0.00020566635066643357, + 0.0001967942516785115, + 0.0001872276043286547, + 0.00017897134239319712, + 0.00017287296941503882, + 0.00016580727242399007, + 0.00016062518989201635, + 0.0001559370430186391, + 0.00015203763905446976, + 0.00014800428471062332, + 0.00014486612053588033, + 0.00014247088984120637, + 0.00013994764594826847, + 0.00013763502647634596, + 0.00013544136891141534, + 0.00013363662583287805, + 0.0001319239236181602, + 0.00013088916603010148, + 0.00012932090612594038, + 0.00012785944272764027, + 0.00012667682312894613, + 0.0001251104404218495, + 0.00012353518104646355 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00030814765159448143, + "first_to_last_ratio": 2.395816319369912, + "imbalance_abs_log_ratio": 0.8737240159497589, + "last_quartile_mean": 0.00012861906361649744, + "mean": 0.00019174550334355445, + "normalized": [ + 2.1925882952885543, + 1.9546455826984732, + 1.7671026277801187, + 1.5900075479395406, + 1.4634227242606306, + 1.3797920635754504, + 1.2989253829752037, + 1.2100426288859099, + 1.1338851875012121, + 1.0726006455438846, + 1.0263304653664347, + 0.9764380445115058, + 0.9333796061570758, + 0.9015750899007978, + 0.8647257408008662, + 0.8376999047754503, + 0.8132500647967916, + 0.7929137132465669, + 0.7718787774931073, + 0.7555124788314888, + 0.7430207611488979, + 0.7298614231256383, + 0.7178005433052705, + 0.7063600791135226, + 0.6969478997034861, + 0.6880157360550423, + 0.6826192205174408, + 0.6744403590744623, + 0.6668184676985713, + 0.660650815377801, + 0.6524817439796045, + 0.644266378571199 + ], + "population_cv": 0.4034880502878489 + }, + "activation_output_rms_by_block": [ + 0.02817646972835064, + 0.028377581387758255, + 0.028678538277745247, + 0.028959807008504868, + 0.029224006459116936, + 0.029430421069264412, + 0.02957010455429554, + 0.029960129410028458, + 0.030406322330236435, + 0.03066711500287056, + 0.030792171135544777, + 0.031063809990882874, + 0.03129913657903671, + 0.03160789608955383, + 0.03180097043514252, + 0.032092027366161346, + 0.03248445317149162, + 0.032903194427490234, + 0.03316524624824524, + 0.033593542873859406, + 0.034045662730932236, + 0.03455357626080513, + 0.03491735830903053, + 0.03521261364221573, + 0.03560211881995201, + 0.0359802283346653, + 0.03664350509643555, + 0.03706779330968857, + 0.0375242717564106, + 0.03805854171514511, + 0.038408540189266205, + 0.039021097123622894 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.029047132236883044, + "first_to_last_ratio": 0.7789886319526242, + "imbalance_abs_log_ratio": 0.24975882634672159, + "last_quartile_mean": 0.03728826204314828, + "mean": 0.03285275783855468, + "normalized": [ + 0.8576591915605899, + 0.863780798166523, + 0.8729415782588962, + 0.8815030735264118, + 0.8895449996231614, + 0.8958280219241153, + 0.9000798258584322, + 0.9119517319446604, + 0.9255333290331198, + 0.93347155674343, + 0.9372781209682287, + 0.945546493951495, + 0.9527095634663979, + 0.96210784631481, + 0.9679848063720901, + 0.9768442431490312, + 0.9887892313676379, + 1.0015352315073034, + 1.0095117862319563, + 1.0225486407851998, + 1.0363106469855512, + 1.051770948138011, + 1.0628440534770849, + 1.0718312847663467, + 1.0836873724546434, + 1.095196589323784, + 1.1153859677933096, + 1.128300810904444, + 1.1421954875390588, + 1.1584580479414466, + 1.1691115972063533, + 1.187757122716477 + ], + "population_cv": 0.09694557255182189 + }, + "bits_per_byte": 8.04236897991864, + "branch_output_rms_by_sublayer": [ + 0.0023512106854468584, + 0.0027520405128598213, + 0.0023248849902302027, + 0.0026936428621411324, + 0.0027838374953716993, + 0.002722234698012471, + 0.002502240240573883, + 0.002735150745138526, + 0.0026338943280279636, + 0.002727407729253173, + 0.0029348398093134165, + 0.002726956969127059, + 0.0029466808773577213, + 0.002693725284188986, + 0.0032394269946962595, + 0.0027635409496724606, + 0.003579996759071946, + 0.0027838009409606457, + 0.0033850702457129955, + 0.0026989516336470842, + 0.0034471398685127497, + 0.002707457635551691, + 0.0036693045403808355, + 0.0027280766516923904, + 0.0035536750219762325, + 0.002806172939017415, + 0.0038618813268840313, + 0.0027198586612939835, + 0.003969449549913406, + 0.002747237915173173, + 0.003774659475311637, + 0.0027643220964819193, + 0.004222253803163767, + 0.002741035772487521, + 0.004245356190949678, + 0.00275977305136621, + 0.004526515491306782, + 0.0027434700168669224, + 0.004249728750437498, + 0.0026767924427986145, + 0.004827029537409544, + 0.002736816881224513, + 0.004658437799662352, + 0.0027651384007185698, + 0.0048249042592942715, + 0.0027371037285774946, + 0.005304279271513224, + 0.0027228749822825193, + 0.005711541976779699, + 0.002716122195124626, + 0.005045780446380377, + 0.002763758646324277, + 0.005378549452871084, + 0.0027021593414247036, + 0.005505796521902084, + 0.0027199271135032177, + 0.005381250753998756, + 0.0027382206171751022, + 0.005084864795207977, + 0.002702005673199892, + 0.005992465652525425, + 0.002712319139391184, + 0.00573702622205019, + 0.002734891138970852 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.010411069784541356, + 0.009271735997530202, + 0.009345779773643629, + 0.008099773113238128, + 0.008225801589217078, + 0.008647498843153682, + 0.007992995665779355, + 0.0076659621960069, + 0.007406648616870928, + 0.006601475371863238, + 0.006568536350729326, + 0.006644454140273569, + 0.005847021988988907, + 0.006236102346072616, + 0.006448411665711761, + 0.005769055287848533, + 0.005701484582581849, + 0.005836584711958408, + 0.005498613632501928, + 0.004978853123018873, + 0.005187902378445178, + 0.005078997897134883, + 0.005158829428828408, + 0.005043411661395455, + 0.00572819320620205, + 0.004439465860302503, + 0.004811331873512697, + 0.004686191415610403, + 0.004548827932369003, + 0.00413026790671143, + 0.004686908144290253, + 0.004087931241888514 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00870757712038879, + "first_to_last_ratio": 1.8766776125890325, + "imbalance_abs_log_ratio": 0.6295029861071718, + "last_quartile_mean": 0.004639889697610857, + "mean": 0.006274566179006908, + "normalized": [ + 1.6592493389222875, + 1.4776696480708191, + 1.489470268862924, + 1.2908897415629936, + 1.3109753494573861, + 1.3781827454599171, + 1.2738722387727572, + 1.2217517478188766, + 1.1804240174646141, + 1.0521006844983298, + 1.0468510751716935, + 1.0589503641708666, + 0.9318607569319367, + 0.9938698817038568, + 1.027706375507919, + 0.9194349255810409, + 0.9086659411861104, + 0.930197330850717, + 0.8763336740154054, + 0.7934975870804967, + 0.8268145128188418, + 0.8094580170542961, + 0.8221810531042816, + 0.8037865116905484, + 0.9129225898305317, + 0.707533514453289, + 0.766799127820212, + 0.7468550465352011, + 0.7249629380893642, + 0.6582555333514927, + 0.7469692741422425, + 0.6515081880187487 + ], + "population_cv": 0.26225105130829346 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.027978211641311646, + 0.028061864897608757, + 0.02817646972835064, + 0.028305010870099068, + 0.028377581387758255, + 0.02853298746049404, + 0.028678538277745247, + 0.028806988149881363, + 0.028959807008504868, + 0.029115403071045876, + 0.029224006459116936, + 0.02931177243590355, + 0.029430421069264412, + 0.02947254478931427, + 0.02957010455429554, + 0.02984699048101902, + 0.029960129410028458, + 0.0302706528455019, + 0.030406322330236435, + 0.030570803210139275, + 0.03066711500287056, + 0.030674004927277565, + 0.030792171135544777, + 0.03094458393752575, + 0.031063809990882874, + 0.031205367296934128, + 0.03129913657903671, + 0.03147619590163231, + 0.03160789608955383, + 0.03167128562927246, + 0.03180097043514252, + 0.03195846453309059, + 0.032092027366161346, + 0.032371606677770615, + 0.03248445317149162, + 0.032751601189374924, + 0.032903194427490234, + 0.03311120718717575, + 0.03316524624824524, + 0.03350966051220894, + 0.033593542873859406, + 0.03394605591893196, + 0.034045662730932236, + 0.03444153815507889, + 0.03455357626080513, + 0.03483813256025314, + 0.03491735830903053, + 0.03510763868689537, + 0.03521261364221573, + 0.03553535044193268, + 0.03560211881995201, + 0.03590916842222214, + 0.0359802283346653, + 0.036534491926431656, + 0.03664350509643555, + 0.036974843591451645, + 0.03706779330968857, + 0.037423327565193176, + 0.0375242717564106, + 0.037971291691064835, + 0.03805854171514511, + 0.03832196444272995, + 0.038408540189266205, + 0.03889527916908264 + ], + "loss_nats": 5.574545383453369, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028061864897608757, + 0.02817646972835064, + 0.028305010870099068, + 0.028377581387758255, + 0.02853298746049404, + 0.028678538277745247, + 0.028806988149881363, + 0.028959807008504868, + 0.029115403071045876, + 0.029224006459116936, + 0.02931177243590355, + 0.029430421069264412, + 0.02947254478931427, + 0.02957010455429554, + 0.02984699048101902, + 0.029960129410028458, + 0.0302706528455019, + 0.030406322330236435, + 0.030570803210139275, + 0.03066711500287056, + 0.030674004927277565, + 0.030792171135544777, + 0.03094458393752575, + 0.031063809990882874, + 0.031205367296934128, + 0.03129913657903671, + 0.03147619590163231, + 0.03160789608955383, + 0.03167128562927246, + 0.03180097043514252, + 0.03195846453309059, + 0.032092027366161346, + 0.032371606677770615, + 0.03248445317149162, + 0.032751601189374924, + 0.032903194427490234, + 0.03311120718717575, + 0.03316524624824524, + 0.03350966051220894, + 0.033593542873859406, + 0.03394605591893196, + 0.034045662730932236, + 0.03444153815507889, + 0.03455357626080513, + 0.03483813256025314, + 0.03491735830903053, + 0.03510763868689537, + 0.03521261364221573, + 0.03553535044193268, + 0.03560211881995201, + 0.03590916842222214, + 0.0359802283346653, + 0.036534491926431656, + 0.03664350509643555, + 0.036974843591451645, + 0.03706779330968857, + 0.037423327565193176, + 0.0375242717564106, + 0.037971291691064835, + 0.03805854171514511, + 0.03832196444272995, + 0.038408540189266205, + 0.03889527916908264, + 0.039021097123622894 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.223192369565368e-05, + 6.862939335405827e-05, + 6.46814878564328e-05, + 6.090753959142603e-05, + 5.703999340767041e-05, + 5.302598947309889e-05, + 4.9269601731793955e-05, + 4.545194315141998e-05, + 4.1738676372915506e-05, + 3.8814047002233565e-05, + 3.584829028113745e-05, + 3.317802838864736e-05, + 3.07806913042441e-05, + 2.884516288759187e-05, + 2.6981158953276463e-05, + 2.530650635890197e-05, + 2.3913053155411035e-05, + 2.265583498228807e-05, + 2.164491161238402e-05, + 2.069839865725953e-05, + 1.9972700101789087e-05, + 1.928487472468987e-05, + 1.8700015061767772e-05, + 1.8183285646955483e-05, + 1.7775813830667175e-05, + 1.7374133676639758e-05, + 1.7024083717842586e-05, + 1.673021324677393e-05, + 1.646951204747893e-05, + 1.624442847969476e-05, + 1.604618228157051e-05, + 1.58860220835777e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.890473403269425e-05, + "first_to_last_ratio": 3.528539860533838, + "imbalance_abs_log_ratio": 1.2608841481311845, + "last_quartile_mean": 1.669379867053067e-05, + "mean": 3.222918428491539e-05, + "normalized": [ + 2.2411961487173366, + 2.1294176342582674, + 2.0069228958644927, + 1.8898256640002307, + 1.7698242966194933, + 1.64527867054697, + 1.528726302727252, + 1.4102728368677142, + 1.2950584167422121, + 1.2043136636380887, + 1.1122928202038285, + 1.029440524939878, + 0.9550564802426835, + 0.8950013327235407, + 0.8371654310191393, + 0.7852046808006393, + 0.7419689230733446, + 0.7029602357293278, + 0.6715935290523237, + 0.6422253344757239, + 0.6197085202413003, + 0.5983668266067783, + 0.5802199303728628, + 0.5641869644049918, + 0.5515440190333021, + 0.5390807760769664, + 0.5282195033961989, + 0.5191013554322059, + 0.5110123762948401, + 0.5040285331483811, + 0.49787739397055714, + 0.49290797877912856 + ], + "population_cv": 0.5370990514498413 + }, + "activation_output_rms_by_block": [ + 0.029305610805749893, + 0.03049159236252308, + 0.0324031226336956, + 0.03488599509000778, + 0.037802305072546005, + 0.04105908423662186, + 0.04460485652089119, + 0.04878050833940506, + 0.053197599947452545, + 0.057943083345890045, + 0.06266378611326218, + 0.06765592098236084, + 0.07297883927822113, + 0.07823853194713593, + 0.08444638550281525, + 0.09052036702632904, + 0.09696295112371445, + 0.10418187081813812, + 0.11156722903251648, + 0.1197449341416359, + 0.12838369607925415, + 0.13761533796787262, + 0.14720149338245392, + 0.15857642889022827, + 0.16927987337112427, + 0.1816006898880005, + 0.19378916919231415, + 0.20805081725120544, + 0.22281049191951752, + 0.2388531118631363, + 0.25510889291763306, + 0.2763713300228119 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03741663438268006, + "first_to_last_ratio": 0.17145265067740043, + "imbalance_abs_log_ratio": 1.763448139840426, + "last_quartile_mean": 0.2182330470532179, + "mean": 0.11303362209582701, + "normalized": [ + 0.2592645467992289, + 0.2697568369230271, + 0.286668002253569, + 0.30863378916082446, + 0.3344341654423688, + 0.36324664710658405, + 0.39461582928906175, + 0.4315575085973109, + 0.47063518766436574, + 0.5126181243379725, + 0.554381828622084, + 0.5985468724076087, + 0.6456383324277757, + 0.6921704400555022, + 0.747090856127956, + 0.8008269163451932, + 0.8578239759627604, + 0.9216892185390237, + 0.9870269302520681, + 1.0593744756769734, + 1.135800956377508, + 1.2174726016583441, + 1.3022806015864932, + 1.4029138052020595, + 1.497606377928977, + 1.6066077200821192, + 1.714438284814271, + 1.8406100184494245, + 1.9711877562467606, + 2.1131156149330748, + 2.256929294023315, + 2.445036484706394 + ], + "population_cv": 0.6334590447760624 + }, + "bits_per_byte": 5.11848367743279, + "branch_output_rms_by_sublayer": [ + 0.003032666863873601, + 0.004902590066194534, + 0.002639081096276641, + 0.004850045777857304, + 0.0032534960191696882, + 0.005442064255475998, + 0.0027798404917120934, + 0.005710980389267206, + 0.0031097980681806803, + 0.006003146059811115, + 0.0030368315055966377, + 0.0059938253834843636, + 0.0034559990745037794, + 0.0061976369470357895, + 0.0037613045424222946, + 0.0062723769806325436, + 0.004607585724443197, + 0.0063410610891878605, + 0.003909118939191103, + 0.006816320586949587, + 0.004674337804317474, + 0.006481682416051626, + 0.004910735413432121, + 0.0065277558751404285, + 0.00493645528331399, + 0.006758649833500385, + 0.004751654341816902, + 0.006488988175988197, + 0.006120393518358469, + 0.006713794078677893, + 0.005455858074128628, + 0.00696563022211194, + 0.005903614219278097, + 0.007100075017660856, + 0.006780208554118872, + 0.007398899644613266, + 0.007146476302295923, + 0.007193166296929121, + 0.007645847275853157, + 0.00761415995657444, + 0.00879710540175438, + 0.007752604782581329, + 0.00870877131819725, + 0.007873632945120335, + 0.008980454877018929, + 0.008168715983629227, + 0.011330032721161842, + 0.008200661279261112, + 0.010545967146754265, + 0.007979212328791618, + 0.011436101980507374, + 0.008451862260699272, + 0.01130830217152834, + 0.008476903662085533, + 0.013203900307416916, + 0.008634857833385468, + 0.01393965259194374, + 0.009264753200113773, + 0.014520598575472832, + 0.009397287853062153, + 0.015960348770022392, + 0.00835501030087471, + 0.02132205106317997, + 0.009183173067867756 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00028524855645157025, + 0.00022469127596424853, + 0.00022670239393809648, + 0.0001993403227560908, + 0.0002100203582478305, + 0.00019183315773228503, + 0.00020704106188439305, + 0.00020121077118127333, + 0.00020796371696885172, + 0.00017825031863398375, + 0.00018469615204762195, + 0.0001848551977445252, + 0.00018214824531137669, + 0.0001649843875278976, + 0.0001966104199015976, + 0.00016919176207710993, + 0.0001681445657707544, + 0.00017333427923610245, + 0.0001624042419733422, + 0.00015992743704390023, + 0.00016485453322609178, + 0.00015449118786750607, + 0.00015016403044324942, + 0.00015560032515501767, + 0.00015156764536984935, + 0.00014581309890539414, + 0.000137484036700603, + 0.0001500871171184727, + 0.00014447415143049674, + 0.00014218802742214665, + 0.00014221421491139387, + 0.00016697872359292516 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002182609872694735, + "first_to_last_ratio": 1.4787241905811908, + "imbalance_abs_log_ratio": 0.3911796826204884, + "last_quartile_mean": 0.0001476008769314102, + "mean": 0.00017764111607924995, + "normalized": [ + 1.6057575112527105, + 1.2648607536557492, + 1.2761819951466593, + 1.122151938446279, + 1.1822733547459552, + 1.0798916487706802, + 1.1655019201299495, + 1.1326813049942133, + 1.170695847666675, + 1.0034294006262718, + 1.0397151072008834, + 1.0406104274984227, + 1.0253721060281789, + 0.9287511313219521, + 1.1067844215405909, + 0.9524358201038854, + 0.9465408092557868, + 0.9757554054027329, + 0.9142266472863736, + 0.9002839014620511, + 0.9280201389443322, + 0.869681475084765, + 0.8453224892837178, + 0.8759251719945322, + 0.8532238972323919, + 0.8208296712138615, + 0.7739426532271284, + 0.8448895189980412, + 0.8132922975221759, + 0.8004229570293466, + 0.8005703749797919, + 0.9399779019539145 + ], + "population_cv": 0.17622384112301237 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028563687577843666, + 0.028535494580864906, + 0.029305610805749893, + 0.029406530782580376, + 0.03049159236252308, + 0.030669530853629112, + 0.0324031226336956, + 0.03261565789580345, + 0.03488599509000778, + 0.035065844655036926, + 0.037802305072546005, + 0.03803478553891182, + 0.04105908423662186, + 0.041278183460235596, + 0.04460485652089119, + 0.04512108862400055, + 0.04878050833940506, + 0.04953810200095177, + 0.053197599947452545, + 0.05369982123374939, + 0.057943083345890045, + 0.0585285909473896, + 0.06266378611326218, + 0.06350461393594742, + 0.06765592098236084, + 0.06854932755231857, + 0.07297883927822113, + 0.07389408349990845, + 0.07823853194713593, + 0.07984820008277893, + 0.08444638550281525, + 0.08570227771997452, + 0.09052036702632904, + 0.09210614860057831, + 0.09696295112371445, + 0.09899988770484924, + 0.10418187081813812, + 0.10663887858390808, + 0.11156722903251648, + 0.11457803845405579, + 0.1197449341416359, + 0.12300471216440201, + 0.12838369607925415, + 0.13199855387210846, + 0.13761533796787262, + 0.14150823652744293, + 0.14720149338245392, + 0.15258266031742096, + 0.15857642889022827, + 0.16335271298885345, + 0.16927987337112427, + 0.17534348368644714, + 0.1816006898880005, + 0.1876170039176941, + 0.19378916919231415, + 0.2015211284160614, + 0.20805081725120544, + 0.21581925451755524, + 0.22281049191951752, + 0.23164939880371094, + 0.2388531118631363, + 0.24887791275978088, + 0.25510889291763306, + 0.26933422684669495 + ], + "loss_nats": 3.5478625297546387, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028535494580864906, + 0.029305610805749893, + 0.029406530782580376, + 0.03049159236252308, + 0.030669530853629112, + 0.0324031226336956, + 0.03261565789580345, + 0.03488599509000778, + 0.035065844655036926, + 0.037802305072546005, + 0.03803478553891182, + 0.04105908423662186, + 0.041278183460235596, + 0.04460485652089119, + 0.04512108862400055, + 0.04878050833940506, + 0.04953810200095177, + 0.053197599947452545, + 0.05369982123374939, + 0.057943083345890045, + 0.0585285909473896, + 0.06266378611326218, + 0.06350461393594742, + 0.06765592098236084, + 0.06854932755231857, + 0.07297883927822113, + 0.07389408349990845, + 0.07823853194713593, + 0.07984820008277893, + 0.08444638550281525, + 0.08570227771997452, + 0.09052036702632904, + 0.09210614860057831, + 0.09696295112371445, + 0.09899988770484924, + 0.10418187081813812, + 0.10663887858390808, + 0.11156722903251648, + 0.11457803845405579, + 0.1197449341416359, + 0.12300471216440201, + 0.12838369607925415, + 0.13199855387210846, + 0.13761533796787262, + 0.14150823652744293, + 0.14720149338245392, + 0.15258266031742096, + 0.15857642889022827, + 0.16335271298885345, + 0.16927987337112427, + 0.17534348368644714, + 0.1816006898880005, + 0.1876170039176941, + 0.19378916919231415, + 0.2015211284160614, + 0.20805081725120544, + 0.21581925451755524, + 0.22281049191951752, + 0.23164939880371094, + 0.2388531118631363, + 0.24887791275978088, + 0.25510889291763306, + 0.26933422684669495, + 0.2763713300228119 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014093925710767508, + 0.00014282441406976432, + 0.00014458753867074847, + 0.0001413008722011, + 0.00013349838263820857, + 0.00012304198753554374, + 0.00011454652121756226, + 0.00010784466576296836, + 0.0001000517513602972, + 9.189492993755266e-05, + 8.563220035284758e-05, + 8.04013543529436e-05, + 7.555109914392233e-05, + 7.068048580549657e-05, + 6.581390334758908e-05, + 6.151734851300716e-05, + 5.7112174545181915e-05, + 5.2947088988730684e-05, + 4.8879119276534766e-05, + 4.527245255303569e-05, + 4.1345181671204045e-05, + 3.832591755781323e-05, + 3.484322223812342e-05, + 3.232451854273677e-05, + 3.010137334058527e-05, + 2.8228545488673262e-05, + 2.6716850697994232e-05, + 2.5478970201220363e-05, + 2.4533494070055895e-05, + 2.4085495169856586e-05, + 2.3973674615263008e-05, + 2.415898779872805e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013107295490044635, + "first_to_last_ratio": 5.058842318548806, + "imbalance_abs_log_ratio": 1.621137666322135, + "last_quartile_mean": 2.5909673922797083e-05, + "mean": 6.995168058665513e-05, + "normalized": [ + 2.01480873548251, + 2.041758151797876, + 2.06696304446375, + 2.019978233776078, + 1.9084371028489115, + 1.7589568471213655, + 1.6375092100276802, + 1.5417022844700914, + 1.430298036032952, + 1.3136915248764403, + 1.2241621592889058, + 1.14938417031087, + 1.0800469482692516, + 1.010418694915251, + 0.9408480653450457, + 0.8794263125215463, + 0.8164517859500485, + 0.756909462999114, + 0.6987554675828556, + 0.6471960669615769, + 0.591053443240527, + 0.5478913049177658, + 0.49810414769034944, + 0.46209781197027316, + 0.4303166569857621, + 0.4035434924784137, + 0.38193293533380923, + 0.3642367129358298, + 0.3507205811826659, + 0.34431617607842635, + 0.34271763614835193, + 0.34536679599570513 + ], + "population_cv": 0.5909500348424317 + }, + "activation_output_rms_by_block": [ + 0.04959565028548241, + 0.05870900675654411, + 0.06652694195508957, + 0.07286322116851807, + 0.07801168411970139, + 0.08261121064424515, + 0.08867742121219635, + 0.09177010506391525, + 0.09494514763355255, + 0.0981190949678421, + 0.10163456201553345, + 0.10214314609766006, + 0.10418473184108734, + 0.10623370856046677, + 0.10849270969629288, + 0.11100933700799942, + 0.11405495554208755, + 0.11736668646335602, + 0.12177617102861404, + 0.12738163769245148, + 0.13297238945960999, + 0.14011575281620026, + 0.15017743408679962, + 0.16121913492679596, + 0.17427310347557068, + 0.189283087849617, + 0.2044389545917511, + 0.22237628698349, + 0.24115440249443054, + 0.26191622018814087, + 0.2883519232273102, + 0.3199674189090729 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07359565515071154, + "first_to_last_ratio": 0.309589437408787, + "imbalance_abs_log_ratio": 1.1725082547452423, + "last_quartile_mean": 0.2377201747149229, + "mean": 0.13694853871129453, + "normalized": [ + 0.3621480795062482, + 0.4286939262660582, + 0.48578059014807806, + 0.5320481828734462, + 0.5696423258970309, + 0.6032281280372068, + 0.6475236760221296, + 0.6701064934864231, + 0.6932906953736058, + 0.7164668998381211, + 0.7421368856647125, + 0.745850573206854, + 0.7607582586968847, + 0.775719913188861, + 0.7922151686847111, + 0.8105916138471667, + 0.8328307597537082, + 0.8570130617514684, + 0.8892111750482725, + 0.9301423650893323, + 0.9709661067646236, + 1.0231270383365143, + 1.0965975650415178, + 1.177224207310945, + 1.2725444544024032, + 1.3821475543353592, + 1.4928158892059098, + 1.6237945222058072, + 1.7609125644108963, + 1.9125156255978355, + 2.105549470923482, + 2.336406229084387 + ], + "population_cv": 0.48939845088217687 + }, + "bits_per_byte": 3.103317553853433, + "branch_output_rms_by_sublayer": [ + 0.011270907707512379, + 0.01699911244213581, + 0.012035840190947056, + 0.01337367296218872, + 0.010826056823134422, + 0.010907532647252083, + 0.01394603680819273, + 0.008780849166214466, + 0.015218054875731468, + 0.009724137373268604, + 0.015542514622211456, + 0.009259181097149849, + 0.025958625599741936, + 0.009840037673711777, + 0.03120059333741665, + 0.009063811041414738, + 0.014079542830586433, + 0.008434661664068699, + 0.01716585084795952, + 0.009029604494571686, + 0.021518073976039886, + 0.01014146488159895, + 0.030795104801654816, + 0.00910214427858591, + 0.018123170360922813, + 0.009148034267127514, + 0.016974516212940216, + 0.00891866721212864, + 0.01868187077343464, + 0.00946173444390297, + 0.017949005588889122, + 0.00928061455488205, + 0.021634839475154877, + 0.01075725443661213, + 0.02261096052825451, + 0.010938199236989021, + 0.017853887751698494, + 0.012016859836876392, + 0.016204316169023514, + 0.012549139559268951, + 0.019116709008812904, + 0.014130656607449055, + 0.021943919360637665, + 0.015834644436836243, + 0.015276721678674221, + 0.01847725547850132, + 0.01698230765759945, + 0.019000399857759476, + 0.017433995380997658, + 0.019877459853887558, + 0.017341874539852142, + 0.022254064679145813, + 0.018439622595906258, + 0.023488705977797508, + 0.013915229588747025, + 0.02547476813197136, + 0.008195439353585243, + 0.027127297595143318, + 0.007334497291594744, + 0.02754027582705021, + 0.014406879432499409, + 0.029805582016706467, + 0.02094510942697525, + 0.03139834478497505 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005239687790313407, + 0.0004396407496139251, + 0.0004479555677531567, + 0.000484081397596248, + 0.0004298002646761996, + 0.0003778969963663625, + 0.0003311985651517251, + 0.0003381159607123189, + 0.00027120323619484496, + 0.00025249768611013433, + 0.0002492173720055997, + 0.0002512649978189958, + 0.00021872457711399188, + 0.0001974354513181695, + 0.0001883229322305711, + 0.00018548264077727127, + 0.00019625685908145198, + 0.00017137558371316787, + 0.00015305597823747372, + 0.00014260825992090635, + 0.00013921721401447933, + 0.00013266258711253505, + 0.00012717821667718618, + 0.00011256530678442586, + 0.0001087245205863228, + 0.00010436458537579927, + 0.00010507936295752309, + 9.703474953173527e-05, + 9.385369360633854e-05, + 9.122967598255955e-05, + 0.00010232602359400015, + 0.0001188300325189452 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0004215822851126596, + "first_to_last_ratio": 4.105774523524949, + "imbalance_abs_log_ratio": 1.4123944031885163, + "last_quartile_mean": 0.00010268033051915298, + "mean": 0.00022447405700517828, + "normalized": [ + 2.3342063934775927, + 1.9585370152764836, + 1.995578347580821, + 2.156513781835738, + 1.9146990543601428, + 1.6834773755509973, + 1.475442505786247, + 1.506258519239571, + 1.2081718476206305, + 1.1248412822347198, + 1.1102279494144423, + 1.11934983120655, + 0.9743868847567816, + 0.8795468570054622, + 0.8389518804225415, + 0.8262987859349487, + 0.874296396206376, + 0.7634538529733728, + 0.6818426159328645, + 0.6352995166725061, + 0.6201928894227087, + 0.5909929587519093, + 0.5665608684314569, + 0.5014624330589309, + 0.48435227677029374, + 0.4649293854629791, + 0.4681136179362547, + 0.4322760092026885, + 0.41810485745430026, + 0.4064152321194739, + 0.45584788264257925, + 0.5293708952576376 + ], + "population_cv": 0.5698198347315865 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.044032808393239975, + 0.044310975819826126, + 0.04959565028548241, + 0.052685707807540894, + 0.05870900675654411, + 0.061390094459056854, + 0.06652694195508957, + 0.0693279355764389, + 0.07286322116851807, + 0.07403621077537537, + 0.07801168411970139, + 0.07902701944112778, + 0.08261121064424515, + 0.08570802956819534, + 0.08867742121219635, + 0.08839071542024612, + 0.09177010506391525, + 0.0920921340584755, + 0.09494514763355255, + 0.09543459117412567, + 0.0981190949678421, + 0.09975367039442062, + 0.10163456201553345, + 0.09967570006847382, + 0.10214314609766006, + 0.10169660300016403, + 0.10418473184108734, + 0.1038530021905899, + 0.10623370856046677, + 0.10576826333999634, + 0.10849270969629288, + 0.10825389623641968, + 0.11100933700799942, + 0.11050844192504883, + 0.11405495554208755, + 0.11365467309951782, + 0.11736668646335602, + 0.117034412920475, + 0.12177617102861404, + 0.12205220758914948, + 0.12738163769245148, + 0.12672542035579681, + 0.13297238945960999, + 0.13195016980171204, + 0.14011575281620026, + 0.13969174027442932, + 0.15017743408679962, + 0.15003392100334167, + 0.16121913492679596, + 0.16189023852348328, + 0.17427310347557068, + 0.1747252345085144, + 0.189283087849617, + 0.18889831006526947, + 0.2044389545917511, + 0.20492415130138397, + 0.22237628698349, + 0.22264817357063293, + 0.24115440249443054, + 0.2428991198539734, + 0.26191622018814087, + 0.267524778842926, + 0.2883519232273102, + 0.2990366220474243 + ], + "loss_nats": 2.1510558128356934, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.044310975819826126, + 0.04959565028548241, + 0.052685707807540894, + 0.05870900675654411, + 0.061390094459056854, + 0.06652694195508957, + 0.0693279355764389, + 0.07286322116851807, + 0.07403621077537537, + 0.07801168411970139, + 0.07902701944112778, + 0.08261121064424515, + 0.08570802956819534, + 0.08867742121219635, + 0.08839071542024612, + 0.09177010506391525, + 0.0920921340584755, + 0.09494514763355255, + 0.09543459117412567, + 0.0981190949678421, + 0.09975367039442062, + 0.10163456201553345, + 0.09967570006847382, + 0.10214314609766006, + 0.10169660300016403, + 0.10418473184108734, + 0.1038530021905899, + 0.10623370856046677, + 0.10576826333999634, + 0.10849270969629288, + 0.10825389623641968, + 0.11100933700799942, + 0.11050844192504883, + 0.11405495554208755, + 0.11365467309951782, + 0.11736668646335602, + 0.117034412920475, + 0.12177617102861404, + 0.12205220758914948, + 0.12738163769245148, + 0.12672542035579681, + 0.13297238945960999, + 0.13195016980171204, + 0.14011575281620026, + 0.13969174027442932, + 0.15017743408679962, + 0.15003392100334167, + 0.16121913492679596, + 0.16189023852348328, + 0.17427310347557068, + 0.1747252345085144, + 0.189283087849617, + 0.18889831006526947, + 0.2044389545917511, + 0.20492415130138397, + 0.22237628698349, + 0.22264817357063293, + 0.24115440249443054, + 0.2428991198539734, + 0.26191622018814087, + 0.267524778842926, + 0.2883519232273102, + 0.2990366220474243, + 0.3199674189090729 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010639085667207837, + 0.00010674773511709645, + 0.000114928770926781, + 0.00011333057773299515, + 0.00011114918743260205, + 0.00010892991122091189, + 0.00010597817890811712, + 0.00010435423610033467, + 0.00010266973549732938, + 0.0001015385496430099, + 0.00010008511890191585, + 9.485893679084256e-05, + 9.186805982608348e-05, + 8.913136116461828e-05, + 8.655320561956614e-05, + 8.490345499012619e-05, + 8.653065015096217e-05, + 8.836571214487776e-05, + 8.356331090908498e-05, + 8.248459926107898e-05, + 7.508730777772143e-05, + 7.080488285282627e-05, + 6.500465678982437e-05, + 6.0319798649288714e-05, + 5.538603363675065e-05, + 5.071960185887292e-05, + 4.5680528273805976e-05, + 4.0155850001610816e-05, + 3.449051291681826e-05, + 2.892977681767661e-05, + 2.4221473722718656e-05, + 2.1527655917452648e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010897618176386459, + "first_to_last_ratio": 2.8953050536910427, + "imbalance_abs_log_ratio": 1.0630904780785062, + "last_quartile_mean": 3.763892914321332e-05, + "mean": 7.927156963205562e-05, + "normalized": [ + 1.3421060938480058, + 1.3466080665814153, + 1.4498107134781197, + 1.429649725103549, + 1.4021317850587363, + 1.3741359194288378, + 1.3369002203440912, + 1.3164144041136308, + 1.29516465958575, + 1.2808949048733107, + 1.262560075023968, + 1.1966325030668217, + 1.1589029995557718, + 1.1243799205481557, + 1.0918568412522767, + 1.0710454628842516, + 1.0915723071032915, + 1.1147213629682524, + 1.054139728744489, + 1.0405319289618822, + 0.9472161094607345, + 0.8931939052231708, + 0.8200248473891447, + 0.760926003222431, + 0.6986872329364575, + 0.6398208348124227, + 0.5762536112989216, + 0.5065605511281904, + 0.4350930992903045, + 0.3649451745683369, + 0.305550575510795, + 0.2715684326344833 + ], + "population_cv": 0.35248901334980626 + }, + "activation_output_rms_by_block": [ + 0.09771591424942017, + 0.13586144149303436, + 0.1991395801305771, + 0.2241763025522232, + 0.22696681320667267, + 0.22763192653656006, + 0.22912801802158356, + 0.2290782928466797, + 0.22818906605243683, + 0.22590669989585876, + 0.22214065492153168, + 0.21565376222133636, + 0.20698806643486023, + 0.20169880986213684, + 0.1979832947254181, + 0.1976672261953354, + 0.2011667788028717, + 0.20515663921833038, + 0.205980584025383, + 0.20982402563095093, + 0.2095191776752472, + 0.2096652388572693, + 0.2105090171098709, + 0.2148621529340744, + 0.2245095670223236, + 0.2386746108531952, + 0.25724881887435913, + 0.2833212912082672, + 0.3200184106826782, + 0.3660464584827423, + 0.43435168266296387, + 0.5434449911117554 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19621228612959385, + "first_to_last_ratio": 0.5884274155428802, + "imbalance_abs_log_ratio": 0.5303016979635413, + "last_quartile_mean": 0.3334519788622856, + "mean": 0.23750704107806087, + "normalized": [ + 0.4114232311003534, + 0.5720312159014314, + 0.8384575852011324, + 0.9438722386278381, + 0.9556214088494162, + 0.9584218030055994, + 0.9647209488255828, + 0.9645115858750019, + 0.960767584054228, + 0.951157906184392, + 0.9353013448073788, + 0.9079889221071891, + 0.8715028636428086, + 0.8492329698800171, + 0.8335891594066359, + 0.8322583839961553, + 0.8469929055145556, + 0.8637918197587332, + 0.8672609582032722, + 0.8834433904719, + 0.8821598581845203, + 0.8827748344031583, + 0.8863274796164187, + 0.9046559291834054, + 0.9452754158498172, + 1.0049159375226715, + 1.0831208106786603, + 1.1928963870807885, + 1.3474059936500937, + 1.5412025547589332, + 1.8287949725254948, + 2.288121601132417 + ], + "population_cv": 0.33570692978684213 + }, + "bits_per_byte": 2.0753701983577404, + "branch_output_rms_by_sublayer": [ + 0.017155639827251434, + 0.059614092111587524, + 0.05783598870038986, + 0.017162656411528587, + 0.08590122312307358, + 0.003484122920781374, + 0.037979111075401306, + 0.0025395466946065426, + 0.02590934932231903, + 0.004171412438154221, + 0.027761045843362808, + 0.005880473647266626, + 0.021641533821821213, + 0.0071118478663265705, + 0.01924360916018486, + 0.00746425986289978, + 0.022704657167196274, + 0.00924856960773468, + 0.018269814550876617, + 0.010196910239756107, + 0.022024810314178467, + 0.012587273493409157, + 0.03360125049948692, + 0.015558412298560143, + 0.026457328349351883, + 0.018742306157946587, + 0.024426763877272606, + 0.01840486004948616, + 0.02595876343548298, + 0.019045589491724968, + 0.023261789232492447, + 0.01923760399222374, + 0.0244763121008873, + 0.01806296780705452, + 0.0326010137796402, + 0.017410961911082268, + 0.037907715886831284, + 0.019881773740053177, + 0.034735601395368576, + 0.020177578553557396, + 0.04502493515610695, + 0.02574281580746174, + 0.034673795104026794, + 0.027684781700372696, + 0.0429018959403038, + 0.03708433732390404, + 0.03208737075328827, + 0.0411839634180069, + 0.027529660612344742, + 0.04565756395459175, + 0.02247321978211403, + 0.048749130219221115, + 0.01957971602678299, + 0.05322333425283432, + 0.02326148748397827, + 0.06035253033041954, + 0.014175337739288807, + 0.07119441032409668, + 0.011726614087820053, + 0.07984230667352676, + 0.013617267832159996, + 0.10812961310148239, + 0.01758233644068241, + 0.16684016585350037 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00037331716215226093, + 0.0002052857320094383, + 0.00022573696371192266, + 0.00015002318346068796, + 0.00011232796492243865, + 0.00011469567077281986, + 0.0001328622706326584, + 0.00011987742098506258, + 0.00012268315232766492, + 0.00013855849890850403, + 0.00016581239426236876, + 0.00020970093409220108, + 0.00019415593885306704, + 0.0001873084275669645, + 0.00019997003245353254, + 0.00019081336552079618, + 0.00019544152058884818, + 0.0002331889851483562, + 0.00028258169221688257, + 0.00026717547757069314, + 0.00031847213451059053, + 0.00028331189819324093, + 0.0003090653888381074, + 0.000275741819401602, + 0.000255221776165695, + 0.00023469865916502512, + 0.00021558263955965886, + 0.0002009260839676739, + 0.00018496297858040776, + 0.0001671377375581853, + 0.00015254274522576858, + 0.00014770686125523138 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017926579608091117, + "first_to_last_ratio": 0.9200315924660546, + "imbalance_abs_log_ratio": 0.08334726989162704, + "last_quartile_mean": 0.00019484743518470574, + "mean": 0.0002052152347055736, + "normalized": [ + 1.8191493564689118, + 1.0003435286077362, + 1.1000010015620527, + 0.7310528561679606, + 0.5473665982138111, + 0.5589042691561181, + 0.6474288852057137, + 0.584154588508271, + 0.5978267281358565, + 0.6751862214678978, + 0.8079926156567426, + 1.021858510617222, + 0.946108797096017, + 0.9127413363617942, + 0.9744404831368078, + 0.9298206626547972, + 0.9523733502010805, + 1.1363141994936055, + 1.377001530233894, + 1.301928085183418, + 1.5518932352537513, + 1.3805597747151384, + 1.5060547979370522, + 1.3436712912528852, + 1.2436785043365166, + 1.1436707391717382, + 1.0505196647264496, + 0.9790992576937408, + 0.9013121216160088, + 0.8144509241625318, + 0.7433305107421713, + 0.7197655742623075 + ], + "population_cv": 0.31225216605899336 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.05658925697207451, + 0.0616849884390831, + 0.09771591424942017, + 0.1262788027524948, + 0.13586144149303436, + 0.19808362424373627, + 0.1991395801305771, + 0.22383953630924225, + 0.2241763025522232, + 0.22702015936374664, + 0.22696681320667267, + 0.227730393409729, + 0.22763192653656006, + 0.22866052389144897, + 0.22912801802158356, + 0.22953395545482635, + 0.2290782928466797, + 0.22848571836948395, + 0.22818906605243683, + 0.22744755446910858, + 0.22590669989585876, + 0.22389201819896698, + 0.22214065492153168, + 0.21693368256092072, + 0.21565376222133636, + 0.209661602973938, + 0.20698806643486023, + 0.20248256623744965, + 0.20169880986213684, + 0.19645103812217712, + 0.1979832947254181, + 0.1954326629638672, + 0.1976672261953354, + 0.19818583130836487, + 0.2011667788028717, + 0.20395968854427338, + 0.20515663921833038, + 0.2042165845632553, + 0.205980584025383, + 0.20741206407546997, + 0.20982402563095093, + 0.20696893334388733, + 0.2095191776752472, + 0.20577766001224518, + 0.2096652388572693, + 0.20420841872692108, + 0.2105090171098709, + 0.20596569776535034, + 0.2148621529340744, + 0.21317537128925323, + 0.2245095670223236, + 0.22439339756965637, + 0.2386746108531952, + 0.23865793645381927, + 0.25724881887435913, + 0.25777244567871094, + 0.2833212912082672, + 0.28324905037879944, + 0.3200184106826782, + 0.3202260434627533, + 0.3660464584827423, + 0.366729199886322, + 0.43435168266296387, + 0.4362008571624756 + ], + "loss_nats": 1.4385370016098022, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.0616849884390831, + 0.09771591424942017, + 0.1262788027524948, + 0.13586144149303436, + 0.19808362424373627, + 0.1991395801305771, + 0.22383953630924225, + 0.2241763025522232, + 0.22702015936374664, + 0.22696681320667267, + 0.227730393409729, + 0.22763192653656006, + 0.22866052389144897, + 0.22912801802158356, + 0.22953395545482635, + 0.2290782928466797, + 0.22848571836948395, + 0.22818906605243683, + 0.22744755446910858, + 0.22590669989585876, + 0.22389201819896698, + 0.22214065492153168, + 0.21693368256092072, + 0.21565376222133636, + 0.209661602973938, + 0.20698806643486023, + 0.20248256623744965, + 0.20169880986213684, + 0.19645103812217712, + 0.1979832947254181, + 0.1954326629638672, + 0.1976672261953354, + 0.19818583130836487, + 0.2011667788028717, + 0.20395968854427338, + 0.20515663921833038, + 0.2042165845632553, + 0.205980584025383, + 0.20741206407546997, + 0.20982402563095093, + 0.20696893334388733, + 0.2095191776752472, + 0.20577766001224518, + 0.2096652388572693, + 0.20420841872692108, + 0.2105090171098709, + 0.20596569776535034, + 0.2148621529340744, + 0.21317537128925323, + 0.2245095670223236, + 0.22439339756965637, + 0.2386746108531952, + 0.23865793645381927, + 0.25724881887435913, + 0.25777244567871094, + 0.2833212912082672, + 0.28324905037879944, + 0.3200184106826782, + 0.3202260434627533, + 0.3660464584827423, + 0.366729199886322, + 0.43435168266296387, + 0.4362008571624756, + 0.5434449911117554 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001095466359402053, + 0.00011157916014781222, + 0.0001209566617035307, + 0.00011857408389914781, + 0.00011687575170071796, + 0.00011450610327301547, + 0.00011203528265468776, + 0.00011082263517891988, + 0.00010936566832242534, + 0.00010819115414051339, + 0.00010705165914259851, + 0.00010176753858104348, + 9.876617696136236e-05, + 9.541869076201692e-05, + 9.238059283234179e-05, + 9.015990508487448e-05, + 9.102027252083644e-05, + 9.192643483402207e-05, + 8.731557318242267e-05, + 8.749608969083056e-05, + 7.95004380051978e-05, + 7.466241368092597e-05, + 6.735928764101118e-05, + 6.187916733324528e-05, + 5.64747242606245e-05, + 5.17304906679783e-05, + 4.670304406317882e-05, + 4.1035571484826505e-05, + 3.507353176246397e-05, + 2.8758000553352758e-05, + 2.3154751033871435e-05, + 1.9675369912874885e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011436203931225464, + "first_to_last_ratio": 3.02339634825199, + "imbalance_abs_log_ratio": 1.1063808181034647, + "last_quartile_mean": 3.78256854673964e-05, + "mean": 8.318008940477739e-05, + "normalized": [ + 1.3169814642434559, + 1.341416689333394, + 1.45415402374626, + 1.4255104165869963, + 1.4050928838507029, + 1.376604715051423, + 1.3469002432720767, + 1.3323216646188754, + 1.3148058520377592, + 1.300685715953312, + 1.286986584273256, + 1.223460317358093, + 1.1873776244786027, + 1.1471337845969738, + 1.1106094438392846, + 1.0839120963928204, + 1.0942555264386233, + 1.1051495074341955, + 1.0497172316985723, + 1.0518874206187772, + 0.955762834280288, + 0.8975995844101339, + 0.8098006160250947, + 0.743918019036071, + 0.6789452219244779, + 0.6219095343387212, + 0.5614690293960716, + 0.49333406321717244, + 0.4216577790844446, + 0.34573178219860223, + 0.2783689125554313, + 0.23653941771003728 + ], + "population_cv": 0.36398106608216985 + }, + "activation_output_rms_by_block": [ + 0.11107808351516724, + 0.1828140765428543, + 0.2578570544719696, + 0.27254435420036316, + 0.2737041115760803, + 0.2742544412612915, + 0.27536308765411377, + 0.2744285464286804, + 0.2742423117160797, + 0.27115195989608765, + 0.26583269238471985, + 0.255132257938385, + 0.23856717348098755, + 0.22743643820285797, + 0.22118839621543884, + 0.22011704742908478, + 0.22253327071666718, + 0.2256278693675995, + 0.22584785521030426, + 0.22925184667110443, + 0.23080435395240784, + 0.23398378491401672, + 0.24050307273864746, + 0.24922524392604828, + 0.2630935311317444, + 0.27895617485046387, + 0.2995731830596924, + 0.3278428912162781, + 0.36820048093795776, + 0.42407870292663574, + 0.5123529434204102, + 0.6704596281051636 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24025546945631504, + "first_to_last_ratio": 0.6112286812568156, + "imbalance_abs_log_ratio": 0.4922841160907453, + "last_quartile_mean": 0.39306969195604324, + "mean": 0.2780639645643532, + "normalized": [ + 0.39946953820209985, + 0.6574533195240588, + 0.9273299935716515, + 0.9801498537480837, + 0.9843206832100536, + 0.9862998310153919, + 0.9902868503134846, + 0.9869259645298935, + 0.9862562095945767, + 0.975142393301143, + 0.9560127390156568, + 0.9175308218672072, + 0.8579578941656613, + 0.817928488357655, + 0.7954586872196038, + 0.7916058011110692, + 0.8002952524442108, + 0.8114243415938267, + 0.8122154755440653, + 0.824457232458282, + 0.8300405063777765, + 0.8414746775282531, + 0.8649199586701105, + 0.896287457875072, + 0.9461619075450383, + 1.0032086512450777, + 1.077353491413524, + 1.179019696888499, + 1.3241574884211362, + 1.5251120496359385, + 1.842572244926166, + 2.411170498685733 + ], + "population_cv": 0.35267348500337414 + }, + "bits_per_byte": 1.9134731260894795, + "branch_output_rms_by_sublayer": [ + 0.024391189217567444, + 0.06930837035179138, + 0.10445628315210342, + 0.019052622839808464, + 0.09584712982177734, + 0.0026203703600913286, + 0.031347230076789856, + 0.0018832145724445581, + 0.020275620743632317, + 0.002480348339304328, + 0.017658807337284088, + 0.005030750762671232, + 0.020567260682582855, + 0.0064065041951835155, + 0.016946211457252502, + 0.006859246641397476, + 0.017460251227021217, + 0.009737920947372913, + 0.022223282605409622, + 0.009925507940351963, + 0.02365109510719776, + 0.012353206984698772, + 0.04038562253117561, + 0.01845649816095829, + 0.03250120207667351, + 0.023668669164180756, + 0.029116366058588028, + 0.023308098316192627, + 0.029388509690761566, + 0.023618459701538086, + 0.026225870475172997, + 0.024062534794211388, + 0.02648484893143177, + 0.02197168581187725, + 0.03580358996987343, + 0.021145176142454147, + 0.04329143092036247, + 0.025955181568861008, + 0.04141230136156082, + 0.02680986560881138, + 0.05275771766901016, + 0.035374872386455536, + 0.044423554092645645, + 0.038993529975414276, + 0.05167620629072189, + 0.05052104964852333, + 0.03969942033290863, + 0.05720135569572449, + 0.0389031358063221, + 0.06300201267004013, + 0.026857485994696617, + 0.06565802544355392, + 0.030672859400510788, + 0.0703272894024849, + 0.02773989364504814, + 0.07888873666524887, + 0.019402364268898964, + 0.09176591783761978, + 0.018159963190555573, + 0.10585808008909225, + 0.02488313429057598, + 0.1433597356081009, + 0.024982891976833344, + 0.2408120483160019 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00037295757895409754, + 0.00022454763424927058, + 0.00021936409959100382, + 0.00010566073392567048, + 9.515815728211845e-05, + 0.00010551160558417538, + 0.00011265453132986426, + 0.00010566938037237213, + 0.00012588745811984293, + 0.0001281493774985002, + 0.0001672472035341268, + 0.0002455532725992925, + 0.00022493633083964273, + 0.0002220131528127045, + 0.0002449305149794571, + 0.0002379224969510024, + 0.00024418235837095776, + 0.00027770634611442576, + 0.000335912973129177, + 0.00033241348484404164, + 0.00039774019686300875, + 0.00036753844348542204, + 0.0003869475308616486, + 0.00034605021914329437, + 0.00032032390763334626, + 0.00028824702486555996, + 0.00027123288540817104, + 0.0002526057589974623, + 0.00022790421762734515, + 0.00020876185354851858, + 0.00018495058171630387, + 0.00016871013519521616 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016769046516107157, + "first_to_last_ratio": 0.6977158937201501, + "imbalance_abs_log_ratio": 0.359943288132801, + "last_quartile_mean": 0.00024034204562399042, + "mean": 0.00023591848270084503, + "normalized": [ + 1.5808747779504164, + 0.9518017905108639, + 0.9298300713012261, + 0.4478696739485771, + 0.40335185354164543, + 0.44723755585512437, + 0.47751464845047825, + 0.44790632409401143, + 0.5336057466912151, + 0.5431934625529073, + 0.7089194607367985, + 1.040839487386263, + 0.9534493790589178, + 0.9410587516122122, + 1.0381997721223044, + 1.0084945199172823, + 1.035028521612661, + 1.1771284001795213, + 1.4238518715599293, + 1.4090184077080408, + 1.6859221554394308, + 1.5579044052749225, + 1.640174718113608, + 1.466821145938367, + 1.357773685072106, + 1.2218077259807991, + 1.1496890040281678, + 1.0707332299935883, + 0.9660295158660277, + 0.8848897770050418, + 0.7839596948867686, + 0.7151204656107762 + ], + "population_cv": 0.3768493091252366 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.060260046273469925, + 0.06848448514938354, + 0.11107808351516724, + 0.17387914657592773, + 0.1828140765428543, + 0.25734034180641174, + 0.2578570544719696, + 0.2722952663898468, + 0.27254435420036316, + 0.27370211482048035, + 0.2737041115760803, + 0.27443060278892517, + 0.2742544412612915, + 0.2751515209674835, + 0.27536308765411377, + 0.2749122083187103, + 0.2744285464286804, + 0.2747279405593872, + 0.2742423117160797, + 0.2733822464942932, + 0.27115195989608765, + 0.26904159784317017, + 0.26583269238471985, + 0.2588832378387451, + 0.255132257938385, + 0.24577246606349945, + 0.23856717348098755, + 0.2310730367898941, + 0.22743643820285797, + 0.22001685202121735, + 0.22118839621543884, + 0.21798942983150482, + 0.22011704742908478, + 0.21957813203334808, + 0.22253327071666718, + 0.22450163960456848, + 0.2256278693675995, + 0.22364993393421173, + 0.22584785521030426, + 0.22638610005378723, + 0.22925184667110443, + 0.22674965858459473, + 0.23080435395240784, + 0.2277289777994156, + 0.23398378491401672, + 0.2319381833076477, + 0.24050307273864746, + 0.23799650371074677, + 0.24922524392604828, + 0.24935781955718994, + 0.2630935311317444, + 0.2632073760032654, + 0.27895617485046387, + 0.2799976170063019, + 0.2995731830596924, + 0.30053073167800903, + 0.3278428912162781, + 0.32797256112098694, + 0.36820048093795776, + 0.3684364855289459, + 0.42407870292663574, + 0.4252319037914276, + 0.5123529434204102, + 0.5140777826309204 + ], + "loss_nats": 1.3263185024261475, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06848448514938354, + 0.11107808351516724, + 0.17387914657592773, + 0.1828140765428543, + 0.25734034180641174, + 0.2578570544719696, + 0.2722952663898468, + 0.27254435420036316, + 0.27370211482048035, + 0.2737041115760803, + 0.27443060278892517, + 0.2742544412612915, + 0.2751515209674835, + 0.27536308765411377, + 0.2749122083187103, + 0.2744285464286804, + 0.2747279405593872, + 0.2742423117160797, + 0.2733822464942932, + 0.27115195989608765, + 0.26904159784317017, + 0.26583269238471985, + 0.2588832378387451, + 0.255132257938385, + 0.24577246606349945, + 0.23856717348098755, + 0.2310730367898941, + 0.22743643820285797, + 0.22001685202121735, + 0.22118839621543884, + 0.21798942983150482, + 0.22011704742908478, + 0.21957813203334808, + 0.22253327071666718, + 0.22450163960456848, + 0.2256278693675995, + 0.22364993393421173, + 0.22584785521030426, + 0.22638610005378723, + 0.22925184667110443, + 0.22674965858459473, + 0.23080435395240784, + 0.2277289777994156, + 0.23398378491401672, + 0.2319381833076477, + 0.24050307273864746, + 0.23799650371074677, + 0.24922524392604828, + 0.24935781955718994, + 0.2630935311317444, + 0.2632073760032654, + 0.27895617485046387, + 0.2799976170063019, + 0.2995731830596924, + 0.30053073167800903, + 0.3278428912162781, + 0.32797256112098694, + 0.36820048093795776, + 0.3684364855289459, + 0.42407870292663574, + 0.4252319037914276, + 0.5123529434204102, + 0.5140777826309204, + 0.6704596281051636 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018927091150544584, + 0.0001941375230671838, + 0.00020724961359519511, + 0.00020481550018303096, + 0.00020258830045349896, + 0.00020023793331347406, + 0.00019735096429940313, + 0.0001957487838808447, + 0.00019380047160666436, + 0.00019302338478155434, + 0.0001922047813422978, + 0.00018312338215764612, + 0.00017868881695903838, + 0.00017335887241642922, + 0.00016814567788969725, + 0.0001643431605771184, + 0.00016659151879139245, + 0.0001678116386756301, + 0.0001584039127919823, + 0.0001587578735779971, + 0.00014523210120387375, + 0.00013619904348161072, + 0.00012270425213500857, + 0.00011189868382643908, + 0.00010116446355823427, + 9.155859152087942e-05, + 8.147361222654581e-05, + 7.049509440548718e-05, + 5.894774221815169e-05, + 4.627152156899683e-05, + 3.385082527529448e-05, + 2.4755347112659365e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019892494128725957, + "first_to_last_ratio": 3.129490087873211, + "imbalance_abs_log_ratio": 1.140870080057971, + "last_quartile_mean": 6.356464973578113e-05, + "mean": 0.00014731888438745955, + "normalized": [ + 1.2847701928535473, + 1.3178047327360134, + 1.4068095509745602, + 1.3902867988353147, + 1.3751685759490058, + 1.3592142931712237, + 1.3396175590113453, + 1.3287419647165597, + 1.3155168287655152, + 1.310241966494184, + 1.304685289611514, + 1.2430407881451104, + 1.212938977253409, + 1.1767593349436627, + 1.141372191276318, + 1.1155607189155992, + 1.1308225655119981, + 1.1391047344227307, + 1.0752451286242999, + 1.077647813028859, + 0.9858349240678649, + 0.9245185642554635, + 0.8329159743857911, + 0.759567819778877, + 0.6867039753855606, + 0.6214993542855888, + 0.553042555034996, + 0.4785204198266929, + 0.40013703920751104, + 0.314090903969238, + 0.2297792670372409, + 0.16803919752440544 + ], + "population_cv": 0.3716245052237534 + }, + "activation_output_rms_by_block": [ + 0.10094190388917923, + 0.17939583957195282, + 0.23316815495491028, + 0.2387540638446808, + 0.23957562446594238, + 0.2399372160434723, + 0.24013783037662506, + 0.23953619599342346, + 0.2386971116065979, + 0.235844686627388, + 0.23058047890663147, + 0.2186841070652008, + 0.2011982500553131, + 0.1896955370903015, + 0.18346232175827026, + 0.1819538027048111, + 0.18335868418216705, + 0.18572981655597687, + 0.1859624832868576, + 0.18881060183048248, + 0.18956796824932098, + 0.19203881919384003, + 0.19704653322696686, + 0.20452122390270233, + 0.21571911871433258, + 0.22943545877933502, + 0.2464856505393982, + 0.26912161707878113, + 0.30166757106781006, + 0.3485357463359833, + 0.4315659701824188, + 0.6013737916946411 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2139308536425233, + "first_to_last_ratio": 0.6473178416327895, + "imbalance_abs_log_ratio": 0.4349178506515776, + "last_quartile_mean": 0.3304881155490875, + "mean": 0.2363282556179911, + "normalized": [ + 0.42712583658360814, + 0.7590960255803447, + 0.9866283417748029, + 1.0102645712860119, + 1.0137409250513, + 1.015270964599827, + 1.0161198446147375, + 1.0135740873093813, + 1.0100235834365736, + 0.9979538249062154, + 0.9756788425644265, + 0.9253405035862031, + 0.8513508024217668, + 0.8026781926446058, + 0.7763029489576774, + 0.7699197974825631, + 0.7758644166466246, + 0.7858976323855102, + 0.7868821390001439, + 0.7989336752676848, + 0.8021383975166516, + 0.8125935626768981, + 0.8337832169568393, + 0.8654116426657729, + 0.9127944441100949, + 0.9708338013978413, + 1.0429800274827308, + 1.138761915603517, + 1.2764769505828182, + 1.4747950702067896, + 1.8261293769290816, + 2.5446546377709565 + ], + "population_cv": 0.36483989965797836 + }, + "bits_per_byte": 1.8402207275901343, + "branch_output_rms_by_sublayer": [ + 0.033478349447250366, + 0.052373696118593216, + 0.10464497655630112, + 0.018782487139105797, + 0.07005563378334045, + 0.002347440691664815, + 0.020065777003765106, + 0.0017286264337599277, + 0.017071161419153214, + 0.002082002116367221, + 0.015527697280049324, + 0.004463158082216978, + 0.01603785529732704, + 0.005855648312717676, + 0.013071421533823013, + 0.006003717426210642, + 0.014386727474629879, + 0.008227710612118244, + 0.014956407248973846, + 0.008421593345701694, + 0.015932755544781685, + 0.010948682203888893, + 0.037222135812044144, + 0.016641829162836075, + 0.029781410470604897, + 0.021082323044538498, + 0.025465231388807297, + 0.02131747640669346, + 0.024801449850201607, + 0.021334858611226082, + 0.024093298241496086, + 0.021644918248057365, + 0.023488737642765045, + 0.020262837409973145, + 0.031102469190955162, + 0.020060395821928978, + 0.03728024289011955, + 0.024732086807489395, + 0.03451274335384369, + 0.025743624195456505, + 0.0439009852707386, + 0.03343674913048744, + 0.0384296216070652, + 0.037262238562107086, + 0.044400904327631, + 0.047452766448259354, + 0.03565425053238869, + 0.054450251162052155, + 0.032797131687402725, + 0.05981304869055748, + 0.022211335599422455, + 0.0634065493941307, + 0.02717488259077072, + 0.068015918135643, + 0.023526398465037346, + 0.07457657158374786, + 0.01840120367705822, + 0.08500036597251892, + 0.017178243026137352, + 0.09797899425029755, + 0.022575926035642624, + 0.13843585550785065, + 0.029057970270514488, + 0.24571193754673004 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004888726102119165, + 0.00034371112095624357, + 0.00027329239576199496, + 0.00013109360072049493, + 0.0001309003833444521, + 0.00015153657998531148, + 0.00016751171475037753, + 0.00015920301567702626, + 0.00019282855273444342, + 0.00018189099839941241, + 0.00022228035276414997, + 0.00039117902483509027, + 0.00035836801505216564, + 0.00037533451853890784, + 0.0004202645380768536, + 0.0004297743688669142, + 0.0004147505595850994, + 0.0004900318658719625, + 0.0005824864404877667, + 0.0005699482810090311, + 0.0006776225188364896, + 0.0006405760434101032, + 0.0006565350519436726, + 0.0006182198906110644, + 0.0005596621809681081, + 0.0005013865499612654, + 0.000471541721969693, + 0.0004314515507572245, + 0.0003811454676239625, + 0.0003426357383233983, + 0.00029149764497300414, + 0.0002412795793653285 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00023076517767597717, + "first_to_last_ratio": 0.5732227450358324, + "imbalance_abs_log_ratio": 0.5564809030131627, + "last_quartile_mean": 0.00040257505424274806, + "mean": 0.000384025402386654, + "normalized": [ + 1.2730215427772602, + 0.8950218366288691, + 0.7116518700677824, + 0.3413670030830513, + 0.34086386611648084, + 0.3946004067531388, + 0.4361995683340739, + 0.4145637624167724, + 0.5021244728500928, + 0.4736431425342962, + 0.5788167954065396, + 1.0186279928462485, + 0.9331883068801387, + 0.9773689870676945, + 1.0943665066554955, + 1.1191300528453012, + 1.0800081375020862, + 1.2760402380324216, + 1.5167914332429844, + 1.4841421360850018, + 1.7645252491766907, + 1.6680564343635333, + 1.7096136033278435, + 1.6098411375105153, + 1.4573571891075452, + 1.305607771895378, + 1.227892006724398, + 1.1234974251073624, + 0.9925006659851322, + 0.8922215462674452, + 0.7590582371931511, + 0.6282906752152749 + ], + "population_cv": 0.4280332727708177 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.061826109886169434, + 0.07441755384206772, + 0.10094190388917923, + 0.17219661176204681, + 0.17939583957195282, + 0.2327817976474762, + 0.23316815495491028, + 0.2385595589876175, + 0.2387540638446808, + 0.23968438804149628, + 0.23957562446594238, + 0.24012914299964905, + 0.2399372160434723, + 0.2401275336742401, + 0.24013783037662506, + 0.24013836681842804, + 0.23953619599342346, + 0.239403635263443, + 0.2386971116065979, + 0.23807503283023834, + 0.235844686627388, + 0.23385408520698547, + 0.23058047890663147, + 0.2232486605644226, + 0.2186841070652008, + 0.20881249010562897, + 0.2011982500553131, + 0.19461779296398163, + 0.1896955370903015, + 0.18348202109336853, + 0.18346232175827026, + 0.18089424073696136, + 0.1819538027048111, + 0.1813284158706665, + 0.18335868418216705, + 0.18508672714233398, + 0.18572981655597687, + 0.1841643899679184, + 0.1859624832868576, + 0.18672171235084534, + 0.18881060183048248, + 0.18648506700992584, + 0.18956796824932098, + 0.1869000494480133, + 0.19203881919384003, + 0.18984925746917725, + 0.19704653322696686, + 0.19547933340072632, + 0.20452122390270233, + 0.20447643101215363, + 0.21571911871433258, + 0.21594053506851196, + 0.22943545877933502, + 0.23029562830924988, + 0.2464856505393982, + 0.24702563881874084, + 0.26912161707878113, + 0.2691715657711029, + 0.30166757106781006, + 0.30165502429008484, + 0.3485357463359833, + 0.3492375612258911, + 0.4315659701824188, + 0.4323083460330963 + ], + "loss_nats": 1.2755438089370728, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07441755384206772, + 0.10094190388917923, + 0.17219661176204681, + 0.17939583957195282, + 0.2327817976474762, + 0.23316815495491028, + 0.2385595589876175, + 0.2387540638446808, + 0.23968438804149628, + 0.23957562446594238, + 0.24012914299964905, + 0.2399372160434723, + 0.2401275336742401, + 0.24013783037662506, + 0.24013836681842804, + 0.23953619599342346, + 0.239403635263443, + 0.2386971116065979, + 0.23807503283023834, + 0.235844686627388, + 0.23385408520698547, + 0.23058047890663147, + 0.2232486605644226, + 0.2186841070652008, + 0.20881249010562897, + 0.2011982500553131, + 0.19461779296398163, + 0.1896955370903015, + 0.18348202109336853, + 0.18346232175827026, + 0.18089424073696136, + 0.1819538027048111, + 0.1813284158706665, + 0.18335868418216705, + 0.18508672714233398, + 0.18572981655597687, + 0.1841643899679184, + 0.1859624832868576, + 0.18672171235084534, + 0.18881060183048248, + 0.18648506700992584, + 0.18956796824932098, + 0.1869000494480133, + 0.19203881919384003, + 0.18984925746917725, + 0.19704653322696686, + 0.19547933340072632, + 0.20452122390270233, + 0.20447643101215363, + 0.21571911871433258, + 0.21594053506851196, + 0.22943545877933502, + 0.23029562830924988, + 0.2464856505393982, + 0.24702563881874084, + 0.26912161707878113, + 0.2691715657711029, + 0.30166757106781006, + 0.30165502429008484, + 0.3485357463359833, + 0.3492375612258911, + 0.4315659701824188, + 0.4323083460330963, + 0.6013737916946411 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.037726566232575, + "cross_entropy_nats": 5.57132750749588, + "step": 0 + }, + { + "bits_per_byte": 5.132850850111892, + "cross_entropy_nats": 3.5578210949897766, + "step": 100 + }, + { + "bits_per_byte": 3.1258965111502706, + "cross_entropy_nats": 2.1667063534259796, + "step": 500 + }, + { + "bits_per_byte": 1.9712387018569424, + "cross_entropy_nats": 1.3663585484027863, + "step": 2000 + }, + { + "bits_per_byte": 1.7961544728728112, + "cross_entropy_nats": 1.2449994087219238, + "step": 4000 + }, + { + "bits_per_byte": 1.7269767303333703, + "cross_entropy_nats": 1.1970490515232086, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "final_optimizer_state": "4b960a87623e39ce5348146d7169b13bd4e5894a99361177367c7518bf6f603e", + "final_public_parameters": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 41.92772524377444, + "measured_steps": 7980, + "median_ms": 41.88774450449273, + "p95_ms": 42.070325661916286, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.02169460149312, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.560214996337891, + "step": 1, + "unclipped_grad_norm": 27.98912239074707 + }, + { + "bits_per_byte": 7.204295458472364, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 4.9936370849609375, + "step": 10, + "unclipped_grad_norm": 12.742182731628418 + }, + { + "bits_per_byte": 6.681083646642096, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.630974292755127, + "step": 20, + "unclipped_grad_norm": 3.4196884632110596 + }, + { + "bits_per_byte": 6.539581824957005, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.53289270401001, + "step": 30, + "unclipped_grad_norm": 3.0530965328216553 + }, + { + "bits_per_byte": 6.404133792582285, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43900728225708, + "step": 40, + "unclipped_grad_norm": 3.002453088760376 + }, + { + "bits_per_byte": 6.077271133589068, + "learning_rate": 3.75e-05, + "loss_nats": 4.2124433517456055, + "step": 50, + "unclipped_grad_norm": 2.940347909927368 + }, + { + "bits_per_byte": 5.8763623767123425, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.073184013366699, + "step": 60, + "unclipped_grad_norm": 2.7971668243408203 + }, + { + "bits_per_byte": 5.7093882046079605, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.957446336746216, + "step": 70, + "unclipped_grad_norm": 2.5766468048095703 + }, + { + "bits_per_byte": 5.512541333979312, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.82100248336792, + "step": 80, + "unclipped_grad_norm": 2.5652287006378174 + }, + { + "bits_per_byte": 5.341233887003791, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.702261209487915, + "step": 90, + "unclipped_grad_norm": 2.3708243370056152 + }, + { + "bits_per_byte": 5.171287854692499, + "learning_rate": 7.5e-05, + "loss_nats": 3.584463596343994, + "step": 100, + "unclipped_grad_norm": 2.273510217666626 + }, + { + "bits_per_byte": 5.001223154352113, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.466583728790283, + "step": 110, + "unclipped_grad_norm": 2.179899215698242 + }, + { + "bits_per_byte": 4.73609126012899, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.282808303833008, + "step": 120, + "unclipped_grad_norm": 1.9098186492919922 + }, + { + "bits_per_byte": 4.5869558165027025, + "learning_rate": 9.75e-05, + "loss_nats": 3.1794354915618896, + "step": 130, + "unclipped_grad_norm": 1.6915351152420044 + }, + { + "bits_per_byte": 4.392154507523416, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0444095134735107, + "step": 140, + "unclipped_grad_norm": 1.478805422782898 + }, + { + "bits_per_byte": 4.261904134823071, + "learning_rate": 0.0001125, + "loss_nats": 2.9541268348693848, + "step": 150, + "unclipped_grad_norm": 1.2380529642105103 + }, + { + "bits_per_byte": 4.155084334510974, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.880084991455078, + "step": 160, + "unclipped_grad_norm": 1.0523717403411865 + }, + { + "bits_per_byte": 4.00081108117281, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.77315092086792, + "step": 170, + "unclipped_grad_norm": 0.9717193841934204 + }, + { + "bits_per_byte": 3.86890313968345, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.6817193031311035, + "step": 180, + "unclipped_grad_norm": 1.2074095010757446 + }, + { + "bits_per_byte": 3.870022058809963, + "learning_rate": 0.0001425, + "loss_nats": 2.682494878768921, + "step": 190, + "unclipped_grad_norm": 0.712932288646698 + }, + { + "bits_per_byte": 3.7502069054333704, + "learning_rate": 0.00015, + "loss_nats": 2.599445343017578, + "step": 200, + "unclipped_grad_norm": 0.6916731595993042 + }, + { + "bits_per_byte": 3.7020534830182927, + "learning_rate": 0.0001575, + "loss_nats": 2.566067934036255, + "step": 210, + "unclipped_grad_norm": 0.8496056795120239 + }, + { + "bits_per_byte": 3.664806168425029, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.540250062942505, + "step": 220, + "unclipped_grad_norm": 0.7601449489593506 + }, + { + "bits_per_byte": 3.680203087226173, + "learning_rate": 0.0001725, + "loss_nats": 2.550922393798828, + "step": 230, + "unclipped_grad_norm": 0.7172863483428955 + }, + { + "bits_per_byte": 3.6667052008558563, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.5415663719177246, + "step": 240, + "unclipped_grad_norm": 0.8026229739189148 + }, + { + "bits_per_byte": 3.532286312663899, + "learning_rate": 0.0001875, + "loss_nats": 2.448394298553467, + "step": 250, + "unclipped_grad_norm": 0.9514944553375244 + }, + { + "bits_per_byte": 3.530420644867339, + "learning_rate": 0.000195, + "loss_nats": 2.44710111618042, + "step": 260, + "unclipped_grad_norm": 0.7874091267585754 + }, + { + "bits_per_byte": 3.4939940314839926, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4218521118164062, + "step": 270, + "unclipped_grad_norm": 0.9165833592414856 + }, + { + "bits_per_byte": 3.5060328170443893, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.430196762084961, + "step": 280, + "unclipped_grad_norm": 0.971961259841919 + }, + { + "bits_per_byte": 3.3964946589527543, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3542706966400146, + "step": 290, + "unclipped_grad_norm": 0.7937634587287903 + }, + { + "bits_per_byte": 3.488020042123622, + "learning_rate": 0.000225, + "loss_nats": 2.4177112579345703, + "step": 300, + "unclipped_grad_norm": 1.0019733905792236 + }, + { + "bits_per_byte": 3.3886120061331084, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.348806858062744, + "step": 310, + "unclipped_grad_norm": 0.8179588913917542 + }, + { + "bits_per_byte": 3.3666969448642643, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3336164951324463, + "step": 320, + "unclipped_grad_norm": 0.8202242255210876 + }, + { + "bits_per_byte": 3.351965942887261, + "learning_rate": 0.0002475, + "loss_nats": 2.3234057426452637, + "step": 330, + "unclipped_grad_norm": 0.8768095374107361 + }, + { + "bits_per_byte": 3.357094121570688, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.326960325241089, + "step": 340, + "unclipped_grad_norm": 0.9328333139419556 + }, + { + "bits_per_byte": 3.3293237389705688, + "learning_rate": 0.0002625, + "loss_nats": 2.307711362838745, + "step": 350, + "unclipped_grad_norm": 0.7516238689422607 + }, + { + "bits_per_byte": 3.3205963073698848, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.301661968231201, + "step": 360, + "unclipped_grad_norm": 0.9922879338264465 + }, + { + "bits_per_byte": 3.3203087523776422, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3014626502990723, + "step": 370, + "unclipped_grad_norm": 0.7593456506729126 + }, + { + "bits_per_byte": 3.3612334000116544, + "learning_rate": 0.000285, + "loss_nats": 2.329829454421997, + "step": 380, + "unclipped_grad_norm": 0.8890619874000549 + }, + { + "bits_per_byte": 3.310503677486652, + "learning_rate": 0.0002925, + "loss_nats": 2.294666290283203, + "step": 390, + "unclipped_grad_norm": 0.8439806699752808 + }, + { + "bits_per_byte": 3.2443041155165395, + "learning_rate": 0.0003, + "loss_nats": 2.2487802505493164, + "step": 400, + "unclipped_grad_norm": 1.1809823513031006 + }, + { + "bits_per_byte": 3.208035726171987, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2236409187316895, + "step": 410, + "unclipped_grad_norm": 0.9897300601005554 + }, + { + "bits_per_byte": 3.232116049015194, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2403321266174316, + "step": 420, + "unclipped_grad_norm": 0.898377537727356 + }, + { + "bits_per_byte": 3.2113598068478635, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.225944995880127, + "step": 430, + "unclipped_grad_norm": 1.0354490280151367 + }, + { + "bits_per_byte": 3.2315632967753207, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2399489879608154, + "step": 440, + "unclipped_grad_norm": 0.9574596285820007 + }, + { + "bits_per_byte": 3.1852710706078806, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.2078616619110107, + "step": 450, + "unclipped_grad_norm": 0.916765570640564 + }, + { + "bits_per_byte": 3.155730642565081, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.1873857975006104, + "step": 460, + "unclipped_grad_norm": 0.9241403937339783 + }, + { + "bits_per_byte": 3.18185446326584, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.205493450164795, + "step": 470, + "unclipped_grad_norm": 0.9457137584686279 + }, + { + "bits_per_byte": 3.13667255709237, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.17417573928833, + "step": 480, + "unclipped_grad_norm": 0.7014725208282471 + }, + { + "bits_per_byte": 3.105025341576501, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1522395610809326, + "step": 490, + "unclipped_grad_norm": 0.840235710144043 + }, + { + "bits_per_byte": 3.1575567543519423, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1886515617370605, + "step": 500, + "unclipped_grad_norm": 1.1171972751617432 + }, + { + "bits_per_byte": 3.1160109053830136, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1598541736602783, + "step": 510, + "unclipped_grad_norm": 0.8018220067024231 + }, + { + "bits_per_byte": 3.0148496782496044, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0897345542907715, + "step": 520, + "unclipped_grad_norm": 1.302595853805542 + }, + { + "bits_per_byte": 3.0793170309602265, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.1344199180603027, + "step": 530, + "unclipped_grad_norm": 0.6869481205940247 + }, + { + "bits_per_byte": 3.040265962324714, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.107351779937744, + "step": 540, + "unclipped_grad_norm": 1.3514107465744019 + }, + { + "bits_per_byte": 2.996858229177544, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.077263832092285, + "step": 550, + "unclipped_grad_norm": 0.9677014350891113 + }, + { + "bits_per_byte": 2.9987245849047075, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0785574913024902, + "step": 560, + "unclipped_grad_norm": 1.1580114364624023 + }, + { + "bits_per_byte": 2.9750642436949075, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.062157392501831, + "step": 570, + "unclipped_grad_norm": 1.1581705808639526 + }, + { + "bits_per_byte": 2.870633970333898, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.989771842956543, + "step": 580, + "unclipped_grad_norm": 0.9525179862976074 + }, + { + "bits_per_byte": 2.9300787417087224, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.030975818634033, + "step": 590, + "unclipped_grad_norm": 0.8967422842979431 + }, + { + "bits_per_byte": 2.847721925638002, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9738904237747192, + "step": 600, + "unclipped_grad_norm": 0.7550538182258606 + }, + { + "bits_per_byte": 2.905717055248704, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.014089584350586, + "step": 610, + "unclipped_grad_norm": 0.6918516755104065 + }, + { + "bits_per_byte": 2.79248952533076, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9356062412261963, + "step": 620, + "unclipped_grad_norm": 0.7874937653541565 + }, + { + "bits_per_byte": 2.8091388217954867, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9471466541290283, + "step": 630, + "unclipped_grad_norm": 0.9318027496337891 + }, + { + "bits_per_byte": 2.8177121569409977, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9530892372131348, + "step": 640, + "unclipped_grad_norm": 0.8821531534194946 + }, + { + "bits_per_byte": 2.7350290897815905, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.895777702331543, + "step": 650, + "unclipped_grad_norm": 1.1635478734970093 + }, + { + "bits_per_byte": 2.7242683072997553, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8883188962936401, + "step": 660, + "unclipped_grad_norm": 0.8677663207054138 + }, + { + "bits_per_byte": 2.7265658235326318, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.889911413192749, + "step": 670, + "unclipped_grad_norm": 0.8130120635032654 + }, + { + "bits_per_byte": 2.7610239234942022, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.9137959480285645, + "step": 680, + "unclipped_grad_norm": 0.9226085543632507 + }, + { + "bits_per_byte": 2.640049951016096, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8299431800842285, + "step": 690, + "unclipped_grad_norm": 0.8291181921958923 + }, + { + "bits_per_byte": 2.583675413924474, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.7908673286437988, + "step": 700, + "unclipped_grad_norm": 0.9929485321044922 + }, + { + "bits_per_byte": 2.6777724536736933, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8560904264450073, + "step": 710, + "unclipped_grad_norm": 1.0362995862960815 + }, + { + "bits_per_byte": 2.7123457820113424, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8800548315048218, + "step": 720, + "unclipped_grad_norm": 0.9176086187362671 + }, + { + "bits_per_byte": 2.545722970462673, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.7645606994628906, + "step": 730, + "unclipped_grad_norm": 1.4210379123687744 + }, + { + "bits_per_byte": 2.557563288043974, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7727677822113037, + "step": 740, + "unclipped_grad_norm": 1.1384987831115723 + }, + { + "bits_per_byte": 2.56630310239552, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7788257598876953, + "step": 750, + "unclipped_grad_norm": 0.9141139984130859 + }, + { + "bits_per_byte": 2.5326694872621855, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7555127143859863, + "step": 760, + "unclipped_grad_norm": 0.9497804045677185 + }, + { + "bits_per_byte": 2.5270525338850063, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7516193389892578, + "step": 770, + "unclipped_grad_norm": 1.0209811925888062 + }, + { + "bits_per_byte": 2.4536180057583974, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7007184028625488, + "step": 780, + "unclipped_grad_norm": 0.8395339250564575 + }, + { + "bits_per_byte": 2.5403350979764427, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7608261108398438, + "step": 790, + "unclipped_grad_norm": 1.0305956602096558 + }, + { + "bits_per_byte": 2.476898265309791, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.7168550491333008, + "step": 800, + "unclipped_grad_norm": 0.7494933605194092 + }, + { + "bits_per_byte": 2.5328731147208075, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7556538581848145, + "step": 810, + "unclipped_grad_norm": 2.0184309482574463 + }, + { + "bits_per_byte": 2.4645244575455134, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7082781791687012, + "step": 820, + "unclipped_grad_norm": 1.0523004531860352 + }, + { + "bits_per_byte": 2.4679443325579205, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7106486558914185, + "step": 830, + "unclipped_grad_norm": 0.9168965816497803 + }, + { + "bits_per_byte": 2.457752984633092, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7035845518112183, + "step": 840, + "unclipped_grad_norm": 1.0069817304611206 + }, + { + "bits_per_byte": 2.4003085434704525, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6637670993804932, + "step": 850, + "unclipped_grad_norm": 0.9418690204620361 + }, + { + "bits_per_byte": 2.440524794565561, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6916428804397583, + "step": 860, + "unclipped_grad_norm": 1.019945502281189 + }, + { + "bits_per_byte": 2.404373525406246, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6665847301483154, + "step": 870, + "unclipped_grad_norm": 1.4541757106781006 + }, + { + "bits_per_byte": 2.4247352393724975, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6806983947753906, + "step": 880, + "unclipped_grad_norm": 0.9109940528869629 + }, + { + "bits_per_byte": 2.4066381929528076, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6681544780731201, + "step": 890, + "unclipped_grad_norm": 0.9647485613822937 + }, + { + "bits_per_byte": 2.3372799976169487, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6200790405273438, + "step": 900, + "unclipped_grad_norm": 0.8496110439300537 + }, + { + "bits_per_byte": 2.3871222896634996, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6546270847320557, + "step": 910, + "unclipped_grad_norm": 1.1169326305389404 + }, + { + "bits_per_byte": 2.3333503660273847, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.617355227470398, + "step": 920, + "unclipped_grad_norm": 0.8540288805961609 + }, + { + "bits_per_byte": 2.3443775496354062, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.624998688697815, + "step": 930, + "unclipped_grad_norm": 1.0937522649765015 + }, + { + "bits_per_byte": 2.285418285283967, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.5841312408447266, + "step": 940, + "unclipped_grad_norm": 0.9093461632728577 + }, + { + "bits_per_byte": 2.261280348252721, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5674000978469849, + "step": 950, + "unclipped_grad_norm": 0.8109434843063354 + }, + { + "bits_per_byte": 2.2762792991997682, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5777965784072876, + "step": 960, + "unclipped_grad_norm": 0.8615142107009888 + }, + { + "bits_per_byte": 2.320710157136921, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.6085937023162842, + "step": 970, + "unclipped_grad_norm": 1.2632875442504883 + }, + { + "bits_per_byte": 2.270013111315582, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5734531879425049, + "step": 980, + "unclipped_grad_norm": 0.9684879183769226 + }, + { + "bits_per_byte": 2.2007698003905, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5254573822021484, + "step": 990, + "unclipped_grad_norm": 0.8230604529380798 + }, + { + "bits_per_byte": 2.245051377386703, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.556151032447815, + "step": 1000, + "unclipped_grad_norm": 0.7837699055671692 + }, + { + "bits_per_byte": 2.2494322914521314, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.559187650680542, + "step": 1010, + "unclipped_grad_norm": 0.8822826743125916 + }, + { + "bits_per_byte": 2.262538057378481, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5682718753814697, + "step": 1020, + "unclipped_grad_norm": 0.9820390939712524 + }, + { + "bits_per_byte": 2.1631948588473424, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4994124174118042, + "step": 1030, + "unclipped_grad_norm": 0.7792394161224365 + }, + { + "bits_per_byte": 2.258897528625017, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5657484531402588, + "step": 1040, + "unclipped_grad_norm": 0.8651536703109741 + }, + { + "bits_per_byte": 2.2449380408197843, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.556072473526001, + "step": 1050, + "unclipped_grad_norm": 0.8636346459388733 + }, + { + "bits_per_byte": 2.190915194496061, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5186266899108887, + "step": 1060, + "unclipped_grad_norm": 0.9602811932563782 + }, + { + "bits_per_byte": 2.2937991718431627, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5899404287338257, + "step": 1070, + "unclipped_grad_norm": 0.82843017578125 + }, + { + "bits_per_byte": 2.212529630091197, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5336086750030518, + "step": 1080, + "unclipped_grad_norm": 0.9238869547843933 + }, + { + "bits_per_byte": 2.140786207370522, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.4838799238204956, + "step": 1090, + "unclipped_grad_norm": 0.8506197333335876 + }, + { + "bits_per_byte": 2.2365177702335415, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5502359867095947, + "step": 1100, + "unclipped_grad_norm": 0.905985951423645 + }, + { + "bits_per_byte": 2.2504714106286445, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5599079132080078, + "step": 1110, + "unclipped_grad_norm": 0.8354071378707886 + }, + { + "bits_per_byte": 2.1399247462723503, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4832828044891357, + "step": 1120, + "unclipped_grad_norm": 0.8565837144851685 + }, + { + "bits_per_byte": 2.1407069233684743, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4838249683380127, + "step": 1130, + "unclipped_grad_norm": 0.7768547534942627 + }, + { + "bits_per_byte": 2.1434902905900377, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4857542514801025, + "step": 1140, + "unclipped_grad_norm": 0.8102613687515259 + }, + { + "bits_per_byte": 2.1476347285105297, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4886269569396973, + "step": 1150, + "unclipped_grad_norm": 0.7917956113815308 + }, + { + "bits_per_byte": 2.0881920209275155, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4474244117736816, + "step": 1160, + "unclipped_grad_norm": 0.8217911720275879 + }, + { + "bits_per_byte": 2.1987029128924305, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5240247249603271, + "step": 1170, + "unclipped_grad_norm": 0.7990338802337646 + }, + { + "bits_per_byte": 2.091903062567833, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4499967098236084, + "step": 1180, + "unclipped_grad_norm": 0.8429117202758789 + }, + { + "bits_per_byte": 2.129434492500322, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4760115146636963, + "step": 1190, + "unclipped_grad_norm": 0.878530740737915 + }, + { + "bits_per_byte": 2.113836010032367, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4651994705200195, + "step": 1200, + "unclipped_grad_norm": 0.7968210577964783 + }, + { + "bits_per_byte": 2.2311123555169234, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5464892387390137, + "step": 1210, + "unclipped_grad_norm": 0.8802653551101685 + }, + { + "bits_per_byte": 2.085374257175779, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4454712867736816, + "step": 1220, + "unclipped_grad_norm": 0.8048909306526184 + }, + { + "bits_per_byte": 2.0275156817898625, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4053667783737183, + "step": 1230, + "unclipped_grad_norm": 0.7767927646636963 + }, + { + "bits_per_byte": 2.034634215691725, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4103009700775146, + "step": 1240, + "unclipped_grad_norm": 0.769820511341095 + }, + { + "bits_per_byte": 2.2082001388383765, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5306077003479004, + "step": 1250, + "unclipped_grad_norm": 0.8053375482559204 + }, + { + "bits_per_byte": 2.1275602255712194, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4747123718261719, + "step": 1260, + "unclipped_grad_norm": 0.7535898685455322 + }, + { + "bits_per_byte": 1.9931237951488738, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3815281391143799, + "step": 1270, + "unclipped_grad_norm": 0.7098326683044434 + }, + { + "bits_per_byte": 2.1454299109264685, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4870986938476562, + "step": 1280, + "unclipped_grad_norm": 0.7225908637046814 + }, + { + "bits_per_byte": 2.0451564582368134, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4175944328308105, + "step": 1290, + "unclipped_grad_norm": 0.7847483158111572 + }, + { + "bits_per_byte": 2.0739233083160316, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4375340938568115, + "step": 1300, + "unclipped_grad_norm": 0.7665325403213501 + }, + { + "bits_per_byte": 2.153918458625104, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4929825067520142, + "step": 1310, + "unclipped_grad_norm": 0.8358681797981262 + }, + { + "bits_per_byte": 2.0810513012636918, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4424748420715332, + "step": 1320, + "unclipped_grad_norm": 0.7432150840759277 + }, + { + "bits_per_byte": 1.9944121171864873, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3824211359024048, + "step": 1330, + "unclipped_grad_norm": 0.9147225022315979 + }, + { + "bits_per_byte": 2.0106892431947467, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.393703579902649, + "step": 1340, + "unclipped_grad_norm": 0.7953330278396606 + }, + { + "bits_per_byte": 2.178856458948164, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.510268211364746, + "step": 1350, + "unclipped_grad_norm": 0.9230644702911377 + }, + { + "bits_per_byte": 2.000966204028218, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3869640827178955, + "step": 1360, + "unclipped_grad_norm": 0.766094446182251 + }, + { + "bits_per_byte": 2.099968876910648, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4555875062942505, + "step": 1370, + "unclipped_grad_norm": 0.802431583404541 + }, + { + "bits_per_byte": 2.0735674762113967, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.437287449836731, + "step": 1380, + "unclipped_grad_norm": 0.7674750685691833 + }, + { + "bits_per_byte": 2.0708314042188203, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4353909492492676, + "step": 1390, + "unclipped_grad_norm": 0.7390276789665222 + }, + { + "bits_per_byte": 2.082916281129648, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4437675476074219, + "step": 1400, + "unclipped_grad_norm": 0.8315935730934143 + }, + { + "bits_per_byte": 2.026164414102033, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.4044301509857178, + "step": 1410, + "unclipped_grad_norm": 0.8864192366600037 + }, + { + "bits_per_byte": 2.0547052789780182, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.424213171005249, + "step": 1420, + "unclipped_grad_norm": 0.6849789023399353 + }, + { + "bits_per_byte": 2.0500001776157135, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4209518432617188, + "step": 1430, + "unclipped_grad_norm": 0.6987867951393127 + }, + { + "bits_per_byte": 2.022067959341132, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4015907049179077, + "step": 1440, + "unclipped_grad_norm": 0.7956437468528748 + }, + { + "bits_per_byte": 2.068799257216225, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4339823722839355, + "step": 1450, + "unclipped_grad_norm": 0.9165704250335693 + }, + { + "bits_per_byte": 2.0134160281241766, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3955936431884766, + "step": 1460, + "unclipped_grad_norm": 0.7257574200630188 + }, + { + "bits_per_byte": 2.0111035494006755, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3939907550811768, + "step": 1470, + "unclipped_grad_norm": 0.77181476354599 + }, + { + "bits_per_byte": 1.915849582359102, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3279657363891602, + "step": 1480, + "unclipped_grad_norm": 0.6826913356781006 + }, + { + "bits_per_byte": 2.0189380470780796, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.399421215057373, + "step": 1490, + "unclipped_grad_norm": 0.850294291973114 + }, + { + "bits_per_byte": 1.925917102775309, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3349440097808838, + "step": 1500, + "unclipped_grad_norm": 0.6999683380126953 + }, + { + "bits_per_byte": 1.9985412486510523, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3852832317352295, + "step": 1510, + "unclipped_grad_norm": 0.7689449787139893 + }, + { + "bits_per_byte": 1.9951115706275462, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3829059600830078, + "step": 1520, + "unclipped_grad_norm": 0.785367488861084 + }, + { + "bits_per_byte": 1.9521630311581806, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.3531363010406494, + "step": 1530, + "unclipped_grad_norm": 0.6442116498947144 + }, + { + "bits_per_byte": 2.0327179849958115, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.4089727401733398, + "step": 1540, + "unclipped_grad_norm": 0.7405364513397217 + }, + { + "bits_per_byte": 1.996995812550399, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3842120170593262, + "step": 1550, + "unclipped_grad_norm": 0.7386215329170227 + }, + { + "bits_per_byte": 2.0870359535484155, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.4466230869293213, + "step": 1560, + "unclipped_grad_norm": 0.7484163641929626 + }, + { + "bits_per_byte": 2.0235489019477115, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4026172161102295, + "step": 1570, + "unclipped_grad_norm": 0.7196536064147949 + }, + { + "bits_per_byte": 2.00750653225788, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3914974927902222, + "step": 1580, + "unclipped_grad_norm": 0.7144435048103333 + }, + { + "bits_per_byte": 2.0128109931584413, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.395174264907837, + "step": 1590, + "unclipped_grad_norm": 0.8628523349761963 + }, + { + "bits_per_byte": 2.0153315708894866, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3969213962554932, + "step": 1600, + "unclipped_grad_norm": 0.715620756149292 + }, + { + "bits_per_byte": 2.048095125792106, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.4196313619613647, + "step": 1610, + "unclipped_grad_norm": 0.7313905358314514 + }, + { + "bits_per_byte": 2.0567931483594926, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.4256603717803955, + "step": 1620, + "unclipped_grad_norm": 0.761407732963562 + }, + { + "bits_per_byte": 1.907960738164025, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3224976062774658, + "step": 1630, + "unclipped_grad_norm": 0.7100295424461365 + }, + { + "bits_per_byte": 2.0356341228238413, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.410994052886963, + "step": 1640, + "unclipped_grad_norm": 0.7265889048576355 + }, + { + "bits_per_byte": 1.9881094689803178, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3780524730682373, + "step": 1650, + "unclipped_grad_norm": 0.7565467953681946 + }, + { + "bits_per_byte": 1.9576370669524927, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3569306135177612, + "step": 1660, + "unclipped_grad_norm": 0.6934499144554138 + }, + { + "bits_per_byte": 2.105078653450433, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4591293334960938, + "step": 1670, + "unclipped_grad_norm": 0.8928083181381226 + }, + { + "bits_per_byte": 2.0211131116635417, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.4009288549423218, + "step": 1680, + "unclipped_grad_norm": 0.6971679925918579 + }, + { + "bits_per_byte": 1.9058530907777012, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.321036696434021, + "step": 1690, + "unclipped_grad_norm": 0.6719903349876404 + }, + { + "bits_per_byte": 1.9394077658742894, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3442950248718262, + "step": 1700, + "unclipped_grad_norm": 0.6790670156478882 + }, + { + "bits_per_byte": 2.021125322431753, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.4009373188018799, + "step": 1710, + "unclipped_grad_norm": 0.6948477029800415 + }, + { + "bits_per_byte": 2.0066268409987162, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.39088773727417, + "step": 1720, + "unclipped_grad_norm": 0.6923770904541016 + }, + { + "bits_per_byte": 1.9082544845316984, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3227012157440186, + "step": 1730, + "unclipped_grad_norm": 0.875411868095398 + }, + { + "bits_per_byte": 1.950284636645457, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3518342971801758, + "step": 1740, + "unclipped_grad_norm": 0.7188331484794617 + }, + { + "bits_per_byte": 2.0503372636114046, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4211854934692383, + "step": 1750, + "unclipped_grad_norm": 0.6843723058700562 + }, + { + "bits_per_byte": 1.9245816574913592, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.334018349647522, + "step": 1760, + "unclipped_grad_norm": 0.7317860722541809 + }, + { + "bits_per_byte": 1.9854330749675908, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.376197338104248, + "step": 1770, + "unclipped_grad_norm": 0.7310082912445068 + }, + { + "bits_per_byte": 1.9060752923626159, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3211907148361206, + "step": 1780, + "unclipped_grad_norm": 0.6923527717590332 + }, + { + "bits_per_byte": 1.8947768921141837, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.313359260559082, + "step": 1790, + "unclipped_grad_norm": 0.7431268692016602 + }, + { + "bits_per_byte": 1.9836379200578849, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.374953031539917, + "step": 1800, + "unclipped_grad_norm": 0.800605058670044 + }, + { + "bits_per_byte": 1.9736849400871523, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3680541515350342, + "step": 1810, + "unclipped_grad_norm": 0.6639943718910217 + }, + { + "bits_per_byte": 2.0327128255162856, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4089691638946533, + "step": 1820, + "unclipped_grad_norm": 0.7219430804252625 + }, + { + "bits_per_byte": 1.8641528013882975, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.2921322584152222, + "step": 1830, + "unclipped_grad_norm": 0.6714999079704285 + }, + { + "bits_per_byte": 1.9760018903595762, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3696601390838623, + "step": 1840, + "unclipped_grad_norm": 0.6850439310073853 + }, + { + "bits_per_byte": 1.8495459708852175, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2820075750350952, + "step": 1850, + "unclipped_grad_norm": 0.7066381573677063 + }, + { + "bits_per_byte": 2.002332434206672, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.387911081314087, + "step": 1860, + "unclipped_grad_norm": 0.7545463442802429 + }, + { + "bits_per_byte": 2.025226076758926, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.4037797451019287, + "step": 1870, + "unclipped_grad_norm": 0.7131425142288208 + }, + { + "bits_per_byte": 1.8761874593650736, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3004740476608276, + "step": 1880, + "unclipped_grad_norm": 0.627357006072998 + }, + { + "bits_per_byte": 1.9627021280030534, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3604414463043213, + "step": 1890, + "unclipped_grad_norm": 0.6769031882286072 + }, + { + "bits_per_byte": 1.87336023656754, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2985143661499023, + "step": 1900, + "unclipped_grad_norm": 0.6752462983131409 + }, + { + "bits_per_byte": 1.8594791728511009, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2888927459716797, + "step": 1910, + "unclipped_grad_norm": 0.6314391493797302 + }, + { + "bits_per_byte": 1.9380384400081196, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3433458805084229, + "step": 1920, + "unclipped_grad_norm": 0.6901462078094482 + }, + { + "bits_per_byte": 1.8913296718602894, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3109698295593262, + "step": 1930, + "unclipped_grad_norm": 0.6903179287910461 + }, + { + "bits_per_byte": 1.8624152606666307, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2909278869628906, + "step": 1940, + "unclipped_grad_norm": 0.6572144627571106 + }, + { + "bits_per_byte": 1.930714558821127, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3382693529129028, + "step": 1950, + "unclipped_grad_norm": 0.7911640405654907 + }, + { + "bits_per_byte": 1.901616298173696, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.3180999755859375, + "step": 1960, + "unclipped_grad_norm": 0.7476807236671448 + }, + { + "bits_per_byte": 1.8545080143279113, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2854470014572144, + "step": 1970, + "unclipped_grad_norm": 0.6583976745605469 + }, + { + "bits_per_byte": 1.8296589290353475, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2682229280471802, + "step": 1980, + "unclipped_grad_norm": 0.6357024908065796 + }, + { + "bits_per_byte": 1.9398576724889465, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.344606876373291, + "step": 1990, + "unclipped_grad_norm": 0.7643471956253052 + }, + { + "bits_per_byte": 1.866766077768158, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.2939436435699463, + "step": 2000, + "unclipped_grad_norm": 0.682447612285614 + }, + { + "bits_per_byte": 1.9452981716663407, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3483779430389404, + "step": 2010, + "unclipped_grad_norm": 0.6716283559799194 + }, + { + "bits_per_byte": 1.922642553102881, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.332674264907837, + "step": 2020, + "unclipped_grad_norm": 0.7301247119903564 + }, + { + "bits_per_byte": 1.9056240098867518, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3208779096603394, + "step": 2030, + "unclipped_grad_norm": 0.7755060791969299 + }, + { + "bits_per_byte": 1.9283054258478407, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3365994691848755, + "step": 2040, + "unclipped_grad_norm": 0.648145854473114 + }, + { + "bits_per_byte": 1.8707481640662356, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.296703815460205, + "step": 2050, + "unclipped_grad_norm": 0.6962859630584717 + }, + { + "bits_per_byte": 1.774047478965604, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2296760082244873, + "step": 2060, + "unclipped_grad_norm": 0.605278730392456 + }, + { + "bits_per_byte": 1.8770535639948187, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3010743856430054, + "step": 2070, + "unclipped_grad_norm": 0.6994346976280212 + }, + { + "bits_per_byte": 1.9817110264376179, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.37361741065979, + "step": 2080, + "unclipped_grad_norm": 0.7077776789665222 + }, + { + "bits_per_byte": 1.8423689628820616, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2770328521728516, + "step": 2090, + "unclipped_grad_norm": 0.6188594698905945 + }, + { + "bits_per_byte": 1.9159211271418608, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3280153274536133, + "step": 2100, + "unclipped_grad_norm": 0.6360442638397217 + }, + { + "bits_per_byte": 1.8300450300865345, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2684905529022217, + "step": 2110, + "unclipped_grad_norm": 0.6518478393554688 + }, + { + "bits_per_byte": 1.8775524856649717, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.3014202117919922, + "step": 2120, + "unclipped_grad_norm": 0.677630603313446 + }, + { + "bits_per_byte": 1.88719968048247, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3081071376800537, + "step": 2130, + "unclipped_grad_norm": 0.6904116868972778 + }, + { + "bits_per_byte": 1.7861939256654409, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2380952835083008, + "step": 2140, + "unclipped_grad_norm": 0.5910601615905762 + }, + { + "bits_per_byte": 1.8339614190119824, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.271205186843872, + "step": 2150, + "unclipped_grad_norm": 0.7349988222122192 + }, + { + "bits_per_byte": 1.8980882460738961, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3156545162200928, + "step": 2160, + "unclipped_grad_norm": 0.6920630931854248 + }, + { + "bits_per_byte": 1.8727663804741106, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2981027364730835, + "step": 2170, + "unclipped_grad_norm": 0.7049499750137329 + }, + { + "bits_per_byte": 1.7856067768953952, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2376883029937744, + "step": 2180, + "unclipped_grad_norm": 0.652675211429596 + }, + { + "bits_per_byte": 1.9184699100276477, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3297820091247559, + "step": 2190, + "unclipped_grad_norm": 0.6759907007217407 + }, + { + "bits_per_byte": 1.9474407315307896, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.349863052368164, + "step": 2200, + "unclipped_grad_norm": 0.6637232899665833 + }, + { + "bits_per_byte": 1.9672947527116937, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.3636248111724854, + "step": 2210, + "unclipped_grad_norm": 0.7057329416275024 + }, + { + "bits_per_byte": 1.8580620358079913, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2879104614257812, + "step": 2220, + "unclipped_grad_norm": 0.6423102021217346 + }, + { + "bits_per_byte": 1.8639622446111417, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2920001745224, + "step": 2230, + "unclipped_grad_norm": 0.664786159992218 + }, + { + "bits_per_byte": 1.8281711071227333, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2671916484832764, + "step": 2240, + "unclipped_grad_norm": 0.6536247730255127 + }, + { + "bits_per_byte": 1.8379201156695426, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.273949146270752, + "step": 2250, + "unclipped_grad_norm": 0.6435070633888245 + }, + { + "bits_per_byte": 1.8619839281782669, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2906289100646973, + "step": 2260, + "unclipped_grad_norm": 0.6457217335700989 + }, + { + "bits_per_byte": 1.9082623957336382, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.322706699371338, + "step": 2270, + "unclipped_grad_norm": 0.6787397861480713 + }, + { + "bits_per_byte": 1.7849188462919439, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2372114658355713, + "step": 2280, + "unclipped_grad_norm": 0.6086795330047607 + }, + { + "bits_per_byte": 1.8290736720744614, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2678172588348389, + "step": 2290, + "unclipped_grad_norm": 0.630083441734314 + }, + { + "bits_per_byte": 1.8820873242029221, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3045635223388672, + "step": 2300, + "unclipped_grad_norm": 0.6462850570678711 + }, + { + "bits_per_byte": 1.834706619838171, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2717217206954956, + "step": 2310, + "unclipped_grad_norm": 0.6394227743148804 + }, + { + "bits_per_byte": 1.8546760413778043, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2855634689331055, + "step": 2320, + "unclipped_grad_norm": 0.6594879031181335 + }, + { + "bits_per_byte": 1.7978681080060082, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2461872100830078, + "step": 2330, + "unclipped_grad_norm": 0.6738982796669006 + }, + { + "bits_per_byte": 1.8462244699491042, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.279705286026001, + "step": 2340, + "unclipped_grad_norm": 0.687351405620575 + }, + { + "bits_per_byte": 1.7549572327371814, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.216443657875061, + "step": 2350, + "unclipped_grad_norm": 0.6065738201141357 + }, + { + "bits_per_byte": 1.8467686230564342, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2800824642181396, + "step": 2360, + "unclipped_grad_norm": 0.676578938961029 + }, + { + "bits_per_byte": 1.8782005162934228, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3018693923950195, + "step": 2370, + "unclipped_grad_norm": 0.6810930967330933 + }, + { + "bits_per_byte": 1.7780921669485956, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2324795722961426, + "step": 2380, + "unclipped_grad_norm": 0.5974810719490051 + }, + { + "bits_per_byte": 1.7797094917973095, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2336006164550781, + "step": 2390, + "unclipped_grad_norm": 0.6242655515670776 + }, + { + "bits_per_byte": 1.8773909939558115, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.301308274269104, + "step": 2400, + "unclipped_grad_norm": 0.7230958342552185 + }, + { + "bits_per_byte": 1.843124310684651, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2775564193725586, + "step": 2410, + "unclipped_grad_norm": 0.6579370498657227 + }, + { + "bits_per_byte": 1.9371601246101633, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.342737078666687, + "step": 2420, + "unclipped_grad_norm": 0.6557210087776184 + }, + { + "bits_per_byte": 1.8373321069862427, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2735415697097778, + "step": 2430, + "unclipped_grad_norm": 0.6670473217964172 + }, + { + "bits_per_byte": 1.7749123797167932, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2302755117416382, + "step": 2440, + "unclipped_grad_norm": 0.6538338661193848 + }, + { + "bits_per_byte": 1.8761522029216466, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.3004496097564697, + "step": 2450, + "unclipped_grad_norm": 0.6531840562820435 + }, + { + "bits_per_byte": 1.8790119304401935, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.3024318218231201, + "step": 2460, + "unclipped_grad_norm": 0.6944200992584229 + }, + { + "bits_per_byte": 1.8171404838616947, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2595458030700684, + "step": 2470, + "unclipped_grad_norm": 0.6366217136383057 + }, + { + "bits_per_byte": 1.8241991678010567, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.264438509941101, + "step": 2480, + "unclipped_grad_norm": 0.6316180229187012 + }, + { + "bits_per_byte": 1.7491533342185142, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2124207019805908, + "step": 2490, + "unclipped_grad_norm": 0.6618465185165405 + }, + { + "bits_per_byte": 1.827854315079844, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2669720649719238, + "step": 2500, + "unclipped_grad_norm": 0.6852855682373047 + }, + { + "bits_per_byte": 1.8052024801347035, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2512710094451904, + "step": 2510, + "unclipped_grad_norm": 0.663596510887146 + }, + { + "bits_per_byte": 1.7913451500240836, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2416658401489258, + "step": 2520, + "unclipped_grad_norm": 0.6331848502159119 + }, + { + "bits_per_byte": 1.7932974970766782, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2430191040039062, + "step": 2530, + "unclipped_grad_norm": 0.6864507794380188 + }, + { + "bits_per_byte": 1.8207522915124645, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2620493173599243, + "step": 2540, + "unclipped_grad_norm": 0.6478230953216553 + }, + { + "bits_per_byte": 1.8469901367107453, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.280236005783081, + "step": 2550, + "unclipped_grad_norm": 0.6545907258987427 + }, + { + "bits_per_byte": 1.7922375679994107, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2422844171524048, + "step": 2560, + "unclipped_grad_norm": 0.6228907704353333 + }, + { + "bits_per_byte": 1.783868376260474, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2364833354949951, + "step": 2570, + "unclipped_grad_norm": 0.6730257868766785 + }, + { + "bits_per_byte": 1.8186578867902572, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.260597586631775, + "step": 2580, + "unclipped_grad_norm": 0.6890478730201721 + }, + { + "bits_per_byte": 1.7885676302126492, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2397406101226807, + "step": 2590, + "unclipped_grad_norm": 0.6227992177009583 + }, + { + "bits_per_byte": 1.8280013602463319, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2670739889144897, + "step": 2600, + "unclipped_grad_norm": 0.6276137828826904 + }, + { + "bits_per_byte": 1.7336515060003441, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2016756534576416, + "step": 2610, + "unclipped_grad_norm": 0.6445866823196411 + }, + { + "bits_per_byte": 1.7935037042750628, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2431620359420776, + "step": 2620, + "unclipped_grad_norm": 0.6606776714324951 + }, + { + "bits_per_byte": 1.8032969123631437, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2499501705169678, + "step": 2630, + "unclipped_grad_norm": 0.6334376931190491 + }, + { + "bits_per_byte": 1.810602563389145, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2550140619277954, + "step": 2640, + "unclipped_grad_norm": 0.6691069006919861 + }, + { + "bits_per_byte": 1.7587798911179091, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2190933227539062, + "step": 2650, + "unclipped_grad_norm": 0.6500447988510132 + }, + { + "bits_per_byte": 1.7476075541525593, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2113492488861084, + "step": 2660, + "unclipped_grad_norm": 0.5952810049057007 + }, + { + "bits_per_byte": 1.7865554331975544, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2383458614349365, + "step": 2670, + "unclipped_grad_norm": 0.6339060664176941 + }, + { + "bits_per_byte": 1.7817725956770598, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2350306510925293, + "step": 2680, + "unclipped_grad_norm": 0.6211492419242859 + }, + { + "bits_per_byte": 1.8492028654967463, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2817697525024414, + "step": 2690, + "unclipped_grad_norm": 0.6305774450302124 + }, + { + "bits_per_byte": 1.8342178451444189, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2713829278945923, + "step": 2700, + "unclipped_grad_norm": 0.6566506624221802 + }, + { + "bits_per_byte": 1.7120219359319322, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.186683177947998, + "step": 2710, + "unclipped_grad_norm": 0.6094632744789124 + }, + { + "bits_per_byte": 1.8346758349436665, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.271700382232666, + "step": 2720, + "unclipped_grad_norm": 0.6520996689796448 + }, + { + "bits_per_byte": 1.7481449999365055, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2117217779159546, + "step": 2730, + "unclipped_grad_norm": 0.598540186882019 + }, + { + "bits_per_byte": 1.7678683143027543, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2253929376602173, + "step": 2740, + "unclipped_grad_norm": 0.6463729739189148 + }, + { + "bits_per_byte": 1.7756850977671197, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2308111190795898, + "step": 2750, + "unclipped_grad_norm": 0.6158115863800049 + }, + { + "bits_per_byte": 1.8688916113501715, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2954169511795044, + "step": 2760, + "unclipped_grad_norm": 0.6701047420501709 + }, + { + "bits_per_byte": 1.7889463360098492, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2400031089782715, + "step": 2770, + "unclipped_grad_norm": 0.6446090936660767 + }, + { + "bits_per_byte": 1.7765037351852266, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2313785552978516, + "step": 2780, + "unclipped_grad_norm": 0.637730598449707 + }, + { + "bits_per_byte": 1.7761525466121648, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2311351299285889, + "step": 2790, + "unclipped_grad_norm": 0.7148120403289795 + }, + { + "bits_per_byte": 1.7209247898364963, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.1928541660308838, + "step": 2800, + "unclipped_grad_norm": 0.6417195200920105 + }, + { + "bits_per_byte": 1.792574482012451, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2425179481506348, + "step": 2810, + "unclipped_grad_norm": 0.6152215003967285 + }, + { + "bits_per_byte": 1.7960922151531988, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2449562549591064, + "step": 2820, + "unclipped_grad_norm": 0.6487079858779907 + }, + { + "bits_per_byte": 1.72477857707703, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.1955254077911377, + "step": 2830, + "unclipped_grad_norm": 0.6001560091972351 + }, + { + "bits_per_byte": 1.807270399528678, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.252704381942749, + "step": 2840, + "unclipped_grad_norm": 0.6889218688011169 + }, + { + "bits_per_byte": 1.7224509638802528, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.1939120292663574, + "step": 2850, + "unclipped_grad_norm": 0.6565119028091431 + }, + { + "bits_per_byte": 1.7068036383394527, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1830661296844482, + "step": 2860, + "unclipped_grad_norm": 0.637535035610199 + }, + { + "bits_per_byte": 1.8267986855688483, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2662403583526611, + "step": 2870, + "unclipped_grad_norm": 0.635680615901947 + }, + { + "bits_per_byte": 1.7536737262137922, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2155539989471436, + "step": 2880, + "unclipped_grad_norm": 0.6707039475440979 + }, + { + "bits_per_byte": 1.7805597740231751, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2341899871826172, + "step": 2890, + "unclipped_grad_norm": 0.6349734663963318 + }, + { + "bits_per_byte": 1.713862494261466, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1879589557647705, + "step": 2900, + "unclipped_grad_norm": 0.6683149337768555 + }, + { + "bits_per_byte": 1.7497669683167927, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.212846040725708, + "step": 2910, + "unclipped_grad_norm": 0.6716704368591309 + }, + { + "bits_per_byte": 1.8262012178397509, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2658262252807617, + "step": 2920, + "unclipped_grad_norm": 0.6724333763122559 + }, + { + "bits_per_byte": 1.761767401746047, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2211641073226929, + "step": 2930, + "unclipped_grad_norm": 0.6612268686294556 + }, + { + "bits_per_byte": 1.8132794733498245, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2568695545196533, + "step": 2940, + "unclipped_grad_norm": 0.6505576968193054 + }, + { + "bits_per_byte": 1.7731756989083804, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2290717363357544, + "step": 2950, + "unclipped_grad_norm": 0.6778621077537537 + }, + { + "bits_per_byte": 1.7513619354208945, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.213951587677002, + "step": 2960, + "unclipped_grad_norm": 0.6551554203033447 + }, + { + "bits_per_byte": 1.8079858473562673, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2532002925872803, + "step": 2970, + "unclipped_grad_norm": 0.6640533208847046 + }, + { + "bits_per_byte": 1.7386847502604952, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2051644325256348, + "step": 2980, + "unclipped_grad_norm": 0.643166184425354 + }, + { + "bits_per_byte": 1.8672367942835695, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2942699193954468, + "step": 2990, + "unclipped_grad_norm": 0.6637417078018188 + }, + { + "bits_per_byte": 1.7982774267150616, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2464709281921387, + "step": 3000, + "unclipped_grad_norm": 0.6690325736999512 + }, + { + "bits_per_byte": 1.762204753627191, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.2214672565460205, + "step": 3010, + "unclipped_grad_norm": 0.6516304612159729 + }, + { + "bits_per_byte": 1.7294503138050674, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1987636089324951, + "step": 3020, + "unclipped_grad_norm": 0.6919088363647461 + }, + { + "bits_per_byte": 1.7609028449601596, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.220564842224121, + "step": 3030, + "unclipped_grad_norm": 0.6205207109451294 + }, + { + "bits_per_byte": 1.7350170482481948, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.2026221752166748, + "step": 3040, + "unclipped_grad_norm": 0.7004191279411316 + }, + { + "bits_per_byte": 1.7472788953067604, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2111214399337769, + "step": 3050, + "unclipped_grad_norm": 0.6576176881790161 + }, + { + "bits_per_byte": 1.768669753455775, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.225948452949524, + "step": 3060, + "unclipped_grad_norm": 0.7110167145729065 + }, + { + "bits_per_byte": 1.7354397816040157, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.2029151916503906, + "step": 3070, + "unclipped_grad_norm": 0.6717135906219482 + }, + { + "bits_per_byte": 1.748681069859245, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2120933532714844, + "step": 3080, + "unclipped_grad_norm": 0.6585274934768677 + }, + { + "bits_per_byte": 1.7073046238014162, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.1834133863449097, + "step": 3090, + "unclipped_grad_norm": 0.6292386651039124 + }, + { + "bits_per_byte": 1.8085337840819162, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.253580093383789, + "step": 3100, + "unclipped_grad_norm": 0.6514226198196411 + }, + { + "bits_per_byte": 1.7739243393875863, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.229590654373169, + "step": 3110, + "unclipped_grad_norm": 0.6451593041419983 + }, + { + "bits_per_byte": 1.6671654209338944, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1555910110473633, + "step": 3120, + "unclipped_grad_norm": 0.6327124834060669 + }, + { + "bits_per_byte": 1.7555804978639082, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.216875672340393, + "step": 3130, + "unclipped_grad_norm": 0.6219494342803955 + }, + { + "bits_per_byte": 1.7189500850392896, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1914854049682617, + "step": 3140, + "unclipped_grad_norm": 0.6722466349601746 + }, + { + "bits_per_byte": 1.6998178750440565, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.178223967552185, + "step": 3150, + "unclipped_grad_norm": 0.6060081720352173 + }, + { + "bits_per_byte": 1.7570095017099274, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2178661823272705, + "step": 3160, + "unclipped_grad_norm": 0.6486148834228516 + }, + { + "bits_per_byte": 1.8187225522669817, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.260642409324646, + "step": 3170, + "unclipped_grad_norm": 0.6782575845718384 + }, + { + "bits_per_byte": 1.680190870962291, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1646195650100708, + "step": 3180, + "unclipped_grad_norm": 0.6483905911445618 + }, + { + "bits_per_byte": 1.7650935462137336, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.223469614982605, + "step": 3190, + "unclipped_grad_norm": 0.7115707993507385 + }, + { + "bits_per_byte": 1.6804184040093824, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1647772789001465, + "step": 3200, + "unclipped_grad_norm": 0.6299659609794617 + }, + { + "bits_per_byte": 1.7133916057634035, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1876325607299805, + "step": 3210, + "unclipped_grad_norm": 0.6491534113883972 + }, + { + "bits_per_byte": 1.710321027514899, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1855041980743408, + "step": 3220, + "unclipped_grad_norm": 0.6104377508163452 + }, + { + "bits_per_byte": 1.6818119794293238, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1657432317733765, + "step": 3230, + "unclipped_grad_norm": 0.6708391308784485 + }, + { + "bits_per_byte": 1.7427318460005987, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2079696655273438, + "step": 3240, + "unclipped_grad_norm": 0.6418254375457764 + }, + { + "bits_per_byte": 1.733884714474914, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2018373012542725, + "step": 3250, + "unclipped_grad_norm": 0.6041762828826904 + }, + { + "bits_per_byte": 1.83323737205185, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2707033157348633, + "step": 3260, + "unclipped_grad_norm": 0.7252490520477295 + }, + { + "bits_per_byte": 1.6948699341787334, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.174794316291809, + "step": 3270, + "unclipped_grad_norm": 0.6451477408409119 + }, + { + "bits_per_byte": 1.6338448142251303, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1324949264526367, + "step": 3280, + "unclipped_grad_norm": 0.6251513361930847 + }, + { + "bits_per_byte": 1.6883934115125416, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1703051328659058, + "step": 3290, + "unclipped_grad_norm": 0.6296114921569824 + }, + { + "bits_per_byte": 1.744128001160303, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.208937406539917, + "step": 3300, + "unclipped_grad_norm": 0.6343309283256531 + }, + { + "bits_per_byte": 1.735473146238283, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2029383182525635, + "step": 3310, + "unclipped_grad_norm": 0.6353676915168762 + }, + { + "bits_per_byte": 1.7632710460625405, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2222063541412354, + "step": 3320, + "unclipped_grad_norm": 0.6778877973556519 + }, + { + "bits_per_byte": 1.6989030993241172, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1775898933410645, + "step": 3330, + "unclipped_grad_norm": 0.629153847694397 + }, + { + "bits_per_byte": 1.7071973066272779, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.18333899974823, + "step": 3340, + "unclipped_grad_norm": 0.6583864688873291 + }, + { + "bits_per_byte": 1.70429750715108, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1813290119171143, + "step": 3350, + "unclipped_grad_norm": 0.6497669816017151 + }, + { + "bits_per_byte": 1.7138317093669615, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.187937617301941, + "step": 3360, + "unclipped_grad_norm": 0.6299119591712952 + }, + { + "bits_per_byte": 1.6462306607749682, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1410801410675049, + "step": 3370, + "unclipped_grad_norm": 0.6380488872528076 + }, + { + "bits_per_byte": 1.7443316286189245, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2090785503387451, + "step": 3380, + "unclipped_grad_norm": 0.702843189239502 + }, + { + "bits_per_byte": 1.7228774808543925, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1942076683044434, + "step": 3390, + "unclipped_grad_norm": 0.6957465410232544 + }, + { + "bits_per_byte": 1.7111405248462603, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1860722303390503, + "step": 3400, + "unclipped_grad_norm": 0.6491814255714417 + }, + { + "bits_per_byte": 1.7448699343161251, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.209451675415039, + "step": 3410, + "unclipped_grad_norm": 0.6583186388015747 + }, + { + "bits_per_byte": 1.7129637129280568, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1873359680175781, + "step": 3420, + "unclipped_grad_norm": 0.6641048192977905 + }, + { + "bits_per_byte": 1.670370317632723, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.157812476158142, + "step": 3430, + "unclipped_grad_norm": 0.629267692565918 + }, + { + "bits_per_byte": 1.6804600238108913, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1648061275482178, + "step": 3440, + "unclipped_grad_norm": 0.6077277064323425 + }, + { + "bits_per_byte": 1.6713492428814338, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1584910154342651, + "step": 3450, + "unclipped_grad_norm": 0.6522519588470459 + }, + { + "bits_per_byte": 1.669551508231965, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.1572449207305908, + "step": 3460, + "unclipped_grad_norm": 0.627102255821228 + }, + { + "bits_per_byte": 1.7251744811393164, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1957998275756836, + "step": 3470, + "unclipped_grad_norm": 0.6381968855857849 + }, + { + "bits_per_byte": 1.7823232841251224, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.235412359237671, + "step": 3480, + "unclipped_grad_norm": 0.6999886631965637 + }, + { + "bits_per_byte": 1.718624865846508, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1912599802017212, + "step": 3490, + "unclipped_grad_norm": 0.6294557452201843 + }, + { + "bits_per_byte": 1.6614911973339777, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1516579389572144, + "step": 3500, + "unclipped_grad_norm": 0.6351874470710754 + }, + { + "bits_per_byte": 1.6797325371977416, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.164301872253418, + "step": 3510, + "unclipped_grad_norm": 0.6315440535545349 + }, + { + "bits_per_byte": 1.717181759423118, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1902596950531006, + "step": 3520, + "unclipped_grad_norm": 0.6176374554634094 + }, + { + "bits_per_byte": 1.6772098956748858, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.162553310394287, + "step": 3530, + "unclipped_grad_norm": 0.6610027551651001 + }, + { + "bits_per_byte": 1.6855345439072493, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1683235168457031, + "step": 3540, + "unclipped_grad_norm": 0.658635139465332 + }, + { + "bits_per_byte": 1.747215777673894, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2110776901245117, + "step": 3550, + "unclipped_grad_norm": 0.6718185544013977 + }, + { + "bits_per_byte": 1.666718266041651, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1552810668945312, + "step": 3560, + "unclipped_grad_norm": 0.6717336773872375 + }, + { + "bits_per_byte": 1.7121595220526225, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1867785453796387, + "step": 3570, + "unclipped_grad_norm": 0.6859359741210938 + }, + { + "bits_per_byte": 1.739672790589702, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.205849289894104, + "step": 3580, + "unclipped_grad_norm": 0.6660984754562378 + }, + { + "bits_per_byte": 1.6956691375572928, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1753482818603516, + "step": 3590, + "unclipped_grad_norm": 0.6079815030097961 + }, + { + "bits_per_byte": 1.7521809168043032, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2145192623138428, + "step": 3600, + "unclipped_grad_norm": 0.7036373615264893 + }, + { + "bits_per_byte": 1.7346027420422663, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.202335000038147, + "step": 3610, + "unclipped_grad_norm": 0.6451415419578552 + }, + { + "bits_per_byte": 1.6701776970637565, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1576789617538452, + "step": 3620, + "unclipped_grad_norm": 0.620152473449707 + }, + { + "bits_per_byte": 1.6714409096343439, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1585545539855957, + "step": 3630, + "unclipped_grad_norm": 0.6908113956451416 + }, + { + "bits_per_byte": 1.6125346159393705, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.117723822593689, + "step": 3640, + "unclipped_grad_norm": 0.604874312877655 + }, + { + "bits_per_byte": 1.6950919637809974, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1749482154846191, + "step": 3650, + "unclipped_grad_norm": 0.6512246131896973 + }, + { + "bits_per_byte": 1.6897813115050047, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1712671518325806, + "step": 3660, + "unclipped_grad_norm": 0.647068440914154 + }, + { + "bits_per_byte": 1.7204222565306753, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1925058364868164, + "step": 3670, + "unclipped_grad_norm": 0.7147674560546875 + }, + { + "bits_per_byte": 1.6652036148355023, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1542311906814575, + "step": 3680, + "unclipped_grad_norm": 0.6392054557800293 + }, + { + "bits_per_byte": 1.7297106955384738, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.198944091796875, + "step": 3690, + "unclipped_grad_norm": 0.7120746374130249 + }, + { + "bits_per_byte": 1.6313792709423611, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1307859420776367, + "step": 3700, + "unclipped_grad_norm": 0.6207358241081238 + }, + { + "bits_per_byte": 1.6932649920808815, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1736818552017212, + "step": 3710, + "unclipped_grad_norm": 0.6575222611427307 + }, + { + "bits_per_byte": 1.7191832935138596, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1916470527648926, + "step": 3720, + "unclipped_grad_norm": 0.6928348541259766 + }, + { + "bits_per_byte": 1.6966805675270171, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1760493516921997, + "step": 3730, + "unclipped_grad_norm": 0.6457412242889404 + }, + { + "bits_per_byte": 1.6651184834233252, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1541721820831299, + "step": 3740, + "unclipped_grad_norm": 0.6149929761886597 + }, + { + "bits_per_byte": 1.690470445987012, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1717448234558105, + "step": 3750, + "unclipped_grad_norm": 0.6539504528045654 + }, + { + "bits_per_byte": 1.7257394441474005, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.196191430091858, + "step": 3760, + "unclipped_grad_norm": 0.6810935735702515 + }, + { + "bits_per_byte": 1.709021182639678, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.184603214263916, + "step": 3770, + "unclipped_grad_norm": 0.6034973859786987 + }, + { + "bits_per_byte": 1.7700676284419878, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2269173860549927, + "step": 3780, + "unclipped_grad_norm": 0.6868588328361511 + }, + { + "bits_per_byte": 1.6607432447853754, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.151139497756958, + "step": 3790, + "unclipped_grad_norm": 0.6561638712882996 + }, + { + "bits_per_byte": 1.6881032767805362, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1701040267944336, + "step": 3800, + "unclipped_grad_norm": 0.6706013679504395 + }, + { + "bits_per_byte": 1.6338372469884925, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1324896812438965, + "step": 3810, + "unclipped_grad_norm": 0.6373327374458313 + }, + { + "bits_per_byte": 1.6310115720348164, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1305310726165771, + "step": 3820, + "unclipped_grad_norm": 0.6431099772453308 + }, + { + "bits_per_byte": 1.699418359346102, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1779470443725586, + "step": 3830, + "unclipped_grad_norm": 0.6568827629089355 + }, + { + "bits_per_byte": 1.6534394855685337, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1460769176483154, + "step": 3840, + "unclipped_grad_norm": 0.6673737168312073 + }, + { + "bits_per_byte": 1.6227488375567638, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1248037815093994, + "step": 3850, + "unclipped_grad_norm": 0.6300650835037231 + }, + { + "bits_per_byte": 1.6095687751252414, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1156680583953857, + "step": 3860, + "unclipped_grad_norm": 0.6291298270225525 + }, + { + "bits_per_byte": 1.5641092889532788, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.084157943725586, + "step": 3870, + "unclipped_grad_norm": 0.619691789150238 + }, + { + "bits_per_byte": 1.670964173726152, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.158224105834961, + "step": 3880, + "unclipped_grad_norm": 0.6558656692504883 + }, + { + "bits_per_byte": 1.6390438497607132, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1360986232757568, + "step": 3890, + "unclipped_grad_norm": 0.6659069061279297 + }, + { + "bits_per_byte": 1.6436039697483404, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1392594575881958, + "step": 3900, + "unclipped_grad_norm": 0.6627041101455688 + }, + { + "bits_per_byte": 1.6594528589559516, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1502450704574585, + "step": 3910, + "unclipped_grad_norm": 0.6734365224838257 + }, + { + "bits_per_byte": 1.6941402117911224, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1742885112762451, + "step": 3920, + "unclipped_grad_norm": 0.6930717825889587 + }, + { + "bits_per_byte": 1.6772669939149722, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.162592887878418, + "step": 3930, + "unclipped_grad_norm": 0.7069801688194275 + }, + { + "bits_per_byte": 1.6121323485190024, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1174449920654297, + "step": 3940, + "unclipped_grad_norm": 0.6367635726928711 + }, + { + "bits_per_byte": 1.7182599186613772, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1910070180892944, + "step": 3950, + "unclipped_grad_norm": 0.690847635269165 + }, + { + "bits_per_byte": 1.6993077745015972, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1778703927993774, + "step": 3960, + "unclipped_grad_norm": 0.6660118699073792 + }, + { + "bits_per_byte": 1.6431734971732308, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1389610767364502, + "step": 3970, + "unclipped_grad_norm": 0.669096827507019 + }, + { + "bits_per_byte": 1.6275112091418058, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.12810480594635, + "step": 3980, + "unclipped_grad_norm": 0.6583772301673889 + }, + { + "bits_per_byte": 1.669720739160414, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1573622226715088, + "step": 3990, + "unclipped_grad_norm": 0.6523779630661011 + }, + { + "bits_per_byte": 1.5952216384249642, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1057233810424805, + "step": 4000, + "unclipped_grad_norm": 0.6264373064041138 + }, + { + "bits_per_byte": 1.6774286576067834, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1627049446105957, + "step": 4010, + "unclipped_grad_norm": 0.6995967030525208 + }, + { + "bits_per_byte": 1.6794896976947233, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1641335487365723, + "step": 4020, + "unclipped_grad_norm": 0.6873654127120972 + }, + { + "bits_per_byte": 1.6416081110850775, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.137876033782959, + "step": 4030, + "unclipped_grad_norm": 0.7148170471191406 + }, + { + "bits_per_byte": 1.62684030482079, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1276397705078125, + "step": 4040, + "unclipped_grad_norm": 0.6498348712921143 + }, + { + "bits_per_byte": 1.644293964143602, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1397377252578735, + "step": 4050, + "unclipped_grad_norm": 0.6837151646614075 + }, + { + "bits_per_byte": 1.6349320885438852, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1332485675811768, + "step": 4060, + "unclipped_grad_norm": 0.6708939671516418 + }, + { + "bits_per_byte": 1.769896849669681, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.2267990112304688, + "step": 4070, + "unclipped_grad_norm": 0.7203540802001953 + }, + { + "bits_per_byte": 1.5942710903136454, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1050645112991333, + "step": 4080, + "unclipped_grad_norm": 0.6567085385322571 + }, + { + "bits_per_byte": 1.6276448396615262, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.128197431564331, + "step": 4090, + "unclipped_grad_norm": 0.6684355139732361 + }, + { + "bits_per_byte": 1.7285615074654084, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1981475353240967, + "step": 4100, + "unclipped_grad_norm": 0.7356466054916382 + }, + { + "bits_per_byte": 1.6342331510507786, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1327641010284424, + "step": 4110, + "unclipped_grad_norm": 0.655012309551239 + }, + { + "bits_per_byte": 1.6424126459258137, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1384336948394775, + "step": 4120, + "unclipped_grad_norm": 0.7107924222946167 + }, + { + "bits_per_byte": 1.6944599275390764, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.17451012134552, + "step": 4130, + "unclipped_grad_norm": 0.6998885869979858 + }, + { + "bits_per_byte": 1.7017107160994525, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.179535984992981, + "step": 4140, + "unclipped_grad_norm": 0.6857012510299683 + }, + { + "bits_per_byte": 1.6170882005862652, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.120880126953125, + "step": 4150, + "unclipped_grad_norm": 0.6785045266151428 + }, + { + "bits_per_byte": 1.6430269679546958, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138859510421753, + "step": 4160, + "unclipped_grad_norm": 0.6962723135948181 + }, + { + "bits_per_byte": 1.6682418603456448, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1563371419906616, + "step": 4170, + "unclipped_grad_norm": 0.7071301937103271 + }, + { + "bits_per_byte": 1.6402630347726797, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1369436979293823, + "step": 4180, + "unclipped_grad_norm": 0.6712477207183838 + }, + { + "bits_per_byte": 1.67100166594404, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.158250093460083, + "step": 4190, + "unclipped_grad_norm": 0.6872346997261047 + }, + { + "bits_per_byte": 1.6579426792987255, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.149198293685913, + "step": 4200, + "unclipped_grad_norm": 0.7030718326568604 + }, + { + "bits_per_byte": 1.6808303024581988, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1650627851486206, + "step": 4210, + "unclipped_grad_norm": 0.658928394317627 + }, + { + "bits_per_byte": 1.659455094730413, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1502466201782227, + "step": 4220, + "unclipped_grad_norm": 0.6767720580101013 + }, + { + "bits_per_byte": 1.6554740403282406, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1474871635437012, + "step": 4230, + "unclipped_grad_norm": 0.7309125065803528 + }, + { + "bits_per_byte": 1.6771517655388943, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.162513017654419, + "step": 4240, + "unclipped_grad_norm": 0.7190505862236023 + }, + { + "bits_per_byte": 1.6138194983239667, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1186144351959229, + "step": 4250, + "unclipped_grad_norm": 0.6701101660728455 + }, + { + "bits_per_byte": 1.6526860295751036, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1455546617507935, + "step": 4260, + "unclipped_grad_norm": 0.6757012009620667 + }, + { + "bits_per_byte": 1.5671267245626668, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.0862494707107544, + "step": 4270, + "unclipped_grad_norm": 0.647323727607727 + }, + { + "bits_per_byte": 1.640895586962553, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.13738214969635, + "step": 4280, + "unclipped_grad_norm": 0.666140079498291 + }, + { + "bits_per_byte": 1.6259669769197085, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1270344257354736, + "step": 4290, + "unclipped_grad_norm": 0.6681356430053711 + }, + { + "bits_per_byte": 1.5856241465935652, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0990709066390991, + "step": 4300, + "unclipped_grad_norm": 0.6779436469078064 + }, + { + "bits_per_byte": 1.5644216094472456, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.0843744277954102, + "step": 4310, + "unclipped_grad_norm": 0.6528535485267639 + }, + { + "bits_per_byte": 1.6990217673532124, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1776721477508545, + "step": 4320, + "unclipped_grad_norm": 0.6877329349517822 + }, + { + "bits_per_byte": 1.6415279671697756, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1378204822540283, + "step": 4330, + "unclipped_grad_norm": 0.7055521011352539 + }, + { + "bits_per_byte": 1.604049851859054, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1118426322937012, + "step": 4340, + "unclipped_grad_norm": 0.6865817308425903 + }, + { + "bits_per_byte": 1.5633534252027368, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0836340188980103, + "step": 4350, + "unclipped_grad_norm": 0.6733583211898804 + }, + { + "bits_per_byte": 1.6505902489916895, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1441019773483276, + "step": 4360, + "unclipped_grad_norm": 0.6889402270317078 + }, + { + "bits_per_byte": 1.6184719729951074, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1218392848968506, + "step": 4370, + "unclipped_grad_norm": 0.6762850284576416 + }, + { + "bits_per_byte": 1.556092145700658, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.0786008834838867, + "step": 4380, + "unclipped_grad_norm": 0.6198620200157166 + }, + { + "bits_per_byte": 1.7130244228038114, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1873780488967896, + "step": 4390, + "unclipped_grad_norm": 0.6756254434585571 + }, + { + "bits_per_byte": 1.6522954569749941, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1452839374542236, + "step": 4400, + "unclipped_grad_norm": 0.6876093745231628 + }, + { + "bits_per_byte": 1.661213789318136, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.151465654373169, + "step": 4410, + "unclipped_grad_norm": 0.6821660399436951 + }, + { + "bits_per_byte": 1.6753433679650718, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1612595319747925, + "step": 4420, + "unclipped_grad_norm": 0.7177299857139587 + }, + { + "bits_per_byte": 1.6206171125993192, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1233261823654175, + "step": 4430, + "unclipped_grad_norm": 0.6967712640762329 + }, + { + "bits_per_byte": 1.6227820302083802, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1248267889022827, + "step": 4440, + "unclipped_grad_norm": 0.6610331535339355 + }, + { + "bits_per_byte": 1.6554300127696198, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1474566459655762, + "step": 4450, + "unclipped_grad_norm": 0.6704992055892944 + }, + { + "bits_per_byte": 1.6458450756717338, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.140812873840332, + "step": 4460, + "unclipped_grad_norm": 0.7582810521125793 + }, + { + "bits_per_byte": 1.6144303806998315, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1190378665924072, + "step": 4470, + "unclipped_grad_norm": 0.7301307916641235 + }, + { + "bits_per_byte": 1.6560006512051826, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1478521823883057, + "step": 4480, + "unclipped_grad_norm": 0.7387185096740723 + }, + { + "bits_per_byte": 1.6943782357799166, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1744534969329834, + "step": 4490, + "unclipped_grad_norm": 0.7154380083084106 + }, + { + "bits_per_byte": 1.6097710267226562, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1158082485198975, + "step": 4500, + "unclipped_grad_norm": 0.6710496544837952 + }, + { + "bits_per_byte": 1.6061318738303991, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.113285779953003, + "step": 4510, + "unclipped_grad_norm": 0.7097434401512146 + }, + { + "bits_per_byte": 1.676021495557424, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1617295742034912, + "step": 4520, + "unclipped_grad_norm": 0.7576807141304016 + }, + { + "bits_per_byte": 1.635137779794317, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1333911418914795, + "step": 4530, + "unclipped_grad_norm": 0.6911670565605164 + }, + { + "bits_per_byte": 1.64362185594403, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.139271855354309, + "step": 4540, + "unclipped_grad_norm": 0.6786331534385681 + }, + { + "bits_per_byte": 1.650524379636409, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1440563201904297, + "step": 4550, + "unclipped_grad_norm": 0.6853471994400024 + }, + { + "bits_per_byte": 1.6363368428361325, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1342222690582275, + "step": 4560, + "unclipped_grad_norm": 0.7256327867507935 + }, + { + "bits_per_byte": 1.6365012582503573, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.134336233139038, + "step": 4570, + "unclipped_grad_norm": 0.697358250617981 + }, + { + "bits_per_byte": 1.6112284077060677, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1168184280395508, + "step": 4580, + "unclipped_grad_norm": 0.6777223348617554 + }, + { + "bits_per_byte": 1.5962301446896239, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1064224243164062, + "step": 4590, + "unclipped_grad_norm": 0.7330402731895447 + }, + { + "bits_per_byte": 1.5998985346325274, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1089651584625244, + "step": 4600, + "unclipped_grad_norm": 0.6789212822914124 + }, + { + "bits_per_byte": 1.596745060746307, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1067793369293213, + "step": 4610, + "unclipped_grad_norm": 0.6819480061531067 + }, + { + "bits_per_byte": 1.6577524664868712, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.14906644821167, + "step": 4620, + "unclipped_grad_norm": 0.7283186912536621 + }, + { + "bits_per_byte": 1.646706020821953, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1414096355438232, + "step": 4630, + "unclipped_grad_norm": 0.7197591662406921 + }, + { + "bits_per_byte": 1.6542598428131492, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.1466455459594727, + "step": 4640, + "unclipped_grad_norm": 0.7083833813667297 + }, + { + "bits_per_byte": 1.6790158854915962, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1638051271438599, + "step": 4650, + "unclipped_grad_norm": 0.684671938419342 + }, + { + "bits_per_byte": 1.6168221434253804, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.12069571018219, + "step": 4660, + "unclipped_grad_norm": 0.7103760838508606 + }, + { + "bits_per_byte": 1.6034386255178876, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1114189624786377, + "step": 4670, + "unclipped_grad_norm": 0.6704027652740479 + }, + { + "bits_per_byte": 1.591610346722147, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1032202243804932, + "step": 4680, + "unclipped_grad_norm": 0.7572834491729736 + }, + { + "bits_per_byte": 1.6094411639983013, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.115579605102539, + "step": 4690, + "unclipped_grad_norm": 0.6773369312286377 + }, + { + "bits_per_byte": 1.6221410508686145, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.124382495880127, + "step": 4700, + "unclipped_grad_norm": 0.7124287486076355 + }, + { + "bits_per_byte": 1.6214163159778787, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.12388014793396, + "step": 4710, + "unclipped_grad_norm": 0.7075444459915161 + }, + { + "bits_per_byte": 1.6415025137374477, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1378028392791748, + "step": 4720, + "unclipped_grad_norm": 0.7044992446899414 + }, + { + "bits_per_byte": 1.654741394235565, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1469793319702148, + "step": 4730, + "unclipped_grad_norm": 0.7124809622764587 + }, + { + "bits_per_byte": 1.6107767812649019, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1165053844451904, + "step": 4740, + "unclipped_grad_norm": 0.7306708693504333 + }, + { + "bits_per_byte": 1.5756537963750956, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.0921599864959717, + "step": 4750, + "unclipped_grad_norm": 0.6872034668922424 + }, + { + "bits_per_byte": 1.550611402582962, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0748019218444824, + "step": 4760, + "unclipped_grad_norm": 0.6966266632080078 + }, + { + "bits_per_byte": 1.5558193812163896, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.0784118175506592, + "step": 4770, + "unclipped_grad_norm": 0.6661347150802612 + }, + { + "bits_per_byte": 1.5972586009417833, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.10713529586792, + "step": 4780, + "unclipped_grad_norm": 0.7209707498550415 + }, + { + "bits_per_byte": 1.6013201432245594, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1099505424499512, + "step": 4790, + "unclipped_grad_norm": 0.7224242091178894 + }, + { + "bits_per_byte": 1.6214809814546032, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.123924970626831, + "step": 4800, + "unclipped_grad_norm": 0.7595614790916443 + }, + { + "bits_per_byte": 1.598943858937588, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.108303427696228, + "step": 4810, + "unclipped_grad_norm": 0.6932185292243958 + }, + { + "bits_per_byte": 1.5882840302718095, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1009145975112915, + "step": 4820, + "unclipped_grad_norm": 0.7010871171951294 + }, + { + "bits_per_byte": 1.5957891811728115, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.106116771697998, + "step": 4830, + "unclipped_grad_norm": 0.6711499691009521 + }, + { + "bits_per_byte": 1.5337598545208206, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.0631213188171387, + "step": 4840, + "unclipped_grad_norm": 0.7005295157432556 + }, + { + "bits_per_byte": 1.6063714456630511, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1134518384933472, + "step": 4850, + "unclipped_grad_norm": 0.6924180388450623 + }, + { + "bits_per_byte": 1.58919708616524, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1015474796295166, + "step": 4860, + "unclipped_grad_norm": 0.6728304028511047 + }, + { + "bits_per_byte": 1.5949746713383253, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.1055521965026855, + "step": 4870, + "unclipped_grad_norm": 0.6908511519432068 + }, + { + "bits_per_byte": 1.562132004416309, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0827873945236206, + "step": 4880, + "unclipped_grad_norm": 0.6754211187362671 + }, + { + "bits_per_byte": 1.5204766024987808, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0539140701293945, + "step": 4890, + "unclipped_grad_norm": 0.686815619468689 + }, + { + "bits_per_byte": 1.522231341485534, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0551303625106812, + "step": 4900, + "unclipped_grad_norm": 0.6894630789756775 + }, + { + "bits_per_byte": 1.5875051208460518, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.100374698638916, + "step": 4910, + "unclipped_grad_norm": 0.7035151124000549 + }, + { + "bits_per_byte": 1.5873929881576891, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.100296974182129, + "step": 4920, + "unclipped_grad_norm": 0.707631528377533 + }, + { + "bits_per_byte": 1.5818637459324498, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0964643955230713, + "step": 4930, + "unclipped_grad_norm": 0.7146247029304504 + }, + { + "bits_per_byte": 1.6602286726939939, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.150782823562622, + "step": 4940, + "unclipped_grad_norm": 0.7488285899162292 + }, + { + "bits_per_byte": 1.5109480757103777, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.047309398651123, + "step": 4950, + "unclipped_grad_norm": 0.6982936263084412 + }, + { + "bits_per_byte": 1.6013034609074257, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1099389791488647, + "step": 4960, + "unclipped_grad_norm": 0.7697318196296692 + }, + { + "bits_per_byte": 1.673659485830474, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1600923538208008, + "step": 4970, + "unclipped_grad_norm": 0.7759159207344055 + }, + { + "bits_per_byte": 1.5500858236019253, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.0744376182556152, + "step": 4980, + "unclipped_grad_norm": 0.7765741944313049 + }, + { + "bits_per_byte": 1.6026017579387892, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1108388900756836, + "step": 4990, + "unclipped_grad_norm": 0.7224560976028442 + }, + { + "bits_per_byte": 1.6136506113608193, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.118497371673584, + "step": 5000, + "unclipped_grad_norm": 0.7415750622749329 + }, + { + "bits_per_byte": 1.5910434919049032, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1028273105621338, + "step": 5010, + "unclipped_grad_norm": 0.6908667087554932 + }, + { + "bits_per_byte": 1.598975331762696, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1083252429962158, + "step": 5020, + "unclipped_grad_norm": 0.730786919593811 + }, + { + "bits_per_byte": 1.5824023955949522, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0968377590179443, + "step": 5030, + "unclipped_grad_norm": 0.7392095923423767 + }, + { + "bits_per_byte": 1.5712477728426413, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0891059637069702, + "step": 5040, + "unclipped_grad_norm": 0.7138375639915466 + }, + { + "bits_per_byte": 1.600523519585763, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.109398365020752, + "step": 5050, + "unclipped_grad_norm": 0.7417458295822144 + }, + { + "bits_per_byte": 1.556087158203783, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0785974264144897, + "step": 5060, + "unclipped_grad_norm": 0.766182541847229 + }, + { + "bits_per_byte": 1.6239489324944845, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1256356239318848, + "step": 5070, + "unclipped_grad_norm": 0.7354997992515564 + }, + { + "bits_per_byte": 1.6499730032577429, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1436741352081299, + "step": 5080, + "unclipped_grad_norm": 0.8061900734901428 + }, + { + "bits_per_byte": 1.621231606610852, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1237521171569824, + "step": 5090, + "unclipped_grad_norm": 0.789067804813385 + }, + { + "bits_per_byte": 1.5443199332490989, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0704410076141357, + "step": 5100, + "unclipped_grad_norm": 0.7130411267280579 + }, + { + "bits_per_byte": 1.5718306220464153, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0895099639892578, + "step": 5110, + "unclipped_grad_norm": 0.6817747950553894 + }, + { + "bits_per_byte": 1.5553593276253317, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0780929327011108, + "step": 5120, + "unclipped_grad_norm": 0.7432519793510437 + }, + { + "bits_per_byte": 1.611734380664906, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1171691417694092, + "step": 5130, + "unclipped_grad_norm": 0.7314459681510925 + }, + { + "bits_per_byte": 1.5509013653323167, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.075002908706665, + "step": 5140, + "unclipped_grad_norm": 0.7262561321258545 + }, + { + "bits_per_byte": 1.5519597465657264, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0757365226745605, + "step": 5150, + "unclipped_grad_norm": 0.7442956566810608 + }, + { + "bits_per_byte": 1.582139434121783, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0966554880142212, + "step": 5160, + "unclipped_grad_norm": 0.7529576420783997 + }, + { + "bits_per_byte": 1.5941560339202183, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1049847602844238, + "step": 5170, + "unclipped_grad_norm": 0.7270960211753845 + }, + { + "bits_per_byte": 1.6180967068509247, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1215791702270508, + "step": 5180, + "unclipped_grad_norm": 0.759490966796875 + }, + { + "bits_per_byte": 1.5759898504748815, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.092392921447754, + "step": 5190, + "unclipped_grad_norm": 0.7248204946517944 + }, + { + "bits_per_byte": 1.5225916451390915, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.05538010597229, + "step": 5200, + "unclipped_grad_norm": 0.7128092646598816 + }, + { + "bits_per_byte": 1.601221769148266, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.1098823547363281, + "step": 5210, + "unclipped_grad_norm": 0.7404454946517944 + }, + { + "bits_per_byte": 1.5843179383602621, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.098165512084961, + "step": 5220, + "unclipped_grad_norm": 0.7370045781135559 + }, + { + "bits_per_byte": 1.614152456736037, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1188452243804932, + "step": 5230, + "unclipped_grad_norm": 0.8054904937744141 + }, + { + "bits_per_byte": 1.5701584347320763, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0883508920669556, + "step": 5240, + "unclipped_grad_norm": 0.7422505617141724 + }, + { + "bits_per_byte": 1.5364362485335477, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.064976453781128, + "step": 5250, + "unclipped_grad_norm": 0.7392460107803345 + }, + { + "bits_per_byte": 1.510300904995181, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0468608140945435, + "step": 5260, + "unclipped_grad_norm": 0.7152841091156006 + }, + { + "bits_per_byte": 1.6197508359869233, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1227257251739502, + "step": 5270, + "unclipped_grad_norm": 0.762697160243988 + }, + { + "bits_per_byte": 1.6105141637570344, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1163233518600464, + "step": 5280, + "unclipped_grad_norm": 0.7536966800689697 + }, + { + "bits_per_byte": 1.5885342650288148, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.101088047027588, + "step": 5290, + "unclipped_grad_norm": 0.7395375967025757 + }, + { + "bits_per_byte": 1.5242677880544004, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.056541919708252, + "step": 5300, + "unclipped_grad_norm": 0.6971617937088013 + }, + { + "bits_per_byte": 1.589467786857698, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1017351150512695, + "step": 5310, + "unclipped_grad_norm": 0.7348577380180359 + }, + { + "bits_per_byte": 1.5472704676073012, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.072486162185669, + "step": 5320, + "unclipped_grad_norm": 0.7314221858978271 + }, + { + "bits_per_byte": 1.6553672391020549, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.1474131345748901, + "step": 5330, + "unclipped_grad_norm": 0.7674097418785095 + }, + { + "bits_per_byte": 1.5389176142201964, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0666964054107666, + "step": 5340, + "unclipped_grad_norm": 0.7431036829948425 + }, + { + "bits_per_byte": 1.593487021408362, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1045210361480713, + "step": 5350, + "unclipped_grad_norm": 0.7701666951179504 + }, + { + "bits_per_byte": 1.5047200679746828, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0429924726486206, + "step": 5360, + "unclipped_grad_norm": 0.7093538045883179 + }, + { + "bits_per_byte": 1.5323434054083145, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0621395111083984, + "step": 5370, + "unclipped_grad_norm": 0.8088909983634949 + }, + { + "bits_per_byte": 1.5949007187984543, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1055009365081787, + "step": 5380, + "unclipped_grad_norm": 0.7591733336448669 + }, + { + "bits_per_byte": 1.5063042001717801, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0440905094146729, + "step": 5390, + "unclipped_grad_norm": 0.7258779406547546 + }, + { + "bits_per_byte": 1.5763400071520381, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0926356315612793, + "step": 5400, + "unclipped_grad_norm": 0.7020314335823059 + }, + { + "bits_per_byte": 1.540318240928823, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0676672458648682, + "step": 5410, + "unclipped_grad_norm": 0.73995441198349 + }, + { + "bits_per_byte": 1.5024278832039832, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0414036512374878, + "step": 5420, + "unclipped_grad_norm": 0.7208534479141235 + }, + { + "bits_per_byte": 1.556975964543442, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0792135000228882, + "step": 5430, + "unclipped_grad_norm": 0.7300911545753479 + }, + { + "bits_per_byte": 1.5377587951186826, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0658931732177734, + "step": 5440, + "unclipped_grad_norm": 0.7053164839744568 + }, + { + "bits_per_byte": 1.5519281017579678, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0757145881652832, + "step": 5450, + "unclipped_grad_norm": 0.7570145726203918 + }, + { + "bits_per_byte": 1.5295854916190785, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.060227870941162, + "step": 5460, + "unclipped_grad_norm": 0.7293761968612671 + }, + { + "bits_per_byte": 1.5453655877663446, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0711658000946045, + "step": 5470, + "unclipped_grad_norm": 0.7067918181419373 + }, + { + "bits_per_byte": 1.5185650153344405, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0525890588760376, + "step": 5480, + "unclipped_grad_norm": 0.7616989612579346 + }, + { + "bits_per_byte": 1.5811864782533522, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0959949493408203, + "step": 5490, + "unclipped_grad_norm": 0.8041682243347168 + }, + { + "bits_per_byte": 1.5613092394145813, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0822170972824097, + "step": 5500, + "unclipped_grad_norm": 0.7645291686058044 + }, + { + "bits_per_byte": 1.5218208188979245, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0548458099365234, + "step": 5510, + "unclipped_grad_norm": 0.7549375891685486 + }, + { + "bits_per_byte": 1.4947696677437132, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.036095380783081, + "step": 5520, + "unclipped_grad_norm": 0.7179425358772278 + }, + { + "bits_per_byte": 1.5059726176209167, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.043860673904419, + "step": 5530, + "unclipped_grad_norm": 0.7216454744338989 + }, + { + "bits_per_byte": 1.625827155024557, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1269375085830688, + "step": 5540, + "unclipped_grad_norm": 0.7688449621200562 + }, + { + "bits_per_byte": 1.5739950237075238, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0910102128982544, + "step": 5550, + "unclipped_grad_norm": 0.7435398697853088 + }, + { + "bits_per_byte": 1.5977501273579493, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.107475996017456, + "step": 5560, + "unclipped_grad_norm": 0.8213235139846802 + }, + { + "bits_per_byte": 1.537488782356828, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0657060146331787, + "step": 5570, + "unclipped_grad_norm": 0.7458736896514893 + }, + { + "bits_per_byte": 1.569628556184768, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0879836082458496, + "step": 5580, + "unclipped_grad_norm": 0.745820939540863 + }, + { + "bits_per_byte": 1.5662062734152489, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0856114625930786, + "step": 5590, + "unclipped_grad_norm": 0.8023808002471924 + }, + { + "bits_per_byte": 1.5614218880508965, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0822951793670654, + "step": 5600, + "unclipped_grad_norm": 0.7724120020866394 + }, + { + "bits_per_byte": 1.5000096351502012, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0397274494171143, + "step": 5610, + "unclipped_grad_norm": 0.7330023050308228 + }, + { + "bits_per_byte": 1.564252034553495, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.084256887435913, + "step": 5620, + "unclipped_grad_norm": 0.7773064374923706 + }, + { + "bits_per_byte": 1.4778873347870674, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0243934392929077, + "step": 5630, + "unclipped_grad_norm": 0.7191233038902283 + }, + { + "bits_per_byte": 1.4838417181252395, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0285207033157349, + "step": 5640, + "unclipped_grad_norm": 0.7019065022468567 + }, + { + "bits_per_byte": 1.5419510442161144, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0687990188598633, + "step": 5650, + "unclipped_grad_norm": 0.7909054160118103 + }, + { + "bits_per_byte": 1.4908660055344292, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0333895683288574, + "step": 5660, + "unclipped_grad_norm": 0.7980607151985168 + }, + { + "bits_per_byte": 1.5729701790910324, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0902998447418213, + "step": 5670, + "unclipped_grad_norm": 0.796278178691864 + }, + { + "bits_per_byte": 1.5668854329035062, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0860822200775146, + "step": 5680, + "unclipped_grad_norm": 0.8280957341194153 + }, + { + "bits_per_byte": 1.592162067066115, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.103602647781372, + "step": 5690, + "unclipped_grad_norm": 0.7912419438362122 + }, + { + "bits_per_byte": 1.5918954939572776, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1034178733825684, + "step": 5700, + "unclipped_grad_norm": 0.7910536527633667 + }, + { + "bits_per_byte": 1.5647875885282816, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0846281051635742, + "step": 5710, + "unclipped_grad_norm": 0.8025376200675964 + }, + { + "bits_per_byte": 1.576262442976499, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.092581868171692, + "step": 5720, + "unclipped_grad_norm": 0.7657404541969299 + }, + { + "bits_per_byte": 1.5180726290050204, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0522477626800537, + "step": 5730, + "unclipped_grad_norm": 0.7721706032752991 + }, + { + "bits_per_byte": 1.6017487239905095, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1102476119995117, + "step": 5740, + "unclipped_grad_norm": 0.7985641956329346 + }, + { + "bits_per_byte": 1.4880221004197618, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0314183235168457, + "step": 5750, + "unclipped_grad_norm": 0.7275866866111755 + }, + { + "bits_per_byte": 1.5012069783655082, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0405573844909668, + "step": 5760, + "unclipped_grad_norm": 0.7711101174354553 + }, + { + "bits_per_byte": 1.5255566260399664, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0574352741241455, + "step": 5770, + "unclipped_grad_norm": 0.8030062913894653 + }, + { + "bits_per_byte": 1.6161017081009161, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.1201963424682617, + "step": 5780, + "unclipped_grad_norm": 0.7891528606414795 + }, + { + "bits_per_byte": 1.436315000489906, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 0.9955776929855347, + "step": 5790, + "unclipped_grad_norm": 0.7522260546684265 + }, + { + "bits_per_byte": 1.5639025658069416, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.084014654159546, + "step": 5800, + "unclipped_grad_norm": 0.77913498878479 + }, + { + "bits_per_byte": 1.4787818165542048, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0250134468078613, + "step": 5810, + "unclipped_grad_norm": 0.744920551776886 + }, + { + "bits_per_byte": 1.485352929678371, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0295681953430176, + "step": 5820, + "unclipped_grad_norm": 0.8005070090293884 + }, + { + "bits_per_byte": 1.551830071646976, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0756466388702393, + "step": 5830, + "unclipped_grad_norm": 0.7964350581169128 + }, + { + "bits_per_byte": 1.5252578921754176, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0572282075881958, + "step": 5840, + "unclipped_grad_norm": 0.8099321722984314 + }, + { + "bits_per_byte": 1.4836750669365535, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0284051895141602, + "step": 5850, + "unclipped_grad_norm": 0.7372169494628906 + }, + { + "bits_per_byte": 1.511822779472666, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0479156970977783, + "step": 5860, + "unclipped_grad_norm": 0.7115837931632996 + }, + { + "bits_per_byte": 1.5625698722454058, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.083090901374817, + "step": 5870, + "unclipped_grad_norm": 0.7878124713897705 + }, + { + "bits_per_byte": 1.503004884997628, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0418035984039307, + "step": 5880, + "unclipped_grad_norm": 0.7451943159103394 + }, + { + "bits_per_byte": 1.5713764158654868, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0891951322555542, + "step": 5890, + "unclipped_grad_norm": 0.8092991709709167 + }, + { + "bits_per_byte": 1.5881084359852786, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.1007928848266602, + "step": 5900, + "unclipped_grad_norm": 0.7848427891731262 + }, + { + "bits_per_byte": 1.5133455138634053, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.048971176147461, + "step": 5910, + "unclipped_grad_norm": 0.7443398237228394 + }, + { + "bits_per_byte": 1.4825475486774968, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0276236534118652, + "step": 5920, + "unclipped_grad_norm": 0.7709684371948242 + }, + { + "bits_per_byte": 1.5970936695796059, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.1070209741592407, + "step": 5930, + "unclipped_grad_norm": 0.8097158074378967 + }, + { + "bits_per_byte": 1.5296348506398763, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0602620840072632, + "step": 5940, + "unclipped_grad_norm": 0.8011659383773804 + }, + { + "bits_per_byte": 1.5315722352018457, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0616049766540527, + "step": 5950, + "unclipped_grad_norm": 0.7703563570976257 + }, + { + "bits_per_byte": 1.4753576419755263, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0226399898529053, + "step": 5960, + "unclipped_grad_norm": 0.7793571949005127 + }, + { + "bits_per_byte": 1.514912963743369, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0500576496124268, + "step": 5970, + "unclipped_grad_norm": 0.7870919108390808 + }, + { + "bits_per_byte": 1.4934732625215095, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0351967811584473, + "step": 5980, + "unclipped_grad_norm": 0.8144509196281433 + }, + { + "bits_per_byte": 1.5080035607449556, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0452684164047241, + "step": 5990, + "unclipped_grad_norm": 0.7810203433036804 + }, + { + "bits_per_byte": 1.5782032671914858, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0939271450042725, + "step": 6000, + "unclipped_grad_norm": 0.790114164352417 + }, + { + "bits_per_byte": 1.5490276143511665, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0737041234970093, + "step": 6010, + "unclipped_grad_norm": 0.8144342303276062 + }, + { + "bits_per_byte": 1.515940732064925, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0507700443267822, + "step": 6020, + "unclipped_grad_norm": 0.7434252500534058 + }, + { + "bits_per_byte": 1.493858503659442, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.035463809967041, + "step": 6030, + "unclipped_grad_norm": 0.8062746524810791 + }, + { + "bits_per_byte": 1.5237423810560147, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0561777353286743, + "step": 6040, + "unclipped_grad_norm": 0.7702685594558716 + }, + { + "bits_per_byte": 1.5275683071071087, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.058829665184021, + "step": 6050, + "unclipped_grad_norm": 0.7716134786605835 + }, + { + "bits_per_byte": 1.5215145177967377, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0546334981918335, + "step": 6060, + "unclipped_grad_norm": 0.7933171987533569 + }, + { + "bits_per_byte": 1.415498478403448, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 0.9811487793922424, + "step": 6070, + "unclipped_grad_norm": 0.7639470100402832 + }, + { + "bits_per_byte": 1.4966260484771265, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0373821258544922, + "step": 6080, + "unclipped_grad_norm": 0.7900221943855286 + }, + { + "bits_per_byte": 1.4925770609278632, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0345755815505981, + "step": 6090, + "unclipped_grad_norm": 0.809319019317627 + }, + { + "bits_per_byte": 1.5936792980120267, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.104654312133789, + "step": 6100, + "unclipped_grad_norm": 0.824783205986023 + }, + { + "bits_per_byte": 1.5281914002511847, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0592615604400635, + "step": 6110, + "unclipped_grad_norm": 0.8245336413383484 + }, + { + "bits_per_byte": 1.48945093228313, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0324087142944336, + "step": 6120, + "unclipped_grad_norm": 0.8355178833007812 + }, + { + "bits_per_byte": 1.5475705773330568, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.072694182395935, + "step": 6130, + "unclipped_grad_norm": 0.7615814208984375 + }, + { + "bits_per_byte": 1.4980500648262705, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0383691787719727, + "step": 6140, + "unclipped_grad_norm": 0.7743073105812073 + }, + { + "bits_per_byte": 1.483834838819205, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0285159349441528, + "step": 6150, + "unclipped_grad_norm": 0.7909993529319763 + }, + { + "bits_per_byte": 1.5470265962083778, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.072317123413086, + "step": 6160, + "unclipped_grad_norm": 0.7707679867744446 + }, + { + "bits_per_byte": 1.478142213075646, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.024570107460022, + "step": 6170, + "unclipped_grad_norm": 0.7621827721595764 + }, + { + "bits_per_byte": 1.4841139666615555, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0287094116210938, + "step": 6180, + "unclipped_grad_norm": 0.7684755921363831 + }, + { + "bits_per_byte": 1.493858503659442, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.035463809967041, + "step": 6190, + "unclipped_grad_norm": 0.8198496699333191 + }, + { + "bits_per_byte": 1.4883674415826944, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0316576957702637, + "step": 6200, + "unclipped_grad_norm": 0.7631723284721375 + }, + { + "bits_per_byte": 1.5053304344025948, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0434155464172363, + "step": 6210, + "unclipped_grad_norm": 0.7891849875450134 + }, + { + "bits_per_byte": 1.5365181122753584, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.065033197402954, + "step": 6220, + "unclipped_grad_norm": 0.7962750196456909 + }, + { + "bits_per_byte": 1.4799394317771624, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0258158445358276, + "step": 6230, + "unclipped_grad_norm": 0.7991603016853333 + }, + { + "bits_per_byte": 1.5356217386990614, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0644118785858154, + "step": 6240, + "unclipped_grad_norm": 0.8105412721633911 + }, + { + "bits_per_byte": 1.4819163723488304, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0271861553192139, + "step": 6250, + "unclipped_grad_norm": 0.7951217293739319 + }, + { + "bits_per_byte": 1.4447946050906966, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.001455307006836, + "step": 6260, + "unclipped_grad_norm": 0.7625995874404907 + }, + { + "bits_per_byte": 1.5022859975170215, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0413053035736084, + "step": 6270, + "unclipped_grad_norm": 0.8279880881309509 + }, + { + "bits_per_byte": 1.5067209141348208, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0443793535232544, + "step": 6280, + "unclipped_grad_norm": 0.7734115719795227 + }, + { + "bits_per_byte": 1.5122324421470212, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0481996536254883, + "step": 6290, + "unclipped_grad_norm": 0.7805989980697632 + }, + { + "bits_per_byte": 1.476189006109797, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0232162475585938, + "step": 6300, + "unclipped_grad_norm": 0.8065475225448608 + }, + { + "bits_per_byte": 1.5409463215697738, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0681025981903076, + "step": 6310, + "unclipped_grad_norm": 0.8388196229934692 + }, + { + "bits_per_byte": 1.506346851869194, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0441200733184814, + "step": 6320, + "unclipped_grad_norm": 0.8094848990440369 + }, + { + "bits_per_byte": 1.5104558613636083, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0469682216644287, + "step": 6330, + "unclipped_grad_norm": 0.7866588234901428 + }, + { + "bits_per_byte": 1.427382135612766, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 0.9893859028816223, + "step": 6340, + "unclipped_grad_norm": 0.7612236738204956 + }, + { + "bits_per_byte": 1.541960503261912, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0688055753707886, + "step": 6350, + "unclipped_grad_norm": 0.783340573310852 + }, + { + "bits_per_byte": 1.5465876964833758, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0720129013061523, + "step": 6360, + "unclipped_grad_norm": 0.8162001371383667 + }, + { + "bits_per_byte": 1.4662069610710688, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0162972211837769, + "step": 6370, + "unclipped_grad_norm": 0.7845266461372375 + }, + { + "bits_per_byte": 1.4477447954835971, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.00350022315979, + "step": 6380, + "unclipped_grad_norm": 0.7937688231468201 + }, + { + "bits_per_byte": 1.5785561755910564, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0941717624664307, + "step": 6390, + "unclipped_grad_norm": 0.826732337474823 + }, + { + "bits_per_byte": 1.5478830698096744, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0729107856750488, + "step": 6400, + "unclipped_grad_norm": 0.8436270952224731 + }, + { + "bits_per_byte": 1.550731790438566, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.074885368347168, + "step": 6410, + "unclipped_grad_norm": 0.8814867734909058 + }, + { + "bits_per_byte": 1.4786806907554975, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0249433517456055, + "step": 6420, + "unclipped_grad_norm": 0.8324126601219177 + }, + { + "bits_per_byte": 1.5252604719151805, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.057229995727539, + "step": 6430, + "unclipped_grad_norm": 0.7683992981910706 + }, + { + "bits_per_byte": 1.4432244034883193, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0003669261932373, + "step": 6440, + "unclipped_grad_norm": 0.8131992816925049 + }, + { + "bits_per_byte": 1.465075831176344, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0155131816864014, + "step": 6450, + "unclipped_grad_norm": 0.758244514465332 + }, + { + "bits_per_byte": 1.4922360193312023, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.034339189529419, + "step": 6460, + "unclipped_grad_norm": 0.798247754573822 + }, + { + "bits_per_byte": 1.498638761440174, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.038777232170105, + "step": 6470, + "unclipped_grad_norm": 0.8465205430984497 + }, + { + "bits_per_byte": 1.456987831071568, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0099070072174072, + "step": 6480, + "unclipped_grad_norm": 0.7985926270484924 + }, + { + "bits_per_byte": 1.542977092711162, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0695102214813232, + "step": 6490, + "unclipped_grad_norm": 0.8188008666038513 + }, + { + "bits_per_byte": 1.582791420351204, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0971074104309082, + "step": 6500, + "unclipped_grad_norm": 0.8359130620956421 + }, + { + "bits_per_byte": 1.5352007251697493, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.0641200542449951, + "step": 6510, + "unclipped_grad_norm": 0.854469895362854 + }, + { + "bits_per_byte": 1.5259941499037615, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0577385425567627, + "step": 6520, + "unclipped_grad_norm": 0.8313364386558533 + }, + { + "bits_per_byte": 1.4871191915027322, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.030792474746704, + "step": 6530, + "unclipped_grad_norm": 0.7717711329460144 + }, + { + "bits_per_byte": 1.4761642406080728, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0231990814208984, + "step": 6540, + "unclipped_grad_norm": 0.8019536733627319 + }, + { + "bits_per_byte": 1.5028273989019376, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0416805744171143, + "step": 6550, + "unclipped_grad_norm": 0.7671847343444824 + }, + { + "bits_per_byte": 1.5359399066031576, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0646324157714844, + "step": 6560, + "unclipped_grad_norm": 0.8164054751396179 + }, + { + "bits_per_byte": 1.504123116193538, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0425786972045898, + "step": 6570, + "unclipped_grad_norm": 0.7928623557090759 + }, + { + "bits_per_byte": 1.4852913598893622, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0295255184173584, + "step": 6580, + "unclipped_grad_norm": 0.8092948794364929 + }, + { + "bits_per_byte": 1.5248497773449203, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0569453239440918, + "step": 6590, + "unclipped_grad_norm": 0.8197707533836365 + }, + { + "bits_per_byte": 1.426588779644336, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9888359904289246, + "step": 6600, + "unclipped_grad_norm": 0.7768723964691162 + }, + { + "bits_per_byte": 1.4662528804388493, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.016329050064087, + "step": 6610, + "unclipped_grad_norm": 0.8190019726753235 + }, + { + "bits_per_byte": 1.5755777800434143, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0921072959899902, + "step": 6620, + "unclipped_grad_norm": 0.8630855083465576 + }, + { + "bits_per_byte": 1.5211641891369303, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0543906688690186, + "step": 6630, + "unclipped_grad_norm": 0.8092710971832275 + }, + { + "bits_per_byte": 1.4689426890983437, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0181934833526611, + "step": 6640, + "unclipped_grad_norm": 0.7870210409164429 + }, + { + "bits_per_byte": 1.4737494322073081, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.021525263786316, + "step": 6650, + "unclipped_grad_norm": 0.8340798020362854 + }, + { + "bits_per_byte": 1.5123179175245, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.048258900642395, + "step": 6660, + "unclipped_grad_norm": 0.8463515043258667 + }, + { + "bits_per_byte": 1.4946623505695749, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0360209941864014, + "step": 6670, + "unclipped_grad_norm": 0.8076959252357483 + }, + { + "bits_per_byte": 1.4999131528830674, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0396605730056763, + "step": 6680, + "unclipped_grad_norm": 0.8417643904685974 + }, + { + "bits_per_byte": 1.4927383806543726, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0346873998641968, + "step": 6690, + "unclipped_grad_norm": 0.8966479301452637 + }, + { + "bits_per_byte": 1.507631906236441, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0450108051300049, + "step": 6700, + "unclipped_grad_norm": 0.8065080642700195 + }, + { + "bits_per_byte": 1.526726795996437, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.058246374130249, + "step": 6710, + "unclipped_grad_norm": 0.8376452326774597 + }, + { + "bits_per_byte": 1.5822010039107919, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0966981649398804, + "step": 6720, + "unclipped_grad_norm": 0.8624657392501831 + }, + { + "bits_per_byte": 1.447227471669802, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0031416416168213, + "step": 6730, + "unclipped_grad_norm": 0.8040223717689514 + }, + { + "bits_per_byte": 1.4848161718250283, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0291961431503296, + "step": 6740, + "unclipped_grad_norm": 0.8131929636001587 + }, + { + "bits_per_byte": 1.4599282184533695, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.011945128440857, + "step": 6750, + "unclipped_grad_norm": 0.8242661356925964 + }, + { + "bits_per_byte": 1.4561461479782454, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0093235969543457, + "step": 6760, + "unclipped_grad_norm": 0.858147919178009 + }, + { + "bits_per_byte": 1.4679376224867013, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0174968242645264, + "step": 6770, + "unclipped_grad_norm": 0.8389740586280823 + }, + { + "bits_per_byte": 1.446337461451587, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.002524733543396, + "step": 6780, + "unclipped_grad_norm": 0.806380033493042 + }, + { + "bits_per_byte": 1.4957231395600967, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0367562770843506, + "step": 6790, + "unclipped_grad_norm": 0.8443998694419861 + }, + { + "bits_per_byte": 1.483685729860907, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0284125804901123, + "step": 6800, + "unclipped_grad_norm": 0.8614793419837952 + }, + { + "bits_per_byte": 1.4736859706091399, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0214812755584717, + "step": 6810, + "unclipped_grad_norm": 0.8073354959487915 + }, + { + "bits_per_byte": 1.4900140034820548, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0327990055084229, + "step": 6820, + "unclipped_grad_norm": 0.8066654801368713 + }, + { + "bits_per_byte": 1.4893948659389489, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.03236985206604, + "step": 6830, + "unclipped_grad_norm": 0.8152909278869629 + }, + { + "bits_per_byte": 1.4517927511369553, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0063060522079468, + "step": 6840, + "unclipped_grad_norm": 0.8145732283592224 + }, + { + "bits_per_byte": 1.5401706798143826, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0675649642944336, + "step": 6850, + "unclipped_grad_norm": 0.8944696187973022 + }, + { + "bits_per_byte": 1.5020013662298435, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0411080121994019, + "step": 6860, + "unclipped_grad_norm": 0.8296581506729126 + }, + { + "bits_per_byte": 1.4315734388056183, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9922910928726196, + "step": 6870, + "unclipped_grad_norm": 0.8298041224479675 + }, + { + "bits_per_byte": 1.4506086505857647, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0054852962493896, + "step": 6880, + "unclipped_grad_norm": 0.8620857000350952 + }, + { + "bits_per_byte": 1.4600797351687795, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0120501518249512, + "step": 6890, + "unclipped_grad_norm": 0.8483932614326477 + }, + { + "bits_per_byte": 1.5166142161257037, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.051236867904663, + "step": 6900, + "unclipped_grad_norm": 0.8512884378433228 + }, + { + "bits_per_byte": 1.5060785589338481, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0439341068267822, + "step": 6910, + "unclipped_grad_norm": 0.8712192177772522 + }, + { + "bits_per_byte": 1.4495721111490145, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.004766821861267, + "step": 6920, + "unclipped_grad_norm": 0.8044716715812683 + }, + { + "bits_per_byte": 1.4331765750856358, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9934023022651672, + "step": 6930, + "unclipped_grad_norm": 0.8494104743003845 + }, + { + "bits_per_byte": 1.5111775005666286, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0474684238433838, + "step": 6940, + "unclipped_grad_norm": 0.8407061696052551 + }, + { + "bits_per_byte": 1.4275953081085107, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9895336627960205, + "step": 6950, + "unclipped_grad_norm": 0.8335278630256653 + }, + { + "bits_per_byte": 1.4337008641967912, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 0.9937657117843628, + "step": 6960, + "unclipped_grad_norm": 0.8460164666175842 + }, + { + "bits_per_byte": 1.404613954404317, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 0.9736042022705078, + "step": 6970, + "unclipped_grad_norm": 0.8476742506027222 + }, + { + "bits_per_byte": 1.5285965913766173, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0595424175262451, + "step": 6980, + "unclipped_grad_norm": 0.9341830611228943 + }, + { + "bits_per_byte": 1.4927865357966141, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034720778465271, + "step": 6990, + "unclipped_grad_norm": 0.8535965085029602 + }, + { + "bits_per_byte": 1.5015640143486995, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0408048629760742, + "step": 7000, + "unclipped_grad_norm": 0.8839671015739441 + }, + { + "bits_per_byte": 1.409006391307353, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9766488075256348, + "step": 7010, + "unclipped_grad_norm": 0.8374313712120056 + }, + { + "bits_per_byte": 1.4918236049344333, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0340533256530762, + "step": 7020, + "unclipped_grad_norm": 0.8843756318092346 + }, + { + "bits_per_byte": 1.4765720114732686, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0234817266464233, + "step": 7030, + "unclipped_grad_norm": 0.8468295931816101 + }, + { + "bits_per_byte": 1.4469567709773439, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0029540061950684, + "step": 7040, + "unclipped_grad_norm": 0.8328400254249573 + }, + { + "bits_per_byte": 1.4604876780166263, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0123329162597656, + "step": 7050, + "unclipped_grad_norm": 0.8563967943191528 + }, + { + "bits_per_byte": 1.4275743262251053, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 0.9895191192626953, + "step": 7060, + "unclipped_grad_norm": 0.830858588218689 + }, + { + "bits_per_byte": 1.4821155282585297, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0273241996765137, + "step": 7070, + "unclipped_grad_norm": 0.8345243334770203 + }, + { + "bits_per_byte": 1.482162651504866, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.0273568630218506, + "step": 7080, + "unclipped_grad_norm": 0.9049980044364929 + }, + { + "bits_per_byte": 1.4400559671114737, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.9981707334518433, + "step": 7090, + "unclipped_grad_norm": 0.86240553855896 + }, + { + "bits_per_byte": 1.4348600272636065, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9945691823959351, + "step": 7100, + "unclipped_grad_norm": 0.8350730538368225 + }, + { + "bits_per_byte": 1.4152483296377678, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 0.9809753894805908, + "step": 7110, + "unclipped_grad_norm": 0.8401304483413696 + }, + { + "bits_per_byte": 1.4797989219514076, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0257184505462646, + "step": 7120, + "unclipped_grad_norm": 0.8903806805610657 + }, + { + "bits_per_byte": 1.5240091261475028, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0563626289367676, + "step": 7130, + "unclipped_grad_norm": 0.8470056056976318 + }, + { + "bits_per_byte": 1.4628170110399121, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0139474868774414, + "step": 7140, + "unclipped_grad_norm": 0.8429069519042969 + }, + { + "bits_per_byte": 1.5096671489267515, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0464215278625488, + "step": 7150, + "unclipped_grad_norm": 0.8920191526412964 + }, + { + "bits_per_byte": 1.5009496923198173, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0403790473937988, + "step": 7160, + "unclipped_grad_norm": 0.8647960424423218 + }, + { + "bits_per_byte": 1.5036095759980617, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0422227382659912, + "step": 7170, + "unclipped_grad_norm": 0.9017168283462524 + }, + { + "bits_per_byte": 1.4616292988530535, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0131242275238037, + "step": 7180, + "unclipped_grad_norm": 0.8551055788993835 + }, + { + "bits_per_byte": 1.5122007973392624, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.048177719116211, + "step": 7190, + "unclipped_grad_norm": 0.8699921369552612 + }, + { + "bits_per_byte": 1.441016404225217, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 0.9988364577293396, + "step": 7200, + "unclipped_grad_norm": 0.8101796507835388 + }, + { + "bits_per_byte": 1.4373777672809125, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 0.9963143467903137, + "step": 7210, + "unclipped_grad_norm": 0.8248227834701538 + }, + { + "bits_per_byte": 1.4388484769197658, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9973337650299072, + "step": 7220, + "unclipped_grad_norm": 0.8497165441513062 + }, + { + "bits_per_byte": 1.4764554072359837, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.023400902748108, + "step": 7230, + "unclipped_grad_norm": 0.869898796081543 + }, + { + "bits_per_byte": 1.5094511387172678, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.046271800994873, + "step": 7240, + "unclipped_grad_norm": 0.8670166730880737 + }, + { + "bits_per_byte": 1.4896672864579155, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0325586795806885, + "step": 7250, + "unclipped_grad_norm": 0.8726136684417725 + }, + { + "bits_per_byte": 1.4660583680607233, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.016194224357605, + "step": 7260, + "unclipped_grad_norm": 0.8553854823112488 + }, + { + "bits_per_byte": 1.4854459722924878, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0296326875686646, + "step": 7270, + "unclipped_grad_norm": 0.8842402100563049 + }, + { + "bits_per_byte": 1.4706281190767991, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0193617343902588, + "step": 7280, + "unclipped_grad_norm": 0.8827332854270935 + }, + { + "bits_per_byte": 1.465586275684105, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.015866994857788, + "step": 7290, + "unclipped_grad_norm": 0.9209113121032715 + }, + { + "bits_per_byte": 1.4604543133823589, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0123097896575928, + "step": 7300, + "unclipped_grad_norm": 0.8443734645843506 + }, + { + "bits_per_byte": 1.44415740936925, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0010136365890503, + "step": 7310, + "unclipped_grad_norm": 0.8828385472297668 + }, + { + "bits_per_byte": 1.4200490533539523, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9843029975891113, + "step": 7320, + "unclipped_grad_norm": 0.878987193107605 + }, + { + "bits_per_byte": 1.4725175204791778, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0206713676452637, + "step": 7330, + "unclipped_grad_norm": 0.9326861500740051 + }, + { + "bits_per_byte": 1.4404713912046327, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9984586834907532, + "step": 7340, + "unclipped_grad_norm": 0.8777600526809692 + }, + { + "bits_per_byte": 1.448391966198794, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0039488077163696, + "step": 7350, + "unclipped_grad_norm": 0.8631419539451599 + }, + { + "bits_per_byte": 1.453881480431684, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.007753849029541, + "step": 7360, + "unclipped_grad_norm": 0.855156421661377 + }, + { + "bits_per_byte": 1.4104983408035878, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 0.9776829481124878, + "step": 7370, + "unclipped_grad_norm": 0.8616177439689636 + }, + { + "bits_per_byte": 1.406926175153842, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9752069115638733, + "step": 7380, + "unclipped_grad_norm": 0.8499169945716858 + }, + { + "bits_per_byte": 1.4229885808224993, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 0.9863405227661133, + "step": 7390, + "unclipped_grad_norm": 0.8316128849983215 + }, + { + "bits_per_byte": 1.4424061960268393, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9997997879981995, + "step": 7400, + "unclipped_grad_norm": 0.8269481062889099 + }, + { + "bits_per_byte": 1.4779136481326494, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.024411678314209, + "step": 7410, + "unclipped_grad_norm": 0.882456362247467 + }, + { + "bits_per_byte": 1.464608382331299, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0151891708374023, + "step": 7420, + "unclipped_grad_norm": 0.8758774995803833 + }, + { + "bits_per_byte": 1.3991048341492285, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9697855710983276, + "step": 7430, + "unclipped_grad_norm": 0.8292478919029236 + }, + { + "bits_per_byte": 1.401245502204518, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9712693691253662, + "step": 7440, + "unclipped_grad_norm": 0.875770092010498 + }, + { + "bits_per_byte": 1.5201309173705464, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0536744594573975, + "step": 7450, + "unclipped_grad_norm": 0.9047821760177612 + }, + { + "bits_per_byte": 1.4276059710328641, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9895410537719727, + "step": 7460, + "unclipped_grad_norm": 0.8659260272979736 + }, + { + "bits_per_byte": 1.4078822267099882, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9758695960044861, + "step": 7470, + "unclipped_grad_norm": 0.8766598105430603 + }, + { + "bits_per_byte": 1.486473568631393, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.0303449630737305, + "step": 7480, + "unclipped_grad_norm": 0.8612073659896851 + }, + { + "bits_per_byte": 1.450941608997835, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.00571608543396, + "step": 7490, + "unclipped_grad_norm": 0.888404905796051 + }, + { + "bits_per_byte": 1.4814055838757678, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.026832103729248, + "step": 7500, + "unclipped_grad_norm": 0.9002490639686584 + }, + { + "bits_per_byte": 1.4606080658722302, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0124163627624512, + "step": 7510, + "unclipped_grad_norm": 0.8679240345954895 + }, + { + "bits_per_byte": 1.4798904167216664, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0257818698883057, + "step": 7520, + "unclipped_grad_norm": 0.9162602424621582 + }, + { + "bits_per_byte": 1.4597773896685629, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.011840581893921, + "step": 7530, + "unclipped_grad_norm": 0.8853410482406616 + }, + { + "bits_per_byte": 1.4621115382060728, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.013458490371704, + "step": 7540, + "unclipped_grad_norm": 0.869069516658783 + }, + { + "bits_per_byte": 1.4571945542179052, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0100502967834473, + "step": 7550, + "unclipped_grad_norm": 0.8861801028251648 + }, + { + "bits_per_byte": 1.38000641874489, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 0.9565475583076477, + "step": 7560, + "unclipped_grad_norm": 0.8688139915466309 + }, + { + "bits_per_byte": 1.4141796294453064, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9802346229553223, + "step": 7570, + "unclipped_grad_norm": 0.8542686700820923 + }, + { + "bits_per_byte": 1.4002238392670674, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9705612063407898, + "step": 7580, + "unclipped_grad_norm": 0.898608922958374 + }, + { + "bits_per_byte": 1.4730534184192665, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.021042823791504, + "step": 7590, + "unclipped_grad_norm": 0.8726481199264526 + }, + { + "bits_per_byte": 1.5103581752179183, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0469005107879639, + "step": 7600, + "unclipped_grad_norm": 0.8932017087936401 + }, + { + "bits_per_byte": 1.4571630813927972, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0100284814834595, + "step": 7610, + "unclipped_grad_norm": 0.8839172124862671 + }, + { + "bits_per_byte": 1.4709910024701196, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.019613265991211, + "step": 7620, + "unclipped_grad_norm": 0.8883293271064758 + }, + { + "bits_per_byte": 1.436151788954237, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.995464563369751, + "step": 7630, + "unclipped_grad_norm": 0.8680411577224731 + }, + { + "bits_per_byte": 1.4213451146108544, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.985201358795166, + "step": 7640, + "unclipped_grad_norm": 0.9127815961837769 + }, + { + "bits_per_byte": 1.3999687889958379, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 0.970384418964386, + "step": 7650, + "unclipped_grad_norm": 0.8844171166419983 + }, + { + "bits_per_byte": 1.4213168234647875, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9851817488670349, + "step": 7660, + "unclipped_grad_norm": 0.8884557485580444 + }, + { + "bits_per_byte": 1.467116749294133, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0169278383255005, + "step": 7670, + "unclipped_grad_norm": 0.9028065204620361 + }, + { + "bits_per_byte": 1.4497504571579594, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0048904418945312, + "step": 7680, + "unclipped_grad_norm": 0.8733177781105042 + }, + { + "bits_per_byte": 1.436977993608982, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9960372447967529, + "step": 7690, + "unclipped_grad_norm": 0.8574365973472595 + }, + { + "bits_per_byte": 1.4762694939904009, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0232720375061035, + "step": 7700, + "unclipped_grad_norm": 0.8926270008087158 + }, + { + "bits_per_byte": 1.4168368473924622, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9820764660835266, + "step": 7710, + "unclipped_grad_norm": 0.8520281314849854 + }, + { + "bits_per_byte": 1.4226952644114528, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9861372113227844, + "step": 7720, + "unclipped_grad_norm": 0.8500117063522339 + }, + { + "bits_per_byte": 1.4242657239878067, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9872257709503174, + "step": 7730, + "unclipped_grad_norm": 0.8397486209869385 + }, + { + "bits_per_byte": 1.4715379072998633, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0199923515319824, + "step": 7740, + "unclipped_grad_norm": 0.9084712266921997 + }, + { + "bits_per_byte": 1.4017008262726773, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9715849757194519, + "step": 7750, + "unclipped_grad_norm": 0.8779424428939819 + }, + { + "bits_per_byte": 1.4074879564828853, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9755963087081909, + "step": 7760, + "unclipped_grad_norm": 0.8754757642745972 + }, + { + "bits_per_byte": 1.4701193943955468, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.0190091133117676, + "step": 7770, + "unclipped_grad_norm": 0.9014414548873901 + }, + { + "bits_per_byte": 1.4337202982363386, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 0.993779182434082, + "step": 7780, + "unclipped_grad_norm": 0.9682468175888062 + }, + { + "bits_per_byte": 1.461853220264477, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.0132794380187988, + "step": 7790, + "unclipped_grad_norm": 0.8698559403419495 + }, + { + "bits_per_byte": 1.446109240473892, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.002366542816162, + "step": 7800, + "unclipped_grad_norm": 0.8869026303291321 + }, + { + "bits_per_byte": 1.4204974121247516, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9846137762069702, + "step": 7810, + "unclipped_grad_norm": 0.893868088722229 + }, + { + "bits_per_byte": 1.471287500560207, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0198187828063965, + "step": 7820, + "unclipped_grad_norm": 0.9250245690345764 + }, + { + "bits_per_byte": 1.457534563918661, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0102859735488892, + "step": 7830, + "unclipped_grad_norm": 0.902686595916748 + }, + { + "bits_per_byte": 1.4368130622468045, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9959229230880737, + "step": 7840, + "unclipped_grad_norm": 0.880992591381073 + }, + { + "bits_per_byte": 1.4160461571551204, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9815284013748169, + "step": 7850, + "unclipped_grad_norm": 0.8318398594856262 + }, + { + "bits_per_byte": 1.463385069735712, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0143412351608276, + "step": 7860, + "unclipped_grad_norm": 0.8814117908477783 + }, + { + "bits_per_byte": 1.4569142224969986, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0098559856414795, + "step": 7870, + "unclipped_grad_norm": 0.8954723477363586 + }, + { + "bits_per_byte": 1.4148711716844258, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.980713963508606, + "step": 7880, + "unclipped_grad_norm": 0.9006734490394592 + }, + { + "bits_per_byte": 1.44079876018055, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 0.9986855983734131, + "step": 7890, + "unclipped_grad_norm": 0.8986613154411316 + }, + { + "bits_per_byte": 1.4040539788931077, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 0.9732160568237305, + "step": 7900, + "unclipped_grad_norm": 0.8486817479133606 + }, + { + "bits_per_byte": 1.4565468675547557, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.009601354598999, + "step": 7910, + "unclipped_grad_norm": 0.960801362991333 + }, + { + "bits_per_byte": 1.4869515084181408, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.030676245689392, + "step": 7920, + "unclipped_grad_norm": 0.9004883766174316 + }, + { + "bits_per_byte": 1.426570807457321, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9888235330581665, + "step": 7930, + "unclipped_grad_norm": 0.9118951559066772 + }, + { + "bits_per_byte": 1.4575464307215704, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0102941989898682, + "step": 7940, + "unclipped_grad_norm": 0.9135404825210571 + }, + { + "bits_per_byte": 1.4061704833859507, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9746831059455872, + "step": 7950, + "unclipped_grad_norm": 0.8805200457572937 + }, + { + "bits_per_byte": 1.4792671515949398, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0253498554229736, + "step": 7960, + "unclipped_grad_norm": 0.9637572765350342 + }, + { + "bits_per_byte": 1.4296003678435947, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9909234642982483, + "step": 7970, + "unclipped_grad_norm": 0.8976891040802002 + }, + { + "bits_per_byte": 1.4664819613297986, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0164878368377686, + "step": 7980, + "unclipped_grad_norm": 0.9626050591468811 + }, + { + "bits_per_byte": 1.4209173077668331, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9849048256874084, + "step": 7990, + "unclipped_grad_norm": 0.9410001635551453 + }, + { + "bits_per_byte": 1.4482875727297202, + "learning_rate": 3e-05, + "loss_nats": 1.0038764476776123, + "step": 8000, + "unclipped_grad_norm": 0.9042860269546509 + } + ] + }, + "depth-32-baseline-seed-2026073003": { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "5f1f6749be2f106aaf597f0478348ce6e226aa739e557d119b2388ec4460aa16", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002897916710935533, + 0.0002662435290403664, + 0.00024211003619711846, + 0.00022891115804668516, + 0.00021424151782412082, + 0.00020374931045807898, + 0.00019545880786608905, + 0.00018862746946979314, + 0.00018024678865913302, + 0.00017406532424502075, + 0.0001698832056717947, + 0.00016633344057481736, + 0.00016165875422302634, + 0.0001582534605404362, + 0.00015452370280399919, + 0.0001515774056315422, + 0.00014964381989557296, + 0.00014728632231708616, + 0.00014459768135566264, + 0.00014173447561915964, + 0.00013920740457251668, + 0.00013723202573601156, + 0.00013503432273864746, + 0.0001331205276073888, + 0.00013141198724042624, + 0.00013009003305342048, + 0.0001286487968172878, + 0.00012732617324218154, + 0.00012632299331016839, + 0.00012548104859888554, + 0.00012477955897338688, + 0.00012385686568450183 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022864168749947567, + "first_to_last_ratio": 1.796936959436678, + "imbalance_abs_log_ratio": 0.5860835261825511, + "last_quartile_mean": 0.00012723968211503234, + "mean": 0.00016535780059712124, + "normalized": [ + 1.7525128542293782, + 1.6101056501655084, + 1.4641585417974736, + 1.3843384298779209, + 1.2956238958819981, + 1.2321723542664615, + 1.1820356049746095, + 1.1407231396924917, + 1.0900410383317047, + 1.052658680851196, + 1.0273673516358577, + 1.0059001751001344, + 0.9776300461137163, + 0.9570365593214796, + 0.9344808787127115, + 0.9166631697094613, + 0.9049698251621409, + 0.8907128770776013, + 0.874453341986335, + 0.8571381277892198, + 0.8418556854882369, + 0.8299095974937676, + 0.816619005883163, + 0.8050453448623477, + 0.7947129604160569, + 0.786718452855893, + 0.7780025880407573, + 0.7700040323613145, + 0.7639373096038118, + 0.7588456555769529, + 0.7546034025779078, + 0.7490234221623898 + ], + "population_cv": 0.25804396357350073 + }, + "activation_output_rms_by_block": [ + 0.028274519369006157, + 0.02845057100057602, + 0.02867918275296688, + 0.02882254309952259, + 0.0290658138692379, + 0.029274335131049156, + 0.029572302475571632, + 0.02980491891503334, + 0.03008633479475975, + 0.03044024296104908, + 0.030671589076519012, + 0.030981140211224556, + 0.0313686840236187, + 0.031716227531433105, + 0.03204340860247612, + 0.03237249702215195, + 0.032637808471918106, + 0.03308072313666344, + 0.03343126177787781, + 0.033881161361932755, + 0.03435666114091873, + 0.03482793644070625, + 0.03533180430531502, + 0.03546451777219772, + 0.036012887954711914, + 0.03628316894173622, + 0.03689942881464958, + 0.03742557764053345, + 0.03777860850095749, + 0.03856590390205383, + 0.038892220705747604, + 0.039198413491249084 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.02899302332662046, + "first_to_last_ratio": 0.7704348189669382, + "imbalance_abs_log_ratio": 0.2608002235603661, + "last_quartile_mean": 0.0376320262439549, + "mean": 0.032990387349855155, + "normalized": [ + 0.8570532703712039, + 0.8623897227657417, + 0.8693193701798957, + 0.8736648889225345, + 0.8810388784163674, + 0.887359545694382, + 0.8963914901017818, + 0.9034425270294439, + 0.9119727657545023, + 0.9227003805150147, + 0.9297129115597873, + 0.9390959821925394, + 0.9508431559460433, + 0.961377846070436, + 0.9712953128638184, + 0.9812705949417742, + 0.9893126784418129, + 1.002738245715344, + 1.0133637239037931, + 1.0270010170729782, + 1.0414142997647942, + 1.0556995306349188, + 1.070972702764284, + 1.0749954947817073, + 1.0916176149374623, + 1.099810334354729, + 1.1184903172957619, + 1.1344388668021435, + 1.1451398887901616, + 1.1690042767025335, + 1.1788955459450936, + 1.1881768187672155 + ], + "population_cv": 0.10167083190861884 + }, + "bits_per_byte": 8.0691508062416, + "branch_output_rms_by_sublayer": [ + 0.002122601494193077, + 0.002755566267296672, + 0.002245423151180148, + 0.0027348215226083994, + 0.0024954702239483595, + 0.002720654010772705, + 0.0023770928382873535, + 0.002733660163357854, + 0.0023630354553461075, + 0.0027867795433849096, + 0.0024109049700200558, + 0.0027537678834050894, + 0.002701795194298029, + 0.0027542903553694487, + 0.0029824089724570513, + 0.002726059639826417, + 0.0029706284403800964, + 0.0027311695739626884, + 0.003079906338825822, + 0.0027289879508316517, + 0.0028494365978986025, + 0.0027595392893999815, + 0.0032860173378139734, + 0.002710360335186124, + 0.0035252906382083893, + 0.0027479089330881834, + 0.003967344760894775, + 0.002725270576775074, + 0.0035556324291974306, + 0.002750981133431196, + 0.004272372927516699, + 0.0027044012676924467, + 0.004134721122682095, + 0.0027556782588362694, + 0.0041320133022964, + 0.0027166390791535378, + 0.0040069944225251675, + 0.0026826481334865093, + 0.004499548580497503, + 0.002729416359215975, + 0.0049178507179021835, + 0.002681323792785406, + 0.004439458716660738, + 0.002806412521749735, + 0.004581923596560955, + 0.0027696131728589535, + 0.004871132783591747, + 0.0027697959449142218, + 0.0049627418629825115, + 0.0028166540432721376, + 0.0051520452834665775, + 0.0027135310228914022, + 0.005508258938789368, + 0.002744151744991541, + 0.005101518705487251, + 0.002700615907087922, + 0.005637416150420904, + 0.0027939171995967627, + 0.005635067820549011, + 0.0027390308678150177, + 0.00545669998973608, + 0.002716719638556242, + 0.005571703426539898, + 0.002707480685785413 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006421208939522194, + 0.005746349410576503, + 0.005404164999391892, + 0.005101115831054325, + 0.00485388847333966, + 0.0045855144167447825, + 0.004662942770173713, + 0.005059104716899598, + 0.00461737020280362, + 0.004446534400320367, + 0.004114212128857249, + 0.004678028852169936, + 0.004404060169276539, + 0.004656150674369456, + 0.004513829588134647, + 0.004839518675960965, + 0.004348161380019098, + 0.0047201153472655015, + 0.004169246752947831, + 0.004575240937826365, + 0.0047925402340120975, + 0.004508705150890575, + 0.004155367099556682, + 0.004478773084339979, + 0.004234175381969994, + 0.004428191743895327, + 0.004153362780335295, + 0.00410615213012631, + 0.004175626130102292, + 0.004125604583408443, + 0.0038633837013440408, + 0.003947405979968418 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005229286194712833, + "first_to_last_ratio": 1.266404707857162, + "imbalance_abs_log_ratio": 0.236181947089728, + "last_quartile_mean": 0.004129237803893765, + "mean": 0.004590188958362615, + "normalized": [ + 1.3988986069568536, + 1.2518764396632391, + 1.1773295279154767, + 1.1113084618795224, + 1.057448509716931, + 0.9989816232707989, + 1.0158498511653973, + 1.1021560904769923, + 1.0059215959707901, + 0.9687039990411438, + 0.8963056131625672, + 1.0191364439686714, + 0.9594507348663767, + 1.0143701526462583, + 0.983364656461723, + 1.0543179637831923, + 0.9472728507390573, + 1.0283052375580706, + 0.9082952337620236, + 0.9967434847079623, + 1.044083430439357, + 0.9822482672911342, + 0.9052714686148694, + 0.9757273883421174, + 0.9224403222564466, + 0.9647079421050512, + 0.9048348157364001, + 0.894549694440255, + 0.9096850190654888, + 0.8987875272307099, + 0.8416611464993293, + 0.859965900265795 + ], + "population_cv": 0.11345956482154346 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028097067028284073, + 0.028159458190202713, + 0.028274519369006157, + 0.02833542972803116, + 0.02845057100057602, + 0.02851351536810398, + 0.02867918275296688, + 0.028714356943964958, + 0.02882254309952259, + 0.028916271403431892, + 0.0290658138692379, + 0.02915445901453495, + 0.029274335131049156, + 0.02944180741906166, + 0.029572302475571632, + 0.02970401756465435, + 0.02980491891503334, + 0.02993646264076233, + 0.03008633479475975, + 0.030266879126429558, + 0.03044024296104908, + 0.03057776764035225, + 0.030671589076519012, + 0.030867453664541245, + 0.030981140211224556, + 0.03126408904790878, + 0.0313686840236187, + 0.031607914716005325, + 0.031716227531433105, + 0.03191319853067398, + 0.03204340860247612, + 0.032251372933387756, + 0.03237249702215195, + 0.03250224143266678, + 0.032637808471918106, + 0.03294040635228157, + 0.03308072313666344, + 0.0333157554268837, + 0.03343126177787781, + 0.03377276286482811, + 0.033881161361932755, + 0.034277379512786865, + 0.03435666114091873, + 0.03472030907869339, + 0.03482793644070625, + 0.0352315790951252, + 0.03533180430531502, + 0.03532300516963005, + 0.03546451777219772, + 0.03583687171339989, + 0.036012887954711914, + 0.03619367629289627, + 0.03628316894173622, + 0.03677845373749733, + 0.03689942881464958, + 0.03733044117689133, + 0.03742557764053345, + 0.0377187617123127, + 0.03777860850095749, + 0.03849533945322037, + 0.03856590390205383, + 0.03877026215195656, + 0.038892220705747604, + 0.03908585384488106 + ], + "loss_nats": 5.593109130859375, + "loss_scale": 1.0, + "output_weights": null, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.028159458190202713, + 0.028274519369006157, + 0.02833542972803116, + 0.02845057100057602, + 0.02851351536810398, + 0.02867918275296688, + 0.028714356943964958, + 0.02882254309952259, + 0.028916271403431892, + 0.0290658138692379, + 0.02915445901453495, + 0.029274335131049156, + 0.02944180741906166, + 0.029572302475571632, + 0.02970401756465435, + 0.02980491891503334, + 0.02993646264076233, + 0.03008633479475975, + 0.030266879126429558, + 0.03044024296104908, + 0.03057776764035225, + 0.030671589076519012, + 0.030867453664541245, + 0.030981140211224556, + 0.03126408904790878, + 0.0313686840236187, + 0.031607914716005325, + 0.031716227531433105, + 0.03191319853067398, + 0.03204340860247612, + 0.032251372933387756, + 0.03237249702215195, + 0.03250224143266678, + 0.032637808471918106, + 0.03294040635228157, + 0.03308072313666344, + 0.0333157554268837, + 0.03343126177787781, + 0.03377276286482811, + 0.033881161361932755, + 0.034277379512786865, + 0.03435666114091873, + 0.03472030907869339, + 0.03482793644070625, + 0.0352315790951252, + 0.03533180430531502, + 0.03532300516963005, + 0.03546451777219772, + 0.03583687171339989, + 0.036012887954711914, + 0.03619367629289627, + 0.03628316894173622, + 0.03677845373749733, + 0.03689942881464958, + 0.03733044117689133, + 0.03742557764053345, + 0.0377187617123127, + 0.03777860850095749, + 0.03849533945322037, + 0.03856590390205383, + 0.03877026215195656, + 0.038892220705747604, + 0.03908585384488106, + 0.039198413491249084 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.023746729828417e-05, + 6.667003617621958e-05, + 6.32333685643971e-05, + 6.012299854774028e-05, + 5.672227416653186e-05, + 5.335300011211075e-05, + 5.0173275667475536e-05, + 4.719031130662188e-05, + 4.408728636917658e-05, + 4.096491466043517e-05, + 3.824968007393181e-05, + 3.570610351744108e-05, + 3.324189310660586e-05, + 3.1057134037837386e-05, + 2.914190736191813e-05, + 2.7255719032837078e-05, + 2.56736693700077e-05, + 2.4158533051377162e-05, + 2.2910870029591024e-05, + 2.187793143093586e-05, + 2.0998755644541234e-05, + 2.0234831026755273e-05, + 1.963501563295722e-05, + 1.908456215460319e-05, + 1.8524928236729465e-05, + 1.8092870959662832e-05, + 1.7702994227875024e-05, + 1.74019587575458e-05, + 1.7127178580267355e-05, + 1.6885946024558507e-05, + 1.666023126745131e-05, + 1.649606565479189e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8462841479922645e-05, + "first_to_last_ratio": 3.3673800283354014, + "imbalance_abs_log_ratio": 1.2141350023382471, + "last_quartile_mean": 1.7361521713610273e-05, + "mean": 3.315230350153797e-05, + "normalized": [ + 2.1186300763392136, + 2.011022738529366, + 1.9073597272498315, + 1.8135390967614395, + 1.710960270495245, + 1.6093301061157226, + 1.5134174813870156, + 1.4234398917237432, + 1.3298408168570044, + 1.235658169530656, + 1.1537563316575383, + 1.0770323551057208, + 1.0027023644092703, + 0.9368016927208878, + 0.8790311466763148, + 0.8221365079977823, + 0.7744158522443748, + 0.7287135583280486, + 0.691079279861448, + 0.6599219094963015, + 0.6334026123876152, + 0.610359730382432, + 0.5922670088986827, + 0.5756632311754095, + 0.5587825363589011, + 0.5457500399277982, + 0.5339898697251538, + 0.5249094910324558, + 0.516621072181992, + 0.5093445776331997, + 0.5025361591142052, + 0.49758429769523016 + ], + "population_cv": 0.5033867184228629 + }, + "activation_output_rms_by_block": [ + 0.029364673420786858, + 0.03058232180774212, + 0.03253435716032982, + 0.035004038363695145, + 0.03789948299527168, + 0.041081421077251434, + 0.04464675113558769, + 0.04816310852766037, + 0.05221380665898323, + 0.056122854351997375, + 0.06031070277094841, + 0.06517986208200455, + 0.07041798532009125, + 0.07559357583522797, + 0.08127844333648682, + 0.08752159774303436, + 0.09348469227552414, + 0.10084687173366547, + 0.10794128477573395, + 0.11579020321369171, + 0.12420836836099625, + 0.13254153728485107, + 0.14211823046207428, + 0.151936337351799, + 0.16282601654529572, + 0.17539408802986145, + 0.19021794199943542, + 0.2045798897743225, + 0.2205442190170288, + 0.23628318309783936, + 0.2528626322746277, + 0.27021101117134094 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03740951931104064, + "first_to_last_ratio": 0.17471705179812935, + "imbalance_abs_log_ratio": 1.74458746043398, + "last_quartile_mean": 0.214114872738719, + "mean": 0.11030317156109959, + "normalized": [ + 0.2662178521722837, + 0.2772569580268309, + 0.29495395916434053, + 0.31734389744455865, + 0.34359377394944846, + 0.37244097785978386, + 0.40476398369793726, + 0.43664300714129206, + 0.47336632229165543, + 0.5088054455524842, + 0.5467721545752731, + 0.590915575314168, + 0.6384039944045038, + 0.685325496677626, + 0.736864064615447, + 0.793464018344695, + 0.8475249709727586, + 0.9142699190458359, + 0.9785873175545334, + 1.0497450034748341, + 1.126063435920283, + 1.2016112991949022, + 1.2884328569224464, + 1.3774430526472923, + 1.476168039783901, + 1.5901092012817277, + 1.7245011118657556, + 1.8547054167069055, + 1.9994367876798904, + 2.142125015570893, + 2.292433016167273, + 2.4497120739784397 + ], + "population_cv": 0.6367597776091559 + }, + "bits_per_byte": 5.184434896455056, + "branch_output_rms_by_sublayer": [ + 0.0026808311231434345, + 0.00477010989561677, + 0.0025234855711460114, + 0.004875619895756245, + 0.0027965111657977104, + 0.005397462286055088, + 0.002715951530262828, + 0.00578050222247839, + 0.0028346737381070852, + 0.005865047685801983, + 0.0025751085486263037, + 0.0059114922769367695, + 0.0034423002507537603, + 0.006014656741172075, + 0.0034009802620857954, + 0.005813058000057936, + 0.0035280236043035984, + 0.006192575208842754, + 0.003798100398853421, + 0.006026907358318567, + 0.0032950215972959995, + 0.006300882901996374, + 0.004497972317039967, + 0.006683246698230505, + 0.004819076508283615, + 0.006906723603606224, + 0.004967817571014166, + 0.006679704412817955, + 0.005188894923776388, + 0.006651155184954405, + 0.005964669398963451, + 0.006957803387194872, + 0.005442580208182335, + 0.006974066607654095, + 0.00629971781745553, + 0.007549752946943045, + 0.007111458573490381, + 0.007369632367044687, + 0.008486796170473099, + 0.0074090128764510155, + 0.008528640493750572, + 0.007484533358365297, + 0.007537186611443758, + 0.007801020983606577, + 0.00965037476271391, + 0.007585691753774881, + 0.009028015658259392, + 0.007420822512358427, + 0.010029759258031845, + 0.00862929131835699, + 0.01243194192647934, + 0.0086217625066638, + 0.0149146169424057, + 0.00852242112159729, + 0.014407102949917316, + 0.008064002729952335, + 0.01547334622591734, + 0.008129187859594822, + 0.015585782937705517, + 0.008083954453468323, + 0.014871877618134022, + 0.009126133285462856, + 0.016274450346827507, + 0.008773879148066044 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00026099601887828554, + 0.00022299818257146475, + 0.00021593606328411676, + 0.0001980902028289896, + 0.00019039206376599677, + 0.0001764249768268048, + 0.00019262760932023515, + 0.00017741019012756782, + 0.0001866267032514166, + 0.00018194420611625288, + 0.00016335812718032435, + 0.00018628060025801724, + 0.00018197064718745634, + 0.00018231380553041538, + 0.00016812738233283717, + 0.0001758928024359223, + 0.00015172497688937343, + 0.0001676233054310719, + 0.0001547068922126687, + 0.0001733627635453279, + 0.0001619137110647745, + 0.00014190991701657472, + 0.0001580141865100647, + 0.00014951069781870242, + 0.00014861801255322757, + 0.00015877260520542335, + 0.00016243531213177502, + 0.00014972906733594748, + 0.0001506252547372875, + 0.00014728724932094392, + 0.00014196626632039671, + 0.00013988030321371655 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00020435941345043265, + "first_to_last_ratio": 1.3631752910956885, + "imbalance_abs_log_ratio": 0.30981675126526836, + "last_quartile_mean": 0.00014991425885233975, + "mean": 0.00017248344072510563, + "normalized": [ + 1.513165656836857, + 1.2928671971871588, + 1.251923444803397, + 1.1484592491675452, + 1.1038280716433115, + 1.0228516783125923, + 1.1167890002103689, + 1.028563608087539, + 1.0819977991327974, + 1.0548502821567973, + 0.9470945529239257, + 1.0799912123442663, + 1.0550035784447904, + 1.0569930931571387, + 0.9747450632132802, + 1.0197663131978583, + 0.8796495260735443, + 0.9718225977311089, + 0.8969376513031868, + 1.0050980129833083, + 0.9387203222761739, + 0.8227451656808188, + 0.9161122125450802, + 0.8668118933050746, + 0.8616364094341472, + 0.9205092647616311, + 0.941744386875116, + 0.8680779250837024, + 0.8732737131406461, + 0.853921099334295, + 0.823071859672921, + 0.8109781589796199 + ], + "population_cv": 0.15033028761203962 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.028610490262508392, + 0.028607122600078583, + 0.029364673420786858, + 0.029366537928581238, + 0.03058232180774212, + 0.03066444955766201, + 0.03253435716032982, + 0.03263945132493973, + 0.035004038363695145, + 0.035203028470277786, + 0.03789948299527168, + 0.03799305483698845, + 0.041081421077251434, + 0.04133254662156105, + 0.04464675113558769, + 0.04496326297521591, + 0.04816310852766037, + 0.048516932874917984, + 0.05221380665898323, + 0.05251964181661606, + 0.056122854351997375, + 0.056522201746702194, + 0.06031070277094841, + 0.060862768441438675, + 0.06517986208200455, + 0.06596556305885315, + 0.07041798532009125, + 0.0712207704782486, + 0.07559357583522797, + 0.07684765011072159, + 0.08127844333648682, + 0.08279602974653244, + 0.08752159774303436, + 0.08871174603700638, + 0.09348469227552414, + 0.09565582871437073, + 0.10084687173366547, + 0.10279740393161774, + 0.10794128477573395, + 0.11048463732004166, + 0.11579020321369171, + 0.11891037970781326, + 0.12420836836099625, + 0.1270110011100769, + 0.13254153728485107, + 0.13657133281230927, + 0.14211823046207428, + 0.14646445214748383, + 0.151936337351799, + 0.15653157234191895, + 0.16282601654529572, + 0.16900379955768585, + 0.17539408802986145, + 0.18384796380996704, + 0.19021794199943542, + 0.1985616832971573, + 0.2045798897743225, + 0.21447749435901642, + 0.2205442190170288, + 0.23017354309558868, + 0.23628318309783936, + 0.24575340747833252, + 0.2528626322746277, + 0.26334208250045776 + ], + "loss_nats": 3.593576431274414, + "loss_scale": 1.0, + "output_weights": null, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.028607122600078583, + 0.029364673420786858, + 0.029366537928581238, + 0.03058232180774212, + 0.03066444955766201, + 0.03253435716032982, + 0.03263945132493973, + 0.035004038363695145, + 0.035203028470277786, + 0.03789948299527168, + 0.03799305483698845, + 0.041081421077251434, + 0.04133254662156105, + 0.04464675113558769, + 0.04496326297521591, + 0.04816310852766037, + 0.048516932874917984, + 0.05221380665898323, + 0.05251964181661606, + 0.056122854351997375, + 0.056522201746702194, + 0.06031070277094841, + 0.060862768441438675, + 0.06517986208200455, + 0.06596556305885315, + 0.07041798532009125, + 0.0712207704782486, + 0.07559357583522797, + 0.07684765011072159, + 0.08127844333648682, + 0.08279602974653244, + 0.08752159774303436, + 0.08871174603700638, + 0.09348469227552414, + 0.09565582871437073, + 0.10084687173366547, + 0.10279740393161774, + 0.10794128477573395, + 0.11048463732004166, + 0.11579020321369171, + 0.11891037970781326, + 0.12420836836099625, + 0.1270110011100769, + 0.13254153728485107, + 0.13657133281230927, + 0.14211823046207428, + 0.14646445214748383, + 0.151936337351799, + 0.15653157234191895, + 0.16282601654529572, + 0.16900379955768585, + 0.17539408802986145, + 0.18384796380996704, + 0.19021794199943542, + 0.1985616832971573, + 0.2045798897743225, + 0.21447749435901642, + 0.2205442190170288, + 0.23017354309558868, + 0.23628318309783936, + 0.24575340747833252, + 0.2528626322746277, + 0.26334208250045776, + 0.27021101117134094 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001506932603660971, + 0.00015340163372457027, + 0.00015237691695801914, + 0.000148982901009731, + 0.0001392244448652491, + 0.00012851000064983964, + 0.00011962141434196383, + 0.0001119004882639274, + 0.00010405662760604173, + 9.720238449517637e-05, + 9.135912114288658e-05, + 8.468296437058598e-05, + 7.889774860814214e-05, + 7.427640230162069e-05, + 6.881110311951488e-05, + 6.43495368422009e-05, + 5.969387711957097e-05, + 5.503925422090106e-05, + 5.034419518779032e-05, + 4.6255379857029766e-05, + 4.248250479577109e-05, + 3.87999571103137e-05, + 3.563905556802638e-05, + 3.2581570849288255e-05, + 3.028466016985476e-05, + 2.807797682180535e-05, + 2.6328352760174312e-05, + 2.5215304049197584e-05, + 2.4513139578630216e-05, + 2.4017937903408892e-05, + 2.3832226361264475e-05, + 2.3948901798576117e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013808888252242468, + "first_to_last_ratio": 5.356993010635397, + "imbalance_abs_log_ratio": 1.678402812181787, + "last_quartile_mean": 2.5777312430363963e-05, + "mean": 7.298128883803656e-05, + "normalized": [ + 2.0648204870774824, + 2.1019310040551114, + 2.087890189171378, + 2.041384899907163, + 1.907673145841839, + 1.7608623072556988, + 1.6390696334156716, + 1.533276406124618, + 1.425798711735127, + 1.3318808977310936, + 1.2518156721736573, + 1.1603380225103768, + 1.0810681732852876, + 1.0177458288857888, + 0.9428595221471582, + 0.881726506433839, + 0.8179339947263247, + 0.7541556897281809, + 0.6898233230645799, + 0.6337977938383883, + 0.5821013231220159, + 0.5316425309563984, + 0.488331408439747, + 0.44643731794864266, + 0.4149647211227507, + 0.3847284320247795, + 0.3607548342781861, + 0.3455036825282238, + 0.3358825250816124, + 0.32909720129375364, + 0.3265525553289974, + 0.3281512587661287 + ], + "population_cv": 0.6061250608697893 + }, + "activation_output_rms_by_block": [ + 0.048818644136190414, + 0.05689374357461929, + 0.06500593572854996, + 0.07087047398090363, + 0.07611831277608871, + 0.0806790366768837, + 0.08478696644306183, + 0.08861920237541199, + 0.0918315127491951, + 0.09433750063180923, + 0.09700070321559906, + 0.09972338378429413, + 0.10201811045408249, + 0.10424874722957611, + 0.10649008303880692, + 0.10806933790445328, + 0.11109450459480286, + 0.11401215940713882, + 0.11832662671804428, + 0.12293888628482819, + 0.12948566675186157, + 0.1387748122215271, + 0.14864373207092285, + 0.16069285571575165, + 0.1741981953382492, + 0.18964411318302155, + 0.20650440454483032, + 0.22468066215515137, + 0.24238750338554382, + 0.2638821303844452, + 0.2911032736301422, + 0.32429978251457214 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.07147403946146369, + "first_to_last_ratio": 0.2983212272448851, + "imbalance_abs_log_ratio": 1.2095844292661684, + "last_quartile_mean": 0.23958750814199448, + "mean": 0.13550565636251122, + "normalized": [ + 0.3602701573253034, + 0.41986249948426085, + 0.4797285771941724, + 0.523007495652488, + 0.5617353165867361, + 0.5953923905659501, + 0.6257079499045831, + 0.6539889533344176, + 0.67769505136762, + 0.6961886548811884, + 0.7158424660598531, + 0.7359352108336301, + 0.7528697560872202, + 0.7693313329348029, + 0.7858718661449788, + 0.7975263970925391, + 0.8198514185828341, + 0.8413830275994386, + 0.8732227856340641, + 0.907260180755379, + 0.9555738869339544, + 1.024125604397429, + 1.0969559209636535, + 1.185875630799193, + 1.285541873412471, + 1.3995291287005507, + 1.523954129209041, + 1.6580906523494112, + 1.7887629925727762, + 1.9473883044298541, + 2.148273964677672, + 2.393256423532534 + ], + "population_cv": 0.5081939166956976 + }, + "bits_per_byte": 3.1154419867739596, + "branch_output_rms_by_sublayer": [ + 0.010533316060900688, + 0.01737932860851288, + 0.0117434561252594, + 0.012238939292728901, + 0.01211429014801979, + 0.011732303537428379, + 0.014587659388780594, + 0.009741203859448433, + 0.012628368102014065, + 0.009234351105988026, + 0.014413262717425823, + 0.00937508512288332, + 0.015033986419439316, + 0.00873780157417059, + 0.016592979431152344, + 0.008810308761894703, + 0.018142949789762497, + 0.00861456897109747, + 0.01592753641307354, + 0.008246068842709064, + 0.013729470781981945, + 0.008380995132029057, + 0.017536180093884468, + 0.008834624662995338, + 0.021067099645733833, + 0.009005429223179817, + 0.022672412917017937, + 0.009119905531406403, + 0.019366111606359482, + 0.0090357456356287, + 0.0166512168943882, + 0.009071468375623226, + 0.014951958321034908, + 0.01051315013319254, + 0.017114169895648956, + 0.010944797657430172, + 0.01693827658891678, + 0.012414379045367241, + 0.01647246815264225, + 0.013282178901135921, + 0.01672390289604664, + 0.014548475854098797, + 0.017381425946950912, + 0.017058325931429863, + 0.02400100603699684, + 0.018193069845438004, + 0.017901180312037468, + 0.02045683190226555, + 0.01391419768333435, + 0.021929442882537842, + 0.014953838661313057, + 0.02380906604230404, + 0.010647366754710674, + 0.02548205293715, + 0.005664720665663481, + 0.026819046586751938, + 0.008504491299390793, + 0.024410424754023552, + 0.008407330140471458, + 0.02826099656522274, + 0.01054995134472847, + 0.033158741891384125, + 0.02235519140958786, + 0.03220592811703682 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005931392614171548, + 0.0004012275990867258, + 0.00038098222807989436, + 0.0003649111519840816, + 0.00034656151495199636, + 0.00031005860528648927, + 0.00027821812913176474, + 0.0002561842642921938, + 0.0002305368063696604, + 0.00020943363497294808, + 0.0001942700943631853, + 0.0001900904574854747, + 0.00018678033884870243, + 0.00017357948080148642, + 0.00015815775362626997, + 0.00014490835987546657, + 0.0001375100278294081, + 0.0001295570650502436, + 0.00012744106699623042, + 0.00011906627060222178, + 0.00011368520556157287, + 0.0001091753996736031, + 0.00010941555658600093, + 9.636520203422524e-05, + 9.391483800802352e-05, + 9.101856383525677e-05, + 7.998410113295477e-05, + 7.102545922046693e-05, + 6.763159533414917e-05, + 6.887797791066819e-05, + 7.001610738319118e-05, + 7.664349539138475e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00036641034427878756, + "first_to_last_ratio": 4.73465560322638, + "imbalance_abs_log_ratio": 1.5549089897216843, + "last_quartile_mean": 7.738901727701191e-05, + "mean": 0.00018688648791009675, + "normalized": [ + 3.173794253667441, + 2.146905341170195, + 2.038575567128047, + 1.9525817840807467, + 1.8543957823141959, + 1.6590744935805388, + 1.4887011481836172, + 1.3708014268823612, + 1.2335659412710565, + 1.1206462131906392, + 1.0395085088047697, + 1.017143933792149, + 0.9994320131830752, + 0.9287963123636216, + 0.8462770925544549, + 0.7753816848715992, + 0.7357943817509076, + 0.6932393374130293, + 0.681916966932018, + 0.6371047577259816, + 0.6083115308810451, + 0.5841802737826761, + 0.5854653153877886, + 0.5156349349375247, + 0.5025234250921448, + 0.4870259206703161, + 0.4279822582541749, + 0.38004598414110224, + 0.3618859559642637, + 0.3685551517443171, + 0.3746451023086966, + 0.4101072059755049 + ], + "population_cv": 0.6522632271008562 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.04385505989193916, + 0.04404326155781746, + 0.048818644136190414, + 0.05170967057347298, + 0.05689374357461929, + 0.059983257204294205, + 0.06500593572854996, + 0.06660757958889008, + 0.07087047398090363, + 0.07236418128013611, + 0.07611831277608871, + 0.07718922197818756, + 0.0806790366768837, + 0.08160495012998581, + 0.08478696644306183, + 0.08585704118013382, + 0.08861920237541199, + 0.08891960233449936, + 0.0918315127491951, + 0.0918683186173439, + 0.09433750063180923, + 0.09460321068763733, + 0.09700070321559906, + 0.09746499359607697, + 0.09972338378429413, + 0.0994088351726532, + 0.10201811045408249, + 0.10191889107227325, + 0.10424874722957611, + 0.10386203229427338, + 0.10649008303880692, + 0.10544375330209732, + 0.10806933790445328, + 0.10778804868459702, + 0.11109450459480286, + 0.11027765274047852, + 0.11401215940713882, + 0.11356210708618164, + 0.11832662671804428, + 0.11750734597444534, + 0.12293888628482819, + 0.1225726455450058, + 0.12948566675186157, + 0.12988287210464478, + 0.1387748122215271, + 0.13870327174663544, + 0.14864373207092285, + 0.14834089577198029, + 0.16069285571575165, + 0.16013014316558838, + 0.1741981953382492, + 0.1741078794002533, + 0.18964411318302155, + 0.18942901492118835, + 0.20650440454483032, + 0.20662079751491547, + 0.22468066215515137, + 0.2257053554058075, + 0.24238750338554382, + 0.24448354542255402, + 0.2638821303844452, + 0.2678608000278473, + 0.2911032736301422, + 0.30181384086608887 + ], + "loss_nats": 2.1594598293304443, + "loss_scale": 1.0, + "output_weights": null, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.04404326155781746, + 0.048818644136190414, + 0.05170967057347298, + 0.05689374357461929, + 0.059983257204294205, + 0.06500593572854996, + 0.06660757958889008, + 0.07087047398090363, + 0.07236418128013611, + 0.07611831277608871, + 0.07718922197818756, + 0.0806790366768837, + 0.08160495012998581, + 0.08478696644306183, + 0.08585704118013382, + 0.08861920237541199, + 0.08891960233449936, + 0.0918315127491951, + 0.0918683186173439, + 0.09433750063180923, + 0.09460321068763733, + 0.09700070321559906, + 0.09746499359607697, + 0.09972338378429413, + 0.0994088351726532, + 0.10201811045408249, + 0.10191889107227325, + 0.10424874722957611, + 0.10386203229427338, + 0.10649008303880692, + 0.10544375330209732, + 0.10806933790445328, + 0.10778804868459702, + 0.11109450459480286, + 0.11027765274047852, + 0.11401215940713882, + 0.11356210708618164, + 0.11832662671804428, + 0.11750734597444534, + 0.12293888628482819, + 0.1225726455450058, + 0.12948566675186157, + 0.12988287210464478, + 0.1387748122215271, + 0.13870327174663544, + 0.14864373207092285, + 0.14834089577198029, + 0.16069285571575165, + 0.16013014316558838, + 0.1741981953382492, + 0.1741078794002533, + 0.18964411318302155, + 0.18942901492118835, + 0.20650440454483032, + 0.20662079751491547, + 0.22468066215515137, + 0.2257053554058075, + 0.24238750338554382, + 0.24448354542255402, + 0.2638821303844452, + 0.2678608000278473, + 0.2911032736301422, + 0.30181384086608887, + 0.32429978251457214 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010946426482405514, + 0.00010930908320005983, + 0.00012598594184964895, + 0.00012082153261872008, + 0.00011562183499336243, + 0.00011443938274169341, + 0.00011117216490674764, + 0.00010893856233451515, + 0.00010224300058325753, + 9.949541708920151e-05, + 9.518271690467373e-05, + 9.17384895728901e-05, + 8.801270450931042e-05, + 8.554522355552763e-05, + 8.286484808195382e-05, + 8.027214062167332e-05, + 7.707033364567906e-05, + 7.422274211421609e-05, + 7.329088111873716e-05, + 7.053956505842507e-05, + 6.641655636485666e-05, + 6.314766505965963e-05, + 5.928793325438164e-05, + 5.485894507728517e-05, + 5.013645932194777e-05, + 4.561002060654573e-05, + 4.094795440323651e-05, + 3.615552486735396e-05, + 3.135877341264859e-05, + 2.6529152819421142e-05, + 2.2670756152365357e-05, + 2.0441382730496116e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00011446909593360033, + "first_to_last_ratio": 3.3439937416939496, + "imbalance_abs_log_ratio": 1.2071658237176983, + "last_quartile_mean": 3.4231253039251897e-05, + "mean": 7.668099857482957e-05, + "normalized": [ + 1.427527899460436, + 1.4255041696332367, + 1.642987756956567, + 1.5756384875558933, + 1.5078290207779732, + 1.4924086131979242, + 1.4498006934307157, + 1.4206721920582042, + 1.3333551007878848, + 1.297523753451233, + 1.2412816561300926, + 1.196365348364192, + 1.1477772348442066, + 1.115598716050207, + 1.0806438312235815, + 1.046832228499728, + 1.0050773343864263, + 0.9679417781940518, + 0.955789341308797, + 0.91990931742482, + 0.866140994500009, + 0.8235112509396527, + 0.7731763325503017, + 0.7154177188205337, + 0.6538315913168741, + 0.5948021211804766, + 0.5340039274955088, + 0.47150566032432917, + 0.4089510308351939, + 0.3459677535828191, + 0.29565024678495777, + 0.26657689793317285 + ], + "population_cv": 0.3996524250074872 + }, + "activation_output_rms_by_block": [ + 0.09914769232273102, + 0.13720469176769257, + 0.1984461098909378, + 0.2206047922372818, + 0.22331808507442474, + 0.2258162647485733, + 0.22567619383335114, + 0.2251615822315216, + 0.22496475279331207, + 0.22249442338943481, + 0.21478813886642456, + 0.205490842461586, + 0.19144563376903534, + 0.188800647854805, + 0.18874111771583557, + 0.190367192029953, + 0.19439123570919037, + 0.198538675904274, + 0.20280276238918304, + 0.2084609866142273, + 0.21558508276939392, + 0.21985429525375366, + 0.22511647641658783, + 0.2311476618051529, + 0.24068132042884827, + 0.2547627091407776, + 0.2762547731399536, + 0.30550503730773926, + 0.34050679206848145, + 0.39206719398498535, + 0.46342816948890686, + 0.575099527835846 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.19442192651331425, + "first_to_last_ratio": 0.5460704265504183, + "imbalance_abs_log_ratio": 0.6050073251991793, + "last_quartile_mean": 0.3560381904244423, + "mean": 0.2414584644138813, + "normalized": [ + 0.41062007316000754, + 0.5682331000519887, + 0.8218643747803495, + 0.9136345365766327, + 0.9248716362729685, + 0.9352178450099977, + 0.9346377414482436, + 0.932506477990246, + 0.9316913090596918, + 0.9214604421904198, + 0.8895448721907656, + 0.851040128000468, + 0.7928719096004871, + 0.7819177029602238, + 0.7816711589464782, + 0.7884055441669947, + 0.8050711172253067, + 0.8222477368362661, + 0.8399074469452479, + 0.8633409771748842, + 0.8928454146045669, + 0.9105263540353832, + 0.9323196723007322, + 0.9572978208332561, + 0.9967814589274413, + 1.0550995168431603, + 1.144108879390653, + 1.2652488205345183, + 1.410208554481745, + 1.6237459098263265, + 1.9192873217919157, + 2.381774145842633 + ], + "population_cv": 0.36935148393532197 + }, + "bits_per_byte": 2.073311222061611, + "branch_output_rms_by_sublayer": [ + 0.01824937015771866, + 0.06101369485259056, + 0.05977629870176315, + 0.0147639699280262, + 0.08494830876588821, + 0.003444893751293421, + 0.03928658738732338, + 0.004586480092257261, + 0.025128526613116264, + 0.007272258400917053, + 0.024004649370908737, + 0.006204464007169008, + 0.019449269399046898, + 0.007294370327144861, + 0.02213231660425663, + 0.010247048921883106, + 0.041971735656261444, + 0.012659681960940361, + 0.02208700217306614, + 0.013693951070308685, + 0.029153602197766304, + 0.01590428315103054, + 0.02919897623360157, + 0.017466455698013306, + 0.036670926958322525, + 0.021206999197602272, + 0.02140921540558338, + 0.021084947511553764, + 0.023707622662186623, + 0.022131068632006645, + 0.02047598734498024, + 0.022354252636432648, + 0.022473404183983803, + 0.02369093894958496, + 0.02454194240272045, + 0.02377566508948803, + 0.020227128639817238, + 0.025439493358135223, + 0.02723041921854019, + 0.027927791699767113, + 0.03427847847342491, + 0.028011852875351906, + 0.03502028062939644, + 0.030455492436885834, + 0.029505647718906403, + 0.032503895461559296, + 0.033919259905815125, + 0.03810396417975426, + 0.036724433302879333, + 0.04227631166577339, + 0.033869534730911255, + 0.04830062761902809, + 0.021475639194250107, + 0.057604189962148666, + 0.010480014607310295, + 0.06417365372180939, + 0.010398799553513527, + 0.06654922664165497, + 0.012935257516801357, + 0.0871838852763176, + 0.010905168019235134, + 0.11424379050731659, + 0.02395492233335972, + 0.16713054478168488 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00035888588058483766, + 0.00018018950310225553, + 0.0002078431087578172, + 0.0001449691382379752, + 0.00012177099939920197, + 0.00012306582379429542, + 0.00012204348242118167, + 0.00015411618529826362, + 0.00023712554584861813, + 0.0001667297606409996, + 0.00020124714337423614, + 0.00019657670547036952, + 0.00021589641757185544, + 0.00019691649893044806, + 0.0002141921139315525, + 0.00021492094770207205, + 0.00022724548428173296, + 0.00022961612017464663, + 0.00022356249125846063, + 0.00024496232741861765, + 0.0002428330999302831, + 0.0002472582804061272, + 0.00023198586071939664, + 0.00023918771709558322, + 0.0002286007324212131, + 0.00020791382420134994, + 0.00018978221308207498, + 0.00017514362262291224, + 0.0001585770655414503, + 0.0001491152323589239, + 0.00013960034115182495, + 0.00014044566543596396 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00017661051519947854, + "first_to_last_ratio": 1.0170643451662862, + "imbalance_abs_log_ratio": 0.016920384648356147, + "last_quartile_mean": 0.00017364733710196415, + "mean": 0.0001978849791614544, + "normalized": [ + 1.813608501795493, + 0.9105769617573595, + 1.0503228170150194, + 0.7325929378383735, + 0.6153625197587583, + 0.6219058380064613, + 0.6167394965416064, + 0.7788169973857398, + 1.1982998752782916, + 0.8425589519099616, + 1.0169904973436037, + 0.9933887165330652, + 1.0910197352357174, + 0.9951058426207471, + 1.0824071379202214, + 1.0860902561316592, + 1.1483715704177997, + 1.160351438232726, + 1.1297597837178734, + 1.2379025859196358, + 1.227142661152444, + 1.2495050481036718, + 1.1723267814588358, + 1.2087209353087376, + 1.155220236472308, + 1.0506801733127658, + 0.959053152423619, + 0.8850779041698386, + 0.8013597909928638, + 0.7535449784556955, + 0.7054620403397319, + 0.7097338364493765 + ], + "population_cv": 0.24613765810690832 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.0564301572740078, + 0.061963923275470734, + 0.09914769232273102, + 0.12944035232067108, + 0.13720469176769257, + 0.1976913958787918, + 0.1984461098909378, + 0.21992529928684235, + 0.2206047922372818, + 0.22257371246814728, + 0.22331808507442474, + 0.22564378380775452, + 0.2258162647485733, + 0.2261907458305359, + 0.22567619383335114, + 0.22598126530647278, + 0.2251615822315216, + 0.22548764944076538, + 0.22496475279331207, + 0.22340621054172516, + 0.22249442338943481, + 0.21654370427131653, + 0.21478813886642456, + 0.20714853703975677, + 0.205490842461586, + 0.19201314449310303, + 0.19144563376903534, + 0.18731488287448883, + 0.188800647854805, + 0.18594272434711456, + 0.18874111771583557, + 0.18583883345127106, + 0.190367192029953, + 0.18933312594890594, + 0.19439123570919037, + 0.1937587559223175, + 0.198538675904274, + 0.19723950326442719, + 0.20280276238918304, + 0.2025505006313324, + 0.2084609866142273, + 0.2101445198059082, + 0.21558508276939392, + 0.21427761018276215, + 0.21985429525375366, + 0.21906179189682007, + 0.22511647641658783, + 0.22344982624053955, + 0.2311476618051529, + 0.2303154021501541, + 0.24068132042884827, + 0.23970884084701538, + 0.2547627091407776, + 0.25386181473731995, + 0.2762547731399536, + 0.2756711542606354, + 0.30550503730773926, + 0.30545076727867126, + 0.34050679206848145, + 0.34091928601264954, + 0.39206719398498535, + 0.3925974369049072, + 0.46342816948890686, + 0.46681874990463257 + ], + "loss_nats": 1.4371098279953003, + "loss_scale": 1.0, + "output_weights": null, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.061963923275470734, + 0.09914769232273102, + 0.12944035232067108, + 0.13720469176769257, + 0.1976913958787918, + 0.1984461098909378, + 0.21992529928684235, + 0.2206047922372818, + 0.22257371246814728, + 0.22331808507442474, + 0.22564378380775452, + 0.2258162647485733, + 0.2261907458305359, + 0.22567619383335114, + 0.22598126530647278, + 0.2251615822315216, + 0.22548764944076538, + 0.22496475279331207, + 0.22340621054172516, + 0.22249442338943481, + 0.21654370427131653, + 0.21478813886642456, + 0.20714853703975677, + 0.205490842461586, + 0.19201314449310303, + 0.19144563376903534, + 0.18731488287448883, + 0.188800647854805, + 0.18594272434711456, + 0.18874111771583557, + 0.18583883345127106, + 0.190367192029953, + 0.18933312594890594, + 0.19439123570919037, + 0.1937587559223175, + 0.198538675904274, + 0.19723950326442719, + 0.20280276238918304, + 0.2025505006313324, + 0.2084609866142273, + 0.2101445198059082, + 0.21558508276939392, + 0.21427761018276215, + 0.21985429525375366, + 0.21906179189682007, + 0.22511647641658783, + 0.22344982624053955, + 0.2311476618051529, + 0.2303154021501541, + 0.24068132042884827, + 0.23970884084701538, + 0.2547627091407776, + 0.25386181473731995, + 0.2762547731399536, + 0.2756711542606354, + 0.30550503730773926, + 0.30545076727867126, + 0.34050679206848145, + 0.34091928601264954, + 0.39206719398498535, + 0.3925974369049072, + 0.46342816948890686, + 0.46681874990463257, + 0.575099527835846 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00010021305934060365, + 0.00010125436529051512, + 0.000114263420982752, + 0.00011024689592886716, + 0.00010690757335396484, + 0.00010680041305022314, + 0.00010475025192135945, + 0.00010345032933400944, + 9.713075269246474e-05, + 9.490915545029566e-05, + 9.159108594758436e-05, + 8.836394408717752e-05, + 8.501022239215672e-05, + 8.242073090514168e-05, + 7.99787012510933e-05, + 7.76422893977724e-05, + 7.462384382961318e-05, + 7.162441761465743e-05, + 7.150838064262643e-05, + 6.863063026685268e-05, + 6.474268593592569e-05, + 6.149821274448186e-05, + 5.7751010899664834e-05, + 5.37736268597655e-05, + 4.920278661302291e-05, + 4.476245885598473e-05, + 4.037534017697908e-05, + 3.581849159672856e-05, + 3.1086256058188155e-05, + 2.5875511710182764e-05, + 2.1458168703247793e-05, + 1.863625948317349e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00010598578865028685, + "first_to_last_ratio": 3.1730458332581706, + "imbalance_abs_log_ratio": 1.1546919572676668, + "last_quartile_mean": 3.3401909149688436e-05, + "mean": 7.300941479115863e-05, + "normalized": [ + 1.3726046103404645, + 1.3868672359606, + 1.565050498070684, + 1.5100367020366523, + 1.4642984560247596, + 1.4628306959550723, + 1.434749918500163, + 1.4169450560578547, + 1.330386676434875, + 1.2999577619103044, + 1.2545106163304838, + 1.2103088942698703, + 1.1643734254729483, + 1.12890551363689, + 1.0954573664214418, + 1.0634558518222064, + 1.0221126144220245, + 0.9810298825094963, + 0.9794405399245307, + 0.9400243853915095, + 0.8867717419886232, + 0.8423326350498187, + 0.7910077222897892, + 0.7365300353876748, + 0.6739238597346121, + 0.613105295858438, + 0.5530155294693376, + 0.4906009957645373, + 0.425784210804997, + 0.35441335592401246, + 0.29390961103616947, + 0.255258305199158 + ], + "population_cv": 0.383084243994172 + }, + "activation_output_rms_by_block": [ + 0.11517385393381119, + 0.17733007669448853, + 0.24607165157794952, + 0.2600470185279846, + 0.2621006965637207, + 0.26366734504699707, + 0.2635972201824188, + 0.26321178674697876, + 0.2646143436431885, + 0.2614729404449463, + 0.2522071599960327, + 0.23732660710811615, + 0.2133668214082718, + 0.2109174132347107, + 0.21198545396327972, + 0.21621736884117126, + 0.2226637899875641, + 0.22916270792484283, + 0.23635683953762054, + 0.24468488991260529, + 0.25434690713882446, + 0.26204079389572144, + 0.2696947753429413, + 0.27905911207199097, + 0.29096299409866333, + 0.30846187472343445, + 0.3323265314102173, + 0.36396095156669617, + 0.403833270072937, + 0.4673614203929901, + 0.5606572031974792, + 0.7190070152282715 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23139995615929365, + "first_to_last_ratio": 0.5371134119256165, + "imbalance_abs_log_ratio": 0.6215460113796702, + "last_quartile_mean": 0.43082140758633614, + "mean": 0.28637152607552707, + "normalized": [ + 0.4021833298588333, + 0.6192308262090269, + 0.859274156722729, + 0.9080756808880515, + 0.9152470573998156, + 0.9207177426482616, + 0.92047286891539, + 0.9191269479688416, + 0.9240246307637429, + 0.9130549535709983, + 0.8806991513867063, + 0.8287367475407597, + 0.7450699597557016, + 0.7365167065495323, + 0.740246269831209, + 0.7550239781316335, + 0.7775346698709116, + 0.8002286786864554, + 0.8253503509119278, + 0.8544316303572463, + 0.8881710784044343, + 0.9150378792429639, + 0.9417653320456605, + 0.9744652895357776, + 1.0160332561203216, + 1.0771387747610122, + 1.1604733751447416, + 1.2709397353656795, + 1.410172567109311, + 1.6320107896122644, + 1.9577966108600218, + 2.510748973830038 + ], + "population_cv": 0.3918463566300126 + }, + "bits_per_byte": 1.9262875534052675, + "branch_output_rms_by_sublayer": [ + 0.021954918280243874, + 0.07429475337266922, + 0.09672636538743973, + 0.017902610823512077, + 0.0940650999546051, + 0.0039193108677864075, + 0.036259762942790985, + 0.004245741758495569, + 0.02603325992822647, + 0.008932216092944145, + 0.02122427336871624, + 0.005480592139065266, + 0.02764410711824894, + 0.00817549042403698, + 0.02432539500296116, + 0.009956736117601395, + 0.05137033388018608, + 0.014394877478480339, + 0.02661585807800293, + 0.015817945823073387, + 0.030927525833249092, + 0.018779242411255836, + 0.03579767048358917, + 0.02186598815023899, + 0.05378979817032814, + 0.028890088200569153, + 0.025482553988695145, + 0.027309151366353035, + 0.0287014190107584, + 0.028864959254860878, + 0.027393119409680367, + 0.02912190742790699, + 0.029581313952803612, + 0.03136448562145233, + 0.0320778414607048, + 0.03278070688247681, + 0.028201179578900337, + 0.033851705491542816, + 0.035640619695186615, + 0.03695352375507355, + 0.04368399456143379, + 0.038013845682144165, + 0.04277802258729935, + 0.04267064854502678, + 0.03937495872378349, + 0.04597146809101105, + 0.04081331565976143, + 0.053922027349472046, + 0.05011079087853432, + 0.0602298267185688, + 0.04528728500008583, + 0.06855001300573349, + 0.028651293367147446, + 0.07740508019924164, + 0.014390980824828148, + 0.08345231413841248, + 0.015424116514623165, + 0.08889877796173096, + 0.01913117803633213, + 0.11831489205360413, + 0.01693635806441307, + 0.153398334980011, + 0.032383937388658524, + 0.24250756204128265 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003320807822012488, + 0.00017565239824668803, + 0.00017577232236880354, + 0.00012480421087066412, + 0.00013414970937401124, + 0.0001166490159724099, + 0.00011491447201106907, + 0.00015823331904477844, + 0.000249176250949333, + 0.0001712841712530518, + 0.00020010962025834788, + 0.00021729074089037623, + 0.0002489951018894741, + 0.00022387049947650086, + 0.00024825867897268754, + 0.000256508706958601, + 0.0002733879022225402, + 0.00028045470195987973, + 0.0002710271146389, + 0.0002966387758490912, + 0.00030658370753967526, + 0.0003196905817287839, + 0.0002949547776417074, + 0.00030211663360406435, + 0.0002930543360663917, + 0.00026176751657805116, + 0.0002415189744957938, + 0.00021641616631898222, + 0.00019981422740886822, + 0.0001881612684369175, + 0.00017039642606795488, + 0.0001626151127852574 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00016653202876120915, + "first_to_last_ratio": 0.7684272928714567, + "imbalance_abs_log_ratio": 0.2634093296250088, + "last_quartile_mean": 0.0002167180035197771, + "mean": 0.00022582338200252827, + "normalized": [ + 1.4705332072191308, + 0.7778308724679426, + 0.7783619251917662, + 0.5526629251760424, + 0.5940470299594988, + 0.5165497697271398, + 0.5088687937982548, + 0.7006950194511165, + 1.103412094618897, + 0.758487317540635, + 0.8861333068517568, + 0.9622154223513644, + 1.1026099232128512, + 0.991352168634134, + 1.0993488662299287, + 1.1358819652950252, + 1.2106270829806252, + 1.2419205640837485, + 1.2001729503628886, + 1.313587517902686, + 1.3576260563498372, + 1.4156664331827458, + 1.306130370673508, + 1.3378447834984684, + 1.2977147603923083, + 1.159169233304284, + 1.069503841250105, + 0.9583425967668807, + 0.8848252365941058, + 0.8332231444253673, + 0.7545561693254915, + 0.7200986511814654 + ], + "population_cv": 0.2750960622596733 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.06031418219208717, + 0.06744465231895447, + 0.11517385393381119, + 0.16882504522800446, + 0.17733007669448853, + 0.2454802393913269, + 0.24607165157794952, + 0.25892511010169983, + 0.2600470185279846, + 0.2610686123371124, + 0.2621006965637207, + 0.2635994553565979, + 0.26366734504699707, + 0.26440703868865967, + 0.2635972201824188, + 0.2638950049877167, + 0.26321178674697876, + 0.2655138075351715, + 0.2646143436431885, + 0.2625099718570709, + 0.2614729404449463, + 0.25542622804641724, + 0.2522071599960327, + 0.24079175293445587, + 0.23732660710811615, + 0.21542933583259583, + 0.2133668214082718, + 0.20939168334007263, + 0.2109174132347107, + 0.20824690163135529, + 0.21198545396327972, + 0.21042729914188385, + 0.21621736884117126, + 0.21581067144870758, + 0.2226637899875641, + 0.2222982496023178, + 0.22916270792484283, + 0.22995305061340332, + 0.23635683953762054, + 0.23728354275226593, + 0.24468488991260529, + 0.24780811369419098, + 0.25434690713882446, + 0.25493237376213074, + 0.26204079389572144, + 0.26210466027259827, + 0.2696947753429413, + 0.2699442505836487, + 0.27905911207199097, + 0.27968016266822815, + 0.29096299409866333, + 0.29178300499916077, + 0.30846187472343445, + 0.30848607420921326, + 0.3323265314102173, + 0.33218035101890564, + 0.36396095156669617, + 0.3638138175010681, + 0.403833270072937, + 0.40415704250335693, + 0.4673614203929901, + 0.4678023159503937, + 0.5606572031974792, + 0.5632834434509277 + ], + "loss_nats": 1.3352007865905762, + "loss_scale": 1.0, + "output_weights": null, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.06744465231895447, + 0.11517385393381119, + 0.16882504522800446, + 0.17733007669448853, + 0.2454802393913269, + 0.24607165157794952, + 0.25892511010169983, + 0.2600470185279846, + 0.2610686123371124, + 0.2621006965637207, + 0.2635994553565979, + 0.26366734504699707, + 0.26440703868865967, + 0.2635972201824188, + 0.2638950049877167, + 0.26321178674697876, + 0.2655138075351715, + 0.2646143436431885, + 0.2625099718570709, + 0.2614729404449463, + 0.25542622804641724, + 0.2522071599960327, + 0.24079175293445587, + 0.23732660710811615, + 0.21542933583259583, + 0.2133668214082718, + 0.20939168334007263, + 0.2109174132347107, + 0.20824690163135529, + 0.21198545396327972, + 0.21042729914188385, + 0.21621736884117126, + 0.21581067144870758, + 0.2226637899875641, + 0.2222982496023178, + 0.22916270792484283, + 0.22995305061340332, + 0.23635683953762054, + 0.23728354275226593, + 0.24468488991260529, + 0.24780811369419098, + 0.25434690713882446, + 0.25493237376213074, + 0.26204079389572144, + 0.26210466027259827, + 0.2696947753429413, + 0.2699442505836487, + 0.27905911207199097, + 0.27968016266822815, + 0.29096299409866333, + 0.29178300499916077, + 0.30846187472343445, + 0.30848607420921326, + 0.3323265314102173, + 0.33218035101890564, + 0.36396095156669617, + 0.3638138175010681, + 0.403833270072937, + 0.40415704250335693, + 0.4673614203929901, + 0.4678023159503937, + 0.5606572031974792, + 0.5632834434509277, + 0.7190070152282715 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00018638069741427898, + 0.00018956589337904006, + 0.00020923581905663013, + 0.00020349136320874095, + 0.00019885810615960509, + 0.0001991111203096807, + 0.00019671664631459862, + 0.00019502552459016442, + 0.00018589819956105202, + 0.00018270809960085899, + 0.00017816871695686132, + 0.00017320679035037756, + 0.0001670292258495465, + 0.00016241244156844914, + 0.00015707056445535272, + 0.0001517482742201537, + 0.0001448393304599449, + 0.00013789460354018956, + 0.00013790397497359663, + 0.00013200659304857254, + 0.00012367918679956347, + 0.00011618347343755886, + 0.00010830583050847054, + 9.980145841836929e-05, + 8.979889389593154e-05, + 8.034214988583699e-05, + 7.120905502233654e-05, + 6.192603905219585e-05, + 5.214559132582508e-05, + 4.129187436774373e-05, + 3.1231873435899615e-05, + 2.3505688659497537e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00019729814630409237, + "first_to_last_ratio": 3.49624785701179, + "imbalance_abs_log_ratio": 1.2516903525947785, + "last_quartile_mean": 5.643139570565836e-05, + "mean": 0.00013714665936959136, + "normalized": [ + 1.3589882412812453, + 1.3822129846282736, + 1.525635549698432, + 1.4837500537316024, + 1.449966824373825, + 1.4518116680706283, + 1.434352445905914, + 1.4220216918634343, + 1.3554701253063843, + 1.3322096246506874, + 1.2991108771867432, + 1.2629311654147486, + 1.217887673074309, + 1.1842245543201309, + 1.1452744469121132, + 1.1064671565292232, + 1.0560908382727927, + 1.0054536083783316, + 1.0055219398524642, + 0.9625213888209478, + 0.9018024016630627, + 0.8471476736772742, + 0.7897081198062669, + 0.7276987924978771, + 0.6547654482340389, + 0.5858119348669365, + 0.519218297767195, + 0.451531516238494, + 0.38021772871113024, + 0.3010782366668357, + 0.22772609686200243, + 0.17139089473665517 + ], + "population_cv": 0.4031953558435787 + }, + "activation_output_rms_by_block": [ + 0.1026567816734314, + 0.17673343420028687, + 0.23246681690216064, + 0.24062344431877136, + 0.2416554093360901, + 0.24258241057395935, + 0.24218299984931946, + 0.24160927534103394, + 0.24130181968212128, + 0.23599836230278015, + 0.22310413420200348, + 0.20329499244689941, + 0.176778644323349, + 0.17310521006584167, + 0.1729687601327896, + 0.17606647312641144, + 0.1812635064125061, + 0.18683874607086182, + 0.19253391027450562, + 0.1985689103603363, + 0.20644955337047577, + 0.2135031372308731, + 0.22015690803527832, + 0.228074312210083, + 0.2383035272359848, + 0.2532499134540558, + 0.27322494983673096, + 0.2992607355117798, + 0.3339390158653259, + 0.3918353319168091, + 0.48234984278678894, + 0.651733934879303 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.21506382152438164, + "first_to_last_ratio": 0.5884305856918151, + "imbalance_abs_log_ratio": 0.5302963104844547, + "last_quartile_mean": 0.3654871564358473, + "mean": 0.2460754751227796, + "normalized": [ + 0.41717599700746566, + 0.7182082412402355, + 0.9446972185512233, + 0.9778440708027163, + 0.9820377638832796, + 0.9858049058034841, + 0.9841817829610285, + 0.9818502848383519, + 0.9806008484255633, + 0.9590486909962422, + 0.906649206267652, + 0.8261489380258848, + 0.7183919658598449, + 0.7034638863522286, + 0.7029093819548116, + 0.7154978489366439, + 0.7366175207913933, + 0.7592741453720182, + 0.7824181185810598, + 0.8069431147547724, + 0.838968423275288, + 0.8676327339176954, + 0.8946722867259763, + 0.9268469850410127, + 0.9684164060521798, + 1.0291554382967116, + 1.1103298681041054, + 1.2161339335521484, + 1.357059315637638, + 1.5923380081712835, + 1.9601703198830358, + 2.648512349937025 + ], + "population_cv": 0.40510862299171807 + }, + "bits_per_byte": 1.8611962476373152, + "branch_output_rms_by_sublayer": [ + 0.026356825605034828, + 0.057217150926589966, + 0.10627161711454391, + 0.01789308711886406, + 0.07707435637712479, + 0.0034156402107328176, + 0.02677960880100727, + 0.0038151287008076906, + 0.02153952419757843, + 0.007133069448173046, + 0.01632758229970932, + 0.004304513800889254, + 0.01752857305109501, + 0.006448030937463045, + 0.01845136471092701, + 0.007711905520409346, + 0.044129498302936554, + 0.011710400693118572, + 0.023254960775375366, + 0.01295715756714344, + 0.02913394197821617, + 0.01598021760582924, + 0.03491654247045517, + 0.01944364421069622, + 0.04857943207025528, + 0.02654583752155304, + 0.023438163101673126, + 0.02489112876355648, + 0.025785095989704132, + 0.0264157485216856, + 0.025251252576708794, + 0.02720639854669571, + 0.027512386441230774, + 0.029633820056915283, + 0.02885596640408039, + 0.031438712030649185, + 0.0244715828448534, + 0.03218274191021919, + 0.031743232160806656, + 0.03524785488843918, + 0.038931526243686676, + 0.037403788417577744, + 0.03771914169192314, + 0.041983578354120255, + 0.03549705073237419, + 0.04549537971615791, + 0.0359940230846405, + 0.052616480737924576, + 0.04251093044877052, + 0.059319522231817245, + 0.03818616643548012, + 0.06705058366060257, + 0.024311494082212448, + 0.07359735667705536, + 0.01473314967006445, + 0.07876058667898178, + 0.015392928384244442, + 0.0848938599228859, + 0.018577447161078453, + 0.11232978105545044, + 0.018359949812293053, + 0.1471690982580185, + 0.03667483106255531, + 0.24789714813232422 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0005542142737822822, + 0.0003797328422683109, + 0.000340619215402376, + 0.0001793765938241596, + 0.00020684039436717747, + 0.00016374338231517116, + 0.00018705111879548875, + 0.0002256278252430331, + 0.000426778694633347, + 0.00027155262856832466, + 0.0003392540980461625, + 0.0003897844238340517, + 0.00045204461946776503, + 0.0004208641344247883, + 0.00047738343422760874, + 0.00048450285457785075, + 0.0005178591522769467, + 0.0005253914687520978, + 0.0005086607978429822, + 0.0005610796431247281, + 0.0005682688177114847, + 0.000574059459111366, + 0.0005453911942509562, + 0.0005508569019095279, + 0.0005214428582378581, + 0.000461966711188474, + 0.00041497592078977314, + 0.00037256322877604213, + 0.00033686502618798456, + 0.00030674152201740854, + 0.00025927963978097604, + 0.00021950976285567438 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002796507057497499, + "first_to_last_ratio": 0.7732247282264533, + "imbalance_abs_log_ratio": 0.25718555049038655, + "last_quartile_mean": 0.00036166808372927383, + "mean": 0.00039825883245600557, + "normalized": [ + 1.3915931766397287, + 0.9534825377922003, + 0.8552709636138532, + 0.4504020481302817, + 0.5193617253674505, + 0.41114815032572916, + 0.46967224214958675, + 0.5665356468094339, + 1.0716113739435822, + 0.681849607436697, + 0.8518432496626144, + 0.9787213542266133, + 1.1350523394046785, + 1.056760328024313, + 1.1986763263570401, + 1.2165526916000595, + 1.3003080159789122, + 1.319221134436827, + 1.2772115930389878, + 1.4088316376177517, + 1.4268831508570738, + 1.441423045337684, + 1.3694390426638032, + 1.383163051306287, + 1.3093064503358134, + 1.159966015918821, + 1.0419754365036318, + 0.9354801410893958, + 0.8458444577627716, + 0.7702064512311685, + 0.6510329932472191, + 0.5511736211899907 + ], + "population_cv": 0.3252608102570666 + }, + "depth_weights": [], + "layer_input_rms_by_sublayer": [ + 0.061885204166173935, + 0.07089564949274063, + 0.1026567816734314, + 0.16996370255947113, + 0.17673343420028687, + 0.23203739523887634, + 0.23246681690216064, + 0.23961465060710907, + 0.24062344431877136, + 0.24108263850212097, + 0.2416554093360901, + 0.24260269105434418, + 0.24258241057395935, + 0.2429492026567459, + 0.24218299984931946, + 0.24250134825706482, + 0.24160927534103394, + 0.242857426404953, + 0.24130181968212128, + 0.23791606724262238, + 0.23599836230278015, + 0.22686637938022614, + 0.22310413420200348, + 0.20835906267166138, + 0.20329499244689941, + 0.1809774935245514, + 0.176778644323349, + 0.17327234148979187, + 0.17310521006584167, + 0.17050857841968536, + 0.1729687601327896, + 0.17174123227596283, + 0.17606647312641144, + 0.1756037324666977, + 0.1812635064125061, + 0.1811160296201706, + 0.18683874607086182, + 0.1874236911535263, + 0.19253391027450562, + 0.19280622899532318, + 0.1985689103603363, + 0.20097090303897858, + 0.20644955337047577, + 0.2074177861213684, + 0.2135031372308731, + 0.21357181668281555, + 0.22015690803527832, + 0.22022640705108643, + 0.228074312210083, + 0.22874467074871063, + 0.2383035272359848, + 0.23907209932804108, + 0.2532499134540558, + 0.2534840703010559, + 0.27322494983673096, + 0.27303415536880493, + 0.2992607355117798, + 0.29903557896614075, + 0.3339390158653259, + 0.3341987431049347, + 0.3918353319168091, + 0.3921641707420349, + 0.48234984278678894, + 0.4832552373409271 + ], + "loss_nats": 1.2900829315185547, + "loss_scale": 1.0, + "output_weights": null, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.07089564949274063, + 0.1026567816734314, + 0.16996370255947113, + 0.17673343420028687, + 0.23203739523887634, + 0.23246681690216064, + 0.23961465060710907, + 0.24062344431877136, + 0.24108263850212097, + 0.2416554093360901, + 0.24260269105434418, + 0.24258241057395935, + 0.2429492026567459, + 0.24218299984931946, + 0.24250134825706482, + 0.24160927534103394, + 0.242857426404953, + 0.24130181968212128, + 0.23791606724262238, + 0.23599836230278015, + 0.22686637938022614, + 0.22310413420200348, + 0.20835906267166138, + 0.20329499244689941, + 0.1809774935245514, + 0.176778644323349, + 0.17327234148979187, + 0.17310521006584167, + 0.17050857841968536, + 0.1729687601327896, + 0.17174123227596283, + 0.17606647312641144, + 0.1756037324666977, + 0.1812635064125061, + 0.1811160296201706, + 0.18683874607086182, + 0.1874236911535263, + 0.19253391027450562, + 0.19280622899532318, + 0.1985689103603363, + 0.20097090303897858, + 0.20644955337047577, + 0.2074177861213684, + 0.2135031372308731, + 0.21357181668281555, + 0.22015690803527832, + 0.22022640705108643, + 0.228074312210083, + 0.22874467074871063, + 0.2383035272359848, + 0.23907209932804108, + 0.2532499134540558, + 0.2534840703010559, + 0.27322494983673096, + 0.27303415536880493, + 0.2992607355117798, + 0.29903557896614075, + 0.3339390158653259, + 0.3341987431049347, + 0.3918353319168091, + 0.3921641707420349, + 0.48234984278678894, + 0.4832552373409271, + 0.651733934879303 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.06581270897968, + "cross_entropy_nats": 5.590795338153839, + "step": 0 + }, + { + "bits_per_byte": 5.193364149696528, + "cross_entropy_nats": 3.599765717983246, + "step": 100 + }, + { + "bits_per_byte": 3.118855971380574, + "cross_entropy_nats": 2.1618262231349945, + "step": 500 + }, + { + "bits_per_byte": 1.9576097217110056, + "cross_entropy_nats": 1.3569116592407227, + "step": 2000 + }, + { + "bits_per_byte": 1.792987261872353, + "cross_entropy_nats": 1.2428040653467178, + "step": 4000 + }, + { + "bits_per_byte": 1.709514213904039, + "cross_entropy_nats": 1.1849449574947357, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": null, + "final_model_state": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "final_optimizer_state": "e774625d2eec7b16bb949a8bd26f25b05b7288a59bf24a716a88b489cd22bcd6", + "final_public_parameters": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22", + "initial_mixer_parameters": null, + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 0, + "total": 18985152 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 42.39461278025134, + "measured_steps": 7980, + "median_ms": 42.272048493032344, + "p95_ms": 43.13050716154976, + "peak_allocated_bytes": 5938562048, + "peak_reserved_bytes": 6325010432, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.087939566883062, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.606132507324219, + "step": 1, + "unclipped_grad_norm": 19.928020477294922 + }, + { + "bits_per_byte": 7.3282798152711734, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.07957649230957, + "step": 10, + "unclipped_grad_norm": 11.731513977050781 + }, + { + "bits_per_byte": 6.791570117070589, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.707557678222656, + "step": 20, + "unclipped_grad_norm": 3.350315809249878 + }, + { + "bits_per_byte": 6.667928349712299, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.621855735778809, + "step": 30, + "unclipped_grad_norm": 3.0359153747558594 + }, + { + "bits_per_byte": 6.473404964696806, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.487022399902344, + "step": 40, + "unclipped_grad_norm": 3.083479404449463 + }, + { + "bits_per_byte": 6.159992036931666, + "learning_rate": 3.75e-05, + "loss_nats": 4.269781112670898, + "step": 50, + "unclipped_grad_norm": 2.770655393600464 + }, + { + "bits_per_byte": 5.940455495174687, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.117609977722168, + "step": 60, + "unclipped_grad_norm": 2.5750629901885986 + }, + { + "bits_per_byte": 5.762018679390299, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.993927001953125, + "step": 70, + "unclipped_grad_norm": 2.5019538402557373 + }, + { + "bits_per_byte": 5.5944951946990615, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8778085708618164, + "step": 80, + "unclipped_grad_norm": 2.5689163208007812 + }, + { + "bits_per_byte": 5.40584020169631, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7470428943634033, + "step": 90, + "unclipped_grad_norm": 2.3143606185913086 + }, + { + "bits_per_byte": 5.179311877251155, + "learning_rate": 7.5e-05, + "loss_nats": 3.5900254249572754, + "step": 100, + "unclipped_grad_norm": 2.227365255355835 + }, + { + "bits_per_byte": 4.996660110659421, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.463420867919922, + "step": 110, + "unclipped_grad_norm": 2.0371005535125732 + }, + { + "bits_per_byte": 4.800645980026251, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.327554225921631, + "step": 120, + "unclipped_grad_norm": 1.8128844499588013 + }, + { + "bits_per_byte": 4.634309863556569, + "learning_rate": 9.75e-05, + "loss_nats": 3.212258815765381, + "step": 130, + "unclipped_grad_norm": 1.5838756561279297 + }, + { + "bits_per_byte": 4.478764626532625, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.104443073272705, + "step": 140, + "unclipped_grad_norm": 1.4334924221038818 + }, + { + "bits_per_byte": 4.278149959988874, + "learning_rate": 0.0001125, + "loss_nats": 2.9653875827789307, + "step": 150, + "unclipped_grad_norm": 1.2058515548706055 + }, + { + "bits_per_byte": 4.130972366860008, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.8633718490600586, + "step": 160, + "unclipped_grad_norm": 1.057314395904541 + }, + { + "bits_per_byte": 4.025422830407184, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.790210485458374, + "step": 170, + "unclipped_grad_norm": 0.9353183507919312 + }, + { + "bits_per_byte": 3.9235808959416594, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.719619035720825, + "step": 180, + "unclipped_grad_norm": 0.8658975958824158 + }, + { + "bits_per_byte": 3.837114554428572, + "learning_rate": 0.0001425, + "loss_nats": 2.6596851348876953, + "step": 190, + "unclipped_grad_norm": 0.8723611235618591 + }, + { + "bits_per_byte": 3.771249326731737, + "learning_rate": 0.00015, + "loss_nats": 2.6140308380126953, + "step": 200, + "unclipped_grad_norm": 0.9528701305389404 + }, + { + "bits_per_byte": 3.687100279456375, + "learning_rate": 0.0001575, + "loss_nats": 2.5557031631469727, + "step": 210, + "unclipped_grad_norm": 0.7721163034439087 + }, + { + "bits_per_byte": 3.664338719579984, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.539926052093506, + "step": 220, + "unclipped_grad_norm": 1.0300326347351074 + }, + { + "bits_per_byte": 3.616636923606072, + "learning_rate": 0.0001725, + "loss_nats": 2.506861686706543, + "step": 230, + "unclipped_grad_norm": 1.116415023803711 + }, + { + "bits_per_byte": 3.58821025521026, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4871578216552734, + "step": 240, + "unclipped_grad_norm": 0.7745823860168457 + }, + { + "bits_per_byte": 3.5119345736913976, + "learning_rate": 0.0001875, + "loss_nats": 2.4342875480651855, + "step": 250, + "unclipped_grad_norm": 0.9986598491668701 + }, + { + "bits_per_byte": 3.522710662629159, + "learning_rate": 0.000195, + "loss_nats": 2.4417569637298584, + "step": 260, + "unclipped_grad_norm": 0.8885983228683472 + }, + { + "bits_per_byte": 3.5021329384534243, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4274935722351074, + "step": 270, + "unclipped_grad_norm": 0.7707400321960449 + }, + { + "bits_per_byte": 3.4629779922967905, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.40035343170166, + "step": 280, + "unclipped_grad_norm": 1.055368423461914 + }, + { + "bits_per_byte": 3.4474741002868097, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3896069526672363, + "step": 290, + "unclipped_grad_norm": 1.053453803062439 + }, + { + "bits_per_byte": 3.4491093113312137, + "learning_rate": 0.000225, + "loss_nats": 2.390740394592285, + "step": 300, + "unclipped_grad_norm": 1.2985762357711792 + }, + { + "bits_per_byte": 3.334430591805289, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.311251163482666, + "step": 310, + "unclipped_grad_norm": 0.8886587619781494 + }, + { + "bits_per_byte": 3.4024717440008403, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3584136962890625, + "step": 320, + "unclipped_grad_norm": 1.0081987380981445 + }, + { + "bits_per_byte": 3.413793361907139, + "learning_rate": 0.0002475, + "loss_nats": 2.3662612438201904, + "step": 330, + "unclipped_grad_norm": 0.8511857986450195 + }, + { + "bits_per_byte": 3.3420762525320464, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3165507316589355, + "step": 340, + "unclipped_grad_norm": 0.9547706842422485 + }, + { + "bits_per_byte": 3.3350242759160675, + "learning_rate": 0.0002625, + "loss_nats": 2.3116626739501953, + "step": 350, + "unclipped_grad_norm": 1.1394811868667603 + }, + { + "bits_per_byte": 3.3268671387856448, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.3060085773468018, + "step": 360, + "unclipped_grad_norm": 1.0605591535568237 + }, + { + "bits_per_byte": 3.320889365806955, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3018651008605957, + "step": 370, + "unclipped_grad_norm": 0.8489253520965576 + }, + { + "bits_per_byte": 3.2193453052927254, + "learning_rate": 0.000285, + "loss_nats": 2.231480121612549, + "step": 380, + "unclipped_grad_norm": 0.923150897026062 + }, + { + "bits_per_byte": 3.297119987596508, + "learning_rate": 0.0002925, + "loss_nats": 2.2853894233703613, + "step": 390, + "unclipped_grad_norm": 0.7830708026885986 + }, + { + "bits_per_byte": 3.2808776020837227, + "learning_rate": 0.0003, + "loss_nats": 2.2741310596466064, + "step": 400, + "unclipped_grad_norm": 0.7351507544517517 + }, + { + "bits_per_byte": 3.2985058237971607, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2863500118255615, + "step": 410, + "unclipped_grad_norm": 0.8769013285636902 + }, + { + "bits_per_byte": 3.225643997897925, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2358460426330566, + "step": 420, + "unclipped_grad_norm": 0.7473549842834473 + }, + { + "bits_per_byte": 3.2585050669635836, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2586236000061035, + "step": 430, + "unclipped_grad_norm": 0.9111914038658142 + }, + { + "bits_per_byte": 3.2092602426461303, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.224489688873291, + "step": 440, + "unclipped_grad_norm": 1.075931191444397 + }, + { + "bits_per_byte": 3.2640807445045557, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.26248836517334, + "step": 450, + "unclipped_grad_norm": 0.8512128591537476 + }, + { + "bits_per_byte": 3.207580316112502, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.223325252532959, + "step": 460, + "unclipped_grad_norm": 1.2196831703186035 + }, + { + "bits_per_byte": 3.1660169249131855, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.1945157051086426, + "step": 470, + "unclipped_grad_norm": 0.7675222158432007 + }, + { + "bits_per_byte": 3.1673281206433637, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1954245567321777, + "step": 480, + "unclipped_grad_norm": 0.9160394668579102 + }, + { + "bits_per_byte": 3.113679164602616, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.158237934112549, + "step": 490, + "unclipped_grad_norm": 0.6955189108848572 + }, + { + "bits_per_byte": 3.0709621137813112, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.128628730773926, + "step": 500, + "unclipped_grad_norm": 0.9366468191146851 + }, + { + "bits_per_byte": 3.1516054667014854, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1845264434814453, + "step": 510, + "unclipped_grad_norm": 1.3161629438400269 + }, + { + "bits_per_byte": 3.151479575401054, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.184439182281494, + "step": 520, + "unclipped_grad_norm": 0.8699809312820435 + }, + { + "bits_per_byte": 3.012768688174164, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.088292121887207, + "step": 530, + "unclipped_grad_norm": 1.1934181451797485 + }, + { + "bits_per_byte": 3.0200408025832473, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0933327674865723, + "step": 540, + "unclipped_grad_norm": 0.8321981430053711 + }, + { + "bits_per_byte": 3.02149439994834, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0943403244018555, + "step": 550, + "unclipped_grad_norm": 0.9657202959060669 + }, + { + "bits_per_byte": 3.0546815481200342, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.1173439025878906, + "step": 560, + "unclipped_grad_norm": 1.2880722284317017 + }, + { + "bits_per_byte": 2.973202359516667, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0608668327331543, + "step": 570, + "unclipped_grad_norm": 0.8365735411643982 + }, + { + "bits_per_byte": 2.986164347946895, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0698513984680176, + "step": 580, + "unclipped_grad_norm": 0.9132604002952576 + }, + { + "bits_per_byte": 2.926336399225948, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.028381824493408, + "step": 590, + "unclipped_grad_norm": 0.7623230814933777 + }, + { + "bits_per_byte": 2.903745446139213, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.012722969055176, + "step": 600, + "unclipped_grad_norm": 0.8222759366035461 + }, + { + "bits_per_byte": 2.915261748406288, + "learning_rate": 0.000299491674159121, + "loss_nats": 2.020705461502075, + "step": 610, + "unclipped_grad_norm": 1.3526110649108887 + }, + { + "bits_per_byte": 2.8888483089214763, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.002397060394287, + "step": 620, + "unclipped_grad_norm": 0.9250975847244263 + }, + { + "bits_per_byte": 2.806001858243749, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.944972276687622, + "step": 630, + "unclipped_grad_norm": 0.7925068140029907 + }, + { + "bits_per_byte": 2.807606972324252, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9460848569869995, + "step": 640, + "unclipped_grad_norm": 0.8585688471794128 + }, + { + "bits_per_byte": 2.785753824809718, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.93093740940094, + "step": 650, + "unclipped_grad_norm": 0.7446964383125305 + }, + { + "bits_per_byte": 2.8113040833698495, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.9486474990844727, + "step": 660, + "unclipped_grad_norm": 1.026633381843567 + }, + { + "bits_per_byte": 2.7895744193986354, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.9335856437683105, + "step": 670, + "unclipped_grad_norm": 0.8819775581359863 + }, + { + "bits_per_byte": 2.7608092891459255, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.913647174835205, + "step": 680, + "unclipped_grad_norm": 0.845567524433136 + }, + { + "bits_per_byte": 2.7436651985946665, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.9017637968063354, + "step": 690, + "unclipped_grad_norm": 1.0123262405395508 + }, + { + "bits_per_byte": 2.7268093509662537, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.890080213546753, + "step": 700, + "unclipped_grad_norm": 0.8187565803527832 + }, + { + "bits_per_byte": 2.696082242632501, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.8687818050384521, + "step": 710, + "unclipped_grad_norm": 0.9888712763786316 + }, + { + "bits_per_byte": 2.7032793726058073, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8737704753875732, + "step": 720, + "unclipped_grad_norm": 0.8442726731300354 + }, + { + "bits_per_byte": 2.680898066370474, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8582569360733032, + "step": 730, + "unclipped_grad_norm": 1.1784851551055908 + }, + { + "bits_per_byte": 2.619564409541223, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.8157436847686768, + "step": 740, + "unclipped_grad_norm": 0.8383970260620117 + }, + { + "bits_per_byte": 2.6352585143630582, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.8266220092773438, + "step": 750, + "unclipped_grad_norm": 1.062637448310852 + }, + { + "bits_per_byte": 2.547899926857294, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7660696506500244, + "step": 760, + "unclipped_grad_norm": 0.8228002786636353 + }, + { + "bits_per_byte": 2.573545291823353, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.7838456630706787, + "step": 770, + "unclipped_grad_norm": 0.8357667326927185 + }, + { + "bits_per_byte": 2.523504187832405, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.749159812927246, + "step": 780, + "unclipped_grad_norm": 0.7918128967285156 + }, + { + "bits_per_byte": 2.5308719245953677, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7542667388916016, + "step": 790, + "unclipped_grad_norm": 1.3666422367095947 + }, + { + "bits_per_byte": 2.443318996694129, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6935796737670898, + "step": 800, + "unclipped_grad_norm": 0.7999250888824463 + }, + { + "bits_per_byte": 2.500851320991359, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7334580421447754, + "step": 810, + "unclipped_grad_norm": 1.1067378520965576 + }, + { + "bits_per_byte": 2.4739762800716316, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.714829683303833, + "step": 820, + "unclipped_grad_norm": 1.090567708015442 + }, + { + "bits_per_byte": 2.467352884221603, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7102386951446533, + "step": 830, + "unclipped_grad_norm": 0.9818849563598633 + }, + { + "bits_per_byte": 2.471093850843171, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.712831735610962, + "step": 840, + "unclipped_grad_norm": 0.9710109829902649 + }, + { + "bits_per_byte": 2.396449768733044, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6610924005508423, + "step": 850, + "unclipped_grad_norm": 1.0234057903289795 + }, + { + "bits_per_byte": 2.473427999380681, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.7144496440887451, + "step": 860, + "unclipped_grad_norm": 0.9857868552207947 + }, + { + "bits_per_byte": 2.465926632097998, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.7092500925064087, + "step": 870, + "unclipped_grad_norm": 1.1294745206832886 + }, + { + "bits_per_byte": 2.391602093753174, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6577322483062744, + "step": 880, + "unclipped_grad_norm": 0.8081791400909424 + }, + { + "bits_per_byte": 2.354239550783832, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6318345069885254, + "step": 890, + "unclipped_grad_norm": 0.8606147170066833 + }, + { + "bits_per_byte": 2.395512979233794, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6604430675506592, + "step": 900, + "unclipped_grad_norm": 0.8592826128005981 + }, + { + "bits_per_byte": 2.3758491708647442, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.646813154220581, + "step": 910, + "unclipped_grad_norm": 0.8169032335281372 + }, + { + "bits_per_byte": 2.353533905967342, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6313453912734985, + "step": 920, + "unclipped_grad_norm": 0.7358964085578918 + }, + { + "bits_per_byte": 2.2994974730142004, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5938901901245117, + "step": 930, + "unclipped_grad_norm": 0.8204946517944336 + }, + { + "bits_per_byte": 2.3548098452540933, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6322298049926758, + "step": 940, + "unclipped_grad_norm": 0.8031407594680786 + }, + { + "bits_per_byte": 2.362864480724602, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.637812852859497, + "step": 950, + "unclipped_grad_norm": 0.8955363035202026 + }, + { + "bits_per_byte": 2.311957100121258, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6025265455245972, + "step": 960, + "unclipped_grad_norm": 0.8827840089797974 + }, + { + "bits_per_byte": 2.2663949403067303, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5709452629089355, + "step": 970, + "unclipped_grad_norm": 1.4933069944381714 + }, + { + "bits_per_byte": 2.401434685868302, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6645476818084717, + "step": 980, + "unclipped_grad_norm": 1.0056283473968506 + }, + { + "bits_per_byte": 2.23815401317385, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.5513701438903809, + "step": 990, + "unclipped_grad_norm": 0.7972294092178345 + }, + { + "bits_per_byte": 2.255509814368321, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5634002685546875, + "step": 1000, + "unclipped_grad_norm": 0.7948418855667114 + }, + { + "bits_per_byte": 2.249676162851055, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.559356689453125, + "step": 1010, + "unclipped_grad_norm": 0.9423647522926331 + }, + { + "bits_per_byte": 2.2157856056373317, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5358655452728271, + "step": 1020, + "unclipped_grad_norm": 1.3369964361190796 + }, + { + "bits_per_byte": 2.21131629248936, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.532767653465271, + "step": 1030, + "unclipped_grad_norm": 0.7792882919311523 + }, + { + "bits_per_byte": 2.300165797595453, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.594353437423706, + "step": 1040, + "unclipped_grad_norm": 0.9419035911560059 + }, + { + "bits_per_byte": 2.21666237319143, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.536473274230957, + "step": 1050, + "unclipped_grad_norm": 0.9680492281913757 + }, + { + "bits_per_byte": 2.242138163263738, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5541317462921143, + "step": 1060, + "unclipped_grad_norm": 0.9887726306915283 + }, + { + "bits_per_byte": 2.276496685270459, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5779472589492798, + "step": 1070, + "unclipped_grad_norm": 0.8514108657836914 + }, + { + "bits_per_byte": 2.2430374605450996, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5547550916671753, + "step": 1080, + "unclipped_grad_norm": 0.800697386264801 + }, + { + "bits_per_byte": 2.1823284447037823, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5126748085021973, + "step": 1090, + "unclipped_grad_norm": 0.9124817848205566 + }, + { + "bits_per_byte": 2.1899661942286, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5179688930511475, + "step": 1100, + "unclipped_grad_norm": 0.8022184371948242 + }, + { + "bits_per_byte": 2.145753238310091, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4873228073120117, + "step": 1110, + "unclipped_grad_norm": 1.139209508895874 + }, + { + "bits_per_byte": 2.188020210534087, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5166200399398804, + "step": 1120, + "unclipped_grad_norm": 0.8927217721939087 + }, + { + "bits_per_byte": 2.2131649340034842, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.5340490341186523, + "step": 1130, + "unclipped_grad_norm": 0.8271659016609192 + }, + { + "bits_per_byte": 2.1401269978697646, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4834229946136475, + "step": 1140, + "unclipped_grad_norm": 0.9781836271286011 + }, + { + "bits_per_byte": 2.1955361963420934, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.5218297243118286, + "step": 1150, + "unclipped_grad_norm": 0.8088869452476501 + }, + { + "bits_per_byte": 2.103003854750424, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4576911926269531, + "step": 1160, + "unclipped_grad_norm": 0.761218786239624 + }, + { + "bits_per_byte": 2.164645876472672, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.5004181861877441, + "step": 1170, + "unclipped_grad_norm": 0.8793766498565674 + }, + { + "bits_per_byte": 2.159191274717907, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4966373443603516, + "step": 1180, + "unclipped_grad_norm": 0.8019644618034363 + }, + { + "bits_per_byte": 2.134348724766076, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4794178009033203, + "step": 1190, + "unclipped_grad_norm": 0.7809260487556458 + }, + { + "bits_per_byte": 2.1456660431061034, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4872623682022095, + "step": 1200, + "unclipped_grad_norm": 0.896268904209137 + }, + { + "bits_per_byte": 2.1662845271700926, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.501554012298584, + "step": 1210, + "unclipped_grad_norm": 0.8574862480163574 + }, + { + "bits_per_byte": 2.1378967268533757, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.481877088546753, + "step": 1220, + "unclipped_grad_norm": 0.8502950668334961 + }, + { + "bits_per_byte": 2.128576127089866, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4754165410995483, + "step": 1230, + "unclipped_grad_norm": 0.7498336434364319 + }, + { + "bits_per_byte": 2.0970609942498597, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.4535719156265259, + "step": 1240, + "unclipped_grad_norm": 0.761203944683075 + }, + { + "bits_per_byte": 2.0385041693014396, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4129834175109863, + "step": 1250, + "unclipped_grad_norm": 0.7150603532791138 + }, + { + "bits_per_byte": 2.0885559362167414, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.447676658630371, + "step": 1260, + "unclipped_grad_norm": 0.7949564456939697 + }, + { + "bits_per_byte": 2.1179862953629884, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.468076229095459, + "step": 1270, + "unclipped_grad_norm": 0.862070620059967 + }, + { + "bits_per_byte": 2.1475409979658098, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.488561987876892, + "step": 1280, + "unclipped_grad_norm": 0.7539991140365601 + }, + { + "bits_per_byte": 2.1029943957046267, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4576846361160278, + "step": 1290, + "unclipped_grad_norm": 0.8074342608451843 + }, + { + "bits_per_byte": 2.049033979083166, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4202821254730225, + "step": 1300, + "unclipped_grad_norm": 0.7586180567741394 + }, + { + "bits_per_byte": 2.076369761524555, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4392298460006714, + "step": 1310, + "unclipped_grad_norm": 0.9284558892250061 + }, + { + "bits_per_byte": 2.051258058724124, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4218237400054932, + "step": 1320, + "unclipped_grad_norm": 0.7597900032997131 + }, + { + "bits_per_byte": 2.060438148661879, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4281868934631348, + "step": 1330, + "unclipped_grad_norm": 0.7334625720977783 + }, + { + "bits_per_byte": 2.04908454198252, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4203171730041504, + "step": 1340, + "unclipped_grad_norm": 0.7958364486694336 + }, + { + "bits_per_byte": 2.0721673654507224, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.436316967010498, + "step": 1350, + "unclipped_grad_norm": 0.7738369107246399 + }, + { + "bits_per_byte": 2.034246910761982, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4100325107574463, + "step": 1360, + "unclipped_grad_norm": 0.8466383218765259 + }, + { + "bits_per_byte": 2.0554857362476335, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4247541427612305, + "step": 1370, + "unclipped_grad_norm": 0.7919691205024719 + }, + { + "bits_per_byte": 2.020279167789508, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.40035080909729, + "step": 1380, + "unclipped_grad_norm": 0.8231279850006104 + }, + { + "bits_per_byte": 2.070644631059983, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4352614879608154, + "step": 1390, + "unclipped_grad_norm": 0.7695969343185425 + }, + { + "bits_per_byte": 2.174483628067326, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.507237195968628, + "step": 1400, + "unclipped_grad_norm": 0.8150069713592529 + }, + { + "bits_per_byte": 2.0594870846026074, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.427527666091919, + "step": 1410, + "unclipped_grad_norm": 0.7077280879020691 + }, + { + "bits_per_byte": 2.048123158964197, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4196507930755615, + "step": 1420, + "unclipped_grad_norm": 0.7647379040718079 + }, + { + "bits_per_byte": 2.0019392818668, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.3876385688781738, + "step": 1430, + "unclipped_grad_norm": 0.6688193082809448 + }, + { + "bits_per_byte": 2.0332329010524948, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.4093296527862549, + "step": 1440, + "unclipped_grad_norm": 0.8427934646606445 + }, + { + "bits_per_byte": 2.0070671165849254, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.39119291305542, + "step": 1450, + "unclipped_grad_norm": 0.6920660138130188 + }, + { + "bits_per_byte": 2.1256969655317715, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.4734208583831787, + "step": 1460, + "unclipped_grad_norm": 0.799986720085144 + }, + { + "bits_per_byte": 2.0930381480635276, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.4507834911346436, + "step": 1470, + "unclipped_grad_norm": 0.6732411980628967 + }, + { + "bits_per_byte": 2.0451248134290543, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.4175724983215332, + "step": 1480, + "unclipped_grad_norm": 0.6982382535934448 + }, + { + "bits_per_byte": 2.0566264971708064, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4255448579788208, + "step": 1490, + "unclipped_grad_norm": 0.7266837358474731 + }, + { + "bits_per_byte": 1.9341151717766374, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3406264781951904, + "step": 1500, + "unclipped_grad_norm": 0.6626343727111816 + }, + { + "bits_per_byte": 1.963616731740342, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3610754013061523, + "step": 1510, + "unclipped_grad_norm": 0.789225697517395 + }, + { + "bits_per_byte": 2.0151121210269856, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3967692852020264, + "step": 1520, + "unclipped_grad_norm": 0.7025586366653442 + }, + { + "bits_per_byte": 2.0371694119480934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4120582342147827, + "step": 1530, + "unclipped_grad_norm": 0.750766396522522 + }, + { + "bits_per_byte": 2.0209939276864937, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.400846242904663, + "step": 1540, + "unclipped_grad_norm": 0.7461745738983154 + }, + { + "bits_per_byte": 2.082936231117148, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4437813758850098, + "step": 1550, + "unclipped_grad_norm": 0.765493631362915 + }, + { + "bits_per_byte": 1.954267066908836, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3545947074890137, + "step": 1560, + "unclipped_grad_norm": 0.727699875831604 + }, + { + "bits_per_byte": 1.9578143810655324, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.357053518295288, + "step": 1570, + "unclipped_grad_norm": 0.7358320951461792 + }, + { + "bits_per_byte": 1.9655836973182597, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3624387979507446, + "step": 1580, + "unclipped_grad_norm": 0.6977834701538086 + }, + { + "bits_per_byte": 1.9752479184181935, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3691375255584717, + "step": 1590, + "unclipped_grad_norm": 0.6583591103553772 + }, + { + "bits_per_byte": 1.9806421542625054, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.372876524925232, + "step": 1600, + "unclipped_grad_norm": 0.727998673915863 + }, + { + "bits_per_byte": 1.9474555220387637, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3498733043670654, + "step": 1610, + "unclipped_grad_norm": 0.7535929083824158 + }, + { + "bits_per_byte": 1.942356580405983, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3463389873504639, + "step": 1620, + "unclipped_grad_norm": 0.6403960585594177 + }, + { + "bits_per_byte": 1.9826001767425787, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3742337226867676, + "step": 1630, + "unclipped_grad_norm": 0.7281255125999451 + }, + { + "bits_per_byte": 1.9377553565647996, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3431496620178223, + "step": 1640, + "unclipped_grad_norm": 0.7114111185073853 + }, + { + "bits_per_byte": 1.9278453722567828, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3362805843353271, + "step": 1650, + "unclipped_grad_norm": 0.6373128890991211 + }, + { + "bits_per_byte": 1.9498221752972869, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3515137434005737, + "step": 1660, + "unclipped_grad_norm": 0.6859769225120544 + }, + { + "bits_per_byte": 1.9095433225172644, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.323594570159912, + "step": 1670, + "unclipped_grad_norm": 0.7934040427207947 + }, + { + "bits_per_byte": 1.938967146322779, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.343989610671997, + "step": 1680, + "unclipped_grad_norm": 0.7104787230491638 + }, + { + "bits_per_byte": 1.9591448388526072, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.357975721359253, + "step": 1690, + "unclipped_grad_norm": 0.6892858743667603 + }, + { + "bits_per_byte": 1.9072468381802934, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3220027685165405, + "step": 1700, + "unclipped_grad_norm": 0.7498061656951904 + }, + { + "bits_per_byte": 1.9434417909329273, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3470911979675293, + "step": 1710, + "unclipped_grad_norm": 0.6478108763694763 + }, + { + "bits_per_byte": 1.8728540916260505, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.2981635332107544, + "step": 1720, + "unclipped_grad_norm": 0.7232697010040283 + }, + { + "bits_per_byte": 1.9589699324966796, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3578544855117798, + "step": 1730, + "unclipped_grad_norm": 0.7008518576622009 + }, + { + "bits_per_byte": 1.9590435410712488, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3579055070877075, + "step": 1740, + "unclipped_grad_norm": 0.6955974698066711 + }, + { + "bits_per_byte": 1.919930042733473, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.330794095993042, + "step": 1750, + "unclipped_grad_norm": 0.6556993126869202 + }, + { + "bits_per_byte": 1.9569465566092785, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3564519882202148, + "step": 1760, + "unclipped_grad_norm": 0.698861837387085 + }, + { + "bits_per_byte": 1.9512123110642108, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3524773120880127, + "step": 1770, + "unclipped_grad_norm": 0.6838504672050476 + }, + { + "bits_per_byte": 2.0216474617597724, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.401299238204956, + "step": 1780, + "unclipped_grad_norm": 0.7214158773422241 + }, + { + "bits_per_byte": 1.8826208143858987, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3049333095550537, + "step": 1790, + "unclipped_grad_norm": 0.7236432433128357 + }, + { + "bits_per_byte": 2.013814339943575, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3958697319030762, + "step": 1800, + "unclipped_grad_norm": 0.7153979539871216 + }, + { + "bits_per_byte": 1.8985195785622602, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3159534931182861, + "step": 1810, + "unclipped_grad_norm": 0.7165744304656982 + }, + { + "bits_per_byte": 1.956814645916067, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3563605546951294, + "step": 1820, + "unclipped_grad_norm": 0.7207518815994263 + }, + { + "bits_per_byte": 1.973846947744265, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.368166446685791, + "step": 1830, + "unclipped_grad_norm": 0.6764873266220093 + }, + { + "bits_per_byte": 1.953527541510126, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3540821075439453, + "step": 1840, + "unclipped_grad_norm": 0.6510032415390015 + }, + { + "bits_per_byte": 1.8838981295338566, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.3058186769485474, + "step": 1850, + "unclipped_grad_norm": 0.6467540860176086 + }, + { + "bits_per_byte": 1.9116946534969073, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3250857591629028, + "step": 1860, + "unclipped_grad_norm": 0.6336910128593445 + }, + { + "bits_per_byte": 1.996913260877985, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3841547966003418, + "step": 1870, + "unclipped_grad_norm": 0.7375073432922363 + }, + { + "bits_per_byte": 1.8879835774051028, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3086504936218262, + "step": 1880, + "unclipped_grad_norm": 0.6048603057861328 + }, + { + "bits_per_byte": 1.9029378128629257, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3190159797668457, + "step": 1890, + "unclipped_grad_norm": 0.6331550478935242 + }, + { + "bits_per_byte": 1.9156196415548983, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3278063535690308, + "step": 1900, + "unclipped_grad_norm": 0.7952298521995544 + }, + { + "bits_per_byte": 1.9644835243006904, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3616762161254883, + "step": 1910, + "unclipped_grad_norm": 0.6600897312164307 + }, + { + "bits_per_byte": 1.8776484519841532, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3014867305755615, + "step": 1920, + "unclipped_grad_norm": 0.645132303237915 + }, + { + "bits_per_byte": 1.9490762865404947, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.350996732711792, + "step": 1930, + "unclipped_grad_norm": 0.6212714314460754 + }, + { + "bits_per_byte": 1.942025857768374, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.3461097478866577, + "step": 1940, + "unclipped_grad_norm": 0.6636208295822144 + }, + { + "bits_per_byte": 1.8837596834999122, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.305722713470459, + "step": 1950, + "unclipped_grad_norm": 0.7458274960517883 + }, + { + "bits_per_byte": 1.8770238109962194, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.301053762435913, + "step": 1960, + "unclipped_grad_norm": 0.6796664595603943 + }, + { + "bits_per_byte": 1.873630765277347, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2987018823623657, + "step": 1970, + "unclipped_grad_norm": 0.7089046239852905 + }, + { + "bits_per_byte": 1.8950018454215123, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3135151863098145, + "step": 1980, + "unclipped_grad_norm": 0.6426982283592224 + }, + { + "bits_per_byte": 1.8681299001895, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.294888973236084, + "step": 1990, + "unclipped_grad_norm": 0.7099292278289795 + }, + { + "bits_per_byte": 1.964639340582372, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3617842197418213, + "step": 2000, + "unclipped_grad_norm": 0.7598643898963928 + }, + { + "bits_per_byte": 1.939805561745735, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3445707559585571, + "step": 2010, + "unclipped_grad_norm": 0.6349508166313171 + }, + { + "bits_per_byte": 1.851296754271001, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.2832211256027222, + "step": 2020, + "unclipped_grad_norm": 0.6008884310722351 + }, + { + "bits_per_byte": 1.890781391169339, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3105897903442383, + "step": 2030, + "unclipped_grad_norm": 0.637248158454895 + }, + { + "bits_per_byte": 1.882627693691933, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3049380779266357, + "step": 2040, + "unclipped_grad_norm": 0.6375738382339478 + }, + { + "bits_per_byte": 1.848167701921203, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2810522317886353, + "step": 2050, + "unclipped_grad_norm": 0.6515361070632935 + }, + { + "bits_per_byte": 1.833815233758749, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.271103858947754, + "step": 2060, + "unclipped_grad_norm": 0.6752470135688782 + }, + { + "bits_per_byte": 1.960125139962525, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3586552143096924, + "step": 2070, + "unclipped_grad_norm": 0.7093353867530823 + }, + { + "bits_per_byte": 1.93064937739645, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.338224172592163, + "step": 2080, + "unclipped_grad_norm": 0.7233569622039795 + }, + { + "bits_per_byte": 1.8411410067549012, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.276181697845459, + "step": 2090, + "unclipped_grad_norm": 0.6208991408348083 + }, + { + "bits_per_byte": 1.7919973082361553, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2421178817749023, + "step": 2100, + "unclipped_grad_norm": 0.6259884834289551 + }, + { + "bits_per_byte": 1.830389339353562, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2687292098999023, + "step": 2110, + "unclipped_grad_norm": 0.633368968963623 + }, + { + "bits_per_byte": 1.8446745622995284, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2786309719085693, + "step": 2120, + "unclipped_grad_norm": 0.6770470142364502 + }, + { + "bits_per_byte": 1.8875441617321482, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.308345913887024, + "step": 2130, + "unclipped_grad_norm": 0.7079503536224365 + }, + { + "bits_per_byte": 1.912643825747019, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3257436752319336, + "step": 2140, + "unclipped_grad_norm": 0.6220520734786987 + }, + { + "bits_per_byte": 1.8089860984536854, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2538936138153076, + "step": 2150, + "unclipped_grad_norm": 0.6506339907646179 + }, + { + "bits_per_byte": 1.9026210208200363, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3187963962554932, + "step": 2160, + "unclipped_grad_norm": 0.6963920593261719 + }, + { + "bits_per_byte": 1.8293292382936435, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2679944038391113, + "step": 2170, + "unclipped_grad_norm": 0.6530651450157166 + }, + { + "bits_per_byte": 1.90049290749826, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3173213005065918, + "step": 2180, + "unclipped_grad_norm": 0.6189562678337097 + }, + { + "bits_per_byte": 1.8805790363548553, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3035180568695068, + "step": 2190, + "unclipped_grad_norm": 0.6570361852645874 + }, + { + "bits_per_byte": 1.8344993807438814, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.271578073501587, + "step": 2200, + "unclipped_grad_norm": 0.5929612517356873 + }, + { + "bits_per_byte": 1.837753980428809, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.273833990097046, + "step": 2210, + "unclipped_grad_norm": 0.6836420893669128 + }, + { + "bits_per_byte": 1.8346952689832141, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2717138528823853, + "step": 2220, + "unclipped_grad_norm": 0.6392546892166138 + }, + { + "bits_per_byte": 1.7850062134785822, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.237272024154663, + "step": 2230, + "unclipped_grad_norm": 0.5976205468177795 + }, + { + "bits_per_byte": 1.8807291772090586, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3036221265792847, + "step": 2240, + "unclipped_grad_norm": 0.6432849168777466 + }, + { + "bits_per_byte": 1.861241479074492, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2901142835617065, + "step": 2250, + "unclipped_grad_norm": 0.6619172096252441 + }, + { + "bits_per_byte": 1.9088667427687702, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3231256008148193, + "step": 2260, + "unclipped_grad_norm": 0.6604200601577759 + }, + { + "bits_per_byte": 1.8085772956925845, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2536102533340454, + "step": 2270, + "unclipped_grad_norm": 0.6277530193328857 + }, + { + "bits_per_byte": 1.8640159031982109, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2920373678207397, + "step": 2280, + "unclipped_grad_norm": 0.6375154256820679 + }, + { + "bits_per_byte": 1.8154523021608253, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.258375644683838, + "step": 2290, + "unclipped_grad_norm": 0.6188551187515259 + }, + { + "bits_per_byte": 1.8127673090155552, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.256514549255371, + "step": 2300, + "unclipped_grad_norm": 0.6347740292549133 + }, + { + "bits_per_byte": 1.811916166876435, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2559245824813843, + "step": 2310, + "unclipped_grad_norm": 0.621824324131012 + }, + { + "bits_per_byte": 1.866308603916863, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2936265468597412, + "step": 2320, + "unclipped_grad_norm": 0.6393704414367676 + }, + { + "bits_per_byte": 1.8101679632304146, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2547128200531006, + "step": 2330, + "unclipped_grad_norm": 0.6721166372299194 + }, + { + "bits_per_byte": 1.82719372971788, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2665141820907593, + "step": 2340, + "unclipped_grad_norm": 0.646355152130127 + }, + { + "bits_per_byte": 1.8870428323048831, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.3079984188079834, + "step": 2350, + "unclipped_grad_norm": 0.6942237019538879 + }, + { + "bits_per_byte": 1.804568896048925, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2508318424224854, + "step": 2360, + "unclipped_grad_norm": 0.6387670040130615 + }, + { + "bits_per_byte": 1.739788190948431, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2059292793273926, + "step": 2370, + "unclipped_grad_norm": 0.6083642840385437 + }, + { + "bits_per_byte": 1.8484354789085964, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2812378406524658, + "step": 2380, + "unclipped_grad_norm": 0.6751434803009033 + }, + { + "bits_per_byte": 1.8404750899307605, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2757201194763184, + "step": 2390, + "unclipped_grad_norm": 0.6037119626998901 + }, + { + "bits_per_byte": 1.8757652419572053, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.3001813888549805, + "step": 2400, + "unclipped_grad_norm": 0.6957288980484009 + }, + { + "bits_per_byte": 1.801620425482533, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2487881183624268, + "step": 2410, + "unclipped_grad_norm": 0.626511812210083 + }, + { + "bits_per_byte": 1.8236299052267009, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.264043927192688, + "step": 2420, + "unclipped_grad_norm": 0.6481806635856628 + }, + { + "bits_per_byte": 1.8322460640522769, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.2700161933898926, + "step": 2430, + "unclipped_grad_norm": 0.634190559387207 + }, + { + "bits_per_byte": 1.778641823500753, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2328605651855469, + "step": 2440, + "unclipped_grad_norm": 0.6495575308799744 + }, + { + "bits_per_byte": 1.9154751761281736, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.327706217765808, + "step": 2450, + "unclipped_grad_norm": 0.682089626789093 + }, + { + "bits_per_byte": 1.7848875454494868, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.237189769744873, + "step": 2460, + "unclipped_grad_norm": 0.6699327230453491 + }, + { + "bits_per_byte": 1.784353367335907, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2368195056915283, + "step": 2470, + "unclipped_grad_norm": 0.6535210609436035 + }, + { + "bits_per_byte": 1.865447314801342, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.293029546737671, + "step": 2480, + "unclipped_grad_norm": 0.6640622019767761 + }, + { + "bits_per_byte": 1.857090849778569, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.287237286567688, + "step": 2490, + "unclipped_grad_norm": 0.6223002076148987 + }, + { + "bits_per_byte": 1.8122928088818246, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2561856508255005, + "step": 2500, + "unclipped_grad_norm": 0.6052747368812561 + }, + { + "bits_per_byte": 1.76881026328153, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.226045846939087, + "step": 2510, + "unclipped_grad_norm": 0.6596174836158752 + }, + { + "bits_per_byte": 1.8006248179166882, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2480980157852173, + "step": 2520, + "unclipped_grad_norm": 0.6490770578384399 + }, + { + "bits_per_byte": 1.8232907554391995, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2638088464736938, + "step": 2530, + "unclipped_grad_norm": 0.6303134560585022 + }, + { + "bits_per_byte": 1.8125131186575798, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.256338357925415, + "step": 2540, + "unclipped_grad_norm": 0.6458812952041626 + }, + { + "bits_per_byte": 1.7674137641565237, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2250778675079346, + "step": 2550, + "unclipped_grad_norm": 0.6546228528022766 + }, + { + "bits_per_byte": 1.7342415784754543, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2020846605300903, + "step": 2560, + "unclipped_grad_norm": 0.602087676525116 + }, + { + "bits_per_byte": 1.774496353684356, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2299871444702148, + "step": 2570, + "unclipped_grad_norm": 0.6244005560874939 + }, + { + "bits_per_byte": 1.762712446412538, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2218191623687744, + "step": 2580, + "unclipped_grad_norm": 0.6090607643127441 + }, + { + "bits_per_byte": 1.7589357073995908, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2192013263702393, + "step": 2590, + "unclipped_grad_norm": 0.6631695032119751 + }, + { + "bits_per_byte": 1.7746397872151756, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2300865650177002, + "step": 2600, + "unclipped_grad_norm": 0.6302726864814758 + }, + { + "bits_per_byte": 1.8022297600145398, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2492104768753052, + "step": 2610, + "unclipped_grad_norm": 0.6297239065170288 + }, + { + "bits_per_byte": 1.7449476704743152, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.209505558013916, + "step": 2620, + "unclipped_grad_norm": 0.617965042591095 + }, + { + "bits_per_byte": 1.738682342503383, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.205162763595581, + "step": 2630, + "unclipped_grad_norm": 0.6048376560211182 + }, + { + "bits_per_byte": 1.7097366304672672, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1850991249084473, + "step": 2640, + "unclipped_grad_norm": 0.5914375185966492 + }, + { + "bits_per_byte": 1.790221415383346, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.240886926651001, + "step": 2650, + "unclipped_grad_norm": 0.6316050887107849 + }, + { + "bits_per_byte": 1.7080907564985102, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1839582920074463, + "step": 2660, + "unclipped_grad_norm": 0.6427422761917114 + }, + { + "bits_per_byte": 1.776176280217984, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2311515808105469, + "step": 2670, + "unclipped_grad_norm": 0.695833146572113 + }, + { + "bits_per_byte": 1.8751694940546166, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2997684478759766, + "step": 2680, + "unclipped_grad_norm": 0.7215304374694824 + }, + { + "bits_per_byte": 1.7935356930481232, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.243184208869934, + "step": 2690, + "unclipped_grad_norm": 0.6506277322769165 + }, + { + "bits_per_byte": 1.7071520751901008, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.1833076477050781, + "step": 2700, + "unclipped_grad_norm": 0.6119946241378784 + }, + { + "bits_per_byte": 1.7912357690581349, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2415900230407715, + "step": 2710, + "unclipped_grad_norm": 0.6407736539840698 + }, + { + "bits_per_byte": 1.8449512823847667, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2788227796554565, + "step": 2720, + "unclipped_grad_norm": 0.699180543422699 + }, + { + "bits_per_byte": 1.7188499911364874, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.1914160251617432, + "step": 2730, + "unclipped_grad_norm": 0.6055090427398682 + }, + { + "bits_per_byte": 1.7885469922945458, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2397263050079346, + "step": 2740, + "unclipped_grad_norm": 0.6273350715637207 + }, + { + "bits_per_byte": 1.7444217475279766, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2091410160064697, + "step": 2750, + "unclipped_grad_norm": 0.623899519443512 + }, + { + "bits_per_byte": 1.855314440977807, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.286005973815918, + "step": 2760, + "unclipped_grad_norm": 0.6717104315757751 + }, + { + "bits_per_byte": 1.79349716893433, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.2431575059890747, + "step": 2770, + "unclipped_grad_norm": 0.6304712891578674 + }, + { + "bits_per_byte": 1.8011758503300526, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.248479962348938, + "step": 2780, + "unclipped_grad_norm": 0.6547600626945496 + }, + { + "bits_per_byte": 1.7772979510669111, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.231929063796997, + "step": 2790, + "unclipped_grad_norm": 0.6679553389549255 + }, + { + "bits_per_byte": 1.784543924113063, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2369515895843506, + "step": 2800, + "unclipped_grad_norm": 0.6488308310508728 + }, + { + "bits_per_byte": 1.723108969502454, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.1943681240081787, + "step": 2810, + "unclipped_grad_norm": 0.5890340209007263 + }, + { + "bits_per_byte": 1.7962786443467342, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2450854778289795, + "step": 2820, + "unclipped_grad_norm": 0.678447961807251 + }, + { + "bits_per_byte": 1.7361824026904413, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.203429937362671, + "step": 2830, + "unclipped_grad_norm": 0.5870603322982788 + }, + { + "bits_per_byte": 1.7695838412451108, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2265820503234863, + "step": 2840, + "unclipped_grad_norm": 0.6424716711044312 + }, + { + "bits_per_byte": 1.7215556221998611, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.193291425704956, + "step": 2850, + "unclipped_grad_norm": 0.621953010559082 + }, + { + "bits_per_byte": 1.750462466156882, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2133281230926514, + "step": 2860, + "unclipped_grad_norm": 0.6744791269302368 + }, + { + "bits_per_byte": 1.7458160108785215, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.210107445716858, + "step": 2870, + "unclipped_grad_norm": 0.6091592311859131 + }, + { + "bits_per_byte": 1.7563069525811528, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2173792123794556, + "step": 2880, + "unclipped_grad_norm": 0.6055625081062317 + }, + { + "bits_per_byte": 1.7055577960166026, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.1822025775909424, + "step": 2890, + "unclipped_grad_norm": 0.5587822794914246 + }, + { + "bits_per_byte": 1.7494334939567697, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.212614893913269, + "step": 2900, + "unclipped_grad_norm": 0.7476353049278259 + }, + { + "bits_per_byte": 1.7069099236176861, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.1831398010253906, + "step": 2910, + "unclipped_grad_norm": 0.6083102226257324 + }, + { + "bits_per_byte": 1.7573092674703812, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2180739641189575, + "step": 2920, + "unclipped_grad_norm": 0.594536542892456 + }, + { + "bits_per_byte": 1.7148656690639488, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.1886543035507202, + "step": 2930, + "unclipped_grad_norm": 0.6017132997512817 + }, + { + "bits_per_byte": 1.7206826382640814, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.1926863193511963, + "step": 2940, + "unclipped_grad_norm": 0.6191802620887756 + }, + { + "bits_per_byte": 1.7777722792179909, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2322578430175781, + "step": 2950, + "unclipped_grad_norm": 0.6343991756439209 + }, + { + "bits_per_byte": 1.7965679191654853, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.245285987854004, + "step": 2960, + "unclipped_grad_norm": 0.6511562466621399 + }, + { + "bits_per_byte": 1.750740734085978, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2135210037231445, + "step": 2970, + "unclipped_grad_norm": 0.6213991641998291 + }, + { + "bits_per_byte": 1.7817877301503355, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2350411415100098, + "step": 2980, + "unclipped_grad_norm": 0.6211501359939575 + }, + { + "bits_per_byte": 1.724867836072828, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.1955872774124146, + "step": 2990, + "unclipped_grad_norm": 0.642371416091919 + }, + { + "bits_per_byte": 1.795713509355999, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2446937561035156, + "step": 3000, + "unclipped_grad_norm": 0.6221276521682739 + }, + { + "bits_per_byte": 1.6848799779380654, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1678698062896729, + "step": 3010, + "unclipped_grad_norm": 0.6145718693733215 + }, + { + "bits_per_byte": 1.752584732068529, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.214799165725708, + "step": 3020, + "unclipped_grad_norm": 0.6433340907096863 + }, + { + "bits_per_byte": 1.6945199494842276, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.1745517253875732, + "step": 3030, + "unclipped_grad_norm": 0.6049435138702393 + }, + { + "bits_per_byte": 1.6808318503020565, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1650638580322266, + "step": 3040, + "unclipped_grad_norm": 0.5896543860435486 + }, + { + "bits_per_byte": 1.8340489581812716, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2712658643722534, + "step": 3050, + "unclipped_grad_norm": 0.6642063856124878 + }, + { + "bits_per_byte": 1.7718811854953362, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2281744480133057, + "step": 3060, + "unclipped_grad_norm": 0.6303152441978455 + }, + { + "bits_per_byte": 1.6793462641639036, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.164034128189087, + "step": 3070, + "unclipped_grad_norm": 0.6531770825386047 + }, + { + "bits_per_byte": 1.767122941493915, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2248762845993042, + "step": 3080, + "unclipped_grad_norm": 0.6228736042976379 + }, + { + "bits_per_byte": 1.7547854220689696, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2163245677947998, + "step": 3090, + "unclipped_grad_norm": 0.6808046102523804 + }, + { + "bits_per_byte": 1.7115178547822532, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1863337755203247, + "step": 3100, + "unclipped_grad_norm": 0.6307847499847412 + }, + { + "bits_per_byte": 1.8621731090942157, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2907600402832031, + "step": 3110, + "unclipped_grad_norm": 0.652417004108429 + }, + { + "bits_per_byte": 1.7445247651358435, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2092124223709106, + "step": 3120, + "unclipped_grad_norm": 0.6338313221931458 + }, + { + "bits_per_byte": 1.6854237870800937, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1682467460632324, + "step": 3130, + "unclipped_grad_norm": 0.6034438610076904 + }, + { + "bits_per_byte": 1.717318657613205, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.190354585647583, + "step": 3140, + "unclipped_grad_norm": 0.6048492193222046 + }, + { + "bits_per_byte": 1.7027365926118492, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1802470684051514, + "step": 3150, + "unclipped_grad_norm": 0.5864679217338562 + }, + { + "bits_per_byte": 1.705547477057551, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.1821954250335693, + "step": 3160, + "unclipped_grad_norm": 0.6051758527755737 + }, + { + "bits_per_byte": 1.7744621291368343, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2299634218215942, + "step": 3170, + "unclipped_grad_norm": 0.6531821489334106 + }, + { + "bits_per_byte": 1.7458908233316468, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2101593017578125, + "step": 3180, + "unclipped_grad_norm": 0.6303388476371765 + }, + { + "bits_per_byte": 1.7361681281304195, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2034200429916382, + "step": 3190, + "unclipped_grad_norm": 0.678484320640564 + }, + { + "bits_per_byte": 1.7289813171161645, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1984385251998901, + "step": 3200, + "unclipped_grad_norm": 0.6228032112121582 + }, + { + "bits_per_byte": 1.6986354943193747, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1774044036865234, + "step": 3210, + "unclipped_grad_norm": 0.6199705004692078 + }, + { + "bits_per_byte": 1.6704626723162361, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1578764915466309, + "step": 3220, + "unclipped_grad_norm": 0.6396132111549377 + }, + { + "bits_per_byte": 1.7446800654695727, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.209320068359375, + "step": 3230, + "unclipped_grad_norm": 0.6783336400985718 + }, + { + "bits_per_byte": 1.7503792265538642, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2132704257965088, + "step": 3240, + "unclipped_grad_norm": 0.6652441620826721 + }, + { + "bits_per_byte": 1.7183588086856232, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.1910755634307861, + "step": 3250, + "unclipped_grad_norm": 0.63749098777771 + }, + { + "bits_per_byte": 1.7194732562632142, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1918480396270752, + "step": 3260, + "unclipped_grad_norm": 0.6323399543762207 + }, + { + "bits_per_byte": 1.76014147776479, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2200371026992798, + "step": 3270, + "unclipped_grad_norm": 0.668753445148468 + }, + { + "bits_per_byte": 1.7466587258677493, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2106915712356567, + "step": 3280, + "unclipped_grad_norm": 0.5981830954551697 + }, + { + "bits_per_byte": 1.7077491989538964, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1837215423583984, + "step": 3290, + "unclipped_grad_norm": 0.6258406639099121 + }, + { + "bits_per_byte": 1.7122833495612435, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1868643760681152, + "step": 3300, + "unclipped_grad_norm": 0.6118162870407104 + }, + { + "bits_per_byte": 1.6971689982554674, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.176387906074524, + "step": 3310, + "unclipped_grad_norm": 0.614425003528595 + }, + { + "bits_per_byte": 1.7078183359795434, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1837694644927979, + "step": 3320, + "unclipped_grad_norm": 0.6319419741630554 + }, + { + "bits_per_byte": 1.7065344854908524, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1828795671463013, + "step": 3330, + "unclipped_grad_norm": 0.6474409699440002 + }, + { + "bits_per_byte": 1.6476438422171078, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1420596837997437, + "step": 3340, + "unclipped_grad_norm": 0.6128159165382385 + }, + { + "bits_per_byte": 1.693542744062025, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1738743782043457, + "step": 3350, + "unclipped_grad_norm": 0.6108464002609253 + }, + { + "bits_per_byte": 1.7204735073606323, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1925413608551025, + "step": 3360, + "unclipped_grad_norm": 0.6041978597640991 + }, + { + "bits_per_byte": 1.6622183399818258, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.152161955833435, + "step": 3370, + "unclipped_grad_norm": 0.5887104272842407 + }, + { + "bits_per_byte": 1.7058931621857851, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1824350357055664, + "step": 3380, + "unclipped_grad_norm": 0.6352498531341553 + }, + { + "bits_per_byte": 1.707757282138487, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1837271451950073, + "step": 3390, + "unclipped_grad_norm": 0.6421219706535339 + }, + { + "bits_per_byte": 1.6979798964542856, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1769499778747559, + "step": 3400, + "unclipped_grad_norm": 0.6395401954650879 + }, + { + "bits_per_byte": 1.6959995162296002, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.1755772829055786, + "step": 3410, + "unclipped_grad_norm": 0.6283812522888184 + }, + { + "bits_per_byte": 1.6799083034669233, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1644237041473389, + "step": 3420, + "unclipped_grad_norm": 0.6197249293327332 + }, + { + "bits_per_byte": 1.71363410130112, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.187800645828247, + "step": 3430, + "unclipped_grad_norm": 0.6102274060249329 + }, + { + "bits_per_byte": 1.7091876618457131, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1847186088562012, + "step": 3440, + "unclipped_grad_norm": 0.6624795198440552 + }, + { + "bits_per_byte": 1.7982115573597812, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2464252710342407, + "step": 3450, + "unclipped_grad_norm": 0.6986591815948486 + }, + { + "bits_per_byte": 1.7759950105039746, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2310259342193604, + "step": 3460, + "unclipped_grad_norm": 0.6196012496948242 + }, + { + "bits_per_byte": 1.7475674821949083, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.211321473121643, + "step": 3470, + "unclipped_grad_norm": 0.6346544623374939 + }, + { + "bits_per_byte": 1.7169992858305527, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1901332139968872, + "step": 3480, + "unclipped_grad_norm": 0.6380975246429443 + }, + { + "bits_per_byte": 1.6316707815355735, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1309880018234253, + "step": 3490, + "unclipped_grad_norm": 0.6554663181304932 + }, + { + "bits_per_byte": 1.641168867394774, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1375715732574463, + "step": 3500, + "unclipped_grad_norm": 0.6088140606880188 + }, + { + "bits_per_byte": 1.6880614849963764, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.1700750589370728, + "step": 3510, + "unclipped_grad_norm": 0.6358106732368469 + }, + { + "bits_per_byte": 1.6616972325497115, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1518007516860962, + "step": 3520, + "unclipped_grad_norm": 0.6153261661529541 + }, + { + "bits_per_byte": 1.6712895649015846, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.158449649810791, + "step": 3530, + "unclipped_grad_norm": 0.6411516666412354 + }, + { + "bits_per_byte": 1.6785265948498915, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.163465976715088, + "step": 3540, + "unclipped_grad_norm": 0.6034340858459473 + }, + { + "bits_per_byte": 1.7376134703282706, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.204421877861023, + "step": 3550, + "unclipped_grad_norm": 0.6545450687408447 + }, + { + "bits_per_byte": 1.6707187544833708, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.158053994178772, + "step": 3560, + "unclipped_grad_norm": 0.6340785026550293 + }, + { + "bits_per_byte": 1.7138805524398064, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1879714727401733, + "step": 3570, + "unclipped_grad_norm": 0.6533586382865906 + }, + { + "bits_per_byte": 1.7826314770354685, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.235625982284546, + "step": 3580, + "unclipped_grad_norm": 0.7123501896858215 + }, + { + "bits_per_byte": 1.7131767994324758, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1874836683273315, + "step": 3590, + "unclipped_grad_norm": 0.6637008786201477 + }, + { + "bits_per_byte": 1.6959864455481348, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1755682229995728, + "step": 3600, + "unclipped_grad_norm": 0.6217906475067139 + }, + { + "bits_per_byte": 1.6716725702650561, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1587151288986206, + "step": 3610, + "unclipped_grad_norm": 0.6237639784812927 + }, + { + "bits_per_byte": 1.6746752153664697, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1607964038848877, + "step": 3620, + "unclipped_grad_norm": 0.6187261343002319 + }, + { + "bits_per_byte": 1.6883108598401275, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1702479124069214, + "step": 3630, + "unclipped_grad_norm": 0.6150439381599426 + }, + { + "bits_per_byte": 1.6436062055228018, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.13926100730896, + "step": 3640, + "unclipped_grad_norm": 0.6298535466194153 + }, + { + "bits_per_byte": 1.7182858880416574, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1910250186920166, + "step": 3650, + "unclipped_grad_norm": 0.6331585049629211 + }, + { + "bits_per_byte": 1.6224437403341332, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1245923042297363, + "step": 3660, + "unclipped_grad_norm": 0.6383513808250427 + }, + { + "bits_per_byte": 1.644355533932611, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1397804021835327, + "step": 3670, + "unclipped_grad_norm": 0.6061356067657471 + }, + { + "bits_per_byte": 1.7035681287287707, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1808234453201294, + "step": 3680, + "unclipped_grad_norm": 0.6479222774505615 + }, + { + "bits_per_byte": 1.6706625161565387, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1580150127410889, + "step": 3690, + "unclipped_grad_norm": 0.6192170977592468 + }, + { + "bits_per_byte": 1.6109532354646872, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1166276931762695, + "step": 3700, + "unclipped_grad_norm": 0.6093165278434753 + }, + { + "bits_per_byte": 1.6773731072105547, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1626664400100708, + "step": 3710, + "unclipped_grad_norm": 0.6192633509635925 + }, + { + "bits_per_byte": 1.6614944650043442, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1516602039337158, + "step": 3720, + "unclipped_grad_norm": 0.6275249719619751 + }, + { + "bits_per_byte": 1.7267034069054867, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.19685959815979, + "step": 3730, + "unclipped_grad_norm": 0.6384285092353821 + }, + { + "bits_per_byte": 1.6506055554476162, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1441125869750977, + "step": 3740, + "unclipped_grad_norm": 0.6508066654205322 + }, + { + "bits_per_byte": 1.6398485565841001, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.136656403541565, + "step": 3750, + "unclipped_grad_norm": 0.6484293937683105 + }, + { + "bits_per_byte": 1.6392961483095287, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1362735033035278, + "step": 3760, + "unclipped_grad_norm": 0.6600143909454346 + }, + { + "bits_per_byte": 1.686776946577082, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.169184684753418, + "step": 3770, + "unclipped_grad_norm": 0.6462109684944153 + }, + { + "bits_per_byte": 1.6300704829692951, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.1298787593841553, + "step": 3780, + "unclipped_grad_norm": 0.6114524006843567 + }, + { + "bits_per_byte": 1.675651388892767, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.161473035812378, + "step": 3790, + "unclipped_grad_norm": 0.6200737953186035 + }, + { + "bits_per_byte": 1.617114685914498, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1208984851837158, + "step": 3800, + "unclipped_grad_norm": 0.6216682195663452 + }, + { + "bits_per_byte": 1.6234193979124778, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.125268578529358, + "step": 3810, + "unclipped_grad_norm": 0.6173998117446899 + }, + { + "bits_per_byte": 1.7138243141129743, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1879324913024902, + "step": 3820, + "unclipped_grad_norm": 0.6088318228721619 + }, + { + "bits_per_byte": 1.6341469877426964, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1327043771743774, + "step": 3830, + "unclipped_grad_norm": 0.6438922882080078 + }, + { + "bits_per_byte": 1.655027229401299, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1471774578094482, + "step": 3840, + "unclipped_grad_norm": 0.669468879699707 + }, + { + "bits_per_byte": 1.6818487837166085, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.1657687425613403, + "step": 3850, + "unclipped_grad_norm": 0.6244412660598755 + }, + { + "bits_per_byte": 1.6164324307385254, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1204255819320679, + "step": 3860, + "unclipped_grad_norm": 0.6409146785736084 + }, + { + "bits_per_byte": 1.6824030838003392, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1661529541015625, + "step": 3870, + "unclipped_grad_norm": 0.6731106042861938 + }, + { + "bits_per_byte": 1.6375393455309653, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1350557804107666, + "step": 3880, + "unclipped_grad_norm": 0.6218310594558716 + }, + { + "bits_per_byte": 1.6796400105315774, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1642377376556396, + "step": 3890, + "unclipped_grad_norm": 0.635161280632019 + }, + { + "bits_per_byte": 1.666564169586478, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1551742553710938, + "step": 3900, + "unclipped_grad_norm": 0.7091010808944702 + }, + { + "bits_per_byte": 1.7026119051899735, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.180160641670227, + "step": 3910, + "unclipped_grad_norm": 0.6815565824508667 + }, + { + "bits_per_byte": 1.6485720325838145, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1427030563354492, + "step": 3920, + "unclipped_grad_norm": 0.628564178943634 + }, + { + "bits_per_byte": 1.6283515163739215, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1286872625350952, + "step": 3930, + "unclipped_grad_norm": 0.6302286982536316 + }, + { + "bits_per_byte": 1.6363571367889342, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1342363357543945, + "step": 3940, + "unclipped_grad_norm": 0.607494056224823 + }, + { + "bits_per_byte": 1.7146744243561893, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1885217428207397, + "step": 3950, + "unclipped_grad_norm": 0.70033860206604 + }, + { + "bits_per_byte": 1.7144123227962744, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1883400678634644, + "step": 3960, + "unclipped_grad_norm": 0.645106315612793 + }, + { + "bits_per_byte": 1.6837865122438798, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.167111873626709, + "step": 3970, + "unclipped_grad_norm": 0.6373703479766846 + }, + { + "bits_per_byte": 1.6677649524548022, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1560065746307373, + "step": 3980, + "unclipped_grad_norm": 0.6565650701522827 + }, + { + "bits_per_byte": 1.7059107044161732, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1824471950531006, + "step": 3990, + "unclipped_grad_norm": 0.6542640924453735 + }, + { + "bits_per_byte": 1.7121254694877515, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1867549419403076, + "step": 4000, + "unclipped_grad_norm": 0.6575196385383606 + }, + { + "bits_per_byte": 1.7220670986035271, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.19364595413208, + "step": 4010, + "unclipped_grad_norm": 0.6806800961494446 + }, + { + "bits_per_byte": 1.5958445595863893, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1061551570892334, + "step": 4020, + "unclipped_grad_norm": 0.6171497106552124 + }, + { + "bits_per_byte": 1.6345342926724393, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1329728364944458, + "step": 4030, + "unclipped_grad_norm": 0.6173087954521179 + }, + { + "bits_per_byte": 1.6363991005557448, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.134265422821045, + "step": 4040, + "unclipped_grad_norm": 0.634988009929657 + }, + { + "bits_per_byte": 1.6319382145576653, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1311733722686768, + "step": 4050, + "unclipped_grad_norm": 0.6698096990585327 + }, + { + "bits_per_byte": 1.6486795217406036, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1427775621414185, + "step": 4060, + "unclipped_grad_norm": 0.6841239929199219 + }, + { + "bits_per_byte": 1.6129987971140494, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1180455684661865, + "step": 4070, + "unclipped_grad_norm": 0.6276201605796814 + }, + { + "bits_per_byte": 1.6508999897458934, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1443166732788086, + "step": 4080, + "unclipped_grad_norm": 0.6372854113578796 + }, + { + "bits_per_byte": 1.6061886281051838, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1133251190185547, + "step": 4090, + "unclipped_grad_norm": 0.6243290305137634 + }, + { + "bits_per_byte": 1.578650766049031, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.0942373275756836, + "step": 4100, + "unclipped_grad_norm": 0.5791712403297424 + }, + { + "bits_per_byte": 1.6840439702722212, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1672903299331665, + "step": 4110, + "unclipped_grad_norm": 0.6848294734954834 + }, + { + "bits_per_byte": 1.6312047085517354, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1306649446487427, + "step": 4120, + "unclipped_grad_norm": 0.6449601054191589 + }, + { + "bits_per_byte": 1.594994965291127, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1055662631988525, + "step": 4130, + "unclipped_grad_norm": 0.6161995530128479 + }, + { + "bits_per_byte": 1.6232881751498696, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1251776218414307, + "step": 4140, + "unclipped_grad_norm": 0.6562530398368835 + }, + { + "bits_per_byte": 1.67032388231699, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1577802896499634, + "step": 4150, + "unclipped_grad_norm": 0.672451376914978 + }, + { + "bits_per_byte": 1.6420112384187, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138155460357666, + "step": 4160, + "unclipped_grad_norm": 0.6645690202713013 + }, + { + "bits_per_byte": 1.597898376402993, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1075787544250488, + "step": 4170, + "unclipped_grad_norm": 0.6766495108604431 + }, + { + "bits_per_byte": 1.5688259131531912, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.0874272584915161, + "step": 4180, + "unclipped_grad_norm": 0.6362206339836121 + }, + { + "bits_per_byte": 1.7152548658028512, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1889240741729736, + "step": 4190, + "unclipped_grad_norm": 0.6535307765007019 + }, + { + "bits_per_byte": 1.6086450563074575, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1150277853012085, + "step": 4200, + "unclipped_grad_norm": 0.6474415063858032 + }, + { + "bits_per_byte": 1.6237748860518113, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1255149841308594, + "step": 4210, + "unclipped_grad_norm": 0.6242862343788147 + }, + { + "bits_per_byte": 1.6672617312183775, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1556577682495117, + "step": 4220, + "unclipped_grad_norm": 0.6519601345062256 + }, + { + "bits_per_byte": 1.635159105643024, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1334059238433838, + "step": 4230, + "unclipped_grad_norm": 0.6224661469459534 + }, + { + "bits_per_byte": 1.6806368219759782, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.164928674697876, + "step": 4240, + "unclipped_grad_norm": 0.6462982296943665 + }, + { + "bits_per_byte": 1.7100040634893587, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1852844953536987, + "step": 4250, + "unclipped_grad_norm": 0.6697248220443726 + }, + { + "bits_per_byte": 1.6765044228410466, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1620643138885498, + "step": 4260, + "unclipped_grad_norm": 0.6539841294288635 + }, + { + "bits_per_byte": 1.6521546031839376, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1451863050460815, + "step": 4270, + "unclipped_grad_norm": 0.6337559819221497 + }, + { + "bits_per_byte": 1.688240002987972, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1701987981796265, + "step": 4280, + "unclipped_grad_norm": 0.6939600706100464 + }, + { + "bits_per_byte": 1.645464994013327, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1405494213104248, + "step": 4290, + "unclipped_grad_norm": 0.6979376673698425 + }, + { + "bits_per_byte": 1.5687516166480184, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0873757600784302, + "step": 4300, + "unclipped_grad_norm": 0.6490140557289124 + }, + { + "bits_per_byte": 1.720270739815265, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1924008131027222, + "step": 4310, + "unclipped_grad_norm": 0.7267009615898132 + }, + { + "bits_per_byte": 1.5912837516681586, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1029938459396362, + "step": 4320, + "unclipped_grad_norm": 0.6906614899635315 + }, + { + "bits_per_byte": 1.5899403951822693, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.102062702178955, + "step": 4330, + "unclipped_grad_norm": 0.6376214027404785 + }, + { + "bits_per_byte": 1.5963857889886546, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1065303087234497, + "step": 4340, + "unclipped_grad_norm": 0.6380553841590881 + }, + { + "bits_per_byte": 1.5301035033634773, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.060586929321289, + "step": 4350, + "unclipped_grad_norm": 0.6225847601890564 + }, + { + "bits_per_byte": 1.67761336697381, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1628329753875732, + "step": 4360, + "unclipped_grad_norm": 0.6995339393615723 + }, + { + "bits_per_byte": 1.6233086410853221, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1251918077468872, + "step": 4370, + "unclipped_grad_norm": 0.6690109372138977 + }, + { + "bits_per_byte": 1.5971554113512656, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1070637702941895, + "step": 4380, + "unclipped_grad_norm": 0.7015417814254761 + }, + { + "bits_per_byte": 1.6390723988807563, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1361184120178223, + "step": 4390, + "unclipped_grad_norm": 0.6816950440406799 + }, + { + "bits_per_byte": 1.5932933689434905, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.104386806488037, + "step": 4400, + "unclipped_grad_norm": 0.667495846748352 + }, + { + "bits_per_byte": 1.5991400911422224, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1084394454956055, + "step": 4410, + "unclipped_grad_norm": 0.6873564124107361 + }, + { + "bits_per_byte": 1.58255941575519, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.0969465970993042, + "step": 4420, + "unclipped_grad_norm": 0.6864304542541504 + }, + { + "bits_per_byte": 1.5926611607189187, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1039485931396484, + "step": 4430, + "unclipped_grad_norm": 0.6576697826385498 + }, + { + "bits_per_byte": 1.6095080652494869, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1156259775161743, + "step": 4440, + "unclipped_grad_norm": 0.6515920162200928 + }, + { + "bits_per_byte": 1.5838350110766395, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.0978307723999023, + "step": 4450, + "unclipped_grad_norm": 0.6458138227462769 + }, + { + "bits_per_byte": 1.5629618207067222, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0833625793457031, + "step": 4460, + "unclipped_grad_norm": 0.6665330529212952 + }, + { + "bits_per_byte": 1.6502230660320973, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1438474655151367, + "step": 4470, + "unclipped_grad_norm": 0.6849002838134766 + }, + { + "bits_per_byte": 1.5654218605446637, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.0850677490234375, + "step": 4480, + "unclipped_grad_norm": 0.630775511264801 + }, + { + "bits_per_byte": 1.533044750658533, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.0626256465911865, + "step": 4490, + "unclipped_grad_norm": 0.6286258697509766 + }, + { + "bits_per_byte": 1.6266820807819962, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1275300979614258, + "step": 4500, + "unclipped_grad_norm": 0.6595945954322815 + }, + { + "bits_per_byte": 1.5885456158837719, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1010959148406982, + "step": 4510, + "unclipped_grad_norm": 0.6231974363327026 + }, + { + "bits_per_byte": 1.5897587815029581, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1019368171691895, + "step": 4520, + "unclipped_grad_norm": 0.6408888697624207 + }, + { + "bits_per_byte": 1.6075569220754484, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1142735481262207, + "step": 4530, + "unclipped_grad_norm": 0.6721575260162354 + }, + { + "bits_per_byte": 1.5571181941957055, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.0793120861053467, + "step": 4540, + "unclipped_grad_norm": 0.6309816837310791 + }, + { + "bits_per_byte": 1.6337729254770696, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1324450969696045, + "step": 4550, + "unclipped_grad_norm": 0.6641245484352112 + }, + { + "bits_per_byte": 1.6370763682348426, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.134734869003296, + "step": 4560, + "unclipped_grad_norm": 0.6858609914779663 + }, + { + "bits_per_byte": 1.5844336826842929, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.0982457399368286, + "step": 4570, + "unclipped_grad_norm": 0.6500639319419861 + }, + { + "bits_per_byte": 1.5406408803818417, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0678908824920654, + "step": 4580, + "unclipped_grad_norm": 0.636735200881958 + }, + { + "bits_per_byte": 1.6489479866586005, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1429636478424072, + "step": 4590, + "unclipped_grad_norm": 0.6933116316795349 + }, + { + "bits_per_byte": 1.6320697812855753, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.131264567375183, + "step": 4600, + "unclipped_grad_norm": 0.6834596395492554 + }, + { + "bits_per_byte": 1.589729888417613, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.101916790008545, + "step": 4610, + "unclipped_grad_norm": 0.6632579565048218 + }, + { + "bits_per_byte": 1.5335797886853673, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.062996506690979, + "step": 4620, + "unclipped_grad_norm": 0.6395919322967529 + }, + { + "bits_per_byte": 1.6054043872172494, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1127815246582031, + "step": 4630, + "unclipped_grad_norm": 0.6490065455436707 + }, + { + "bits_per_byte": 1.5840957367753474, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.0980114936828613, + "step": 4640, + "unclipped_grad_norm": 0.6492030024528503 + }, + { + "bits_per_byte": 1.5576186637097162, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.0796589851379395, + "step": 4650, + "unclipped_grad_norm": 0.6488379836082458 + }, + { + "bits_per_byte": 1.593877078060519, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1047914028167725, + "step": 4660, + "unclipped_grad_norm": 0.6992486119270325 + }, + { + "bits_per_byte": 1.6105064245377456, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1163179874420166, + "step": 4670, + "unclipped_grad_norm": 0.6828540563583374 + }, + { + "bits_per_byte": 1.5810454524796447, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.0958971977233887, + "step": 4680, + "unclipped_grad_norm": 0.6880136728286743 + }, + { + "bits_per_byte": 1.5551754781715592, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.077965497970581, + "step": 4690, + "unclipped_grad_norm": 0.6890891194343567 + }, + { + "bits_per_byte": 1.6623815515174944, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1522750854492188, + "step": 4700, + "unclipped_grad_norm": 0.6787605285644531 + }, + { + "bits_per_byte": 1.574140349047503, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.0911109447479248, + "step": 4710, + "unclipped_grad_norm": 0.6684292554855347 + }, + { + "bits_per_byte": 1.5498508953008467, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.0742747783660889, + "step": 4720, + "unclipped_grad_norm": 0.6828400492668152 + }, + { + "bits_per_byte": 1.6704473658603094, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1578658819198608, + "step": 4730, + "unclipped_grad_norm": 0.6912305355072021 + }, + { + "bits_per_byte": 1.6014281483293014, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.110025405883789, + "step": 4740, + "unclipped_grad_norm": 0.6723529100418091 + }, + { + "bits_per_byte": 1.639597977861793, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1364827156066895, + "step": 4750, + "unclipped_grad_norm": 0.7447375059127808 + }, + { + "bits_per_byte": 1.523300557625948, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0558714866638184, + "step": 4760, + "unclipped_grad_norm": 0.6460341811180115 + }, + { + "bits_per_byte": 1.6314774730360038, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1308540105819702, + "step": 4770, + "unclipped_grad_norm": 0.6882368922233582 + }, + { + "bits_per_byte": 1.5726310293035308, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.0900647640228271, + "step": 4780, + "unclipped_grad_norm": 0.6284239888191223 + }, + { + "bits_per_byte": 1.60157536547844, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1101274490356445, + "step": 4790, + "unclipped_grad_norm": 0.6737696528434753 + }, + { + "bits_per_byte": 1.577140070443852, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0931901931762695, + "step": 4800, + "unclipped_grad_norm": 0.6438981890678406 + }, + { + "bits_per_byte": 1.5444138357764698, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.0705060958862305, + "step": 4810, + "unclipped_grad_norm": 0.690973699092865 + }, + { + "bits_per_byte": 1.6096637095485178, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1157338619232178, + "step": 4820, + "unclipped_grad_norm": 0.6775097250938416 + }, + { + "bits_per_byte": 1.6126245628657718, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.117786169052124, + "step": 4830, + "unclipped_grad_norm": 0.7250232696533203 + }, + { + "bits_per_byte": 1.6110075819823597, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1166653633117676, + "step": 4840, + "unclipped_grad_norm": 0.6946505308151245 + }, + { + "bits_per_byte": 1.560937584906067, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.0819594860076904, + "step": 4850, + "unclipped_grad_norm": 0.7250073552131653 + }, + { + "bits_per_byte": 1.5705242418304615, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.08860445022583, + "step": 4860, + "unclipped_grad_norm": 0.6968505382537842 + }, + { + "bits_per_byte": 1.5589560008028256, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.0805859565734863, + "step": 4870, + "unclipped_grad_norm": 0.659229040145874 + }, + { + "bits_per_byte": 1.7343542271117696, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.202162742614746, + "step": 4880, + "unclipped_grad_norm": 0.7630480527877808 + }, + { + "bits_per_byte": 1.5109135071975544, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0472854375839233, + "step": 4890, + "unclipped_grad_norm": 0.6373679041862488 + }, + { + "bits_per_byte": 1.6030399697331876, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.111142635345459, + "step": 4900, + "unclipped_grad_norm": 0.6882696151733398 + }, + { + "bits_per_byte": 1.6017267102111992, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1102323532104492, + "step": 4910, + "unclipped_grad_norm": 0.7222312688827515 + }, + { + "bits_per_byte": 1.533604038239139, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.0630133152008057, + "step": 4920, + "unclipped_grad_norm": 0.6319286227226257 + }, + { + "bits_per_byte": 1.5357895937663035, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.064528226852417, + "step": 4930, + "unclipped_grad_norm": 0.6290988922119141 + }, + { + "bits_per_byte": 1.5644580697692285, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.084399700164795, + "step": 4940, + "unclipped_grad_norm": 0.6635708808898926 + }, + { + "bits_per_byte": 1.596738869370876, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.1067750453948975, + "step": 4950, + "unclipped_grad_norm": 0.7171201109886169 + }, + { + "bits_per_byte": 1.6163304450265636, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1203548908233643, + "step": 4960, + "unclipped_grad_norm": 0.739273726940155 + }, + { + "bits_per_byte": 1.566945798813959, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.086124062538147, + "step": 4970, + "unclipped_grad_norm": 0.6495019197463989 + }, + { + "bits_per_byte": 1.623716067985216, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.125474214553833, + "step": 4980, + "unclipped_grad_norm": 0.7210032939910889 + }, + { + "bits_per_byte": 1.574822088275523, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.091583490371704, + "step": 4990, + "unclipped_grad_norm": 0.6862797737121582 + }, + { + "bits_per_byte": 1.5840169687212522, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.097956895828247, + "step": 5000, + "unclipped_grad_norm": 0.6914816498756409 + }, + { + "bits_per_byte": 1.6270380848692823, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.127776861190796, + "step": 5010, + "unclipped_grad_norm": 0.7159916758537292 + }, + { + "bits_per_byte": 1.5917599716283977, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1033239364624023, + "step": 5020, + "unclipped_grad_norm": 0.7277734875679016 + }, + { + "bits_per_byte": 1.5241281381418998, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0564451217651367, + "step": 5030, + "unclipped_grad_norm": 0.6716130375862122 + }, + { + "bits_per_byte": 1.5837488477685573, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0977710485458374, + "step": 5040, + "unclipped_grad_norm": 0.6623762845993042 + }, + { + "bits_per_byte": 1.554003760371231, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0771533250808716, + "step": 5050, + "unclipped_grad_norm": 0.7006968855857849 + }, + { + "bits_per_byte": 1.621560093474, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1239798069000244, + "step": 5060, + "unclipped_grad_norm": 0.6823760271072388 + }, + { + "bits_per_byte": 1.5589410383122004, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.0805755853652954, + "step": 5070, + "unclipped_grad_norm": 0.685271680355072 + }, + { + "bits_per_byte": 1.6273230601217619, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1279743909835815, + "step": 5080, + "unclipped_grad_norm": 0.725353479385376 + }, + { + "bits_per_byte": 1.5689380458415538, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0875049829483032, + "step": 5090, + "unclipped_grad_norm": 0.6860542297363281 + }, + { + "bits_per_byte": 1.5571642855461367, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0793440341949463, + "step": 5100, + "unclipped_grad_norm": 0.7099642753601074 + }, + { + "bits_per_byte": 1.593192587110085, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.1043169498443604, + "step": 5110, + "unclipped_grad_norm": 0.683228075504303 + }, + { + "bits_per_byte": 1.5025353723607726, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.041478157043457, + "step": 5120, + "unclipped_grad_norm": 0.6536243557929993 + }, + { + "bits_per_byte": 1.6259317204762818, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1270099878311157, + "step": 5130, + "unclipped_grad_norm": 0.7276868224143982 + }, + { + "bits_per_byte": 1.5937286570328242, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1046885251998901, + "step": 5140, + "unclipped_grad_norm": 0.7137117981910706 + }, + { + "bits_per_byte": 1.5363523209999266, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0649182796478271, + "step": 5150, + "unclipped_grad_norm": 0.6770874857902527 + }, + { + "bits_per_byte": 1.5548322008004372, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0777275562286377, + "step": 5160, + "unclipped_grad_norm": 0.6993823051452637 + }, + { + "bits_per_byte": 1.6209285731800318, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.123542070388794, + "step": 5170, + "unclipped_grad_norm": 0.7498713135719299 + }, + { + "bits_per_byte": 1.5887963665887297, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1012697219848633, + "step": 5180, + "unclipped_grad_norm": 0.7006720304489136 + }, + { + "bits_per_byte": 1.4814539110006604, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0268656015396118, + "step": 5190, + "unclipped_grad_norm": 0.6951001286506653 + }, + { + "bits_per_byte": 1.5501159205658264, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0744584798812866, + "step": 5200, + "unclipped_grad_norm": 0.696289598941803 + }, + { + "bits_per_byte": 1.519997458833477, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.053581953048706, + "step": 5210, + "unclipped_grad_norm": 0.6860307455062866 + }, + { + "bits_per_byte": 1.5330389032484038, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0626215934753418, + "step": 5220, + "unclipped_grad_norm": 0.6861689686775208 + }, + { + "bits_per_byte": 1.6136053799236425, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1184660196304321, + "step": 5230, + "unclipped_grad_norm": 0.6980286836624146 + }, + { + "bits_per_byte": 1.5805682006235005, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0955663919448853, + "step": 5240, + "unclipped_grad_norm": 0.7267829775810242 + }, + { + "bits_per_byte": 1.5736680846882336, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0907835960388184, + "step": 5250, + "unclipped_grad_norm": 0.6905968189239502 + }, + { + "bits_per_byte": 1.5160989561037188, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.050879716873169, + "step": 5260, + "unclipped_grad_norm": 0.7048349380493164 + }, + { + "bits_per_byte": 1.5706724908755052, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0887072086334229, + "step": 5270, + "unclipped_grad_norm": 0.7007996439933777 + }, + { + "bits_per_byte": 1.4955876172312168, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0366623401641846, + "step": 5280, + "unclipped_grad_norm": 0.7013132572174072 + }, + { + "bits_per_byte": 1.524417756925953, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.0566458702087402, + "step": 5290, + "unclipped_grad_norm": 0.6891999840736389 + }, + { + "bits_per_byte": 1.4950008124264729, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0362555980682373, + "step": 5300, + "unclipped_grad_norm": 0.6865445971488953 + }, + { + "bits_per_byte": 1.4986692023693766, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0387983322143555, + "step": 5310, + "unclipped_grad_norm": 0.6979682445526123 + }, + { + "bits_per_byte": 1.5140539104023092, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0494621992111206, + "step": 5320, + "unclipped_grad_norm": 0.7112694978713989 + }, + { + "bits_per_byte": 1.5563401446832021, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.078772783279419, + "step": 5330, + "unclipped_grad_norm": 0.7123240828514099 + }, + { + "bits_per_byte": 1.620746271570117, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1234157085418701, + "step": 5340, + "unclipped_grad_norm": 0.7256788611412048 + }, + { + "bits_per_byte": 1.4811187168141287, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0266332626342773, + "step": 5350, + "unclipped_grad_norm": 0.6512540578842163 + }, + { + "bits_per_byte": 1.5334795227999143, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.062927007675171, + "step": 5360, + "unclipped_grad_norm": 0.7001609206199646 + }, + { + "bits_per_byte": 1.6031871868823262, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.1112446784973145, + "step": 5370, + "unclipped_grad_norm": 0.7270175814628601 + }, + { + "bits_per_byte": 1.5079560935333174, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.045235514640808, + "step": 5380, + "unclipped_grad_norm": 0.6764644980430603 + }, + { + "bits_per_byte": 1.5113635857948622, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0475974082946777, + "step": 5390, + "unclipped_grad_norm": 0.7015690803527832 + }, + { + "bits_per_byte": 1.567986121869028, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0868451595306396, + "step": 5400, + "unclipped_grad_norm": 0.7621549963951111 + }, + { + "bits_per_byte": 1.456367317667255, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.009476900100708, + "step": 5410, + "unclipped_grad_norm": 0.7006528377532959 + }, + { + "bits_per_byte": 1.6203708334432836, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1231554746627808, + "step": 5420, + "unclipped_grad_norm": 0.7240427136421204 + }, + { + "bits_per_byte": 1.543870886547696, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0701297521591187, + "step": 5430, + "unclipped_grad_norm": 0.7349725961685181 + }, + { + "bits_per_byte": 1.5391332804643783, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0668458938598633, + "step": 5440, + "unclipped_grad_norm": 0.6845248937606812 + }, + { + "bits_per_byte": 1.5639070373558641, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0840177536010742, + "step": 5450, + "unclipped_grad_norm": 0.7343961596488953 + }, + { + "bits_per_byte": 1.5791133993798518, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0945580005645752, + "step": 5460, + "unclipped_grad_norm": 0.6990759968757629 + }, + { + "bits_per_byte": 1.5450971228483479, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0709797143936157, + "step": 5470, + "unclipped_grad_norm": 0.73023521900177 + }, + { + "bits_per_byte": 1.4896457886265577, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0325437784194946, + "step": 5480, + "unclipped_grad_norm": 0.7134303450584412 + }, + { + "bits_per_byte": 1.5553481487530256, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.07808518409729, + "step": 5490, + "unclipped_grad_norm": 0.713401734828949 + }, + { + "bits_per_byte": 1.5796455137016214, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0949268341064453, + "step": 5500, + "unclipped_grad_norm": 0.7012893557548523 + }, + { + "bits_per_byte": 1.56096613402611, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0819792747497559, + "step": 5510, + "unclipped_grad_norm": 0.7354693412780762 + }, + { + "bits_per_byte": 1.558417523122974, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0802127122879028, + "step": 5520, + "unclipped_grad_norm": 0.8108682036399841 + }, + { + "bits_per_byte": 1.572987033390817, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0903115272521973, + "step": 5530, + "unclipped_grad_norm": 0.813023567199707 + }, + { + "bits_per_byte": 1.603627462468535, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1115498542785645, + "step": 5540, + "unclipped_grad_norm": 0.7556710243225098 + }, + { + "bits_per_byte": 1.5354511319094055, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.064293622970581, + "step": 5550, + "unclipped_grad_norm": 0.6996996998786926 + }, + { + "bits_per_byte": 1.531031521747533, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.061230182647705, + "step": 5560, + "unclipped_grad_norm": 0.7085531949996948 + }, + { + "bits_per_byte": 1.5727634559446952, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0901565551757812, + "step": 5570, + "unclipped_grad_norm": 0.7508514523506165 + }, + { + "bits_per_byte": 1.4695356852785186, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0186045169830322, + "step": 5580, + "unclipped_grad_norm": 0.6981281042098999 + }, + { + "bits_per_byte": 1.5444360215384312, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0705214738845825, + "step": 5590, + "unclipped_grad_norm": 0.7099512815475464 + }, + { + "bits_per_byte": 1.5326933901028204, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0623821020126343, + "step": 5600, + "unclipped_grad_norm": 0.7117766737937927 + }, + { + "bits_per_byte": 1.5419961036706404, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0688302516937256, + "step": 5610, + "unclipped_grad_norm": 0.7040424942970276 + }, + { + "bits_per_byte": 1.4775775940328635, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0241787433624268, + "step": 5620, + "unclipped_grad_norm": 0.7147260308265686 + }, + { + "bits_per_byte": 1.4921469323180554, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0342774391174316, + "step": 5630, + "unclipped_grad_norm": 0.7185394763946533 + }, + { + "bits_per_byte": 1.4712231790487844, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0197741985321045, + "step": 5640, + "unclipped_grad_norm": 0.7179238796234131 + }, + { + "bits_per_byte": 1.5902296700010203, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.1022632122039795, + "step": 5650, + "unclipped_grad_norm": 0.7511428594589233 + }, + { + "bits_per_byte": 1.541337238135185, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0683735609054565, + "step": 5660, + "unclipped_grad_norm": 0.7174038887023926 + }, + { + "bits_per_byte": 1.5341244577406499, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0633740425109863, + "step": 5670, + "unclipped_grad_norm": 0.7094559669494629 + }, + { + "bits_per_byte": 1.5432753106277581, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0697169303894043, + "step": 5680, + "unclipped_grad_norm": 0.7527599334716797 + }, + { + "bits_per_byte": 1.5847210656938846, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.098444938659668, + "step": 5690, + "unclipped_grad_norm": 0.7476640939712524 + }, + { + "bits_per_byte": 1.5107903676195364, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.047200083732605, + "step": 5700, + "unclipped_grad_norm": 0.7240637540817261 + }, + { + "bits_per_byte": 1.5529789157547396, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0764429569244385, + "step": 5710, + "unclipped_grad_norm": 0.7409384250640869 + }, + { + "bits_per_byte": 1.5779322225337262, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0937392711639404, + "step": 5720, + "unclipped_grad_norm": 0.7523439526557922 + }, + { + "bits_per_byte": 1.4812139952027068, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0266993045806885, + "step": 5730, + "unclipped_grad_norm": 0.6902204155921936 + }, + { + "bits_per_byte": 1.5538140635073292, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.077021837234497, + "step": 5740, + "unclipped_grad_norm": 0.7577133178710938 + }, + { + "bits_per_byte": 1.5073047952345, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0447840690612793, + "step": 5750, + "unclipped_grad_norm": 0.7320472002029419 + }, + { + "bits_per_byte": 1.5094478710469015, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0462695360183716, + "step": 5760, + "unclipped_grad_norm": 0.7613655924797058 + }, + { + "bits_per_byte": 1.4626881960344158, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0138581991195679, + "step": 5770, + "unclipped_grad_norm": 0.7212797403335571 + }, + { + "bits_per_byte": 1.5541401426133652, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0772478580474854, + "step": 5780, + "unclipped_grad_norm": 0.7401144504547119 + }, + { + "bits_per_byte": 1.4976115090665703, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0380651950836182, + "step": 5790, + "unclipped_grad_norm": 0.6966454386711121 + }, + { + "bits_per_byte": 1.508392929466509, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.045538306236267, + "step": 5800, + "unclipped_grad_norm": 0.7519288063049316 + }, + { + "bits_per_byte": 1.536275616737642, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0648651123046875, + "step": 5810, + "unclipped_grad_norm": 0.7322137355804443 + }, + { + "bits_per_byte": 1.5318357126229676, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.0617876052856445, + "step": 5820, + "unclipped_grad_norm": 0.7490501999855042 + }, + { + "bits_per_byte": 1.5069907549140245, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0445663928985596, + "step": 5830, + "unclipped_grad_norm": 0.7249871492385864 + }, + { + "bits_per_byte": 1.4694756633333674, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.018562912940979, + "step": 5840, + "unclipped_grad_norm": 0.7599647641181946 + }, + { + "bits_per_byte": 1.4762989030236986, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0232924222946167, + "step": 5850, + "unclipped_grad_norm": 0.7097748517990112 + }, + { + "bits_per_byte": 1.502543627528014, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0414838790893555, + "step": 5860, + "unclipped_grad_norm": 0.7764433026313782 + }, + { + "bits_per_byte": 1.5481047554466367, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0730644464492798, + "step": 5870, + "unclipped_grad_norm": 0.770169734954834 + }, + { + "bits_per_byte": 1.5544658777540994, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0774736404418945, + "step": 5880, + "unclipped_grad_norm": 0.731796145439148 + }, + { + "bits_per_byte": 1.528056737835559, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0591682195663452, + "step": 5890, + "unclipped_grad_norm": 0.7195150852203369 + }, + { + "bits_per_byte": 1.5162458292875556, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0509815216064453, + "step": 5900, + "unclipped_grad_norm": 0.8138624429702759 + }, + { + "bits_per_byte": 1.5072279189895643, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.04473078250885, + "step": 5910, + "unclipped_grad_norm": 0.7780150771141052 + }, + { + "bits_per_byte": 1.514708304388842, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0499157905578613, + "step": 5920, + "unclipped_grad_norm": 0.7354658246040344 + }, + { + "bits_per_byte": 1.5573187259666117, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.079451084136963, + "step": 5930, + "unclipped_grad_norm": 0.7978370785713196 + }, + { + "bits_per_byte": 1.5533937379086207, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.076730489730835, + "step": 5940, + "unclipped_grad_norm": 0.8458540439605713 + }, + { + "bits_per_byte": 1.5137727187681485, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.049267292022705, + "step": 5950, + "unclipped_grad_norm": 0.7532646059989929 + }, + { + "bits_per_byte": 1.5086940710881698, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0457470417022705, + "step": 5960, + "unclipped_grad_norm": 0.7333019375801086 + }, + { + "bits_per_byte": 1.5303000795334134, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0607231855392456, + "step": 5970, + "unclipped_grad_norm": 0.8049731254577637 + }, + { + "bits_per_byte": 1.4828514420215715, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0278342962265015, + "step": 5980, + "unclipped_grad_norm": 0.7353474497795105 + }, + { + "bits_per_byte": 1.4545697550004368, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0082309246063232, + "step": 5990, + "unclipped_grad_norm": 0.737419068813324 + }, + { + "bits_per_byte": 1.5419821730759207, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.068820595741272, + "step": 6000, + "unclipped_grad_norm": 0.7629180550575256 + }, + { + "bits_per_byte": 1.5277340983825405, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.058944582939148, + "step": 6010, + "unclipped_grad_norm": 0.7620755434036255 + }, + { + "bits_per_byte": 1.4955477172562166, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0366346836090088, + "step": 6020, + "unclipped_grad_norm": 0.7292536497116089 + }, + { + "bits_per_byte": 1.5533901262729524, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0767279863357544, + "step": 6030, + "unclipped_grad_norm": 0.767517626285553 + }, + { + "bits_per_byte": 1.495873108431649, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0368602275848389, + "step": 6040, + "unclipped_grad_norm": 0.7536795735359192 + }, + { + "bits_per_byte": 1.5050292927809341, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.043206810951233, + "step": 6050, + "unclipped_grad_norm": 0.7510635256767273 + }, + { + "bits_per_byte": 1.556982155918873, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.079217791557312, + "step": 6060, + "unclipped_grad_norm": 0.7961422801017761 + }, + { + "bits_per_byte": 1.5009287104364122, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.0403645038604736, + "step": 6070, + "unclipped_grad_norm": 0.7310256958007812 + }, + { + "bits_per_byte": 1.5023740526342633, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0413663387298584, + "step": 6080, + "unclipped_grad_norm": 0.7653129696846008 + }, + { + "bits_per_byte": 1.5019286175685285, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0410575866699219, + "step": 6090, + "unclipped_grad_norm": 0.7687012553215027 + }, + { + "bits_per_byte": 1.5089062976793344, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0458941459655762, + "step": 6100, + "unclipped_grad_norm": 0.7417535185813904 + }, + { + "bits_per_byte": 1.475459455704837, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0227105617523193, + "step": 6110, + "unclipped_grad_norm": 0.7481279969215393 + }, + { + "bits_per_byte": 1.4820735644917191, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0272951126098633, + "step": 6120, + "unclipped_grad_norm": 0.7211357951164246 + }, + { + "bits_per_byte": 1.5036838725032344, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0422742366790771, + "step": 6130, + "unclipped_grad_norm": 0.7536711096763611 + }, + { + "bits_per_byte": 1.5397933498783898, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0673034191131592, + "step": 6140, + "unclipped_grad_norm": 0.7828236818313599 + }, + { + "bits_per_byte": 1.4767353949915882, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0235949754714966, + "step": 6150, + "unclipped_grad_norm": 0.7367910742759705 + }, + { + "bits_per_byte": 1.563501158299828, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0837364196777344, + "step": 6160, + "unclipped_grad_norm": 0.7933995723724365 + }, + { + "bits_per_byte": 1.433251129564785, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 0.9934539794921875, + "step": 6170, + "unclipped_grad_norm": 0.728959321975708 + }, + { + "bits_per_byte": 1.4945863342378936, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.03596830368042, + "step": 6180, + "unclipped_grad_norm": 0.7535345554351807 + }, + { + "bits_per_byte": 1.5074559679846085, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0448888540267944, + "step": 6190, + "unclipped_grad_norm": 0.7919867038726807 + }, + { + "bits_per_byte": 1.4628087558726706, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.013941764831543, + "step": 6200, + "unclipped_grad_norm": 0.7595178484916687 + }, + { + "bits_per_byte": 1.4945321597028718, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0359307527542114, + "step": 6210, + "unclipped_grad_norm": 0.7536692023277283 + }, + { + "bits_per_byte": 1.4821234394604692, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.027329683303833, + "step": 6220, + "unclipped_grad_norm": 0.7611709833145142 + }, + { + "bits_per_byte": 1.4502368240945993, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0052275657653809, + "step": 6230, + "unclipped_grad_norm": 0.7465225458145142 + }, + { + "bits_per_byte": 1.4792031740488187, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0253055095672607, + "step": 6240, + "unclipped_grad_norm": 0.8008144497871399 + }, + { + "bits_per_byte": 1.4709335602647315, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.019573450088501, + "step": 6250, + "unclipped_grad_norm": 0.7485248446464539 + }, + { + "bits_per_byte": 1.5311833824282448, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0613354444503784, + "step": 6260, + "unclipped_grad_norm": 0.7729804515838623 + }, + { + "bits_per_byte": 1.494225858601685, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0357184410095215, + "step": 6270, + "unclipped_grad_norm": 0.7719584703445435 + }, + { + "bits_per_byte": 1.606436799070379, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1134971380233765, + "step": 6280, + "unclipped_grad_norm": 0.814758837223053 + }, + { + "bits_per_byte": 1.469587624039079, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0186405181884766, + "step": 6290, + "unclipped_grad_norm": 0.7629152536392212 + }, + { + "bits_per_byte": 1.4932348945674137, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0350315570831299, + "step": 6300, + "unclipped_grad_norm": 0.7575296759605408 + }, + { + "bits_per_byte": 1.5423225267419782, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.069056510925293, + "step": 6310, + "unclipped_grad_norm": 0.7936079502105713 + }, + { + "bits_per_byte": 1.497591903044372, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0380516052246094, + "step": 6320, + "unclipped_grad_norm": 0.7546678781509399 + }, + { + "bits_per_byte": 1.438224007914483, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 0.9969009160995483, + "step": 6330, + "unclipped_grad_norm": 0.7539783716201782 + }, + { + "bits_per_byte": 1.5101449167308483, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.046752691268921, + "step": 6340, + "unclipped_grad_norm": 0.7760982513427734 + }, + { + "bits_per_byte": 1.5028686747381446, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0417091846466064, + "step": 6350, + "unclipped_grad_norm": 0.8006681203842163 + }, + { + "bits_per_byte": 1.4519638738745637, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0064246654510498, + "step": 6360, + "unclipped_grad_norm": 0.7430040240287781 + }, + { + "bits_per_byte": 1.5063248380898837, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.044104814529419, + "step": 6370, + "unclipped_grad_norm": 0.8066008687019348 + }, + { + "bits_per_byte": 1.461864399136783, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0132871866226196, + "step": 6380, + "unclipped_grad_norm": 0.739668071269989 + }, + { + "bits_per_byte": 1.518872692296834, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.052802324295044, + "step": 6390, + "unclipped_grad_norm": 0.8261381983757019 + }, + { + "bits_per_byte": 1.4734527621345699, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0213196277618408, + "step": 6400, + "unclipped_grad_norm": 0.7401824593544006 + }, + { + "bits_per_byte": 1.4482483606853236, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0038492679595947, + "step": 6410, + "unclipped_grad_norm": 0.7752094268798828 + }, + { + "bits_per_byte": 1.4492296936911466, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0045294761657715, + "step": 6420, + "unclipped_grad_norm": 0.7460422515869141 + }, + { + "bits_per_byte": 1.4798391658917094, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0257463455200195, + "step": 6430, + "unclipped_grad_norm": 0.7759443521499634 + }, + { + "bits_per_byte": 1.4925646781770012, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0345669984817505, + "step": 6440, + "unclipped_grad_norm": 0.7866507768630981 + }, + { + "bits_per_byte": 1.473837315341899, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0215861797332764, + "step": 6450, + "unclipped_grad_norm": 0.7885896563529968 + }, + { + "bits_per_byte": 1.452434934355277, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0067511796951294, + "step": 6460, + "unclipped_grad_norm": 0.7844802737236023 + }, + { + "bits_per_byte": 1.4767341911130323, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0235941410064697, + "step": 6470, + "unclipped_grad_norm": 0.7592610716819763 + }, + { + "bits_per_byte": 1.5086992305676956, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.045750617980957, + "step": 6480, + "unclipped_grad_norm": 0.8035558462142944 + }, + { + "bits_per_byte": 1.4908353926225757, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0333683490753174, + "step": 6490, + "unclipped_grad_norm": 0.8066015243530273 + }, + { + "bits_per_byte": 1.4884610001447638, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0317225456237793, + "step": 6500, + "unclipped_grad_norm": 0.7627172470092773 + }, + { + "bits_per_byte": 1.4363769142442164, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 0.995620608329773, + "step": 6510, + "unclipped_grad_norm": 0.7699123024940491 + }, + { + "bits_per_byte": 1.4159479550614777, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9814603328704834, + "step": 6520, + "unclipped_grad_norm": 0.7812637090682983 + }, + { + "bits_per_byte": 1.498887964301274, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.038949966430664, + "step": 6530, + "unclipped_grad_norm": 0.8116927742958069 + }, + { + "bits_per_byte": 1.4673561491441343, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0170937776565552, + "step": 6540, + "unclipped_grad_norm": 0.7371414303779602 + }, + { + "bits_per_byte": 1.5117546743429242, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0478684902191162, + "step": 6550, + "unclipped_grad_norm": 0.8187429308891296 + }, + { + "bits_per_byte": 1.4773364743563537, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0240116119384766, + "step": 6560, + "unclipped_grad_norm": 0.7700304388999939 + }, + { + "bits_per_byte": 1.4043874532531306, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 0.9734472036361694, + "step": 6570, + "unclipped_grad_norm": 0.7884607315063477 + }, + { + "bits_per_byte": 1.4975829599465271, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0380454063415527, + "step": 6580, + "unclipped_grad_norm": 0.778003454208374 + }, + { + "bits_per_byte": 1.4674713775202122, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0171736478805542, + "step": 6590, + "unclipped_grad_norm": 0.7916907668113708 + }, + { + "bits_per_byte": 1.3998400599816672, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9702951908111572, + "step": 6600, + "unclipped_grad_norm": 0.7509376406669617 + }, + { + "bits_per_byte": 1.5187203156681697, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.052696704864502, + "step": 6610, + "unclipped_grad_norm": 0.7783279418945312 + }, + { + "bits_per_byte": 1.511926828976438, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0479878187179565, + "step": 6620, + "unclipped_grad_norm": 0.8419784903526306 + }, + { + "bits_per_byte": 1.5028970518755371, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0417288541793823, + "step": 6630, + "unclipped_grad_norm": 0.8222247958183289 + }, + { + "bits_per_byte": 1.4773347545298452, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.024010419845581, + "step": 6640, + "unclipped_grad_norm": 0.8660594820976257 + }, + { + "bits_per_byte": 1.4614148364874275, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0129755735397339, + "step": 6650, + "unclipped_grad_norm": 0.7894479036331177 + }, + { + "bits_per_byte": 1.5265555012761776, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0581276416778564, + "step": 6660, + "unclipped_grad_norm": 0.810829758644104 + }, + { + "bits_per_byte": 1.4984096805492246, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0386184453964233, + "step": 6670, + "unclipped_grad_norm": 0.8341873288154602 + }, + { + "bits_per_byte": 1.5059076081788905, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0438156127929688, + "step": 6680, + "unclipped_grad_norm": 0.822468638420105 + }, + { + "bits_per_byte": 1.4780378196065724, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0244977474212646, + "step": 6690, + "unclipped_grad_norm": 0.810896635055542 + }, + { + "bits_per_byte": 1.524204498438883, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0564980506896973, + "step": 6700, + "unclipped_grad_norm": 0.8146765828132629 + }, + { + "bits_per_byte": 1.451385668202363, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.00602388381958, + "step": 6710, + "unclipped_grad_norm": 0.8321484327316284 + }, + { + "bits_per_byte": 1.4556473982907434, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0089778900146484, + "step": 6720, + "unclipped_grad_norm": 0.781524658203125 + }, + { + "bits_per_byte": 1.448410712307738, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0039618015289307, + "step": 6730, + "unclipped_grad_norm": 0.8034297227859497 + }, + { + "bits_per_byte": 1.4779710903380376, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.024451494216919, + "step": 6740, + "unclipped_grad_norm": 0.8218453526496887 + }, + { + "bits_per_byte": 1.4435792036970492, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0006128549575806, + "step": 6750, + "unclipped_grad_norm": 0.7989961504936218 + }, + { + "bits_per_byte": 1.4296412137231747, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 0.9909517765045166, + "step": 6760, + "unclipped_grad_norm": 0.7599804997444153 + }, + { + "bits_per_byte": 1.4469161830717403, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.0029258728027344, + "step": 6770, + "unclipped_grad_norm": 0.7921444773674011 + }, + { + "bits_per_byte": 1.438630832875099, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 0.9971829056739807, + "step": 6780, + "unclipped_grad_norm": 0.7756091356277466 + }, + { + "bits_per_byte": 1.5384264317693321, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0663559436798096, + "step": 6790, + "unclipped_grad_norm": 0.8340134024620056 + }, + { + "bits_per_byte": 1.5144365718004789, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.049727439880371, + "step": 6800, + "unclipped_grad_norm": 0.8667078614234924 + }, + { + "bits_per_byte": 1.4971179188585941, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0377230644226074, + "step": 6810, + "unclipped_grad_norm": 0.8064154386520386 + }, + { + "bits_per_byte": 1.4405763866129846, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 0.9985314607620239, + "step": 6820, + "unclipped_grad_norm": 0.7713363170623779 + }, + { + "bits_per_byte": 1.4385394240961653, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 0.9971195459365845, + "step": 6830, + "unclipped_grad_norm": 0.8126972317695618 + }, + { + "bits_per_byte": 1.4959006256557872, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.036879301071167, + "step": 6840, + "unclipped_grad_norm": 0.7896748185157776 + }, + { + "bits_per_byte": 1.465242826330332, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0156289339065552, + "step": 6850, + "unclipped_grad_norm": 0.8369152545928955 + }, + { + "bits_per_byte": 1.4691382333723746, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0183290243148804, + "step": 6860, + "unclipped_grad_norm": 0.8024821877479553 + }, + { + "bits_per_byte": 1.4717907217966317, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.020167589187622, + "step": 6870, + "unclipped_grad_norm": 0.808424174785614 + }, + { + "bits_per_byte": 1.5114406340224489, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0476508140563965, + "step": 6880, + "unclipped_grad_norm": 0.8209573030471802 + }, + { + "bits_per_byte": 1.4045318326885299, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9735472798347473, + "step": 6890, + "unclipped_grad_norm": 0.7931460738182068 + }, + { + "bits_per_byte": 1.436284731543354, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 0.9955567121505737, + "step": 6900, + "unclipped_grad_norm": 0.7861348390579224 + }, + { + "bits_per_byte": 1.6150507221214936, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.119467854499817, + "step": 6910, + "unclipped_grad_norm": 0.8759272694587708 + }, + { + "bits_per_byte": 1.4220056999728183, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 0.9856592416763306, + "step": 6920, + "unclipped_grad_norm": 0.7773904204368591 + }, + { + "bits_per_byte": 1.4332720254568647, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9934684634208679, + "step": 6930, + "unclipped_grad_norm": 0.8008426427841187 + }, + { + "bits_per_byte": 1.46887819560427, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0181487798690796, + "step": 6940, + "unclipped_grad_norm": 0.8342399597167969 + }, + { + "bits_per_byte": 1.4565233059315876, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 1.0095850229263306, + "step": 6950, + "unclipped_grad_norm": 0.8141571283340454 + }, + { + "bits_per_byte": 1.5009811651449254, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0404008626937866, + "step": 6960, + "unclipped_grad_norm": 0.8327651619911194 + }, + { + "bits_per_byte": 1.4767801104808127, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0236259698867798, + "step": 6970, + "unclipped_grad_norm": 0.801481306552887 + }, + { + "bits_per_byte": 1.517935386849632, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0521526336669922, + "step": 6980, + "unclipped_grad_norm": 0.8559384942054749 + }, + { + "bits_per_byte": 1.462364696668143, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0136339664459229, + "step": 6990, + "unclipped_grad_norm": 0.8250771164894104 + }, + { + "bits_per_byte": 1.4113358963132898, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 0.9782634973526001, + "step": 7000, + "unclipped_grad_norm": 0.8160101771354675 + }, + { + "bits_per_byte": 1.4572850170922589, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.010113000869751, + "step": 7010, + "unclipped_grad_norm": 0.8388007283210754 + }, + { + "bits_per_byte": 1.4311967968002286, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 0.9920300245285034, + "step": 7020, + "unclipped_grad_norm": 0.8063672780990601 + }, + { + "bits_per_byte": 1.4271649215247264, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 0.9892353415489197, + "step": 7030, + "unclipped_grad_norm": 0.7880839109420776 + }, + { + "bits_per_byte": 1.5230513547648479, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0556987524032593, + "step": 7040, + "unclipped_grad_norm": 0.8723810911178589 + }, + { + "bits_per_byte": 1.48768088684045, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.031181812286377, + "step": 7050, + "unclipped_grad_norm": 0.8570515513420105 + }, + { + "bits_per_byte": 1.4927676177050193, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0347076654434204, + "step": 7060, + "unclipped_grad_norm": 0.8982753157615662 + }, + { + "bits_per_byte": 1.4459658069430723, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0022671222686768, + "step": 7070, + "unclipped_grad_norm": 0.8039788603782654 + }, + { + "bits_per_byte": 1.364939964607379, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9461042881011963, + "step": 7080, + "unclipped_grad_norm": 0.7935104966163635 + }, + { + "bits_per_byte": 1.4353330655448047, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.9948970675468445, + "step": 7090, + "unclipped_grad_norm": 0.810722291469574 + }, + { + "bits_per_byte": 1.4301969756594377, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9913370013237, + "step": 7100, + "unclipped_grad_norm": 0.7980725169181824 + }, + { + "bits_per_byte": 1.4812796925753362, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0267448425292969, + "step": 7110, + "unclipped_grad_norm": 0.8589518666267395 + }, + { + "bits_per_byte": 1.491397775890897, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0337581634521484, + "step": 7120, + "unclipped_grad_norm": 0.881410539150238 + }, + { + "bits_per_byte": 1.4146181852050066, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 0.9805386066436768, + "step": 7130, + "unclipped_grad_norm": 0.8223039507865906 + }, + { + "bits_per_byte": 1.522814190689308, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0555343627929688, + "step": 7140, + "unclipped_grad_norm": 0.8451608419418335 + }, + { + "bits_per_byte": 1.5467134158011566, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.072100043296814, + "step": 7150, + "unclipped_grad_norm": 0.8498587012290955 + }, + { + "bits_per_byte": 1.4723551688567635, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0205588340759277, + "step": 7160, + "unclipped_grad_norm": 0.8513677716255188 + }, + { + "bits_per_byte": 1.4676500674944588, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0172975063323975, + "step": 7170, + "unclipped_grad_norm": 0.8321399092674255 + }, + { + "bits_per_byte": 1.3859237398218012, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 0.9606491327285767, + "step": 7180, + "unclipped_grad_norm": 0.7659841179847717 + }, + { + "bits_per_byte": 1.3902817801946648, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9636698961257935, + "step": 7190, + "unclipped_grad_norm": 0.7895154356956482 + }, + { + "bits_per_byte": 1.4776350362382518, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0242185592651367, + "step": 7200, + "unclipped_grad_norm": 0.8478621244430542 + }, + { + "bits_per_byte": 1.4778720283311406, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.0243828296661377, + "step": 7210, + "unclipped_grad_norm": 0.8023228049278259 + }, + { + "bits_per_byte": 1.4423936412933265, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9997910857200623, + "step": 7220, + "unclipped_grad_norm": 0.8181896805763245 + }, + { + "bits_per_byte": 1.4464050506333759, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0025715827941895, + "step": 7230, + "unclipped_grad_norm": 0.8425759077072144 + }, + { + "bits_per_byte": 1.4082518174266925, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 0.9761257767677307, + "step": 7240, + "unclipped_grad_norm": 0.7972860336303711 + }, + { + "bits_per_byte": 1.4478307868090285, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0035598278045654, + "step": 7250, + "unclipped_grad_norm": 0.8181965947151184 + }, + { + "bits_per_byte": 1.4351106059859136, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9947428703308105, + "step": 7260, + "unclipped_grad_norm": 0.8426879048347473 + }, + { + "bits_per_byte": 1.5111732010003571, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.047465443611145, + "step": 7270, + "unclipped_grad_norm": 0.862413763999939 + }, + { + "bits_per_byte": 1.457399041589781, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0101920366287231, + "step": 7280, + "unclipped_grad_norm": 0.8488914370536804 + }, + { + "bits_per_byte": 1.4536087159474156, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0075647830963135, + "step": 7290, + "unclipped_grad_norm": 0.8383923172950745 + }, + { + "bits_per_byte": 1.4305855704590622, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 0.9916063547134399, + "step": 7300, + "unclipped_grad_norm": 0.8480672240257263 + }, + { + "bits_per_byte": 1.4800558640317965, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0258965492248535, + "step": 7310, + "unclipped_grad_norm": 0.8378974795341492 + }, + { + "bits_per_byte": 1.4690943777964045, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.018298625946045, + "step": 7320, + "unclipped_grad_norm": 0.8481025695800781 + }, + { + "bits_per_byte": 1.4814797083982898, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0268834829330444, + "step": 7330, + "unclipped_grad_norm": 0.9106674194335938 + }, + { + "bits_per_byte": 1.449738418372399, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 1.0048820972442627, + "step": 7340, + "unclipped_grad_norm": 0.8405259847640991 + }, + { + "bits_per_byte": 1.486937577823421, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0306665897369385, + "step": 7350, + "unclipped_grad_norm": 0.8228537440299988 + }, + { + "bits_per_byte": 1.4739011209053692, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0216304063796997, + "step": 7360, + "unclipped_grad_norm": 0.8316923975944519 + }, + { + "bits_per_byte": 1.4767537971352307, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0236077308654785, + "step": 7370, + "unclipped_grad_norm": 0.8346049785614014 + }, + { + "bits_per_byte": 1.5318123229824503, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0617713928222656, + "step": 7380, + "unclipped_grad_norm": 0.9013696908950806 + }, + { + "bits_per_byte": 1.5039485538029123, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0424576997756958, + "step": 7390, + "unclipped_grad_norm": 0.8856703639030457 + }, + { + "bits_per_byte": 1.4314098833046476, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9921777248382568, + "step": 7400, + "unclipped_grad_norm": 0.8249487280845642 + }, + { + "bits_per_byte": 1.4394619390353935, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9977589845657349, + "step": 7410, + "unclipped_grad_norm": 0.8362482786178589 + }, + { + "bits_per_byte": 1.518494330464936, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0525400638580322, + "step": 7420, + "unclipped_grad_norm": 0.8421095609664917 + }, + { + "bits_per_byte": 1.4326227909498577, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9930184483528137, + "step": 7430, + "unclipped_grad_norm": 0.8718878626823425 + }, + { + "bits_per_byte": 1.4438445729273304, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0007967948913574, + "step": 7440, + "unclipped_grad_norm": 0.8476481437683105 + }, + { + "bits_per_byte": 1.3938656406646694, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 0.9661540389060974, + "step": 7450, + "unclipped_grad_norm": 0.8507251739501953 + }, + { + "bits_per_byte": 1.420028243453198, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9842885732650757, + "step": 7460, + "unclipped_grad_norm": 0.8349717259407043 + }, + { + "bits_per_byte": 1.4775118966602339, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0241332054138184, + "step": 7470, + "unclipped_grad_norm": 0.8582044243812561 + }, + { + "bits_per_byte": 1.3999588999934132, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 0.9703775644302368, + "step": 7480, + "unclipped_grad_norm": 0.84687340259552 + }, + { + "bits_per_byte": 1.4226986180731447, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 0.9861395359039307, + "step": 7490, + "unclipped_grad_norm": 0.831066370010376 + }, + { + "bits_per_byte": 1.4892197876003705, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0322484970092773, + "step": 7500, + "unclipped_grad_norm": 0.8232710361480713 + }, + { + "bits_per_byte": 1.461985990870943, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.013371467590332, + "step": 7510, + "unclipped_grad_norm": 0.8441055417060852 + }, + { + "bits_per_byte": 1.4446456681150495, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.001352071762085, + "step": 7520, + "unclipped_grad_norm": 0.8653393387794495 + }, + { + "bits_per_byte": 1.4775328785436392, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0241477489471436, + "step": 7530, + "unclipped_grad_norm": 0.8701198697090149 + }, + { + "bits_per_byte": 1.5027517265355579, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.041628122329712, + "step": 7540, + "unclipped_grad_norm": 0.8485988974571228 + }, + { + "bits_per_byte": 1.4225563884208812, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 0.9860409498214722, + "step": 7550, + "unclipped_grad_norm": 0.8343487977981567 + }, + { + "bits_per_byte": 1.4745732291049412, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0220962762832642, + "step": 7560, + "unclipped_grad_norm": 0.8472090363502502 + }, + { + "bits_per_byte": 1.480253472097638, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0260335206985474, + "step": 7570, + "unclipped_grad_norm": 0.8703269958496094 + }, + { + "bits_per_byte": 1.4382653697420156, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9969295859336853, + "step": 7580, + "unclipped_grad_norm": 0.847757875919342 + }, + { + "bits_per_byte": 1.4346089325933469, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9943951368331909, + "step": 7590, + "unclipped_grad_norm": 0.845406711101532 + }, + { + "bits_per_byte": 1.414465206637064, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9804325699806213, + "step": 7600, + "unclipped_grad_norm": 0.8379301428794861 + }, + { + "bits_per_byte": 1.3641118681434745, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9455302953720093, + "step": 7610, + "unclipped_grad_norm": 0.7969970107078552 + }, + { + "bits_per_byte": 1.4648639485504813, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0153663158416748, + "step": 7620, + "unclipped_grad_norm": 0.8826617002487183 + }, + { + "bits_per_byte": 1.4355323934371547, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9950352311134338, + "step": 7630, + "unclipped_grad_norm": 0.8636281490325928 + }, + { + "bits_per_byte": 1.4410591419139565, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.998866081237793, + "step": 7640, + "unclipped_grad_norm": 0.8592743277549744 + }, + { + "bits_per_byte": 1.5176450801349755, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0519514083862305, + "step": 7650, + "unclipped_grad_norm": 0.9256166219711304 + }, + { + "bits_per_byte": 1.4474489853241133, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0032951831817627, + "step": 7660, + "unclipped_grad_norm": 0.8606430292129517 + }, + { + "bits_per_byte": 1.4195236463555665, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 0.9839388132095337, + "step": 7670, + "unclipped_grad_norm": 0.8635953664779663 + }, + { + "bits_per_byte": 1.3989710316468573, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9696928262710571, + "step": 7680, + "unclipped_grad_norm": 0.8459085822105408 + }, + { + "bits_per_byte": 1.446349844202449, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0025333166122437, + "step": 7690, + "unclipped_grad_norm": 0.8500027656555176 + }, + { + "bits_per_byte": 1.456846289349908, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.009808897972107, + "step": 7700, + "unclipped_grad_norm": 0.896350085735321 + }, + { + "bits_per_byte": 1.3794107568336267, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9561346769332886, + "step": 7710, + "unclipped_grad_norm": 0.8459534049034119 + }, + { + "bits_per_byte": 1.3797531742914946, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9563720226287842, + "step": 7720, + "unclipped_grad_norm": 0.8360687494277954 + }, + { + "bits_per_byte": 1.4376540574095236, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.996505856513977, + "step": 7730, + "unclipped_grad_norm": 0.82816481590271 + }, + { + "bits_per_byte": 1.4181697129366488, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9830003380775452, + "step": 7740, + "unclipped_grad_norm": 0.83390212059021 + }, + { + "bits_per_byte": 1.4407892151434272, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.998678982257843, + "step": 7750, + "unclipped_grad_norm": 0.8495392203330994 + }, + { + "bits_per_byte": 1.4190407190719436, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9836040735244751, + "step": 7760, + "unclipped_grad_norm": 0.850292444229126 + }, + { + "bits_per_byte": 1.4212888762840223, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9851623773574829, + "step": 7770, + "unclipped_grad_norm": 0.888531506061554 + }, + { + "bits_per_byte": 1.5506865590013892, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0748540163040161, + "step": 7780, + "unclipped_grad_norm": 0.9023171067237854 + }, + { + "bits_per_byte": 1.4346873566821403, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9944494962692261, + "step": 7790, + "unclipped_grad_norm": 0.8470334410667419 + }, + { + "bits_per_byte": 1.4727918328073042, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0208615064620972, + "step": 7800, + "unclipped_grad_norm": 0.9052314758300781 + }, + { + "bits_per_byte": 1.3950921349392975, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9670041799545288, + "step": 7810, + "unclipped_grad_norm": 0.841830849647522 + }, + { + "bits_per_byte": 1.4118079886899082, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.978590726852417, + "step": 7820, + "unclipped_grad_norm": 0.8733639121055603 + }, + { + "bits_per_byte": 1.4649716096899212, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0154409408569336, + "step": 7830, + "unclipped_grad_norm": 0.851719081401825 + }, + { + "bits_per_byte": 1.4172854641372379, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9823874235153198, + "step": 7840, + "unclipped_grad_norm": 0.8457031846046448 + }, + { + "bits_per_byte": 1.3998510668713222, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9703028202056885, + "step": 7850, + "unclipped_grad_norm": 0.8359724283218384 + }, + { + "bits_per_byte": 1.413704183406996, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 0.9799050688743591, + "step": 7860, + "unclipped_grad_norm": 0.8466342091560364 + }, + { + "bits_per_byte": 1.4588555626599382, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0112016201019287, + "step": 7870, + "unclipped_grad_norm": 0.8837466239929199 + }, + { + "bits_per_byte": 1.4292317230314702, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9906679391860962, + "step": 7880, + "unclipped_grad_norm": 0.8551597595214844 + }, + { + "bits_per_byte": 1.5078522160121963, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0451635122299194, + "step": 7890, + "unclipped_grad_norm": 0.8927572965621948 + }, + { + "bits_per_byte": 1.4757206973514978, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.022891640663147, + "step": 7900, + "unclipped_grad_norm": 0.8807728290557861 + }, + { + "bits_per_byte": 1.4780766876856672, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0245246887207031, + "step": 7910, + "unclipped_grad_norm": 0.9293215870857239 + }, + { + "bits_per_byte": 1.3870213330996077, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 0.9614099264144897, + "step": 7920, + "unclipped_grad_norm": 0.8720042109489441 + }, + { + "bits_per_byte": 1.3890644869918578, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.962826132774353, + "step": 7930, + "unclipped_grad_norm": 0.8701853156089783 + }, + { + "bits_per_byte": 1.4495222361802644, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0047322511672974, + "step": 7940, + "unclipped_grad_norm": 0.8989008069038391 + }, + { + "bits_per_byte": 1.4133007980993975, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9796254634857178, + "step": 7950, + "unclipped_grad_norm": 0.839148759841919 + }, + { + "bits_per_byte": 1.4208300265715204, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9848443269729614, + "step": 7960, + "unclipped_grad_norm": 0.8453875184059143 + }, + { + "bits_per_byte": 1.41087549875693, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9779443740844727, + "step": 7970, + "unclipped_grad_norm": 0.8587523102760315 + }, + { + "bits_per_byte": 1.446222061092858, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0024447441101074, + "step": 7980, + "unclipped_grad_norm": 0.8559814691543579 + }, + { + "bits_per_byte": 1.4566966644436572, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.0097051858901978, + "step": 7990, + "unclipped_grad_norm": 0.860873818397522 + }, + { + "bits_per_byte": 1.3805011268400968, + "learning_rate": 3e-05, + "loss_nats": 0.9568904638290405, + "step": 8000, + "unclipped_grad_norm": 0.8578172326087952 + } + ] + }, + "depth-32-block-seed-2026073001": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "c7554d9beea6fe9e63617aafd88403f7fc93b4b305a0c6d603a98a8572e5b5f1", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "population_cv": 0.2714186609524326 + }, + "activation_output_rms_by_block": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.0056560162047389895, + "first_to_last_ratio": 0.5480088979804595, + "imbalance_abs_log_ratio": 0.6014637549753238, + "last_quartile_mean": 0.010321029869373888, + "mean": 0.007962599374877755, + "normalized": [ + 0.4381177659237371, + 0.606836484864952, + 0.747504205668035, + 0.8697330807712821, + 0.4836113263475722, + 0.6961220406611074, + 0.845112084785299, + 0.9955457339856744, + 0.512559862622497, + 0.7298129234151854, + 0.9462750278110585, + 1.077778999737589, + 0.6278355146046978, + 0.8919850673170736, + 1.072884836415099, + 1.3120964842531853, + 0.6331203314377778, + 0.9591008609440675, + 1.1788918244911069, + 1.384035094304029, + 0.7292868277593074, + 1.0508372411895934, + 1.2868063479276561, + 1.554601902603762, + 0.8104324041933391, + 1.1463105029730196, + 1.3862254440922965, + 1.5837857082010076, + 0.8804162846646224, + 1.2300986632667847, + 1.5630327667805075, + 1.7692063559870779 + ], + "population_cv": 0.3451071044649734 + }, + "bits_per_byte": 7.959341947457508, + "branch_output_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0027073738165199757, + 0.0021613112185150385, + 0.002722575329244137, + 0.0025601957459002733, + 0.0026850115973502398, + 0.0025323182344436646, + 0.0026861224323511124, + 0.0027253651060163975, + 0.00271838391199708, + 0.002904455177485943, + 0.0026645169127732515, + 0.0029208383057266474, + 0.0026859452482312918, + 0.00299624796025455, + 0.0027359756641089916, + 0.0030937574338167906, + 0.0026767742820084095, + 0.0033804215490818024, + 0.0027451682835817337, + 0.0034370317589491606, + 0.0026999011170119047, + 0.0038969130255281925, + 0.002663193503394723, + 0.004255065228790045, + 0.002626648638397455, + 0.004119740333408117, + 0.0026531771291047335, + 0.003970506135374308, + 0.0026517712976783514, + 0.004385598469525576, + 0.002676903735846281, + 0.0041999719105660915, + 0.002670086920261383, + 0.005232699681073427, + 0.0026355870068073273, + 0.004664790350943804, + 0.0026610493659973145, + 0.004898969549685717, + 0.0026388936676084995, + 0.005131533369421959, + 0.0026469926815479994, + 0.005069608334451914, + 0.00264862016774714, + 0.0051318020559847355, + 0.002641494618728757, + 0.0058372304774820805, + 0.002674198243767023, + 0.0059690922498703, + 0.002557571977376938, + 0.00554921617731452, + 0.00252733426168561, + 0.0058550527319312096, + 0.0025468147359788418, + 0.006255472078919411, + 0.0026856842450797558, + 0.006502739619463682, + 0.002551089273765683, + 0.006122888531535864, + 0.0025778315030038357, + 0.007071305997669697, + 0.0026001501828432083, + 0.006241494789719582, + 0.0025662274565547705 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0056993408498569815, + 0.005586029321860134, + 0.005830344684528519, + 0.00540288712216694, + 0.00538025811909819, + 0.0052226633069546126, + 0.005297605878918106, + 0.0052023753769565, + 0.0048107019260181325, + 0.005248672753719287, + 0.004962353375385338, + 0.005501319898161136, + 0.005588445553926757, + 0.005229606380765812, + 0.004884302670286647, + 0.005031817046498285, + 0.005000470961685071, + 0.0053937604587735305, + 0.004980398173225761, + 0.004658922297714218, + 0.00496598185096497, + 0.004578385934052906, + 0.004596161734371138, + 0.004990442293511754, + 0.005201771295872185, + 0.00459813521112035, + 0.004706502430387731, + 0.004781878207244352, + 0.0048378722082161795, + 0.004473110720597102, + 0.005084432926394394, + 0.004685705425265166 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005452688082542498, + "first_to_last_ratio": 1.1368823875795573, + "imbalance_abs_log_ratio": 0.12828976841498158, + "last_quartile_mean": 0.004796176053137182, + "mean": 0.005075395512328069, + "normalized": [ + 1.122935313319594, + 1.1006096585560166, + 1.1487468652180286, + 1.064525337787646, + 1.060066768398564, + 1.0290160233362764, + 1.0437818818356701, + 1.025018713185997, + 0.947847692723257, + 1.0341406380981206, + 0.9777274230809889, + 1.0839194472230789, + 1.1010857262951226, + 1.0303840100861434, + 0.9623491722808084, + 0.9914137793352396, + 0.9852376922229987, + 1.0627271206100406, + 0.9812827711906272, + 0.9179427074003902, + 0.9784423379227619, + 0.9020747098294245, + 0.9055770576316904, + 0.9832617539638114, + 1.024899691706223, + 0.905965889742295, + 0.9273173724009683, + 0.9421685848185097, + 0.9532010257062827, + 0.8813324419214177, + 1.00178063247374, + 0.9232197596982638 + ], + "population_cv": 0.06921897980134169 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02802751213312149, + 0.014061377383768559, + 0.01413465291261673, + 0.014165290631353855, + 0.014225807972252369, + 0.014281988143920898, + 0.01433455292135477, + 0.014350161887705326, + 0.01440478302538395, + 0.00966146495193243, + 0.009703479707241058, + 0.009741888381540775, + 0.00977881159633398, + 0.009820535778999329, + 0.009858808480203152, + 0.00991273857653141, + 0.009950309991836548, + 0.0074927168898284435, + 0.00752287870272994, + 0.007546746172010899, + 0.007571155205368996, + 0.00766373984515667, + 0.007693085819482803, + 0.007762390188872814, + 0.007788818795233965, + 0.006325984839349985, + 0.006353048607707024, + 0.006375366821885109, + 0.006389216054230928, + 0.006445344537496567, + 0.0064707002602517605, + 0.006555322092026472, + 0.00656962301582098, + 0.005531130358576775, + 0.005553483963012695, + 0.005611324217170477, + 0.0056249531917274, + 0.0056777093559503555, + 0.005688921548426151, + 0.00574630219489336, + 0.005768724251538515, + 0.004946626722812653, + 0.004966245498508215, + 0.0050392793491482735, + 0.005057469941675663, + 0.005084683652967215, + 0.005098348017781973, + 0.005203457549214363, + 0.005216081161051989, + 0.004667261149734259, + 0.0046758754178881645, + 0.004717131145298481, + 0.004727107938379049, + 0.004768412560224533, + 0.004777040798217058, + 0.004826710093766451, + 0.004837613552808762, + 0.0044052014127373695, + 0.0044186655431985855, + 0.004504534415900707, + 0.004510571248829365, + 0.004574235528707504, + 0.0045849340967834, + 0.004644096828997135 + ], + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0022265110164880753, + 0.0034885562490671873, + 0.0040087271481752396, + 0.004831995815038681, + 0.005379301495850086, + 0.005952076520770788, + 0.006410556845366955, + 0.006925336085259914, + 0.0027253651060163975, + 0.00385080324485898, + 0.004838948138058186, + 0.005542940925806761, + 0.006175771821290255, + 0.006729288958013058, + 0.00742313964292407, + 0.007927131839096546, + 0.0030937574338167906, + 0.004081308841705322, + 0.005150026176124811, + 0.005811207927763462, + 0.007019025273621082, + 0.007534808944910765, + 0.008143599145114422, + 0.008581922389566898, + 0.004255065228790045, + 0.00499920267611742, + 0.0066224560141563416, + 0.007102519739419222, + 0.008129785768687725, + 0.008542952127754688, + 0.010099578648805618, + 0.010447698645293713, + 0.0041999719105660915, + 0.005041283555328846, + 0.00719039561226964, + 0.007636935915797949, + 0.009030314162373543, + 0.009387043304741383, + 0.010736005380749702, + 0.011020516976714134, + 0.005131533369421959, + 0.005807018838822842, + 0.00787449348717928, + 0.008367395959794521, + 0.009912566281855106, + 0.01024632342159748, + 0.012113703414797783, + 0.012378672137856483, + 0.0059690922498703, + 0.006453148555010557, + 0.00876442901790142, + 0.009127611294388771, + 0.010806876234710217, + 0.011037957854568958, + 0.012353328987956047, + 0.012611051090061665, + 0.006502739619463682, + 0.007010402157902718, + 0.009470963850617409, + 0.009794782847166061, + 0.012123330496251583, + 0.012445803731679916, + 0.01384514570236206, + 0.014087481424212456 + ] + }, + { + "activation_grad_rms_by_block": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "population_cv": 0.5072011380661324 + }, + "activation_output_rms_by_block": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013632480811793357, + "first_to_last_ratio": 0.32005493218522185, + "imbalance_abs_log_ratio": 1.1392626348419668, + "last_quartile_mean": 0.04259419068694115, + "mean": 0.025403792154975235, + "normalized": [ + 0.23092935972382672, + 0.3992639012357397, + 0.5833875213879504, + 0.7700561688360058, + 0.2642123711098415, + 0.4687595594389148, + 0.6729013896370168, + 0.9035435305603466, + 0.2950160146386087, + 0.5418894364067922, + 0.8025599005701267, + 1.0558320747576517, + 0.36189473544610173, + 0.6606003816028687, + 0.9564482474371727, + 1.2361787191078082, + 0.389077999650742, + 0.7376930678145923, + 1.1018562250488806, + 1.4948807199009364, + 0.4969232792693754, + 0.9255949835565679, + 1.3737978229029388, + 1.8632120600111322, + 0.6868422339014575, + 1.1618129342705898, + 1.7580201402629136, + 2.3690874226600394, + 0.7468506348084945, + 1.4805931380763324, + 2.2249315569778956, + 2.985352468990339 + ], + "population_cv": 0.6518934572295025 + }, + "bits_per_byte": 5.150682269327325, + "branch_output_rms_by_sublayer": [ + 0.002919268561527133, + 0.005079356953501701, + 0.0026904908008873463, + 0.0052208867855370045, + 0.0028405722696334124, + 0.005689430516213179, + 0.0029144894797354937, + 0.005868426524102688, + 0.0030634324066340923, + 0.00590060418471694, + 0.0032732728868722916, + 0.0059102908708155155, + 0.003848863998427987, + 0.0058776624500751495, + 0.003392934100702405, + 0.006663382984697819, + 0.0037359180860221386, + 0.006277414970099926, + 0.004302032291889191, + 0.006413500756025314, + 0.004344160668551922, + 0.006856496445834637, + 0.004835603758692741, + 0.006499431096017361, + 0.005531937349587679, + 0.006718362681567669, + 0.005483300890773535, + 0.0070298356004059315, + 0.005397035740315914, + 0.006969383917748928, + 0.005739198997616768, + 0.0068367342464625835, + 0.005659298039972782, + 0.007033169735223055, + 0.007230010814964771, + 0.007157569285482168, + 0.006938732694834471, + 0.0075136395171284676, + 0.007343437522649765, + 0.007708695251494646, + 0.007882392033934593, + 0.007887008599936962, + 0.00830139685422182, + 0.008205465972423553, + 0.008992243558168411, + 0.00814912747591734, + 0.009393857792019844, + 0.008287571370601654, + 0.012831038795411587, + 0.008026733994483948, + 0.00905293133109808, + 0.008120592683553696, + 0.012243949808180332, + 0.008479002863168716, + 0.012258118949830532, + 0.00907627958804369, + 0.013431218452751637, + 0.00841403380036354, + 0.01483316533267498, + 0.008773921057581902, + 0.015388047322630882, + 0.008821850642561913, + 0.014634119346737862, + 0.009164483286440372 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00039664652254938686, + 0.00033413090452743513, + 0.0003161152806455274, + 0.0003097753114302562, + 0.00028318594507881776, + 0.00028174654548060826, + 0.00029366070958977545, + 0.0002617199768643518, + 0.00023850476284541366, + 0.0002578321956415913, + 0.00023580491915923377, + 0.00024129375767696618, + 0.00024677521358199467, + 0.00023282043614108414, + 0.00020802360655182162, + 0.00020816264493602737, + 0.00019546204845585216, + 0.00020244473639840523, + 0.00019508742079131332, + 0.00018509966437476374, + 0.00018490270602469487, + 0.0001750081453781441, + 0.00017332318168420213, + 0.00016951527760705314, + 0.0001849276373753642, + 0.00014904354377519617, + 0.00017214209100244258, + 0.0001582098123018746, + 0.0001606899226932654, + 0.0001683991686835421, + 0.00016911992547040899, + 0.00016138660409180947 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00030962264952076986, + "first_to_last_ratio": 1.8709465967014842, + "imbalance_abs_log_ratio": 0.6264445042344079, + "last_quartile_mean": 0.00016548983817423794, + "mean": 0.0002234675193377695, + "normalized": [ + 1.774962749507496, + 1.4952101563466982, + 1.4145916220053505, + 1.3862207463001954, + 1.2672353723620313, + 1.2607941696204648, + 1.314109139708619, + 1.1711768119140573, + 1.0672905107292818, + 1.1537792892929548, + 1.0552089174213117, + 1.0797710485712801, + 1.104300143095945, + 1.0418535849461763, + 0.9308896754583627, + 0.9315118615577912, + 0.8746776669606858, + 0.9059246596477921, + 0.8730012369110346, + 0.828306793413619, + 0.8274254199117672, + 0.7831480203331949, + 0.775607937107967, + 0.7585678585836542, + 0.827536985792765, + 0.6669584206996899, + 0.7703226469447271, + 0.7079767689314242, + 0.719075072607683, + 0.7535733568018358, + 0.7567986881117407, + 0.7221926684024036 + ], + "population_cv": 0.268339881652204 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6930912733078003, + "mean_weights": [ + 0.5023526549339294, + 0.49764734506607056 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692903995513916, + "mean_weights": [ + 0.5103387832641602, + 0.48966121673583984 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928629875183105, + "mean_weights": [ + 0.5053091049194336, + 0.4946908950805664 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927823424339294, + "mean_weights": [ + 0.5126834511756897, + 0.4873165488243103 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928155422210693, + "mean_weights": [ + 0.502953290939331, + 0.49704670906066895 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692751407623291, + "mean_weights": [ + 0.5129156708717346, + 0.48708435893058777 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925709247589111, + "mean_weights": [ + 0.5112693905830383, + 0.4887305796146393 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929247379302979, + "mean_weights": [ + 0.5095131397247314, + 0.49048691987991333 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0983518362045288, + "mean_weights": [ + 0.3398353159427643, + 0.32833361625671387, + 0.33183106780052185 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098447322845459, + "mean_weights": [ + 0.3408135175704956, + 0.3303682208061218, + 0.3288182020187378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980432033538818, + "mean_weights": [ + 0.3472674489021301, + 0.3313220739364624, + 0.32141047716140747 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985240936279297, + "mean_weights": [ + 0.3373557925224304, + 0.33420175313949585, + 0.32844245433807373 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098268985748291, + "mean_weights": [ + 0.3445510268211365, + 0.33061009645462036, + 0.32483887672424316 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985302925109863, + "mean_weights": [ + 0.3371829390525818, + 0.3332410156726837, + 0.3295760750770569 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983161926269531, + "mean_weights": [ + 0.3423102796077728, + 0.3327076733112335, + 0.32498204708099365 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098489761352539, + "mean_weights": [ + 0.33793729543685913, + 0.3344462811946869, + 0.327616423368454 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3861088752746582, + "mean_weights": [ + 0.2561652958393097, + 0.2513129711151123, + 0.2458309531211853, + 0.2466908097267151 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861844539642334, + "mean_weights": [ + 0.2536832094192505, + 0.25214672088623047, + 0.24721932411193848, + 0.24695077538490295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3860987424850464, + "mean_weights": [ + 0.25705933570861816, + 0.2498370260000229, + 0.2462591528892517, + 0.24684447050094604 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862221240997314, + "mean_weights": [ + 0.2514326870441437, + 0.2528519928455353, + 0.24950753152370453, + 0.24620777368545532 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386016845703125, + "mean_weights": [ + 0.25482749938964844, + 0.2536851465702057, + 0.24858617782592773, + 0.24290117621421814 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862277269363403, + "mean_weights": [ + 0.25171467661857605, + 0.2530350983142853, + 0.24868467450141907, + 0.24656552076339722 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859888315200806, + "mean_weights": [ + 0.25458407402038574, + 0.2546530067920685, + 0.248786062002182, + 0.24197682738304138 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862541913986206, + "mean_weights": [ + 0.2501963675022125, + 0.2521768808364868, + 0.24995486438274384, + 0.24767187237739563 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6090729236602783, + "mean_weights": [ + 0.20360878109931946, + 0.20631682872772217, + 0.19986985623836517, + 0.19557830691337585, + 0.19462622702121735 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093083620071411, + "mean_weights": [ + 0.2017374038696289, + 0.20362424850463867, + 0.20011676847934723, + 0.1968517303466797, + 0.1976698487997055 + ], + "sources": 5 + }, + { + "entropy_mean": 1.608644723892212, + "mean_weights": [ + 0.20431089401245117, + 0.20831988751888275, + 0.20422273874282837, + 0.19545811414718628, + 0.18768835067749023 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609345555305481, + "mean_weights": [ + 0.2002599537372589, + 0.20386311411857605, + 0.2001732438802719, + 0.19764204323291779, + 0.19806164503097534 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6086196899414062, + "mean_weights": [ + 0.20515945553779602, + 0.21025606989860535, + 0.20179016888141632, + 0.1938556730747223, + 0.18893863260746002 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609370470046997, + "mean_weights": [ + 0.2002836912870407, + 0.20340535044670105, + 0.20010054111480713, + 0.1980183869600296, + 0.1981920301914215 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6083989143371582, + "mean_weights": [ + 0.20363344252109528, + 0.21095217764377594, + 0.20392383635044098, + 0.19413790106773376, + 0.18735264241695404 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093392372131348, + "mean_weights": [ + 0.20084109902381897, + 0.20331618189811707, + 0.2011999487876892, + 0.19900763034820557, + 0.19563516974449158 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7908520698547363, + "mean_weights": [ + 0.17160072922706604, + 0.17730243504047394, + 0.16903036832809448, + 0.16099891066551208, + 0.15854957699775696, + 0.1625179797410965 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916775941848755, + "mean_weights": [ + 0.16706335544586182, + 0.16981256008148193, + 0.16858771443367004, + 0.1652718484401703, + 0.16462719440460205, + 0.16463729739189148 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898297309875488, + "mean_weights": [ + 0.17284834384918213, + 0.18101516366004944, + 0.1735592484474182, + 0.16231873631477356, + 0.15689018368721008, + 0.153368279337883 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917213439941406, + "mean_weights": [ + 0.16615921258926392, + 0.16885727643966675, + 0.16764026880264282, + 0.16661542654037476, + 0.16609220206737518, + 0.16463561356067657 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7890949249267578, + "mean_weights": [ + 0.17241932451725006, + 0.18413974344730377, + 0.17484498023986816, + 0.1621202528476715, + 0.15553176403045654, + 0.15094394981861115 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917208671569824, + "mean_weights": [ + 0.16663745045661926, + 0.168935626745224, + 0.16597260534763336, + 0.16514870524406433, + 0.16661295294761658, + 0.16669265925884247 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789101004600525, + "mean_weights": [ + 0.17229264974594116, + 0.18365857005119324, + 0.1747075319290161, + 0.1640234887599945, + 0.155386283993721, + 0.14993149042129517 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917447090148926, + "mean_weights": [ + 0.1658121943473816, + 0.167683407664299, + 0.16629494726657867, + 0.1669958382844925, + 0.16702724993228912, + 0.1661863625049591 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9436898231506348, + "mean_weights": [ + 0.14778709411621094, + 0.15856821835041046, + 0.1500256359577179, + 0.1408001184463501, + 0.13533659279346466, + 0.130964457988739, + 0.13651786744594574 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458942413330078, + "mean_weights": [ + 0.142832413315773, + 0.1429906189441681, + 0.14157448709011078, + 0.14285019040107727, + 0.14392998814582825, + 0.14335544407367706, + 0.14246684312820435 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9425597190856934, + "mean_weights": [ + 0.1483290046453476, + 0.1615946888923645, + 0.15280696749687195, + 0.14239417016506195, + 0.13462956249713898, + 0.1307675987482071, + 0.12947802245616913 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458987712860107, + "mean_weights": [ + 0.1426449418067932, + 0.14291533827781677, + 0.1416938602924347, + 0.14274153113365173, + 0.14325542747974396, + 0.1432831734418869, + 0.14346569776535034 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941068410873413, + "mean_weights": [ + 0.14912807941436768, + 0.16456490755081177, + 0.15460123121738434, + 0.14255177974700928, + 0.13539987802505493, + 0.1283380389213562, + 0.1254161298274994 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458861351013184, + "mean_weights": [ + 0.14264336228370667, + 0.1430070698261261, + 0.14314037561416626, + 0.14221979677677155, + 0.14132913947105408, + 0.1431397795677185, + 0.14452049136161804 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415628910064697, + "mean_weights": [ + 0.14895881712436676, + 0.16342055797576904, + 0.15406624972820282, + 0.14293311536312103, + 0.1363803744316101, + 0.12938986718654633, + 0.1248510330915451 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458723068237305, + "mean_weights": [ + 0.14191991090774536, + 0.1418565809726715, + 0.14164230227470398, + 0.14289972186088562, + 0.14397987723350525, + 0.14332248270511627, + 0.14437910914421082 + ], + "sources": 7 + }, + { + "entropy_mean": 2.073223114013672, + "mean_weights": [ + 0.13280923664569855, + 0.14827430248260498, + 0.13796664774417877, + 0.12756125628948212, + 0.12019773572683334, + 0.11310221254825592, + 0.10922321677207947, + 0.11086538434028625 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079413890838623, + "mean_weights": [ + 0.12475188821554184, + 0.12460128962993622, + 0.12424701452255249, + 0.12420951575040817, + 0.12484367191791534, + 0.12469655275344849, + 0.12630701065063477, + 0.12634305655956268 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072925567626953, + "mean_weights": [ + 0.13228744268417358, + 0.14843900501728058, + 0.13848665356636047, + 0.12843599915504456, + 0.12169650197029114, + 0.11442162096500397, + 0.11023421585559845, + 0.10599856078624725 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793263912200928, + "mean_weights": [ + 0.12420415878295898, + 0.12273682653903961, + 0.12360996007919312, + 0.1238352432847023, + 0.1241140365600586, + 0.12607696652412415, + 0.1275939792394638, + 0.12782883644104004 + ], + "sources": 8 + }, + { + "entropy_mean": 2.073427200317383, + "mean_weights": [ + 0.13135744631290436, + 0.1465209424495697, + 0.13772746920585632, + 0.12966200709342957, + 0.12215200066566467, + 0.11654195189476013, + 0.11090746521949768, + 0.10513069480657578 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079422950744629, + "mean_weights": [ + 0.12477636337280273, + 0.12493626773357391, + 0.1242394968867302, + 0.12470392882823944, + 0.12560302019119263, + 0.12461308389902115, + 0.12591606378555298, + 0.12521176040172577 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0725789070129395, + "mean_weights": [ + 0.13186156749725342, + 0.14827941358089447, + 0.13866659998893738, + 0.12988534569740295, + 0.12183371186256409, + 0.11524046957492828, + 0.1099783331155777, + 0.10425455868244171 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793380737304688, + "mean_weights": [ + 0.12417487055063248, + 0.1225123256444931, + 0.12277349829673767, + 0.1243998259305954, + 0.12567630410194397, + 0.1261572688817978, + 0.12747105956077576, + 0.12683483958244324 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1906280517578125, + "mean_weights": [ + 0.11854428052902222, + 0.13291707634925842, + 0.12378416955471039, + 0.11745279282331467, + 0.11140251904726028, + 0.10409101843833923, + 0.09838292002677917, + 0.09486069530248642, + 0.09856453537940979 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197185516357422, + "mean_weights": [ + 0.11084984987974167, + 0.11021925508975983, + 0.11018812656402588, + 0.11046157777309418, + 0.11131840199232101, + 0.11167057603597641, + 0.11249847710132599, + 0.1118280440568924, + 0.11096569895744324 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1899101734161377, + "mean_weights": [ + 0.1190595030784607, + 0.13352498412132263, + 0.1239522397518158, + 0.11779220402240753, + 0.11234381794929504, + 0.10513705015182495, + 0.09944089502096176, + 0.0946362093091011, + 0.09411308169364929 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197190284729004, + "mean_weights": [ + 0.11067462712526321, + 0.11021950840950012, + 0.11038416624069214, + 0.11103623360395432, + 0.11016053706407547, + 0.11144962906837463, + 0.11242866516113281, + 0.11139006912708282, + 0.11225657165050507 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1896841526031494, + "mean_weights": [ + 0.11968795955181122, + 0.13428261876106262, + 0.12373492866754532, + 0.11790731549263, + 0.11133000254631042, + 0.10504785925149918, + 0.09994535893201828, + 0.09480959177017212, + 0.09325435757637024 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971938610076904, + "mean_weights": [ + 0.11116788536310196, + 0.1117495447397232, + 0.11143069714307785, + 0.11117202788591385, + 0.11217275261878967, + 0.11135637760162354, + 0.11160033196210861, + 0.10969291627407074, + 0.10965746641159058 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188610792160034, + "mean_weights": [ + 0.11987313628196716, + 0.1360916942358017, + 0.12462760508060455, + 0.11755925416946411, + 0.11237476021051407, + 0.10438287258148193, + 0.09920360147953033, + 0.09431086480617523, + 0.09157621115446091 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028571506962180138, + 0.014360599219799042, + 0.01502692885696888, + 0.014914286322891712, + 0.015751885250210762, + 0.015590262599289417, + 0.016715344041585922, + 0.016720104962587357, + 0.01783851347863674, + 0.01203431747853756, + 0.012985615991055965, + 0.01319553330540657, + 0.014028912410140038, + 0.014193638227880001, + 0.015184907242655754, + 0.015358141623437405, + 0.016659293323755264, + 0.012656479142606258, + 0.013609340414404869, + 0.013699201866984367, + 0.014725575223565102, + 0.01495310477912426, + 0.016002535820007324, + 0.01618308201432228, + 0.017315881326794624, + 0.01409733947366476, + 0.014967667870223522, + 0.01521329116076231, + 0.016154561191797256, + 0.016374915838241577, + 0.01737469807267189, + 0.017517324537038803, + 0.018479224294424057, + 0.015613679774105549, + 0.01647813245654106, + 0.016780734062194824, + 0.017671043053269386, + 0.017834074795246124, + 0.01892261393368244, + 0.01904122717678547, + 0.020303834229707718, + 0.017550688236951828, + 0.01861351728439331, + 0.018688000738620758, + 0.01984807848930359, + 0.019847052171826363, + 0.021230308338999748, + 0.021104788407683372, + 0.022779423743486404, + 0.019968591630458832, + 0.02138628251850605, + 0.021149026229977608, + 0.02279111184179783, + 0.02249406836926937, + 0.024287279695272446, + 0.02368815243244171, + 0.02611272968351841, + 0.02278727851808071, + 0.024725768715143204, + 0.024227207526564598, + 0.026407435536384583, + 0.02562388963997364, + 0.028016721829771996, + 0.0269329734146595 + ], + "loss_nats": 3.570180892944336, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1930694580078125, + "mean_weights": [ + 0.10071488469839096, + 0.09411877393722534, + 0.10181896388530731, + 0.10989432036876678, + 0.1145806610584259, + 0.11861918121576309, + 0.11984598636627197, + 0.12094368040561676, + 0.11946355551481247 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.002919268561527133, + 0.005866481456905603, + 0.0065401275642216206, + 0.010142817161977291, + 0.010618876665830612, + 0.01482025533914566, + 0.015120964497327805, + 0.01956234686076641, + 0.0030634324066340923, + 0.0067119961604475975, + 0.007598300464451313, + 0.011908270418643951, + 0.012669830583035946, + 0.017094247043132782, + 0.017725100740790367, + 0.02295343205332756, + 0.0037359180860221386, + 0.007494525518268347, + 0.009108707308769226, + 0.013766046613454819, + 0.015153576619923115, + 0.02038806490600109, + 0.021781234070658684, + 0.02682213857769966, + 0.005531937349587679, + 0.009193498641252518, + 0.0119093032553792, + 0.016781754791736603, + 0.01904802769422531, + 0.024297412484884262, + 0.02626260370016098, + 0.03140362724661827, + 0.005659298039972782, + 0.009884056635200977, + 0.013823030516505241, + 0.018740201368927956, + 0.02223494090139866, + 0.02799132652580738, + 0.0319429449737072, + 0.03797563910484314, + 0.007882392033934593, + 0.012623735703527927, + 0.01784195750951767, + 0.02351362258195877, + 0.028768405318260193, + 0.034899674355983734, + 0.040818072855472565, + 0.047332651913166046, + 0.012831038795411587, + 0.01744839735329151, + 0.023826664313673973, + 0.029514454305171967, + 0.03824474662542343, + 0.044660378247499466, + 0.053081437945365906, + 0.0601838044822216, + 0.013431218452751637, + 0.018972838297486305, + 0.03125866502523422, + 0.0376126803457737, + 0.04969283565878868, + 0.0565216988325119, + 0.06860727071762085, + 0.07583927363157272 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "population_cv": 0.5868596491376813 + }, + "activation_output_rms_by_block": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.036073930095881224, + "first_to_last_ratio": 0.5177916953272103, + "imbalance_abs_log_ratio": 0.6581822501725932, + "last_quartile_mean": 0.0696688077878207, + "mean": 0.04647241474594921, + "normalized": [ + 0.46329906589537145, + 0.8300990481085756, + 1.06603234209322, + 1.2414755802942754, + 0.4606131805713009, + 0.5586764120892566, + 0.7186330436105184, + 0.8711228081179162, + 0.4630334112674251, + 0.7541501214633191, + 0.8015850238486076, + 0.9304351337283648, + 0.4571868851775181, + 0.6517799422454194, + 0.8562272308804426, + 1.1237390363826774, + 0.8427218914862018, + 0.8798161433899512, + 1.0235622313138022, + 1.1926338466716728, + 0.4420161942758587, + 0.7919574083446129, + 1.1031381372276345, + 1.482919724824311, + 0.6253151219931478, + 1.1530933116414785, + 1.6957169376190813, + 2.3138065984257823, + 0.6272181919070765, + 1.163956934601208, + 1.8566997944110037, + 2.5573392660929675 + ], + "population_cv": 0.5056023373965206 + }, + "bits_per_byte": 3.0609520355704922, + "branch_output_rms_by_sublayer": [ + 0.010549778118729591, + 0.017611408606171608, + 0.013795096427202225, + 0.014558257535099983, + 0.012347182258963585, + 0.01080919150263071, + 0.012820021249353886, + 0.010327119380235672, + 0.01913420483469963, + 0.008616750128567219, + 0.018582463264465332, + 0.007770510856062174, + 0.013614931143820286, + 0.007606723811477423, + 0.013612493872642517, + 0.008487755432724953, + 0.019129201769828796, + 0.00865182839334011, + 0.02525762841105461, + 0.009020541794598103, + 0.027986859902739525, + 0.008682658895850182, + 0.014458909630775452, + 0.008706990629434586, + 0.018231205642223358, + 0.009003226645290852, + 0.016914349049329758, + 0.009712434373795986, + 0.022478405386209488, + 0.00981286820024252, + 0.03932488337159157, + 0.011337134055793285, + 0.03611915931105614, + 0.010840619914233685, + 0.0266753900796175, + 0.010956712998449802, + 0.018954744562506676, + 0.012231985107064247, + 0.014813725836575031, + 0.014082341454923153, + 0.013802599161863327, + 0.014436324127018452, + 0.016533223912119865, + 0.016183899715542793, + 0.016679473221302032, + 0.017385432496666908, + 0.01826235093176365, + 0.02254617027938366, + 0.015529165975749493, + 0.022119678556919098, + 0.011229476891458035, + 0.022372635081410408, + 0.008122270926833153, + 0.024847008287906647, + 0.008894859813153744, + 0.026897024363279343, + 0.009738311171531677, + 0.026162855327129364, + 0.007465044502168894, + 0.024674534797668457, + 0.008605006150901318, + 0.030845578759908676, + 0.009693782776594162, + 0.029862264171242714 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0006151943096300597, + 0.0003715258289130363, + 0.0003398511463891951, + 0.0003030496529751275, + 0.00028984271802693696, + 0.0002579116693919001, + 0.00022400787473702318, + 0.00028477639847268295, + 0.000278695861793727, + 0.00022792247112618847, + 0.00023417680672174036, + 0.00019296669269337194, + 0.00020274366236211453, + 0.00017465616055496573, + 0.00015881266908702134, + 0.00017808900330691153, + 0.00016226510475368574, + 0.00014508607560593053, + 0.00013101353173913878, + 0.00011930204103094746, + 0.00012786456583361604, + 0.00012508100764064805, + 0.00012354146515800217, + 0.00012514897623388166, + 0.00011397604830293932, + 9.556324385288696e-05, + 9.289013346881659e-05, + 9.744025747256133e-05, + 8.302691520063822e-05, + 7.404706874242776e-05, + 8.914255818565412e-05, + 8.643783624226864e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003357699498169952, + "first_to_last_ratio": 3.666991625028822, + "imbalance_abs_log_ratio": 1.2993716052111575, + "last_quartile_mean": 9.156550768352412e-05, + "mean": 0.00019143905486393895, + "normalized": [ + 3.2135256312631473, + 1.940700288021637, + 1.7752445896201103, + 1.5830085098911153, + 1.514020836724963, + 1.3472259857070703, + 1.1701263093689624, + 1.487556437609251, + 1.4557941794677367, + 1.1905745736583337, + 1.223244686870066, + 1.0079797606111183, + 1.0590506859021536, + 0.9123329650739171, + 0.8295729896906037, + 0.9302647436986373, + 0.8476071137574935, + 0.7578708310540252, + 0.6843615678747147, + 0.6231854890619823, + 0.6679126467924371, + 0.6533724674390322, + 0.645330521746498, + 0.6537275078109245, + 0.5953646625760102, + 0.49918363795102527, + 0.4852203931681348, + 0.5089883960292994, + 0.4336989401647838, + 0.386791863316788, + 0.46564457941463516, + 0.45151620866339115 + ], + "population_cv": 0.578773657781326 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.658259391784668, + "mean_weights": [ + 0.3887256383895874, + 0.6112743616104126 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6694332361221313, + "mean_weights": [ + 0.42782124876976013, + 0.5721787214279175 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6890689730644226, + "mean_weights": [ + 0.504529595375061, + 0.49547040462493896 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740086078643799, + "mean_weights": [ + 0.45225459337234497, + 0.547745406627655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.679557204246521, + "mean_weights": [ + 0.4461178183555603, + 0.5538821816444397 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6740646362304688, + "mean_weights": [ + 0.44189155101776123, + 0.5581084489822388 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6792672276496887, + "mean_weights": [ + 0.4455937147140503, + 0.5544062852859497 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674970269203186, + "mean_weights": [ + 0.44674086570739746, + 0.5532591342926025 + ], + "sources": 2 + }, + { + "entropy_mean": 1.093062162399292, + "mean_weights": [ + 0.3082374036312103, + 0.33985745906829834, + 0.35190510749816895 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0853323936462402, + "mean_weights": [ + 0.30180174112319946, + 0.3268374502658844, + 0.3713608384132385 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0921990871429443, + "mean_weights": [ + 0.3034760355949402, + 0.3359980285167694, + 0.3605259656906128 + ], + "sources": 3 + }, + { + "entropy_mean": 1.076724648475647, + "mean_weights": [ + 0.29931002855300903, + 0.30481812357902527, + 0.3958718776702881 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856893062591553, + "mean_weights": [ + 0.2801036834716797, + 0.3310527801513672, + 0.3888435363769531 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0752873420715332, + "mean_weights": [ + 0.29463040828704834, + 0.31052330136299133, + 0.3948462903499603 + ], + "sources": 3 + }, + { + "entropy_mean": 1.088913917541504, + "mean_weights": [ + 0.29400017857551575, + 0.32166045904159546, + 0.3843393623828888 + ], + "sources": 3 + }, + { + "entropy_mean": 1.054490327835083, + "mean_weights": [ + 0.2850497364997864, + 0.2890807092189789, + 0.42586958408355713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3700100183486938, + "mean_weights": [ + 0.21610887348651886, + 0.2282472848892212, + 0.2728016972541809, + 0.28284209966659546 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3620216846466064, + "mean_weights": [ + 0.22791308164596558, + 0.2214205265045166, + 0.2562018632888794, + 0.2944645285606384 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3784711360931396, + "mean_weights": [ + 0.23347607254981995, + 0.23307976126670837, + 0.2570483684539795, + 0.2763957977294922 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3619427680969238, + "mean_weights": [ + 0.24715131521224976, + 0.22003202140331268, + 0.23014183342456818, + 0.3026748299598694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804662227630615, + "mean_weights": [ + 0.24302782118320465, + 0.23108476400375366, + 0.24228686094284058, + 0.2836005389690399 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3502743244171143, + "mean_weights": [ + 0.2455797642469406, + 0.21441739797592163, + 0.2208172082901001, + 0.31918564438819885 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3820215463638306, + "mean_weights": [ + 0.24522465467453003, + 0.22846508026123047, + 0.25027886033058167, + 0.27603137493133545 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3491160869598389, + "mean_weights": [ + 0.24498313665390015, + 0.21178750693798065, + 0.2205154001712799, + 0.3227139711380005 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6032840013504028, + "mean_weights": [ + 0.20074918866157532, + 0.17959122359752655, + 0.18619459867477417, + 0.2124267816543579, + 0.22103822231292725 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5699318647384644, + "mean_weights": [ + 0.1938496232032776, + 0.17019830644130707, + 0.16257275640964508, + 0.20397180318832397, + 0.2694075107574463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6068798303604126, + "mean_weights": [ + 0.19717518985271454, + 0.1942940056324005, + 0.19104912877082825, + 0.20343774557113647, + 0.21404394507408142 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5761134624481201, + "mean_weights": [ + 0.1940101534128189, + 0.18250694870948792, + 0.16849923133850098, + 0.18937936425209045, + 0.26560431718826294 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6027262210845947, + "mean_weights": [ + 0.20452985167503357, + 0.18702910840511322, + 0.17977532744407654, + 0.20095866918563843, + 0.22770702838897705 + ], + "sources": 5 + }, + { + "entropy_mean": 1.577937126159668, + "mean_weights": [ + 0.1962222307920456, + 0.1818992793560028, + 0.17167484760284424, + 0.18149182200431824, + 0.2687118649482727 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5977532863616943, + "mean_weights": [ + 0.2153339385986328, + 0.18402469158172607, + 0.17019470036029816, + 0.1919252574443817, + 0.23852142691612244 + ], + "sources": 5 + }, + { + "entropy_mean": 1.578121542930603, + "mean_weights": [ + 0.20422281324863434, + 0.1835813820362091, + 0.16642534732818604, + 0.17722904682159424, + 0.26854145526885986 + ], + "sources": 5 + }, + { + "entropy_mean": 1.782798409461975, + "mean_weights": [ + 0.15961389243602753, + 0.16475999355316162, + 0.14714640378952026, + 0.15768738090991974, + 0.1760568618774414, + 0.19473549723625183 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7707297801971436, + "mean_weights": [ + 0.16306114196777344, + 0.1578725278377533, + 0.139655202627182, + 0.14299748837947845, + 0.18966570496559143, + 0.20674791932106018 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781312108039856, + "mean_weights": [ + 0.15919369459152222, + 0.16844786703586578, + 0.14742718636989594, + 0.1539241373538971, + 0.1669306755065918, + 0.20407646894454956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7736209630966187, + "mean_weights": [ + 0.15695160627365112, + 0.16567477583885193, + 0.1478763222694397, + 0.14320015907287598, + 0.17375096678733826, + 0.2125461995601654 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781569480895996, + "mean_weights": [ + 0.15368077158927917, + 0.1725173443555832, + 0.14978237450122833, + 0.15674246847629547, + 0.16688688099384308, + 0.20039016008377075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7793617248535156, + "mean_weights": [ + 0.1678246557712555, + 0.1683025360107422, + 0.14878368377685547, + 0.14351718127727509, + 0.16739681363105774, + 0.20417512953281403 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7772380113601685, + "mean_weights": [ + 0.1536770462989807, + 0.18459689617156982, + 0.14832255244255066, + 0.15016385912895203, + 0.1652815043926239, + 0.19795814156532288 + ], + "sources": 6 + }, + { + "entropy_mean": 1.780207633972168, + "mean_weights": [ + 0.16951784491539001, + 0.17200809717178345, + 0.15038040280342102, + 0.14669056236743927, + 0.16062122583389282, + 0.20078186690807343 + ], + "sources": 6 + }, + { + "entropy_mean": 1.937079668045044, + "mean_weights": [ + 0.13008031249046326, + 0.15163566172122955, + 0.1323973685503006, + 0.13641127943992615, + 0.13806134462356567, + 0.1444631665945053, + 0.16695083677768707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939245343208313, + "mean_weights": [ + 0.14692582190036774, + 0.14671875536441803, + 0.13336658477783203, + 0.1307726800441742, + 0.131200909614563, + 0.147438645362854, + 0.16357660293579102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9312366247177124, + "mean_weights": [ + 0.12334205210208893, + 0.16950437426567078, + 0.13870283961296082, + 0.1300538182258606, + 0.12885476648807526, + 0.14288470149040222, + 0.1666574478149414 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9345850944519043, + "mean_weights": [ + 0.1465124636888504, + 0.153631791472435, + 0.13159149885177612, + 0.13120220601558685, + 0.1253916621208191, + 0.14393125474452972, + 0.1677391082048416 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9284589290618896, + "mean_weights": [ + 0.12412485480308533, + 0.1698783040046692, + 0.13807369768619537, + 0.13034150004386902, + 0.13028821349143982, + 0.1419651210308075, + 0.16532832384109497 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9329564571380615, + "mean_weights": [ + 0.1454196274280548, + 0.1566689908504486, + 0.1286500096321106, + 0.12666445970535278, + 0.12782412767410278, + 0.14222374558448792, + 0.17254900932312012 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9259672164916992, + "mean_weights": [ + 0.12082186341285706, + 0.17791953682899475, + 0.1424643099308014, + 0.13298648595809937, + 0.12929362058639526, + 0.1344166398048401, + 0.16209754347801208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9201786518096924, + "mean_weights": [ + 0.13756656646728516, + 0.17355048656463623, + 0.14306491613388062, + 0.13006147742271423, + 0.11658851057291031, + 0.139857679605484, + 0.15931037068367004 + ], + "sources": 7 + }, + { + "entropy_mean": 2.043663501739502, + "mean_weights": [ + 0.10298450291156769, + 0.1751018464565277, + 0.1414714753627777, + 0.12628036737442017, + 0.11767096072435379, + 0.12069195508956909, + 0.12702946364879608, + 0.08876943588256836 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0607244968414307, + "mean_weights": [ + 0.1220984011888504, + 0.1498527079820633, + 0.12913095951080322, + 0.1179434061050415, + 0.10775719583034515, + 0.12020602822303772, + 0.13844498991966248, + 0.11456632614135742 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0388383865356445, + "mean_weights": [ + 0.1025313287973404, + 0.18260905146598816, + 0.14208021759986877, + 0.12606360018253326, + 0.11357533931732178, + 0.11596501618623734, + 0.12797299027442932, + 0.08920245617628098 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0634398460388184, + "mean_weights": [ + 0.12216806411743164, + 0.15090107917785645, + 0.12766355276107788, + 0.1134132519364357, + 0.10988184064626694, + 0.11335282772779465, + 0.13999530673027039, + 0.12262405455112457 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0386834144592285, + "mean_weights": [ + 0.10029368102550507, + 0.17931804060935974, + 0.14185777306556702, + 0.13114282488822937, + 0.11872631311416626, + 0.11815309524536133, + 0.12660562992095947, + 0.08390261977910995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0626261234283447, + "mean_weights": [ + 0.11644256860017776, + 0.15398627519607544, + 0.1359611600637436, + 0.12036904692649841, + 0.10964122414588928, + 0.11886263638734818, + 0.13317911326885223, + 0.1115579903125763 + ], + "sources": 8 + }, + { + "entropy_mean": 2.020051956176758, + "mean_weights": [ + 0.09590239822864532, + 0.1913527548313141, + 0.14906750619411469, + 0.1341754049062729, + 0.11847006529569626, + 0.11603336781263351, + 0.12013453245162964, + 0.07486395537853241 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0452136993408203, + "mean_weights": [ + 0.11420649290084839, + 0.15673109889030457, + 0.13583534955978394, + 0.12758757174015045, + 0.1112617775797844, + 0.13417255878448486, + 0.12519711256027222, + 0.09500804543495178 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1403071880340576, + "mean_weights": [ + 0.08488726615905762, + 0.1673685610294342, + 0.13726134598255157, + 0.12220048159360886, + 0.10616928339004517, + 0.1037440225481987, + 0.10401669889688492, + 0.06658264249563217, + 0.10776971280574799 + ], + "sources": 9 + }, + { + "entropy_mean": 2.18704891204834, + "mean_weights": [ + 0.10885682702064514, + 0.12684625387191772, + 0.11236055195331573, + 0.10673345625400543, + 0.10181289166212082, + 0.11441421508789062, + 0.11840946972370148, + 0.10466625541448593, + 0.1059001013636589 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1283018589019775, + "mean_weights": [ + 0.08667369186878204, + 0.1812124401330948, + 0.14178472757339478, + 0.12440058588981628, + 0.11160320788621902, + 0.10330478847026825, + 0.11197598278522491, + 0.06831350177526474, + 0.07073107361793518 + ], + "sources": 9 + }, + { + "entropy_mean": 2.173035144805908, + "mean_weights": [ + 0.10752685368061066, + 0.13639645278453827, + 0.11839058995246887, + 0.11235916614532471, + 0.10138258337974548, + 0.11953261494636536, + 0.11256720125675201, + 0.09452059864997864, + 0.097323939204216 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1028335094451904, + "mean_weights": [ + 0.08200174570083618, + 0.19083696603775024, + 0.15512439608573914, + 0.1322997808456421, + 0.11209174990653992, + 0.10435720533132553, + 0.1024213582277298, + 0.06072331964969635, + 0.06014348566532135 + ], + "sources": 9 + }, + { + "entropy_mean": 2.147671937942505, + "mean_weights": [ + 0.09926989674568176, + 0.14984291791915894, + 0.12505242228507996, + 0.13004425168037415, + 0.1122550517320633, + 0.12226949632167816, + 0.10689133405685425, + 0.07532045990228653, + 0.07905416935682297 + ], + "sources": 9 + }, + { + "entropy_mean": 2.109609603881836, + "mean_weights": [ + 0.08386656641960144, + 0.18956288695335388, + 0.1487923562526703, + 0.12990039587020874, + 0.11159682273864746, + 0.10738210380077362, + 0.10535432398319244, + 0.0630122721195221, + 0.06053228676319122 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04393457621335983, + 0.018067941069602966, + 0.024121688678860664, + 0.027584951370954514, + 0.030756350606679916, + 0.03231721371412277, + 0.03544071316719055, + 0.036101218312978745, + 0.03840360417962074, + 0.025345133617520332, + 0.026969486847519875, + 0.027166159823536873, + 0.027978016063570976, + 0.028526438400149345, + 0.029923241585493088, + 0.02979987859725952, + 0.03152087330818176, + 0.02248864620923996, + 0.024161184206604958, + 0.023894935846328735, + 0.025710709393024445, + 0.02451697364449501, + 0.02641460672020912, + 0.025183402001857758, + 0.02742084674537182, + 0.020499059930443764, + 0.021717891097068787, + 0.02105838619172573, + 0.0224718376994133, + 0.0216353852301836, + 0.023445574566721916, + 0.022715488448739052, + 0.024849263951182365, + 0.01881815865635872, + 0.01992032676935196, + 0.0193756353110075, + 0.02058842033147812, + 0.02001863345503807, + 0.020979218184947968, + 0.021088071167469025, + 0.02214338816702366, + 0.018417155370116234, + 0.01903437450528145, + 0.019219009205698967, + 0.02008410543203354, + 0.020342160016298294, + 0.021635910496115685, + 0.02150510996580124, + 0.023218298330903053, + 0.02099662832915783, + 0.022257626056671143, + 0.02211105264723301, + 0.024358727037906647, + 0.02328667603433132, + 0.025900650769472122, + 0.024160878732800484, + 0.027197737246751785, + 0.02260657213628292, + 0.02823900803923607, + 0.024989914149045944, + 0.02913021855056286, + 0.02536177635192871, + 0.029048049822449684, + 0.027097374200820923 + ], + "loss_nats": 2.121690273284912, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1622862815856934, + "mean_weights": [ + 0.12356589734554291, + 0.09235234558582306, + 0.08736804127693176, + 0.08735492080450058, + 0.08889827132225037, + 0.10504580289125443, + 0.14004790782928467, + 0.14956718683242798, + 0.12579965591430664 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.010549778118729591, + 0.021530626341700554, + 0.02910016104578972, + 0.03857670724391937, + 0.04255657270550728, + 0.04954109713435173, + 0.05177471786737442, + 0.057694368064403534, + 0.01913420483469963, + 0.02140580676496029, + 0.02246284671127796, + 0.025963041931390762, + 0.029843268916010857, + 0.03339661285281181, + 0.036163609474897385, + 0.040483180433511734, + 0.019129201769828796, + 0.02151828072965145, + 0.0314205139875412, + 0.035047177225351334, + 0.03302976116538048, + 0.03725159168243408, + 0.03871368616819382, + 0.043239567428827286, + 0.018231205642223358, + 0.02124657854437828, + 0.02580643631517887, + 0.03028978779911995, + 0.0349617674946785, + 0.03979094699025154, + 0.04659247398376465, + 0.05222286656498909, + 0.03611915931105614, + 0.03916332125663757, + 0.03664228320121765, + 0.04088718071579933, + 0.04177439957857132, + 0.04756740853190422, + 0.04776441678404808, + 0.05542457476258278, + 0.013802599161863327, + 0.020541559904813766, + 0.02693362347781658, + 0.03680417314171791, + 0.03932841867208481, + 0.05126549303531647, + 0.0514267161488533, + 0.06891486048698425, + 0.015529165975749493, + 0.02905990369617939, + 0.03558400645852089, + 0.05358703061938286, + 0.05707564577460289, + 0.07880406081676483, + 0.08332383632659912, + 0.10752817988395691, + 0.009738311171531677, + 0.029148343950510025, + 0.03187238425016403, + 0.054091889411211014, + 0.05786185711622238, + 0.08628532290458679, + 0.09122034162282944, + 0.11884573101997375 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "population_cv": 0.5384894543542103 + }, + "activation_output_rms_by_block": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.14976411033421755, + "first_to_last_ratio": 1.036636701844262, + "imbalance_abs_log_ratio": 0.035981532131895505, + "last_quartile_mean": 0.14447116339579225, + "mean": 0.12951817945577204, + "normalized": [ + 0.7297249643070838, + 1.5686735699681784, + 1.7647660354854744, + 1.8765703797812938, + 0.636469098825333, + 0.749653246535227, + 0.8711234054002422, + 1.0535576234146717, + 0.42020993720306404, + 0.6840666215997374, + 0.9431112804194469, + 1.192130706187048, + 0.4699981697046636, + 0.7312780916470571, + 0.9733494312076132, + 1.2651958587237788, + 0.6835788065012903, + 0.9366911620158862, + 1.2091399212742004, + 1.6196823469097898, + 0.34397948019882096, + 0.5722161994862464, + 0.7727653276358534, + 1.008461533628578, + 0.3829837776042769, + 0.608007158744095, + 0.8769750006177236, + 1.2205488119398642, + 0.5370088925855727, + 1.0271058497013745, + 1.7339584411053826, + 2.5370188696411313 + ], + "population_cv": 0.4929365839545065 + }, + "bits_per_byte": 2.073383110809672, + "branch_output_rms_by_sublayer": [ + 0.04804428666830063, + 0.07808538526296616, + 0.12356939166784286, + 0.03881331905722618, + 0.05923242121934891, + 0.01480295043438673, + 0.04508936405181885, + 0.02500341460108757, + 0.0799286738038063, + 0.015180142596364021, + 0.03290615975856781, + 0.013721407391130924, + 0.03170505166053772, + 0.016902483999729156, + 0.03961387276649475, + 0.03325798362493515, + 0.04508088901638985, + 0.026172807440161705, + 0.0312613770365715, + 0.027887089177966118, + 0.029994890093803406, + 0.03240044787526131, + 0.03058313950896263, + 0.03682815283536911, + 0.0497574545443058, + 0.029301028698682785, + 0.03759258985519409, + 0.0338764488697052, + 0.030690157786011696, + 0.0393521711230278, + 0.03000541403889656, + 0.046694882214069366, + 0.079553984105587, + 0.029503758996725082, + 0.0414140485227108, + 0.03615153953433037, + 0.04122982919216156, + 0.030687781050801277, + 0.05830265209078789, + 0.05279085040092468, + 0.03440643846988678, + 0.024021649733185768, + 0.033897366374731064, + 0.03336174041032791, + 0.02957344427704811, + 0.04255227372050285, + 0.03406442329287529, + 0.0520954355597496, + 0.025225287303328514, + 0.03984410688281059, + 0.01915411651134491, + 0.04249191656708717, + 0.014526215381920338, + 0.05309459939599037, + 0.011566666886210442, + 0.06231515854597092, + 0.0242468249052763, + 0.06388300657272339, + 0.013597902841866016, + 0.0749494656920433, + 0.020146220922470093, + 0.10156872868537903, + 0.02710210345685482, + 0.11044161766767502 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002879944983597963, + 0.00022853159560210997, + 0.00014071406888316784, + 0.00014318671869519129, + 0.00013886536811003923, + 0.00010286844600842235, + 9.677924556041764e-05, + 0.0001337288949114584, + 0.00010758657043361882, + 9.13787469069405e-05, + 9.373485859236601e-05, + 9.381017142422257e-05, + 0.00010148426099734976, + 9.552468088000852e-05, + 0.00010200246393672037, + 0.00010753255944705701, + 0.00013288008273706816, + 0.00012966970691820486, + 0.0001395588859415414, + 0.00019785772756123382, + 0.0004949688435464942, + 0.0005919211037074493, + 0.000516172452953512, + 0.0004666614668327092, + 0.0003441119880144902, + 0.00027788329882105144, + 0.00022798541204965527, + 0.00021216050695355764, + 0.00012431994828037904, + 0.00011340006302052245, + 0.00011536177905162644, + 0.00011797314476595472 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015908360451632537, + "first_to_last_ratio": 0.8300756844691923, + "imbalance_abs_log_ratio": 0.18623839624123317, + "last_quartile_mean": 0.00019164951761965464, + "mean": 0.00019589404874701053, + "normalized": [ + 1.4701544033720488, + 1.1666081591751138, + 0.7183172219024098, + 0.730939605419618, + 0.7088799736299207, + 0.5251228746681984, + 0.4940387223575344, + 0.682659303673712, + 0.5492079576779928, + 0.4664702551783621, + 0.4784977348312449, + 0.4788821917983569, + 0.5180568866000248, + 0.4876344201930039, + 0.5207022091235275, + 0.5489322423772612, + 0.6783262870260932, + 0.661937959563377, + 0.7124202436684344, + 1.01002418821186, + 2.526717104028653, + 3.0216390313719645, + 2.6349572958192744, + 2.3822135987162336, + 1.7566229817369148, + 1.4185387488719823, + 1.1638200012093756, + 1.0830370208313709, + 0.6346285100316312, + 0.5788846764149236, + 0.5888988450109153, + 0.6022293455086652 + ], + "population_cv": 0.7015245062301059 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4738388657569885, + "mean_weights": [ + 0.8067666292190552, + 0.19323334097862244 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6344536542892456, + "mean_weights": [ + 0.4895974397659302, + 0.5104025602340698 + ], + "sources": 2 + }, + { + "entropy_mean": 0.41773349046707153, + "mean_weights": [ + 0.8461704254150391, + 0.15382957458496094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868985891342163, + "mean_weights": [ + 0.5008196830749512, + 0.49918028712272644 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5841774344444275, + "mean_weights": [ + 0.684928297996521, + 0.315071702003479 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6896515488624573, + "mean_weights": [ + 0.5092520117759705, + 0.49074798822402954 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5611414909362793, + "mean_weights": [ + 0.7280503511428833, + 0.2719496488571167 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6817618608474731, + "mean_weights": [ + 0.5313050746917725, + 0.46869492530822754 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0218309164047241, + "mean_weights": [ + 0.4974328279495239, + 0.24236315488815308, + 0.2602040767669678 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0932960510253906, + "mean_weights": [ + 0.3533782958984375, + 0.29999595880508423, + 0.34662577509880066 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0456881523132324, + "mean_weights": [ + 0.4338536858558655, + 0.23105540871620178, + 0.3350909352302551 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0891339778900146, + "mean_weights": [ + 0.33998236060142517, + 0.2821658253669739, + 0.37785181403160095 + ], + "sources": 3 + }, + { + "entropy_mean": 1.038641333580017, + "mean_weights": [ + 0.34165239334106445, + 0.23904743790626526, + 0.4193001389503479 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0754427909851074, + "mean_weights": [ + 0.3781457543373108, + 0.24021697044372559, + 0.3816372752189636 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0433809757232666, + "mean_weights": [ + 0.37802231311798096, + 0.21331989765167236, + 0.4086577892303467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0922105312347412, + "mean_weights": [ + 0.3460429906845093, + 0.3074926435947418, + 0.3464643955230713 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3404325246810913, + "mean_weights": [ + 0.2880150079727173, + 0.16260021924972534, + 0.2385387420654297, + 0.3108460605144501 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3765960931777954, + "mean_weights": [ + 0.28703436255455017, + 0.2552259564399719, + 0.2101576328277588, + 0.24758204817771912 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3274900913238525, + "mean_weights": [ + 0.33539247512817383, + 0.1987970769405365, + 0.20543015003204346, + 0.2603802978992462 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3747133016586304, + "mean_weights": [ + 0.27988845109939575, + 0.23608103394508362, + 0.20066317915916443, + 0.2833673357963562 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3239198923110962, + "mean_weights": [ + 0.3361120820045471, + 0.20296622812747955, + 0.1885247528553009, + 0.2723969519138336 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3692121505737305, + "mean_weights": [ + 0.28167375922203064, + 0.21946553885936737, + 0.20140981674194336, + 0.2974509000778198 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3342361450195312, + "mean_weights": [ + 0.36468517780303955, + 0.20593920350074768, + 0.18943950533866882, + 0.23993611335754395 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3645737171173096, + "mean_weights": [ + 0.2597482204437256, + 0.22607669234275818, + 0.209133118391037, + 0.30504196882247925 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5910131931304932, + "mean_weights": [ + 0.2617476284503937, + 0.18299485743045807, + 0.17935843765735626, + 0.17898105084896088, + 0.19691801071166992 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5884685516357422, + "mean_weights": [ + 0.2046908289194107, + 0.17294223606586456, + 0.15044087171554565, + 0.20923088490962982, + 0.26269516348838806 + ], + "sources": 5 + }, + { + "entropy_mean": 1.599259853363037, + "mean_weights": [ + 0.21657392382621765, + 0.20580387115478516, + 0.19366897642612457, + 0.1731838881969452, + 0.21076934039592743 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5786192417144775, + "mean_weights": [ + 0.20599734783172607, + 0.20593789219856262, + 0.13473063707351685, + 0.17552703619003296, + 0.2778070569038391 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580681324005127, + "mean_weights": [ + 0.21277281641960144, + 0.18522506952285767, + 0.18489867448806763, + 0.17205308377742767, + 0.2450503408908844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5705549716949463, + "mean_weights": [ + 0.2125053107738495, + 0.20224159955978394, + 0.12915469706058502, + 0.1714920997619629, + 0.28460633754730225 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5599372386932373, + "mean_weights": [ + 0.2665032148361206, + 0.181569442152977, + 0.15844659507274628, + 0.14596089720726013, + 0.2475198209285736 + ], + "sources": 5 + }, + { + "entropy_mean": 1.562828540802002, + "mean_weights": [ + 0.21806953847408295, + 0.22324788570404053, + 0.11591021716594696, + 0.17848318815231323, + 0.26428914070129395 + ], + "sources": 5 + }, + { + "entropy_mean": 1.761409044265747, + "mean_weights": [ + 0.1612042486667633, + 0.1651904582977295, + 0.1752963364124298, + 0.14996090531349182, + 0.2072838991880417, + 0.14106415212154388 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7366840839385986, + "mean_weights": [ + 0.1851401925086975, + 0.1663871705532074, + 0.09063514322042465, + 0.11971023678779602, + 0.18503433465957642, + 0.2530929446220398 + ], + "sources": 6 + }, + { + "entropy_mean": 1.777672290802002, + "mean_weights": [ + 0.15483248233795166, + 0.16576308012008667, + 0.16957050561904907, + 0.14225167036056519, + 0.16980668902397156, + 0.19777557253837585 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7414222955703735, + "mean_weights": [ + 0.18020270764827728, + 0.1900298297405243, + 0.09944288432598114, + 0.11960913985967636, + 0.15577411651611328, + 0.25494131445884705 + ], + "sources": 6 + }, + { + "entropy_mean": 1.733978509902954, + "mean_weights": [ + 0.15204599499702454, + 0.14166373014450073, + 0.1616208702325821, + 0.1256796270608902, + 0.1571422517299652, + 0.26184749603271484 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7344707250595093, + "mean_weights": [ + 0.1829337775707245, + 0.18416841328144073, + 0.09014187753200531, + 0.12335793673992157, + 0.15779981017112732, + 0.2615981996059418 + ], + "sources": 6 + }, + { + "entropy_mean": 1.735896348953247, + "mean_weights": [ + 0.17276549339294434, + 0.15101772546768188, + 0.13220663368701935, + 0.11932596564292908, + 0.15458261966705322, + 0.27010154724121094 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7644665241241455, + "mean_weights": [ + 0.19160008430480957, + 0.1915181279182434, + 0.11520625650882721, + 0.1341419219970703, + 0.1580837219953537, + 0.2094498574733734 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6835086345672607, + "mean_weights": [ + 0.10065847635269165, + 0.09339895844459534, + 0.09682720899581909, + 0.08155890554189682, + 0.09126688539981842, + 0.10349369794130325, + 0.4327958822250366 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7937740087509155, + "mean_weights": [ + 0.13723835349082947, + 0.12937892973423004, + 0.07450413703918457, + 0.08537043631076813, + 0.09880411624908447, + 0.12944120168685913, + 0.3452628552913666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5790722370147705, + "mean_weights": [ + 0.09500400722026825, + 0.08892714232206345, + 0.09081116318702698, + 0.07538282871246338, + 0.07685283571481705, + 0.08106325566768646, + 0.49195876717567444 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8034920692443848, + "mean_weights": [ + 0.142653226852417, + 0.13419847190380096, + 0.07437048852443695, + 0.09070360660552979, + 0.1013825386762619, + 0.12073953449726105, + 0.33595213294029236 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5278494358062744, + "mean_weights": [ + 0.10045769065618515, + 0.08844951540231705, + 0.08561251312494278, + 0.06598107516765594, + 0.06767018139362335, + 0.07361933588981628, + 0.51820969581604 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8435695171356201, + "mean_weights": [ + 0.14839640259742737, + 0.1482572704553604, + 0.08113027364015579, + 0.09534022957086563, + 0.10612697899341583, + 0.12550383806228638, + 0.2952450215816498 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6421003341674805, + "mean_weights": [ + 0.11422763019800186, + 0.10543526709079742, + 0.09285669028759003, + 0.07746599614620209, + 0.07571054995059967, + 0.08126918226480484, + 0.4530346691608429 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8733552694320679, + "mean_weights": [ + 0.14798465371131897, + 0.16052685678005219, + 0.09668852388858795, + 0.10477185249328613, + 0.10857058316469193, + 0.11694246530532837, + 0.26451510190963745 + ], + "sources": 7 + }, + { + "entropy_mean": 1.784506916999817, + "mean_weights": [ + 0.08589775860309601, + 0.08786045759916306, + 0.07714233547449112, + 0.06314891576766968, + 0.05684572085738182, + 0.056072384119033813, + 0.29875272512435913, + 0.2742796838283539 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9675533771514893, + "mean_weights": [ + 0.11549945175647736, + 0.12592051923274994, + 0.07530814409255981, + 0.08277833461761475, + 0.08296641707420349, + 0.09016790241003036, + 0.17876628041267395, + 0.24859298765659332 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7119876146316528, + "mean_weights": [ + 0.08804735541343689, + 0.07527872174978256, + 0.05666808784008026, + 0.05041901767253876, + 0.05142981559038162, + 0.05313319340348244, + 0.2591783106327057, + 0.3658455014228821 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9094492197036743, + "mean_weights": [ + 0.10337945818901062, + 0.10583682358264923, + 0.08642245084047318, + 0.08043928444385529, + 0.0717030018568039, + 0.0645938515663147, + 0.21096080541610718, + 0.2766643166542053 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7860023975372314, + "mean_weights": [ + 0.09247160702943802, + 0.0784597396850586, + 0.07236532866954803, + 0.05986309051513672, + 0.05389959365129471, + 0.06059400737285614, + 0.2579019069671631, + 0.3244447112083435 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9232951402664185, + "mean_weights": [ + 0.1160542219877243, + 0.11346358060836792, + 0.08816123008728027, + 0.08138670027256012, + 0.07161372900009155, + 0.06013647839426994, + 0.21822679042816162, + 0.2509572505950928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8294003009796143, + "mean_weights": [ + 0.08831655979156494, + 0.10072161257266998, + 0.0818098857998848, + 0.07514279335737228, + 0.06651484966278076, + 0.06491033732891083, + 0.31431835889816284, + 0.20826560258865356 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9393219947814941, + "mean_weights": [ + 0.11041727662086487, + 0.12080886960029602, + 0.09995828568935394, + 0.09934850037097931, + 0.08031868189573288, + 0.075287364423275, + 0.2604207992553711, + 0.15344023704528809 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9925593137741089, + "mean_weights": [ + 0.08698172867298126, + 0.08857265114784241, + 0.07299254834651947, + 0.060478877276182175, + 0.054115522652864456, + 0.056821923702955246, + 0.2503892779350281, + 0.14502599835395813, + 0.18462148308753967 + ], + "sources": 9 + }, + { + "entropy_mean": 2.068819999694824, + "mean_weights": [ + 0.09429103136062622, + 0.09028280526399612, + 0.09188265353441238, + 0.0735517293214798, + 0.06746295094490051, + 0.060166649520397186, + 0.21415996551513672, + 0.15325874090194702, + 0.15494348108768463 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0139355659484863, + "mean_weights": [ + 0.0956040769815445, + 0.09831400215625763, + 0.0714857280254364, + 0.0677608773112297, + 0.05654233694076538, + 0.05388312786817551, + 0.24272149801254272, + 0.17641445994377136, + 0.137273907661438 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0349321365356445, + "mean_weights": [ + 0.10256819427013397, + 0.0938093364238739, + 0.0912778377532959, + 0.08394120633602142, + 0.06161238253116608, + 0.053358763456344604, + 0.2261163890361786, + 0.15197426080703735, + 0.13534162938594818 + ], + "sources": 9 + }, + { + "entropy_mean": 2.046657085418701, + "mean_weights": [ + 0.08380690217018127, + 0.1217331811785698, + 0.10242430865764618, + 0.09225839376449585, + 0.06788390129804611, + 0.06336672604084015, + 0.2572976052761078, + 0.11757771670818329, + 0.09365126490592957 + ], + "sources": 9 + }, + { + "entropy_mean": 2.094015121459961, + "mean_weights": [ + 0.10284869372844696, + 0.13057856261730194, + 0.10152992606163025, + 0.10079468786716461, + 0.0796234980225563, + 0.07438632845878601, + 0.2018478661775589, + 0.1111527606844902, + 0.09723767638206482 + ], + "sources": 9 + }, + { + "entropy_mean": 2.057201385498047, + "mean_weights": [ + 0.09265108406543732, + 0.11481083929538727, + 0.09007447212934494, + 0.08496075868606567, + 0.06770835071802139, + 0.05925413966178894, + 0.24551761150360107, + 0.15022093057632446, + 0.09480181336402893 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05595625191926956, + 0.047033343464136124, + 0.0632273480296135, + 0.060984931886196136, + 0.11160287261009216, + 0.09079429507255554, + 0.12218338996171951, + 0.08531255275011063, + 0.12365243583917618, + 0.0786692351102829, + 0.0926642045378685, + 0.08076046407222748, + 0.093480683863163, + 0.08904207497835159, + 0.08749273419380188, + 0.08403915911912918, + 0.09927791357040405, + 0.06012389436364174, + 0.07820390164852142, + 0.06841155141592026, + 0.07669360190629959, + 0.07131828367710114, + 0.07840896397829056, + 0.07358616590499878, + 0.08920575678348541, + 0.06597626209259033, + 0.06850729882717133, + 0.07235056161880493, + 0.07440610229969025, + 0.07261744141578674, + 0.07867594808340073, + 0.07352335751056671, + 0.09058429300785065, + 0.07520031183958054, + 0.07234266400337219, + 0.07378771901130676, + 0.07771008461713791, + 0.07715436071157455, + 0.08416424691677094, + 0.08649503439664841, + 0.08947562426328659, + 0.04556623473763466, + 0.056279782205820084, + 0.04546353965997696, + 0.057985786348581314, + 0.05167657509446144, + 0.06238175556063652, + 0.058827146887779236, + 0.06515856087207794, + 0.04798740893602371, + 0.05041429027915001, + 0.046526361256837845, + 0.05083387345075607, + 0.051699329167604446, + 0.05600840970873833, + 0.059578344225883484, + 0.06128782033920288, + 0.05090610682964325, + 0.056301049888134, + 0.05821637064218521, + 0.06424970924854279, + 0.06188955530524254, + 0.06844019144773483, + 0.06796324998140335 + ], + "loss_nats": 1.4371596574783325, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.8435449600219727, + "mean_weights": [ + 0.1112452894449234, + 0.06898629665374756, + 0.05835210159420967, + 0.042265649884939194, + 0.041034020483493805, + 0.04826357960700989, + 0.10493715852499008, + 0.31774088740348816, + 0.20717504620552063 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.04804428666830063, + 0.09451264888048172, + 0.18500657379627228, + 0.2031717449426651, + 0.22299833595752716, + 0.22856928408145905, + 0.2331935465335846, + 0.2430499792098999, + 0.0799286738038063, + 0.08243431895971298, + 0.09402555972337723, + 0.09709372371435165, + 0.10843002796173096, + 0.11282631754875183, + 0.12457633018493652, + 0.13645486533641815, + 0.04508088901638985, + 0.05442482605576515, + 0.0760042667388916, + 0.0885990634560585, + 0.10498681664466858, + 0.12215005606412888, + 0.13159531354904175, + 0.15440259873867035, + 0.0497574545443058, + 0.060873307287693024, + 0.07735268771648407, + 0.09471380710601807, + 0.10224150121212006, + 0.1260664463043213, + 0.13362281024456024, + 0.163865864276886, + 0.079553984105587, + 0.08853588253259659, + 0.10536550730466843, + 0.12131853401660919, + 0.14260481297969818, + 0.15660560131072998, + 0.19080305099487305, + 0.2097783088684082, + 0.03440643846988678, + 0.04455159604549408, + 0.060867179185152054, + 0.07411240041255951, + 0.08364810794591904, + 0.10008715838193893, + 0.11119203269481659, + 0.13061410188674927, + 0.025225287303328514, + 0.04960336163640022, + 0.05530283972620964, + 0.07874798029661179, + 0.08037404716014862, + 0.11358420550823212, + 0.11502843350172043, + 0.1580832600593567, + 0.0242468249052763, + 0.06955241411924362, + 0.07164724916219711, + 0.13302887976169586, + 0.1371813863515854, + 0.22457914054393768, + 0.23144805431365967, + 0.32859006524086 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "population_cv": 0.6027314599912318 + }, + "activation_output_rms_by_block": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.22688884753733873, + "first_to_last_ratio": 1.3228372499356764, + "imbalance_abs_log_ratio": 0.2797788616441042, + "last_quartile_mean": 0.17151682684198022, + "mean": 0.17372163233812898, + "normalized": [ + 0.9268367409782202, + 1.919387740182172, + 2.055289136269615, + 2.170611306021289, + 0.6406640960204442, + 0.7450635822219737, + 0.8704616111421293, + 1.1200724150644035, + 0.4345583639666029, + 0.677281348405711, + 0.9287495476341928, + 1.2372530952113672, + 0.4794804996179952, + 0.7087278040538177, + 0.9675973641757485, + 1.32977546738468, + 0.6641820538556276, + 0.930123337383164, + 1.1558946518222766, + 1.5614450026347633, + 0.3131140120506511, + 0.5373702635317168, + 0.742418848247821, + 0.9851745133746388, + 0.3553571676266622, + 0.5579738073870804, + 0.7927337262517246, + 1.0584297759573136, + 0.4595608055861626, + 0.8937492173058054, + 1.5039930398498347, + 2.276669658784396 + ], + "population_cv": 0.5173518438224235 + }, + "bits_per_byte": 1.9433676664104056, + "branch_output_rms_by_sublayer": [ + 0.08528144657611847, + 0.13266965746879578, + 0.2063225358724594, + 0.045500293374061584, + 0.07479097694158554, + 0.018729008734226227, + 0.053203217685222626, + 0.03815881907939911, + 0.1070452407002449, + 0.02209169790148735, + 0.03943018242716789, + 0.01640710048377514, + 0.041509829461574554, + 0.02340921014547348, + 0.06257687509059906, + 0.05481670796871185, + 0.061002109199762344, + 0.037687551230192184, + 0.04113767296075821, + 0.04149388521909714, + 0.040399398654699326, + 0.048693906515836716, + 0.03907898813486099, + 0.06190986558794975, + 0.06714949756860733, + 0.04065176844596863, + 0.04437125846743584, + 0.049332331866025925, + 0.0412614606320858, + 0.0589165985584259, + 0.046277787536382675, + 0.08173932880163193, + 0.10073095560073853, + 0.043543290346860886, + 0.05776162073016167, + 0.05228494107723236, + 0.05185067281126976, + 0.04403071105480194, + 0.07827401161193848, + 0.0687810555100441, + 0.04170985147356987, + 0.030229199677705765, + 0.041845910251140594, + 0.04699210822582245, + 0.038240060210227966, + 0.06082639843225479, + 0.046402644366025925, + 0.07667754590511322, + 0.030182205140590668, + 0.050203632563352585, + 0.025695007294416428, + 0.05539015680551529, + 0.024306239560246468, + 0.06946423649787903, + 0.017157377675175667, + 0.07630576938390732, + 0.030125634744763374, + 0.07239200174808502, + 0.01829664409160614, + 0.09628339111804962, + 0.024464696645736694, + 0.12468792498111725, + 0.032671861350536346, + 0.15056027472019196 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002458031044227025, + 0.00017276993155813553, + 0.00010939491852011586, + 0.000125258738441463, + 0.00011215544691093282, + 7.31087164016768e-05, + 9.049562402296399e-05, + 0.00013630654451635742, + 9.25970647966133e-05, + 7.580293890923953e-05, + 7.957502622146584e-05, + 8.776523713070475e-05, + 0.00010318467043462154, + 0.00010632361913360551, + 0.00011315585331317908, + 0.00012408116098017353, + 0.00012849590289080713, + 0.00014012462333747852, + 0.00013373629199672077, + 0.00019202792118308513, + 0.0005640243510171648, + 0.0006705621295490249, + 0.000579819507581129, + 0.0005486985299520294, + 0.0004559472160958734, + 0.00038131537611027277, + 0.00030455316817913544, + 0.000254991251417659, + 0.00017923215935365346, + 0.0001463414879582755, + 0.00013146832622599984, + 0.0001387805380572583 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0001331616280992935, + "first_to_last_ratio": 0.5346167023449758, + "imbalance_abs_log_ratio": 0.6262052330699333, + "last_quartile_mean": 0.000249078690424766, + "mean": 0.00021243429301935995, + "normalized": [ + 1.1570782707870122, + 0.8132864477883067, + 0.5149588466403882, + 0.5896352074852985, + 0.5279535865742339, + 0.34414743195447317, + 0.4259934812630124, + 0.6416409637964399, + 0.4358856730733892, + 0.3568300476907056, + 0.37458653727914765, + 0.4131406275478619, + 0.4857251104237578, + 0.5005012026185237, + 0.5326628375526298, + 0.584091952464876, + 0.6048736344046712, + 0.6596139509580424, + 0.6295419166835406, + 0.9039403123373536, + 2.655053207279312, + 3.156562530551101, + 2.7294063464992813, + 2.5829093888434693, + 2.146297613325176, + 1.7949803239890372, + 1.433634672869791, + 1.200329983419488, + 0.8437063376453977, + 0.6888788334421074, + 0.6188658354422025, + 0.6532868873699723 + ], + "population_cv": 0.7868568919954686 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.24809812009334564, + "mean_weights": [ + 0.9288002252578735, + 0.07119979709386826 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6067080497741699, + "mean_weights": [ + 0.5604246854782104, + 0.43957534432411194 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3847596049308777, + "mean_weights": [ + 0.8648147583007812, + 0.13518527150154114 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6843481659889221, + "mean_weights": [ + 0.49438929557800293, + 0.5056107044219971 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5728601813316345, + "mean_weights": [ + 0.6954652070999146, + 0.3045347332954407 + ], + "sources": 2 + }, + { + "entropy_mean": 0.686518132686615, + "mean_weights": [ + 0.5263072848320007, + 0.4736926555633545 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044870376586914, + "mean_weights": [ + 0.777843713760376, + 0.2221563160419464 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6724032163619995, + "mean_weights": [ + 0.5694063901901245, + 0.4305936098098755 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9718273282051086, + "mean_weights": [ + 0.5544795989990234, + 0.20597517490386963, + 0.23954525589942932 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856361389160156, + "mean_weights": [ + 0.3703838884830475, + 0.26754841208457947, + 0.36206769943237305 + ], + "sources": 3 + }, + { + "entropy_mean": 1.022249460220337, + "mean_weights": [ + 0.46441033482551575, + 0.21309645473957062, + 0.32249322533607483 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0742855072021484, + "mean_weights": [ + 0.38770636916160583, + 0.24731171131134033, + 0.36498188972473145 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0207322835922241, + "mean_weights": [ + 0.3404029607772827, + 0.22699853777885437, + 0.4325985312461853 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0628657341003418, + "mean_weights": [ + 0.4176652133464813, + 0.22035521268844604, + 0.361979603767395 + ], + "sources": 3 + }, + { + "entropy_mean": 1.029329538345337, + "mean_weights": [ + 0.42153143882751465, + 0.19259268045425415, + 0.3858758807182312 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0849454402923584, + "mean_weights": [ + 0.37970077991485596, + 0.2838303744792938, + 0.3364688754081726 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3144235610961914, + "mean_weights": [ + 0.3205832540988922, + 0.13048580288887024, + 0.2396528720855713, + 0.30927807092666626 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3647149801254272, + "mean_weights": [ + 0.3188267648220062, + 0.2379944771528244, + 0.19448311626911163, + 0.24869562685489655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2778496742248535, + "mean_weights": [ + 0.37896889448165894, + 0.16353651881217957, + 0.1984466314315796, + 0.2590479254722595 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3567583560943604, + "mean_weights": [ + 0.29770219326019287, + 0.21492166817188263, + 0.17857596278190613, + 0.3088001608848572 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2773678302764893, + "mean_weights": [ + 0.3856483995914459, + 0.18032220005989075, + 0.1928499937057495, + 0.24117937684059143 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3655540943145752, + "mean_weights": [ + 0.29819539189338684, + 0.20063453912734985, + 0.20929867029190063, + 0.29187142848968506 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2754871845245361, + "mean_weights": [ + 0.42651012539863586, + 0.1713070571422577, + 0.1726072132587433, + 0.22957557439804077 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3596348762512207, + "mean_weights": [ + 0.2750997543334961, + 0.22746169567108154, + 0.18702520430088043, + 0.31041333079338074 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5752925872802734, + "mean_weights": [ + 0.28211507201194763, + 0.1675817370414734, + 0.1759152114391327, + 0.16564446687698364, + 0.20874354243278503 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5704567432403564, + "mean_weights": [ + 0.21000899374485016, + 0.18079039454460144, + 0.13544103503227234, + 0.1749160885810852, + 0.29884350299835205 + ], + "sources": 5 + }, + { + "entropy_mean": 1.579707384109497, + "mean_weights": [ + 0.21335941553115845, + 0.18788449466228485, + 0.20250459015369415, + 0.1511598825454712, + 0.24509160220623016 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5663042068481445, + "mean_weights": [ + 0.21676909923553467, + 0.19467836618423462, + 0.13713446259498596, + 0.15313217043876648, + 0.29828590154647827 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5384478569030762, + "mean_weights": [ + 0.21904908120632172, + 0.15864285826683044, + 0.18178997933864594, + 0.14981383085250854, + 0.29070425033569336 + ], + "sources": 5 + }, + { + "entropy_mean": 1.552668571472168, + "mean_weights": [ + 0.22710487246513367, + 0.1922147125005722, + 0.13047492504119873, + 0.14686013758182526, + 0.30334532260894775 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5210195779800415, + "mean_weights": [ + 0.2812899947166443, + 0.14511609077453613, + 0.15909329056739807, + 0.13197475671768188, + 0.282525897026062 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5376527309417725, + "mean_weights": [ + 0.23905543982982635, + 0.21673071384429932, + 0.0950920507311821, + 0.16199833154678345, + 0.2871234714984894 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7209875583648682, + "mean_weights": [ + 0.1703755110502243, + 0.14151470363140106, + 0.17087554931640625, + 0.12742051482200623, + 0.25328749418258667, + 0.1365262269973755 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7200093269348145, + "mean_weights": [ + 0.18992580473423004, + 0.16036251187324524, + 0.08439569175243378, + 0.11250905692577362, + 0.1775888204574585, + 0.27521812915802 + ], + "sources": 6 + }, + { + "entropy_mean": 1.753357172012329, + "mean_weights": [ + 0.15539541840553284, + 0.1388990879058838, + 0.15745995938777924, + 0.12427005916833878, + 0.17903099954128265, + 0.24494445323944092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7199265956878662, + "mean_weights": [ + 0.19207912683486938, + 0.17904852330684662, + 0.08410340547561646, + 0.11219910532236099, + 0.15951165556907654, + 0.2730581760406494 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6756346225738525, + "mean_weights": [ + 0.15863081812858582, + 0.11242729425430298, + 0.15970338881015778, + 0.09774985909461975, + 0.15905189514160156, + 0.3124367296695709 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7271867990493774, + "mean_weights": [ + 0.19560003280639648, + 0.18333327770233154, + 0.08475087583065033, + 0.11598999053239822, + 0.15889793634414673, + 0.2614278793334961 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6916961669921875, + "mean_weights": [ + 0.19065657258033752, + 0.12983909249305725, + 0.11857748031616211, + 0.09883742034435272, + 0.1529499739408493, + 0.3091394901275635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7504222393035889, + "mean_weights": [ + 0.20809361338615417, + 0.19103100895881653, + 0.10743936896324158, + 0.1194620355963707, + 0.15873855352401733, + 0.2152354121208191 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5379457473754883, + "mean_weights": [ + 0.09571269154548645, + 0.07078731805086136, + 0.0824701339006424, + 0.060012347996234894, + 0.08145605027675629, + 0.09452696144580841, + 0.5150344967842102 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7319592237472534, + "mean_weights": [ + 0.13678354024887085, + 0.11740625649690628, + 0.06668869405984879, + 0.07360264658927917, + 0.09287553280591965, + 0.12232495844364166, + 0.390318363904953 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4663677215576172, + "mean_weights": [ + 0.09485072642564774, + 0.07167200744152069, + 0.0841417908668518, + 0.05784624442458153, + 0.07052354514598846, + 0.0724712610244751, + 0.5484944581985474 + ], + "sources": 7 + }, + { + "entropy_mean": 1.749481201171875, + "mean_weights": [ + 0.14530614018440247, + 0.12230368703603745, + 0.0660901889204979, + 0.07630561292171478, + 0.09453228116035461, + 0.12125740200281143, + 0.3742046654224396 + ], + "sources": 7 + }, + { + "entropy_mean": 1.472255825996399, + "mean_weights": [ + 0.10688216984272003, + 0.07465942203998566, + 0.08327433466911316, + 0.054589346051216125, + 0.0637175589799881, + 0.07299347221851349, + 0.5438836812973022 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7924723625183105, + "mean_weights": [ + 0.15522946417331696, + 0.13724805414676666, + 0.06889298558235168, + 0.08198928833007812, + 0.09816981852054596, + 0.123870849609375, + 0.3345995247364044 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5755163431167603, + "mean_weights": [ + 0.12274101376533508, + 0.08900541067123413, + 0.08551642298698425, + 0.06285787373781204, + 0.07429315149784088, + 0.07904880493879318, + 0.4865373373031616 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8390922546386719, + "mean_weights": [ + 0.15884171426296234, + 0.14726482331752777, + 0.08053631335496902, + 0.09097073972225189, + 0.10738551616668701, + 0.12150256335735321, + 0.29349833726882935 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6550344228744507, + "mean_weights": [ + 0.0844201147556305, + 0.06695197522640228, + 0.061310961842536926, + 0.0447731651365757, + 0.04748314991593361, + 0.045562684535980225, + 0.2905467748641968, + 0.3589511513710022 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8679088354110718, + "mean_weights": [ + 0.11674155294895172, + 0.1088850349187851, + 0.05779466778039932, + 0.06273136287927628, + 0.07066237181425095, + 0.08129343390464783, + 0.17753683030605316, + 0.32435476779937744 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5923268795013428, + "mean_weights": [ + 0.08656083047389984, + 0.05770102143287659, + 0.05105586349964142, + 0.039375390857458115, + 0.04410157352685928, + 0.04500962793827057, + 0.2357216477394104, + 0.4404740333557129 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8469831943511963, + "mean_weights": [ + 0.10824646055698395, + 0.09116974472999573, + 0.0701109915971756, + 0.0629766583442688, + 0.07049155235290527, + 0.07297641038894653, + 0.18795298039913177, + 0.33607521653175354 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6590619087219238, + "mean_weights": [ + 0.0944472923874855, + 0.0654030293226242, + 0.060137998312711716, + 0.0449567511677742, + 0.044405773282051086, + 0.05244143307209015, + 0.2202538251876831, + 0.4179539084434509 + ], + "sources": 8 + }, + { + "entropy_mean": 1.884438157081604, + "mean_weights": [ + 0.11613816022872925, + 0.09775452315807343, + 0.08394932001829147, + 0.07057420164346695, + 0.06920696794986725, + 0.06340071558952332, + 0.21605131030082703, + 0.2829247713088989 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7424441576004028, + "mean_weights": [ + 0.08763163536787033, + 0.07771545648574829, + 0.07502467930316925, + 0.06006643921136856, + 0.06314834952354431, + 0.054570816457271576, + 0.3030073344707489, + 0.278835266828537 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8953055143356323, + "mean_weights": [ + 0.1135263741016388, + 0.10274903476238251, + 0.0858360230922699, + 0.08475605398416519, + 0.08078759908676147, + 0.07204630225896835, + 0.2779633104801178, + 0.18233531713485718 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801485300064087, + "mean_weights": [ + 0.07824120670557022, + 0.06023171544075012, + 0.054100219160318375, + 0.039871927350759506, + 0.04207881540060043, + 0.045095670968294144, + 0.2047339826822281, + 0.15412676334381104, + 0.321519672870636 + ], + "sources": 9 + }, + { + "entropy_mean": 2.008213520050049, + "mean_weights": [ + 0.09438058733940125, + 0.07153058052062988, + 0.07423336803913116, + 0.06403130292892456, + 0.0645662397146225, + 0.05577655881643295, + 0.21496367454528809, + 0.1609123945236206, + 0.1996053010225296 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8966764211654663, + "mean_weights": [ + 0.09610478579998016, + 0.07267358899116516, + 0.060228414833545685, + 0.046177469193935394, + 0.04206448048353195, + 0.04096413776278496, + 0.21562573313713074, + 0.21000833809375763, + 0.21615305542945862 + ], + "sources": 9 + }, + { + "entropy_mean": 1.981720209121704, + "mean_weights": [ + 0.10501760244369507, + 0.06946457922458649, + 0.07816961407661438, + 0.07385893911123276, + 0.06364607810974121, + 0.05328303575515747, + 0.23084522783756256, + 0.15472577512264252, + 0.17098914086818695 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0028607845306396, + "mean_weights": [ + 0.0842779204249382, + 0.10298711061477661, + 0.09207960218191147, + 0.07771934568881989, + 0.06265503168106079, + 0.0552704893052578, + 0.2620389461517334, + 0.1382768750190735, + 0.12469469010829926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0655057430267334, + "mean_weights": [ + 0.10472239553928375, + 0.11792458593845367, + 0.0928182601928711, + 0.10183010995388031, + 0.08287280797958374, + 0.06570011377334595, + 0.21871674060821533, + 0.11616810411214828, + 0.09924688935279846 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0258567333221436, + "mean_weights": [ + 0.09674200415611267, + 0.09482249617576599, + 0.08996410667896271, + 0.06836152076721191, + 0.06293784081935883, + 0.05016916245222092, + 0.22868779301643372, + 0.1848844438791275, + 0.12343062460422516 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059412047266960144, + 0.056020382791757584, + 0.08935670554637909, + 0.0742802768945694, + 0.17747226357460022, + 0.1285802572965622, + 0.17946027219295502, + 0.10598119348287582, + 0.17260213196277618, + 0.10133238881826401, + 0.12674333155155182, + 0.10985831171274185, + 0.12412907183170319, + 0.12611564993858337, + 0.11767174303531647, + 0.1125042662024498, + 0.13582420349121094, + 0.07848236709833145, + 0.10855290293693542, + 0.0899009183049202, + 0.10545514523983002, + 0.09790628403425217, + 0.11005345731973648, + 0.09713682532310486, + 0.13049271702766418, + 0.09147685021162033, + 0.09666567295789719, + 0.10080859810113907, + 0.10358717292547226, + 0.09896469116210938, + 0.11041814833879471, + 0.10007023066282272, + 0.13209649920463562, + 0.10974904894828796, + 0.10239993035793304, + 0.10291008651256561, + 0.11153820157051086, + 0.10674569010734558, + 0.11912636458873749, + 0.12148815393447876, + 0.127323716878891, + 0.055056143552064896, + 0.07468672841787338, + 0.05981758236885071, + 0.07975661009550095, + 0.07101839780807495, + 0.08746611326932907, + 0.08339047431945801, + 0.09389390796422958, + 0.06143829599022865, + 0.06575146317481995, + 0.06018613278865814, + 0.0681111291050911, + 0.06816941499710083, + 0.07568862289190292, + 0.08121132105588913, + 0.08424623310565948, + 0.05916989967226982, + 0.0719028189778328, + 0.07458337396383286, + 0.08439797908067703, + 0.08333029597997665, + 0.08961156010627747, + 0.09104464948177338 + ], + "loss_nats": 1.347039818763733, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7394249439239502, + "mean_weights": [ + 0.10602211207151413, + 0.05902107432484627, + 0.049884356558322906, + 0.034865088760852814, + 0.03282830864191055, + 0.043302543461322784, + 0.08693542331457138, + 0.3314756751060486, + 0.2556654214859009 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08528144657611847, + 0.16101159155368805, + 0.3133305609226227, + 0.3334391713142395, + 0.3510810136795044, + 0.3570481836795807, + 0.36267346143722534, + 0.37708213925361633, + 0.1070452407002449, + 0.11129721254110336, + 0.1260577291250229, + 0.12943366169929504, + 0.14502130448818207, + 0.1512180119752884, + 0.17329032719135284, + 0.1945808082818985, + 0.061002109199762344, + 0.07549218833446503, + 0.0987050011754036, + 0.11765842139720917, + 0.13416865468025208, + 0.16134388744831085, + 0.17569874227046967, + 0.21493762731552124, + 0.06714949756860733, + 0.08329613506793976, + 0.09735818952322006, + 0.12312135100364685, + 0.13312570750713348, + 0.16809259355068207, + 0.17941679060459137, + 0.23101076483726501, + 0.10073095560073853, + 0.11538279056549072, + 0.13780468702316284, + 0.16158254444599152, + 0.17965665459632874, + 0.20080390572547913, + 0.24453195929527283, + 0.27125677466392517, + 0.04170985147356987, + 0.0543946772813797, + 0.07352267950773239, + 0.09335283935070038, + 0.10462787002325058, + 0.12897421419620514, + 0.14256280660629272, + 0.17114612460136414, + 0.030182205140590668, + 0.06173322722315788, + 0.06915032118558884, + 0.09693212062120438, + 0.10093247890472412, + 0.13771499693393707, + 0.1392749696969986, + 0.18387214839458466, + 0.030125634744763374, + 0.07983565330505371, + 0.08267941325902939, + 0.15526357293128967, + 0.15948866307735443, + 0.26127612590789795, + 0.2670789062976837, + 0.39550676941871643 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "population_cv": 0.6420341358971389 + }, + "activation_output_rms_by_block": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23796202801167965, + "first_to_last_ratio": 1.7210004980371292, + "imbalance_abs_log_ratio": 0.542905806617582, + "last_quartile_mean": 0.13826958695426583, + "mean": 0.15974999987520278, + "normalized": [ + 1.0588703589137207, + 2.3439518181213, + 2.5018026404306735, + 2.625406629974646, + 0.6192476728139908, + 0.7301872144203714, + 0.8733231516990753, + 1.1639318005158055, + 0.41362918444307983, + 0.6526334563206251, + 0.9108721190988135, + 1.2435622068278698, + 0.40590548584340497, + 0.6529420199592338, + 0.9214830517691831, + 1.2794736788600816, + 0.6292358348878756, + 0.8817240486108118, + 1.0928942572485663, + 1.5151234871428112, + 0.30958389010160414, + 0.5326764501933169, + 0.737604494775539, + 0.9796364773984299, + 0.3464072479049888, + 0.5325333617345424, + 0.7475662125351719, + 0.9760454607844148, + 0.41495429177381643, + 0.7752670281456541, + 1.247487625087717, + 1.8840373416628649 + ], + "population_cv": 0.591409330949146 + }, + "bits_per_byte": 1.8740129107275643, + "branch_output_rms_by_sublayer": [ + 0.0933690071105957, + 0.13583384454250336, + 0.24537226557731628, + 0.0415157787501812, + 0.07698935270309448, + 0.018140379339456558, + 0.0523640513420105, + 0.039518751204013824, + 0.09380833059549332, + 0.022877270355820656, + 0.03647654876112938, + 0.015532293356955051, + 0.039120711386203766, + 0.02314678393304348, + 0.06371714919805527, + 0.05432315170764923, + 0.05156813934445381, + 0.03743875026702881, + 0.03728010877966881, + 0.04006272554397583, + 0.03983496502041817, + 0.045583780854940414, + 0.040284499526023865, + 0.05995568633079529, + 0.046871740370988846, + 0.038651175796985626, + 0.04314620420336723, + 0.04559725895524025, + 0.04033593460917473, + 0.05632574483752251, + 0.04471700266003609, + 0.07952121645212173, + 0.08397205173969269, + 0.042337141931056976, + 0.0528092160820961, + 0.05044656619429588, + 0.04443390294909477, + 0.04418695345520973, + 0.07560808956623077, + 0.06310147047042847, + 0.03708760067820549, + 0.029010163620114326, + 0.03599191829562187, + 0.04586305841803551, + 0.033304885029792786, + 0.06033914536237717, + 0.04061548039317131, + 0.07656749337911606, + 0.025487789884209633, + 0.045884523540735245, + 0.021565141156315804, + 0.051604680716991425, + 0.02342291921377182, + 0.06502768397331238, + 0.017244853079319, + 0.06792232394218445, + 0.024166326969861984, + 0.060306549072265625, + 0.016819356009364128, + 0.08025261014699936, + 0.021590016782283783, + 0.09663844108581543, + 0.026201345026493073, + 0.12230555713176727 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003065916678101079, + 0.00023586292031870838, + 0.0001322871105193671, + 0.00016391009158296018, + 0.0001640597280926294, + 0.00011452252186811768, + 0.00012670462239247783, + 0.00021513822087305528, + 0.00015124888511926465, + 0.00012797922905715274, + 0.0001317390201682035, + 0.00014536342393434786, + 0.00017054401801644127, + 0.00018163687748006386, + 0.00019727429589294548, + 0.0002118946661868627, + 0.0002209032617907253, + 0.0002444386191807451, + 0.00023590577825104205, + 0.00032383840230939837, + 0.0010316902394974224, + 0.0010821684520459678, + 0.0009765210521856827, + 0.0008761640307174725, + 0.0008326776627851861, + 0.0006767392019523396, + 0.00052552098450176, + 0.0004066875433884613, + 0.0003364842674135882, + 0.00025378360531775436, + 0.000205309661176443, + 0.00020659565641729775 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018238461043217796, + "first_to_last_ratio": 0.42368241008054586, + "imbalance_abs_log_ratio": 0.8587711372443304, + "last_quartile_mean": 0.0004304748228691038, + "mean": 0.00035038080369512474, + "normalized": [ + 0.875024158220954, + 0.6731616510701846, + 0.37755239192405493, + 0.46780556998089007, + 0.46823263821091626, + 0.3268515873596963, + 0.3616197607181892, + 0.6140125788975943, + 0.43167001024082974, + 0.3652575361078021, + 0.37598812143496585, + 0.4148726825252455, + 0.486739045683667, + 0.5183984840622454, + 0.5630282647122382, + 0.6047553517550511, + 0.630466222638552, + 0.69763701836085, + 0.6732839692219831, + 0.9242469875466649, + 2.944482770223686, + 3.0885494885375917, + 2.7870278333951726, + 2.500605117282182, + 2.3764933866346176, + 1.9314391508194257, + 1.499856667259172, + 1.160701554136312, + 0.9603387624693381, + 0.7243079604856947, + 0.5859614996348036, + 0.5896317784494322 + ], + "population_cv": 0.8248093894360681 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22886337339878082, + "mean_weights": [ + 0.9364937543869019, + 0.06350618600845337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.603729784488678, + "mean_weights": [ + 0.5866844654083252, + 0.4133155643939972 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3967982828617096, + "mean_weights": [ + 0.858893632888794, + 0.14110638201236725 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6852496862411499, + "mean_weights": [ + 0.48102498054504395, + 0.518975019454956 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5890734195709229, + "mean_weights": [ + 0.6876576542854309, + 0.3123423457145691 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873323321342468, + "mean_weights": [ + 0.5168514847755432, + 0.4831485152244568 + ], + "sources": 2 + }, + { + "entropy_mean": 0.487301766872406, + "mean_weights": [ + 0.7939615249633789, + 0.2060384750366211 + ], + "sources": 2 + }, + { + "entropy_mean": 0.674697756767273, + "mean_weights": [ + 0.5704737305641174, + 0.42952626943588257 + ], + "sources": 2 + }, + { + "entropy_mean": 0.9691112041473389, + "mean_weights": [ + 0.5506306886672974, + 0.18125343322753906, + 0.2681158185005188 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0745112895965576, + "mean_weights": [ + 0.342629998922348, + 0.24457570910453796, + 0.412794291973114 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0074045658111572, + "mean_weights": [ + 0.4399106502532959, + 0.18251104652881622, + 0.3775783181190491 + ], + "sources": 3 + }, + { + "entropy_mean": 1.060497522354126, + "mean_weights": [ + 0.3786153495311737, + 0.2188822329044342, + 0.4025024175643921 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9941019415855408, + "mean_weights": [ + 0.32296958565711975, + 0.19686520099639893, + 0.4801652133464813 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0503522157669067, + "mean_weights": [ + 0.40130966901779175, + 0.19853851199150085, + 0.4001518487930298 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0129402875900269, + "mean_weights": [ + 0.4093267321586609, + 0.16906049847602844, + 0.42161279916763306 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0774019956588745, + "mean_weights": [ + 0.38510820269584656, + 0.25884830951690674, + 0.3560434877872467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2740377187728882, + "mean_weights": [ + 0.29550763964653015, + 0.09541245549917221, + 0.24569378793239594, + 0.3633860945701599 + ], + "sources": 4 + }, + { + "entropy_mean": 1.360302448272705, + "mean_weights": [ + 0.3109527826309204, + 0.20984822511672974, + 0.1935039907693863, + 0.28569501638412476 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2722704410552979, + "mean_weights": [ + 0.350364625453949, + 0.12631991505622864, + 0.22418734431266785, + 0.29912811517715454 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3558036088943481, + "mean_weights": [ + 0.2989768385887146, + 0.1941717267036438, + 0.1926535815000534, + 0.314197838306427 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2696614265441895, + "mean_weights": [ + 0.3667730689048767, + 0.15310388803482056, + 0.19384080171585083, + 0.2862822115421295 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3606936931610107, + "mean_weights": [ + 0.2982274293899536, + 0.18611131608486176, + 0.21284276247024536, + 0.3028184771537781 + ], + "sources": 4 + }, + { + "entropy_mean": 1.269316554069519, + "mean_weights": [ + 0.4081852436065674, + 0.14420126378536224, + 0.18228203058242798, + 0.2653314471244812 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3636136054992676, + "mean_weights": [ + 0.27427661418914795, + 0.21341991424560547, + 0.19867971539497375, + 0.3136237859725952 + ], + "sources": 4 + }, + { + "entropy_mean": 1.570773959159851, + "mean_weights": [ + 0.26587405800819397, + 0.13419361412525177, + 0.1907704770565033, + 0.17973750829696655, + 0.2294243425130844 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5560038089752197, + "mean_weights": [ + 0.20477107167243958, + 0.15721666812896729, + 0.13811248540878296, + 0.1752161681652069, + 0.32468360662460327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5659973621368408, + "mean_weights": [ + 0.20531430840492249, + 0.15986156463623047, + 0.21557101607322693, + 0.15222224593162537, + 0.26703086495399475 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5552496910095215, + "mean_weights": [ + 0.2162160575389862, + 0.17982497811317444, + 0.1307673156261444, + 0.1554446816444397, + 0.31774696707725525 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5068919658660889, + "mean_weights": [ + 0.20646695792675018, + 0.1197461187839508, + 0.19215428829193115, + 0.15179987251758575, + 0.3298327624797821 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5435223579406738, + "mean_weights": [ + 0.22410975396633148, + 0.16986176371574402, + 0.13080401718616486, + 0.15279728174209595, + 0.3224271535873413 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4949920177459717, + "mean_weights": [ + 0.27174079418182373, + 0.12178683280944824, + 0.1529826819896698, + 0.1350194811820984, + 0.31847020983695984 + ], + "sources": 5 + }, + { + "entropy_mean": 1.544264793395996, + "mean_weights": [ + 0.24016115069389343, + 0.20730282366275787, + 0.09692574292421341, + 0.1695065051317215, + 0.2861037850379944 + ], + "sources": 5 + }, + { + "entropy_mean": 1.686682939529419, + "mean_weights": [ + 0.16755425930023193, + 0.10962115228176117, + 0.1783454418182373, + 0.13310004770755768, + 0.2885603904724121, + 0.12281867861747742 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7070834636688232, + "mean_weights": [ + 0.181177020072937, + 0.14557164907455444, + 0.08318156003952026, + 0.11424209922552109, + 0.17698991298675537, + 0.2988377511501312 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7419235706329346, + "mean_weights": [ + 0.1532188206911087, + 0.11420352011919022, + 0.1659742295742035, + 0.1250365972518921, + 0.18140509724617004, + 0.26016172766685486 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7127439975738525, + "mean_weights": [ + 0.18731769919395447, + 0.16715484857559204, + 0.08496320992708206, + 0.11362075805664062, + 0.1561521738767624, + 0.2907913327217102 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6076834201812744, + "mean_weights": [ + 0.15032559633255005, + 0.08624927699565887, + 0.15736976265907288, + 0.08584291487932205, + 0.14632999897003174, + 0.3738824725151062 + ], + "sources": 6 + }, + { + "entropy_mean": 1.709290862083435, + "mean_weights": [ + 0.18991519510746002, + 0.17091524600982666, + 0.07822999358177185, + 0.116765096783638, + 0.1540035903453827, + 0.2901708781719208 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6690887212753296, + "mean_weights": [ + 0.19029225409030914, + 0.11002412438392639, + 0.11471512168645859, + 0.0999990925192833, + 0.1507222205400467, + 0.3342471718788147 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7458693981170654, + "mean_weights": [ + 0.20528709888458252, + 0.1722964346408844, + 0.10435444861650467, + 0.12075909972190857, + 0.16116347908973694, + 0.2361394464969635 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4612125158309937, + "mean_weights": [ + 0.08934466540813446, + 0.054379623383283615, + 0.07567165046930313, + 0.05758609622716904, + 0.07884950935840607, + 0.08982288837432861, + 0.5543456077575684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7241814136505127, + "mean_weights": [ + 0.13360008597373962, + 0.10609637945890427, + 0.06363911926746368, + 0.07426095008850098, + 0.09383077174425125, + 0.13373199105262756, + 0.39484071731567383 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3979300260543823, + "mean_weights": [ + 0.08959861099720001, + 0.05414731800556183, + 0.07668881118297577, + 0.05603286623954773, + 0.06979228556156158, + 0.07073447853326797, + 0.5830056071281433 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7459560632705688, + "mean_weights": [ + 0.14318783581256866, + 0.1094653457403183, + 0.06349228322505951, + 0.07859592139720917, + 0.09567482024431229, + 0.13266830146312714, + 0.37691545486450195 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4055393934249878, + "mean_weights": [ + 0.10154954344034195, + 0.05619275942444801, + 0.07791627943515778, + 0.053915880620479584, + 0.06398355215787888, + 0.06915920972824097, + 0.5772827863693237 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7853620052337646, + "mean_weights": [ + 0.15284112095832825, + 0.12206703424453735, + 0.06508444249629974, + 0.08347312361001968, + 0.09811970591545105, + 0.13767032325267792, + 0.3407442569732666 + ], + "sources": 7 + }, + { + "entropy_mean": 1.509842872619629, + "mean_weights": [ + 0.11788781732320786, + 0.06822571903467178, + 0.08556114137172699, + 0.06067090481519699, + 0.07228855788707733, + 0.07258446514606476, + 0.5227814316749573 + ], + "sources": 7 + }, + { + "entropy_mean": 1.839367389678955, + "mean_weights": [ + 0.15906387567520142, + 0.13139380514621735, + 0.07717745006084442, + 0.09607303887605667, + 0.11078508198261261, + 0.12967455387115479, + 0.29583218693733215 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5841834545135498, + "mean_weights": [ + 0.07896879315376282, + 0.048752427101135254, + 0.055707160383462906, + 0.04139826446771622, + 0.04449019208550453, + 0.04285799711942673, + 0.2801477909088135, + 0.40767738223075867 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8125314712524414, + "mean_weights": [ + 0.10889512300491333, + 0.08656036853790283, + 0.048123255372047424, + 0.06561170518398285, + 0.06917259097099304, + 0.08286689221858978, + 0.16869932413101196, + 0.3700707256793976 + ], + "sources": 8 + }, + { + "entropy_mean": 1.50089693069458, + "mean_weights": [ + 0.07974998652935028, + 0.04166398197412491, + 0.04607272148132324, + 0.03586764261126518, + 0.04223340004682541, + 0.04095568507909775, + 0.22249138355255127, + 0.49096518754959106 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8059929609298706, + "mean_weights": [ + 0.10711923241615295, + 0.07651708275079727, + 0.058284737169742584, + 0.06609588116407394, + 0.07139427959918976, + 0.07493151724338531, + 0.17425751686096191, + 0.37139976024627686 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5622732639312744, + "mean_weights": [ + 0.08775827288627625, + 0.047765426337718964, + 0.053928621113300323, + 0.04293380677700043, + 0.04174989461898804, + 0.04734005779027939, + 0.2016349583864212, + 0.4768889844417572 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8757455348968506, + "mean_weights": [ + 0.11563669890165329, + 0.08108697831630707, + 0.07280858606100082, + 0.07553227990865707, + 0.07606013864278793, + 0.06935927271842957, + 0.19856664538383484, + 0.3109493851661682 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6924852132797241, + "mean_weights": [ + 0.08638577163219452, + 0.059912849217653275, + 0.06609142571687698, + 0.056745607405900955, + 0.06277773529291153, + 0.05352145433425903, + 0.2830069661140442, + 0.3315581977367401 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8980522155761719, + "mean_weights": [ + 0.11603894829750061, + 0.08804173022508621, + 0.07975024729967117, + 0.09026999771595001, + 0.08430293202400208, + 0.07433462888002396, + 0.26602888107299805, + 0.2012326419353485 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7512003183364868, + "mean_weights": [ + 0.07744461297988892, + 0.046086352318525314, + 0.05025433003902435, + 0.037484876811504364, + 0.04005061089992523, + 0.041623204946517944, + 0.198423832654953, + 0.17380405962467194, + 0.33482813835144043 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0160694122314453, + "mean_weights": [ + 0.0938527062535286, + 0.06208505481481552, + 0.06560768187046051, + 0.07029052078723907, + 0.06935097277164459, + 0.056832704693078995, + 0.1942581683397293, + 0.1706070452928543, + 0.21711516380310059 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7837426662445068, + "mean_weights": [ + 0.08836328238248825, + 0.04953558370471001, + 0.04813014343380928, + 0.0373263955116272, + 0.037015400826931, + 0.03390160948038101, + 0.1840541511774063, + 0.23140482604503632, + 0.29026859998703003 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9973840713500977, + "mean_weights": [ + 0.10261696577072144, + 0.06039813905954361, + 0.06671924889087677, + 0.08377155661582947, + 0.07002593576908112, + 0.05768897384405136, + 0.20831166207790375, + 0.1594475358724594, + 0.19101998209953308 + ], + "sources": 9 + }, + { + "entropy_mean": 1.969987154006958, + "mean_weights": [ + 0.08120635896921158, + 0.07894632965326309, + 0.08615730702877045, + 0.07609778642654419, + 0.058728985488414764, + 0.05076971277594566, + 0.24503237009048462, + 0.1544169783592224, + 0.16864418983459473 + ], + "sources": 9 + }, + { + "entropy_mean": 2.071648359298706, + "mean_weights": [ + 0.10569609701633453, + 0.09922641515731812, + 0.08180510997772217, + 0.10646798461675644, + 0.0826798751950264, + 0.06615597754716873, + 0.20955613255500793, + 0.13320739567279816, + 0.11520501971244812 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781701564788818, + "mean_weights": [ + 0.09446848928928375, + 0.07056106626987457, + 0.08033449202775955, + 0.0658978745341301, + 0.058234505355358124, + 0.04316519945859909, + 0.2036968618631363, + 0.22251945734024048, + 0.16112206876277924 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0626930519938469, + 0.059514161199331284, + 0.09030644595623016, + 0.08157797157764435, + 0.20229491591453552, + 0.14292451739311218, + 0.20300470292568207, + 0.10997853428125381, + 0.19049881398677826, + 0.0991024449467659, + 0.12695962190628052, + 0.1065012738108635, + 0.12132706493139267, + 0.12155740708112717, + 0.11585728824138641, + 0.1095810979604721, + 0.13427142798900604, + 0.06964563578367233, + 0.10485932976007462, + 0.08311934769153595, + 0.10419609397649765, + 0.09363401681184769, + 0.10852954536676407, + 0.0941167026758194, + 0.1286265105009079, + 0.08564268052577972, + 0.09096142649650574, + 0.09493856877088547, + 0.10080574452877045, + 0.09063401818275452, + 0.10566592961549759, + 0.09473779052495956, + 0.1287592649459839, + 0.10505300015211105, + 0.09650235623121262, + 0.09355910867452621, + 0.1065765842795372, + 0.09618847072124481, + 0.11446042358875275, + 0.11416846513748169, + 0.12395420670509338, + 0.047205954790115356, + 0.07221997529268265, + 0.053352247923612595, + 0.07677769660949707, + 0.06512638926506042, + 0.08416415005922318, + 0.07829347252845764, + 0.09094838052988052, + 0.05368548259139061, + 0.059439923614263535, + 0.05310077592730522, + 0.062182098627090454, + 0.061235878616571426, + 0.06896103918552399, + 0.07320034503936768, + 0.0776243805885315, + 0.05304349213838577, + 0.06285123527050018, + 0.0639304369688034, + 0.07231374084949493, + 0.07404901832342148, + 0.08016443997621536, + 0.08157160878181458 + ], + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.6036605834960938, + "mean_weights": [ + 0.09759747982025146, + 0.04215940088033676, + 0.041145484894514084, + 0.029554029926657677, + 0.02899969555437565, + 0.03647579252719879, + 0.06642461568117142, + 0.341248482465744, + 0.31639501452445984 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.0933690071105957, + 0.16915453970432281, + 0.35791492462158203, + 0.374446302652359, + 0.3940892517566681, + 0.39966297149658203, + 0.4057151973247528, + 0.4194087088108063, + 0.09380833059549332, + 0.09892481565475464, + 0.11286329478025436, + 0.11664740741252899, + 0.13231655955314636, + 0.13951337337493896, + 0.1621972620487213, + 0.18593810498714447, + 0.05156813934445381, + 0.06607726216316223, + 0.08533531427383423, + 0.10425819456577301, + 0.12003090232610703, + 0.14551182091236115, + 0.16063299775123596, + 0.19865906238555908, + 0.046871740370988846, + 0.06484340131282806, + 0.079384446144104, + 0.10430748760700226, + 0.11459959298372269, + 0.14720691740512848, + 0.15625107288360596, + 0.20439592003822327, + 0.08397205173969269, + 0.10052042454481125, + 0.11654024571180344, + 0.14085541665554047, + 0.15294702351093292, + 0.1745898574590683, + 0.21572710573673248, + 0.2420409768819809, + 0.03708760067820549, + 0.049456026405096054, + 0.06553087383508682, + 0.08509506285190582, + 0.09390194714069366, + 0.11783231794834137, + 0.12852425873279572, + 0.15649692714214325, + 0.025487789884209633, + 0.05533855780959129, + 0.06059965491294861, + 0.08507220447063446, + 0.08860157430171967, + 0.11942370235919952, + 0.12100546807050705, + 0.1559232622385025, + 0.024166326969861984, + 0.06628894805908203, + 0.06877554953098297, + 0.12384890764951706, + 0.12682415544986725, + 0.19928614795207977, + 0.2025238573551178, + 0.30097496509552 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.14386622172245, + "measured_steps": 7980, + "median_ms": 109.04977201425936, + "p95_ms": 109.64786536205793, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + }, + { + "bits_per_byte": 6.611855126224988, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.582988739013672, + "step": 30, + "unclipped_grad_norm": 3.075686454772949 + }, + { + "bits_per_byte": 6.3915955694037825, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43031644821167, + "step": 40, + "unclipped_grad_norm": 3.2318925857543945 + }, + { + "bits_per_byte": 6.093941755902502, + "learning_rate": 3.75e-05, + "loss_nats": 4.223998546600342, + "step": 50, + "unclipped_grad_norm": 2.8704335689544678 + }, + { + "bits_per_byte": 5.8858000966610895, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.079725742340088, + "step": 60, + "unclipped_grad_norm": 2.669306755065918 + }, + { + "bits_per_byte": 5.715495996470702, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9616799354553223, + "step": 70, + "unclipped_grad_norm": 3.1737496852874756 + }, + { + "bits_per_byte": 5.565897919513593, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8579864501953125, + "step": 80, + "unclipped_grad_norm": 2.5544967651367188 + }, + { + "bits_per_byte": 5.354669171689194, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.711573839187622, + "step": 90, + "unclipped_grad_norm": 2.425950050354004 + }, + { + "bits_per_byte": 5.180876231443403, + "learning_rate": 7.5e-05, + "loss_nats": 3.5911097526550293, + "step": 100, + "unclipped_grad_norm": 2.2783501148223877 + }, + { + "bits_per_byte": 4.992272489270609, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4603796005249023, + "step": 110, + "unclipped_grad_norm": 2.1105899810791016 + }, + { + "bits_per_byte": 4.775783480086919, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3103208541870117, + "step": 120, + "unclipped_grad_norm": 1.8969556093215942 + }, + { + "bits_per_byte": 4.582375918510225, + "learning_rate": 9.75e-05, + "loss_nats": 3.1762609481811523, + "step": 130, + "unclipped_grad_norm": 1.7015109062194824 + }, + { + "bits_per_byte": 4.427903165297061, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0691885948181152, + "step": 140, + "unclipped_grad_norm": 1.446854829788208 + }, + { + "bits_per_byte": 4.271120341117507, + "learning_rate": 0.0001125, + "loss_nats": 2.960515022277832, + "step": 150, + "unclipped_grad_norm": 1.2307006120681763 + }, + { + "bits_per_byte": 4.166711049639903, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.888144016265869, + "step": 160, + "unclipped_grad_norm": 1.0598198175430298 + }, + { + "bits_per_byte": 4.026318516052877, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7908313274383545, + "step": 170, + "unclipped_grad_norm": 0.9654788374900818 + }, + { + "bits_per_byte": 3.94755837315965, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.736238956451416, + "step": 180, + "unclipped_grad_norm": 0.8419675230979919 + }, + { + "bits_per_byte": 3.838975750676209, + "learning_rate": 0.0001425, + "loss_nats": 2.660975217819214, + "step": 190, + "unclipped_grad_norm": 0.817121684551239 + }, + { + "bits_per_byte": 3.774983758012572, + "learning_rate": 0.00015, + "loss_nats": 2.616619348526001, + "step": 200, + "unclipped_grad_norm": 0.955824613571167 + }, + { + "bits_per_byte": 3.7344016998190783, + "learning_rate": 0.0001575, + "loss_nats": 2.5884900093078613, + "step": 210, + "unclipped_grad_norm": 0.8557147979736328 + }, + { + "bits_per_byte": 3.63166201591605, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5172762870788574, + "step": 220, + "unclipped_grad_norm": 0.6891153454780579 + }, + { + "bits_per_byte": 3.6208855830129867, + "learning_rate": 0.0001725, + "loss_nats": 2.5098066329956055, + "step": 230, + "unclipped_grad_norm": 1.036860466003418 + }, + { + "bits_per_byte": 3.4987607026353063, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4251561164855957, + "step": 240, + "unclipped_grad_norm": 0.6406477689743042 + }, + { + "bits_per_byte": 3.5387748741156506, + "learning_rate": 0.0001875, + "loss_nats": 2.4528918266296387, + "step": 250, + "unclipped_grad_norm": 1.106186866760254 + }, + { + "bits_per_byte": 3.5260464381252943, + "learning_rate": 0.000195, + "loss_nats": 2.4440691471099854, + "step": 260, + "unclipped_grad_norm": 0.8556521534919739 + }, + { + "bits_per_byte": 3.5219934949750615, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4412598609924316, + "step": 270, + "unclipped_grad_norm": 0.9917709827423096 + }, + { + "bits_per_byte": 3.474740573719901, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4085066318511963, + "step": 280, + "unclipped_grad_norm": 1.2901222705841064 + }, + { + "bits_per_byte": 3.363479149466621, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.331386089324951, + "step": 290, + "unclipped_grad_norm": 0.7895627617835999 + }, + { + "bits_per_byte": 3.4037392561376993, + "learning_rate": 0.000225, + "loss_nats": 2.3592922687530518, + "step": 300, + "unclipped_grad_norm": 1.5281883478164673 + }, + { + "bits_per_byte": 3.3541442751430894, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.324915647506714, + "step": 310, + "unclipped_grad_norm": 0.8648361563682556 + }, + { + "bits_per_byte": 3.3434108379027414, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174757957458496, + "step": 320, + "unclipped_grad_norm": 1.4613573551177979 + }, + { + "bits_per_byte": 3.3325662998699364, + "learning_rate": 0.0002475, + "loss_nats": 2.3099589347839355, + "step": 330, + "unclipped_grad_norm": 1.1215400695800781 + }, + { + "bits_per_byte": 3.4180712583647006, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3692264556884766, + "step": 340, + "unclipped_grad_norm": 1.1166281700134277 + }, + { + "bits_per_byte": 3.3577222022116393, + "learning_rate": 0.0002625, + "loss_nats": 2.3273956775665283, + "step": 350, + "unclipped_grad_norm": 1.0810235738754272 + }, + { + "bits_per_byte": 3.2923828974611427, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2821059226989746, + "step": 360, + "unclipped_grad_norm": 1.128420114517212 + }, + { + "bits_per_byte": 3.291989401155969, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2818331718444824, + "step": 370, + "unclipped_grad_norm": 1.0846341848373413 + }, + { + "bits_per_byte": 3.3389695579268603, + "learning_rate": 0.000285, + "loss_nats": 2.3143973350524902, + "step": 380, + "unclipped_grad_norm": 1.0868512392044067 + }, + { + "bits_per_byte": 3.256548592327368, + "learning_rate": 0.0002925, + "loss_nats": 2.257267475128174, + "step": 390, + "unclipped_grad_norm": 1.1476690769195557 + }, + { + "bits_per_byte": 3.2058776878689605, + "learning_rate": 0.0003, + "loss_nats": 2.2221450805664062, + "step": 400, + "unclipped_grad_norm": 1.1263198852539062 + }, + { + "bits_per_byte": 3.195324144481415, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.214829921722412, + "step": 410, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 3.227808915506986, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.237346649169922, + "step": 420, + "unclipped_grad_norm": 1.2929860353469849 + }, + { + "bits_per_byte": 3.1876103786249166, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2094831466674805, + "step": 430, + "unclipped_grad_norm": 1.071694016456604 + }, + { + "bits_per_byte": 3.2374495749837515, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2440290451049805, + "step": 440, + "unclipped_grad_norm": 0.9089367985725403 + }, + { + "bits_per_byte": 3.155426921203657, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1871752738952637, + "step": 450, + "unclipped_grad_norm": 1.1129212379455566 + }, + { + "bits_per_byte": 3.188086770567806, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.209813356399536, + "step": 460, + "unclipped_grad_norm": 1.3905333280563354 + }, + { + "bits_per_byte": 3.128408790718412, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.168447732925415, + "step": 470, + "unclipped_grad_norm": 1.059533953666687 + }, + { + "bits_per_byte": 3.136691131218663, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1741886138916016, + "step": 480, + "unclipped_grad_norm": 1.0161854028701782 + }, + { + "bits_per_byte": 3.0887341129908705, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1409473419189453, + "step": 490, + "unclipped_grad_norm": 1.385905385017395 + }, + { + "bits_per_byte": 3.033835874974255, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1028947830200195, + "step": 500, + "unclipped_grad_norm": 1.1716443300247192 + }, + { + "bits_per_byte": 3.0607071322756636, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.121520519256592, + "step": 510, + "unclipped_grad_norm": 0.947066605091095 + }, + { + "bits_per_byte": 2.997100036784657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0774314403533936, + "step": 520, + "unclipped_grad_norm": 1.352820634841919 + }, + { + "bits_per_byte": 3.0643050093317137, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.124014377593994, + "step": 530, + "unclipped_grad_norm": 1.2646857500076294 + }, + { + "bits_per_byte": 3.008989197438803, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.085672378540039, + "step": 540, + "unclipped_grad_norm": 1.0451104640960693 + }, + { + "bits_per_byte": 2.978105584892765, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.064265489578247, + "step": 550, + "unclipped_grad_norm": 1.2619587182998657 + }, + { + "bits_per_byte": 2.968300166036473, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.057468891143799, + "step": 560, + "unclipped_grad_norm": 0.9861598610877991 + }, + { + "bits_per_byte": 2.9221661639078262, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.025491237640381, + "step": 570, + "unclipped_grad_norm": 1.1261875629425049 + }, + { + "bits_per_byte": 2.96999178739036, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0586414337158203, + "step": 580, + "unclipped_grad_norm": 0.8640331625938416 + }, + { + "bits_per_byte": 2.883698804389342, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9988276958465576, + "step": 590, + "unclipped_grad_norm": 1.0034456253051758 + }, + { + "bits_per_byte": 2.9404706214044567, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0381789207458496, + "step": 600, + "unclipped_grad_norm": 0.8714175224304199 + }, + { + "bits_per_byte": 2.8380133330141457, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.967160940170288, + "step": 610, + "unclipped_grad_norm": 0.9427422881126404 + }, + { + "bits_per_byte": 2.894644124255553, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0064144134521484, + "step": 620, + "unclipped_grad_norm": 1.0238139629364014 + }, + { + "bits_per_byte": 2.81883675149499, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9538687467575073, + "step": 630, + "unclipped_grad_norm": 0.9875481128692627 + }, + { + "bits_per_byte": 2.7787291934352267, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9260683059692383, + "step": 640, + "unclipped_grad_norm": 1.0125378370285034 + }, + { + "bits_per_byte": 2.728423580127252, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8911991119384766, + "step": 650, + "unclipped_grad_norm": 1.5507988929748535 + }, + { + "bits_per_byte": 2.649404087396524, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8364269733428955, + "step": 660, + "unclipped_grad_norm": 0.9040125012397766 + }, + { + "bits_per_byte": 2.7024973674923345, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8732284307479858, + "step": 670, + "unclipped_grad_norm": 1.205861210823059 + }, + { + "bits_per_byte": 2.641403626461037, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8308814764022827, + "step": 680, + "unclipped_grad_norm": 1.3370649814605713 + }, + { + "bits_per_byte": 2.7325929555321187, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8940891027450562, + "step": 690, + "unclipped_grad_norm": 1.6124995946884155 + }, + { + "bits_per_byte": 2.6130706886099455, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8112425804138184, + "step": 700, + "unclipped_grad_norm": 1.08383309841156 + }, + { + "bits_per_byte": 2.628106443844277, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.821664571762085, + "step": 710, + "unclipped_grad_norm": 1.0767779350280762 + }, + { + "bits_per_byte": 2.6725746220166666, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.852487564086914, + "step": 720, + "unclipped_grad_norm": 1.296352505683899 + }, + { + "bits_per_byte": 2.608218370098502, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8078792095184326, + "step": 730, + "unclipped_grad_norm": 1.0864766836166382 + }, + { + "bits_per_byte": 2.555699340073923, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7714757919311523, + "step": 740, + "unclipped_grad_norm": 1.0339913368225098 + }, + { + "bits_per_byte": 2.515936435246489, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743914246559143, + "step": 750, + "unclipped_grad_norm": 1.1017143726348877 + }, + { + "bits_per_byte": 2.4657125136976736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.709101676940918, + "step": 760, + "unclipped_grad_norm": 1.7173939943313599 + }, + { + "bits_per_byte": 2.601046349592221, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.802907943725586, + "step": 770, + "unclipped_grad_norm": 0.9487742781639099 + }, + { + "bits_per_byte": 2.4794518637098015, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7186250686645508, + "step": 780, + "unclipped_grad_norm": 0.9247332811355591 + }, + { + "bits_per_byte": 2.466098098800908, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7093689441680908, + "step": 790, + "unclipped_grad_norm": 1.9121034145355225 + }, + { + "bits_per_byte": 2.4025570446478333, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.66532564163208, + "step": 800, + "unclipped_grad_norm": 1.1767630577087402 + }, + { + "bits_per_byte": 2.455517898102479, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7020353078842163, + "step": 810, + "unclipped_grad_norm": 1.1299240589141846 + }, + { + "bits_per_byte": 2.400107839716896, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6636279821395874, + "step": 820, + "unclipped_grad_norm": 1.0006046295166016 + }, + { + "bits_per_byte": 2.4500607166079513, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6982526779174805, + "step": 830, + "unclipped_grad_norm": 1.0554603338241577 + }, + { + "bits_per_byte": 2.5153210813217015, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7434877157211304, + "step": 840, + "unclipped_grad_norm": 1.0736713409423828 + }, + { + "bits_per_byte": 2.399556119372928, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6632455587387085, + "step": 850, + "unclipped_grad_norm": 0.8279580473899841 + }, + { + "bits_per_byte": 2.414819407654351, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6738252639770508, + "step": 860, + "unclipped_grad_norm": 1.2342168092727661 + }, + { + "bits_per_byte": 2.3222158652452247, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6096373796463013, + "step": 870, + "unclipped_grad_norm": 1.0648512840270996 + }, + { + "bits_per_byte": 2.3629336177502487, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6378607749938965, + "step": 880, + "unclipped_grad_norm": 0.9258757829666138 + }, + { + "bits_per_byte": 2.2961121665146167, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5915436744689941, + "step": 890, + "unclipped_grad_norm": 1.0717874765396118 + }, + { + "bits_per_byte": 2.3158979105831285, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6052581071853638, + "step": 900, + "unclipped_grad_norm": 0.9598665833473206 + }, + { + "bits_per_byte": 2.3301258633063577, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6151201725006104, + "step": 910, + "unclipped_grad_norm": 1.0995051860809326 + }, + { + "bits_per_byte": 2.3123376977276173, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.602790355682373, + "step": 920, + "unclipped_grad_norm": 1.1837356090545654 + }, + { + "bits_per_byte": 2.3340250539667196, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6178228855133057, + "step": 930, + "unclipped_grad_norm": 1.1502224206924438 + }, + { + "bits_per_byte": 2.330418921743428, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6153233051300049, + "step": 940, + "unclipped_grad_norm": 1.0088714361190796 + }, + { + "bits_per_byte": 2.2995397807463123, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5939195156097412, + "step": 950, + "unclipped_grad_norm": 0.9483612179756165 + }, + { + "bits_per_byte": 2.3536873144919115, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6314517259597778, + "step": 960, + "unclipped_grad_norm": 0.9582105278968811 + }, + { + "bits_per_byte": 2.2526236015215417, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5613996982574463, + "step": 970, + "unclipped_grad_norm": 1.2002545595169067 + }, + { + "bits_per_byte": 2.3230659754884395, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6102266311645508, + "step": 980, + "unclipped_grad_norm": 0.9191399812698364 + }, + { + "bits_per_byte": 2.285470911975131, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.584167718887329, + "step": 990, + "unclipped_grad_norm": 1.053741216659546 + }, + { + "bits_per_byte": 2.248061417742104, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5582374334335327, + "step": 1000, + "unclipped_grad_norm": 0.9024814963340759 + }, + { + "bits_per_byte": 2.2402749032242903, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.552840232849121, + "step": 1010, + "unclipped_grad_norm": 1.048907995223999 + }, + { + "bits_per_byte": 2.2041726491204714, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278160572052002, + "step": 1020, + "unclipped_grad_norm": 1.0486118793487549 + }, + { + "bits_per_byte": 2.216881995036582, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5366255044937134, + "step": 1030, + "unclipped_grad_norm": 0.9243784546852112 + }, + { + "bits_per_byte": 2.2610579746851553, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5672459602355957, + "step": 1040, + "unclipped_grad_norm": 1.102870225906372 + }, + { + "bits_per_byte": 2.2528270569975124, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5615407228469849, + "step": 1050, + "unclipped_grad_norm": 1.0764544010162354 + }, + { + "bits_per_byte": 2.14889054582713, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.489497423171997, + "step": 1060, + "unclipped_grad_norm": 0.971049427986145 + }, + { + "bits_per_byte": 2.2350196293618754, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5491975545883179, + "step": 1070, + "unclipped_grad_norm": 1.0203380584716797 + }, + { + "bits_per_byte": 2.217042282867186, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5367366075515747, + "step": 1080, + "unclipped_grad_norm": 1.023221492767334 + }, + { + "bits_per_byte": 2.186512266651322, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5155748128890991, + "step": 1090, + "unclipped_grad_norm": 1.111006498336792 + }, + { + "bits_per_byte": 2.2323391077655277, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5473395586013794, + "step": 1100, + "unclipped_grad_norm": 0.9547863602638245 + }, + { + "bits_per_byte": 2.116467344590568, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4670233726501465, + "step": 1110, + "unclipped_grad_norm": 1.1085364818572998 + }, + { + "bits_per_byte": 2.143238164023873, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.485579490661621, + "step": 1120, + "unclipped_grad_norm": 0.901618242263794 + }, + { + "bits_per_byte": 2.077939275196329, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4403177499771118, + "step": 1130, + "unclipped_grad_norm": 0.8683149814605713 + }, + { + "bits_per_byte": 2.224320932617002, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5417817831039429, + "step": 1140, + "unclipped_grad_norm": 1.030465006828308 + }, + { + "bits_per_byte": 2.0642131678483175, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4308035373687744, + "step": 1150, + "unclipped_grad_norm": 0.827274739742279 + }, + { + "bits_per_byte": 2.103388407957753, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4579577445983887, + "step": 1160, + "unclipped_grad_norm": 0.9158841967582703 + }, + { + "bits_per_byte": 2.1566975262803965, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4949088096618652, + "step": 1170, + "unclipped_grad_norm": 0.9397322535514832 + }, + { + "bits_per_byte": 2.1322636071070153, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4779725074768066, + "step": 1180, + "unclipped_grad_norm": 0.8510220050811768 + }, + { + "bits_per_byte": 2.068760561119781, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4339555501937866, + "step": 1190, + "unclipped_grad_norm": 0.9270830154418945 + }, + { + "bits_per_byte": 2.0310528489701576, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4078185558319092, + "step": 1200, + "unclipped_grad_norm": 0.8815417885780334 + }, + { + "bits_per_byte": 2.1008361854189492, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.456188678741455, + "step": 1210, + "unclipped_grad_norm": 1.2037686109542847 + }, + { + "bits_per_byte": 2.1123913557651197, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.464198112487793, + "step": 1220, + "unclipped_grad_norm": 0.8964646458625793 + }, + { + "bits_per_byte": 2.0711254945517954, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4355947971343994, + "step": 1230, + "unclipped_grad_norm": 0.8661414980888367 + }, + { + "bits_per_byte": 2.0159510523978943, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.397350788116455, + "step": 1240, + "unclipped_grad_norm": 0.9392534494400024 + }, + { + "bits_per_byte": 2.104896867788471, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4590033292770386, + "step": 1250, + "unclipped_grad_norm": 0.8430818915367126 + }, + { + "bits_per_byte": 2.08180441329182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.442996859550476, + "step": 1260, + "unclipped_grad_norm": 0.8917548656463623 + }, + { + "bits_per_byte": 2.087352401626003, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4468424320220947, + "step": 1270, + "unclipped_grad_norm": 0.9866762161254883 + }, + { + "bits_per_byte": 1.9607164163161914, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.359065055847168, + "step": 1280, + "unclipped_grad_norm": 0.8596939444541931 + }, + { + "bits_per_byte": 2.088993460080536, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4479799270629883, + "step": 1290, + "unclipped_grad_norm": 1.007126808166504 + }, + { + "bits_per_byte": 2.034349412421896, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4101035594940186, + "step": 1300, + "unclipped_grad_norm": 0.8474438786506653 + }, + { + "bits_per_byte": 2.0247273270714237, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4034340381622314, + "step": 1310, + "unclipped_grad_norm": 0.8226730823516846 + }, + { + "bits_per_byte": 2.020511000402871, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4005115032196045, + "step": 1320, + "unclipped_grad_norm": 0.9414357542991638 + }, + { + "bits_per_byte": 2.085571349293668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4456079006195068, + "step": 1330, + "unclipped_grad_norm": 0.9917200207710266 + }, + { + "bits_per_byte": 2.0422444479924042, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4155759811401367, + "step": 1340, + "unclipped_grad_norm": 0.8823419213294983 + }, + { + "bits_per_byte": 2.0447034559344406, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4172804355621338, + "step": 1350, + "unclipped_grad_norm": 0.8861486315727234 + }, + { + "bits_per_byte": 2.0988803987133373, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548330307006836, + "step": 1360, + "unclipped_grad_norm": 0.9090827107429504 + }, + { + "bits_per_byte": 2.082760980795919, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4436599016189575, + "step": 1370, + "unclipped_grad_norm": 0.8366561532020569 + }, + { + "bits_per_byte": 2.0317923743688677, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4083311557769775, + "step": 1380, + "unclipped_grad_norm": 0.8379060626029968 + }, + { + "bits_per_byte": 2.0432778917414387, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4162923097610474, + "step": 1390, + "unclipped_grad_norm": 0.8665340542793274 + }, + { + "bits_per_byte": 1.9746810636009497, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3687446117401123, + "step": 1400, + "unclipped_grad_norm": 0.8112426996231079 + }, + { + "bits_per_byte": 2.0034977886489185, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.388718843460083, + "step": 1410, + "unclipped_grad_norm": 0.8745980262756348 + }, + { + "bits_per_byte": 2.019574210903621, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.3998621702194214, + "step": 1420, + "unclipped_grad_norm": 0.8596643209457397 + }, + { + "bits_per_byte": 2.0775151659793014, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4400237798690796, + "step": 1430, + "unclipped_grad_norm": 0.913244903087616 + }, + { + "bits_per_byte": 2.027370700415185, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.405266284942627, + "step": 1440, + "unclipped_grad_norm": 1.0067864656448364 + }, + { + "bits_per_byte": 1.9631205617926026, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3607314825057983, + "step": 1450, + "unclipped_grad_norm": 0.8125852942466736 + }, + { + "bits_per_byte": 1.9959419028659118, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.383481502532959, + "step": 1460, + "unclipped_grad_norm": 0.9857028126716614 + }, + { + "bits_per_byte": 1.9364156117145783, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3422210216522217, + "step": 1470, + "unclipped_grad_norm": 0.8698804974555969 + }, + { + "bits_per_byte": 1.996789089404062, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3840687274932861, + "step": 1480, + "unclipped_grad_norm": 0.8397032022476196 + }, + { + "bits_per_byte": 2.017818268038312, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.398645043373108, + "step": 1490, + "unclipped_grad_norm": 0.8181726932525635 + }, + { + "bits_per_byte": 1.996446843928845, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.38383150100708, + "step": 1500, + "unclipped_grad_norm": 0.8242432475090027 + }, + { + "bits_per_byte": 2.0162594172908914, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3975645303726196, + "step": 1510, + "unclipped_grad_norm": 0.9299814701080322 + }, + { + "bits_per_byte": 1.980743108078562, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3729465007781982, + "step": 1520, + "unclipped_grad_norm": 0.8446369171142578 + }, + { + "bits_per_byte": 2.047233664693934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4190342426300049, + "step": 1530, + "unclipped_grad_norm": 0.9717140197753906 + }, + { + "bits_per_byte": 1.993557363411699, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3818286657333374, + "step": 1540, + "unclipped_grad_norm": 0.8417568802833557 + }, + { + "bits_per_byte": 2.049996737962696, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209494590759277, + "step": 1550, + "unclipped_grad_norm": 0.8546327352523804 + }, + { + "bits_per_byte": 1.966114779744124, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3628069162368774, + "step": 1560, + "unclipped_grad_norm": 0.9258830547332764 + }, + { + "bits_per_byte": 2.0427641795633114, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4159362316131592, + "step": 1570, + "unclipped_grad_norm": 0.8567822575569153 + }, + { + "bits_per_byte": 1.9830621221427962, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.374553918838501, + "step": 1580, + "unclipped_grad_norm": 0.8522606492042542 + }, + { + "bits_per_byte": 1.9876789964052082, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3777540922164917, + "step": 1590, + "unclipped_grad_norm": 0.8731296062469482 + }, + { + "bits_per_byte": 2.0122188568915207, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3947638273239136, + "step": 1600, + "unclipped_grad_norm": 0.8627029657363892 + }, + { + "bits_per_byte": 1.9498011934138815, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3514991998672485, + "step": 1610, + "unclipped_grad_norm": 0.7740172147750854 + }, + { + "bits_per_byte": 1.9095543294069197, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3236021995544434, + "step": 1620, + "unclipped_grad_norm": 0.7910023331642151 + }, + { + "bits_per_byte": 1.9407452749500496, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3452221155166626, + "step": 1630, + "unclipped_grad_norm": 0.8484107851982117 + }, + { + "bits_per_byte": 1.9911346438089947, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3801493644714355, + "step": 1640, + "unclipped_grad_norm": 0.8722788691520691 + }, + { + "bits_per_byte": 2.0368601871418424, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4118438959121704, + "step": 1650, + "unclipped_grad_norm": 0.8482462763786316 + }, + { + "bits_per_byte": 1.9665676100638458, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3631207942962646, + "step": 1660, + "unclipped_grad_norm": 0.8444085121154785 + }, + { + "bits_per_byte": 1.9945896032821775, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3825441598892212, + "step": 1670, + "unclipped_grad_norm": 0.8560684323310852 + }, + { + "bits_per_byte": 2.0061820938635853, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3905794620513916, + "step": 1680, + "unclipped_grad_norm": 0.8526912927627563 + }, + { + "bits_per_byte": 2.0514603103215387, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4219639301300049, + "step": 1690, + "unclipped_grad_norm": 0.8917502760887146 + }, + { + "bits_per_byte": 1.9422809080396035, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3462865352630615, + "step": 1700, + "unclipped_grad_norm": 0.7474871873855591 + }, + { + "bits_per_byte": 1.878015978909047, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3017414808273315, + "step": 1710, + "unclipped_grad_norm": 0.8072315454483032 + }, + { + "bits_per_byte": 1.8905051870320533, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3103983402252197, + "step": 1720, + "unclipped_grad_norm": 0.845556914806366 + }, + { + "bits_per_byte": 1.8762399140735866, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3005104064941406, + "step": 1730, + "unclipped_grad_norm": 0.9344940185546875 + }, + { + "bits_per_byte": 1.9632232354351677, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3608026504516602, + "step": 1740, + "unclipped_grad_norm": 0.7860697507858276 + }, + { + "bits_per_byte": 1.989613629244764, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3790950775146484, + "step": 1750, + "unclipped_grad_norm": 0.8348115086555481 + }, + { + "bits_per_byte": 1.9869124697303127, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3772227764129639, + "step": 1760, + "unclipped_grad_norm": 0.8862095475196838 + }, + { + "bits_per_byte": 1.9928130224987648, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3813127279281616, + "step": 1770, + "unclipped_grad_norm": 0.816246509552002 + }, + { + "bits_per_byte": 1.8514405317671223, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.2833207845687866, + "step": 1780, + "unclipped_grad_norm": 0.8118871450424194 + }, + { + "bits_per_byte": 1.9588221993995885, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3577520847320557, + "step": 1790, + "unclipped_grad_norm": 0.839783251285553 + }, + { + "bits_per_byte": 1.8827697513615458, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3050365447998047, + "step": 1800, + "unclipped_grad_norm": 0.8949951529502869 + }, + { + "bits_per_byte": 1.92937567388416, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.33734130859375, + "step": 1810, + "unclipped_grad_norm": 0.7295374274253845 + }, + { + "bits_per_byte": 1.8974312723476003, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3151991367340088, + "step": 1820, + "unclipped_grad_norm": 0.7755441069602966 + }, + { + "bits_per_byte": 1.8886371114783813, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3091034889221191, + "step": 1830, + "unclipped_grad_norm": 0.7666956186294556 + }, + { + "bits_per_byte": 1.8876932706904463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3084492683410645, + "step": 1840, + "unclipped_grad_norm": 0.8784642219543457 + }, + { + "bits_per_byte": 1.8964055678178544, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.314488172531128, + "step": 1850, + "unclipped_grad_norm": 0.8603965044021606 + }, + { + "bits_per_byte": 1.8927124123732264, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3119282722473145, + "step": 1860, + "unclipped_grad_norm": 0.8056252002716064 + }, + { + "bits_per_byte": 1.8818489562488263, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3043982982635498, + "step": 1870, + "unclipped_grad_norm": 0.8151036500930786 + }, + { + "bits_per_byte": 1.8811881989042114, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3039402961730957, + "step": 1880, + "unclipped_grad_norm": 0.79389888048172 + }, + { + "bits_per_byte": 1.8949089747900463, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.313450813293457, + "step": 1890, + "unclipped_grad_norm": 0.8192400932312012 + }, + { + "bits_per_byte": 1.9233574849825177, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3331698179244995, + "step": 1900, + "unclipped_grad_norm": 0.7950363159179688 + }, + { + "bits_per_byte": 1.8611783614416253, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2900705337524414, + "step": 1910, + "unclipped_grad_norm": 0.8037525415420532 + }, + { + "bits_per_byte": 1.855479888287937, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2861206531524658, + "step": 1920, + "unclipped_grad_norm": 0.7507341504096985 + }, + { + "bits_per_byte": 1.9602173226633874, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3587191104888916, + "step": 1930, + "unclipped_grad_norm": 0.8329893350601196 + }, + { + "bits_per_byte": 1.8066889261861108, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2523013353347778, + "step": 1940, + "unclipped_grad_norm": 0.7595474123954773 + }, + { + "bits_per_byte": 1.9033319970987033, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.319289207458496, + "step": 1950, + "unclipped_grad_norm": 0.7690061330795288 + }, + { + "bits_per_byte": 1.816397862775269, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259031057357788, + "step": 1960, + "unclipped_grad_norm": 0.7758741974830627 + }, + { + "bits_per_byte": 1.909067618504978, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3232648372650146, + "step": 1970, + "unclipped_grad_norm": 0.7503845691680908 + }, + { + "bits_per_byte": 1.9405849871194454, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3451110124588013, + "step": 1980, + "unclipped_grad_norm": 0.8655099868774414 + }, + { + "bits_per_byte": 1.9436471382180576, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.347233533859253, + "step": 1990, + "unclipped_grad_norm": 0.8436980247497559 + }, + { + "bits_per_byte": 1.9015801818170146, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3180749416351318, + "step": 2000, + "unclipped_grad_norm": 0.8021923303604126 + }, + { + "bits_per_byte": 1.8746163978494417, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.2993850708007812, + "step": 2010, + "unclipped_grad_norm": 0.7083144187927246 + }, + { + "bits_per_byte": 1.9072580170525995, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3220105171203613, + "step": 2020, + "unclipped_grad_norm": 0.7678324580192566 + }, + { + "bits_per_byte": 1.983610230851096, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3749338388442993, + "step": 2030, + "unclipped_grad_norm": 0.8317387104034424 + }, + { + "bits_per_byte": 1.909348466173837, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.323459506034851, + "step": 2040, + "unclipped_grad_norm": 0.8302199840545654 + }, + { + "bits_per_byte": 1.961244747019642, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.359431266784668, + "step": 2050, + "unclipped_grad_norm": 0.7990598082542419 + }, + { + "bits_per_byte": 1.9220292629699043, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3322491645812988, + "step": 2060, + "unclipped_grad_norm": 0.8404503464698792 + }, + { + "bits_per_byte": 1.8663830724046866, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2936781644821167, + "step": 2070, + "unclipped_grad_norm": 0.7720745801925659 + }, + { + "bits_per_byte": 1.8672120287818452, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2942527532577515, + "step": 2080, + "unclipped_grad_norm": 0.8028935790061951 + }, + { + "bits_per_byte": 1.857879218250124, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2877837419509888, + "step": 2090, + "unclipped_grad_norm": 0.7713416814804077 + }, + { + "bits_per_byte": 1.8603779541845098, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.289515733718872, + "step": 2100, + "unclipped_grad_norm": 0.7443708181381226 + }, + { + "bits_per_byte": 1.897931741861611, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3155460357666016, + "step": 2110, + "unclipped_grad_norm": 0.7534405589103699 + }, + { + "bits_per_byte": 1.8710701155886507, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969269752502441, + "step": 2120, + "unclipped_grad_norm": 0.7505779266357422 + }, + { + "bits_per_byte": 1.8349618420920515, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.271898627281189, + "step": 2130, + "unclipped_grad_norm": 0.7835680246353149 + }, + { + "bits_per_byte": 1.9616896661374241, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3597396612167358, + "step": 2140, + "unclipped_grad_norm": 0.7786996960639954 + }, + { + "bits_per_byte": 1.8168395142226847, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2593371868133545, + "step": 2150, + "unclipped_grad_norm": 0.7194197773933411 + }, + { + "bits_per_byte": 1.8739360344826286, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.2989134788513184, + "step": 2160, + "unclipped_grad_norm": 0.8012002110481262 + }, + { + "bits_per_byte": 1.9634492206384013, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3609592914581299, + "step": 2170, + "unclipped_grad_norm": 0.8270084261894226 + }, + { + "bits_per_byte": 1.8175344961148212, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2598189115524292, + "step": 2180, + "unclipped_grad_norm": 0.7824540734291077 + }, + { + "bits_per_byte": 1.8458584908680682, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.279451608657837, + "step": 2190, + "unclipped_grad_norm": 0.7973959445953369 + }, + { + "bits_per_byte": 1.8557901449900935, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2863357067108154, + "step": 2200, + "unclipped_grad_norm": 0.7451788783073425 + }, + { + "bits_per_byte": 1.8380872828061812, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2740650177001953, + "step": 2210, + "unclipped_grad_norm": 0.7804975509643555 + }, + { + "bits_per_byte": 1.8298283319464475, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2683403491973877, + "step": 2220, + "unclipped_grad_norm": 0.7694668173789978 + }, + { + "bits_per_byte": 1.9031531351418058, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3191652297973633, + "step": 2230, + "unclipped_grad_norm": 0.8418469429016113 + }, + { + "bits_per_byte": 1.8400618156207371, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2754336595535278, + "step": 2240, + "unclipped_grad_norm": 0.7987614870071411 + }, + { + "bits_per_byte": 1.852736936989326, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2842193841934204, + "step": 2250, + "unclipped_grad_norm": 0.7795421481132507 + }, + { + "bits_per_byte": 1.8339105121473271, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.271169900894165, + "step": 2260, + "unclipped_grad_norm": 0.7681040167808533 + }, + { + "bits_per_byte": 1.825916414569922, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2656288146972656, + "step": 2270, + "unclipped_grad_norm": 0.7933830618858337 + }, + { + "bits_per_byte": 1.8451832869807807, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2789835929870605, + "step": 2280, + "unclipped_grad_norm": 0.7935850024223328 + }, + { + "bits_per_byte": 1.8711039961708706, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2969504594802856, + "step": 2290, + "unclipped_grad_norm": 0.7818295955657959 + }, + { + "bits_per_byte": 1.7944778140095496, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2438372373580933, + "step": 2300, + "unclipped_grad_norm": 0.7245032787322998 + }, + { + "bits_per_byte": 1.8221434591752936, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2630136013031006, + "step": 2310, + "unclipped_grad_norm": 0.7396021485328674 + }, + { + "bits_per_byte": 1.7928696042413317, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.242722511291504, + "step": 2320, + "unclipped_grad_norm": 0.7618010640144348 + }, + { + "bits_per_byte": 1.8702191454321815, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2963371276855469, + "step": 2330, + "unclipped_grad_norm": 0.7263991236686707 + }, + { + "bits_per_byte": 1.7994940319872652, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.247314214706421, + "step": 2340, + "unclipped_grad_norm": 0.7788637280464172 + }, + { + "bits_per_byte": 1.849551646312696, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2820115089416504, + "step": 2350, + "unclipped_grad_norm": 0.7732409238815308 + }, + { + "bits_per_byte": 1.8613248906601605, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2901721000671387, + "step": 2360, + "unclipped_grad_norm": 0.8177725672721863 + }, + { + "bits_per_byte": 1.8822589628884834, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3046824932098389, + "step": 2370, + "unclipped_grad_norm": 0.7931574583053589 + }, + { + "bits_per_byte": 1.8382422391746087, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2741724252700806, + "step": 2380, + "unclipped_grad_norm": 0.8604201674461365 + }, + { + "bits_per_byte": 1.8797817247854554, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3029654026031494, + "step": 2390, + "unclipped_grad_norm": 0.7636454105377197 + }, + { + "bits_per_byte": 1.817860919186159, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2600451707839966, + "step": 2400, + "unclipped_grad_norm": 0.779283881187439 + }, + { + "bits_per_byte": 1.8354538844561699, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2722396850585938, + "step": 2410, + "unclipped_grad_norm": 0.7781111001968384 + }, + { + "bits_per_byte": 1.8069649583407457, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2524926662445068, + "step": 2420, + "unclipped_grad_norm": 0.7548633813858032 + }, + { + "bits_per_byte": 1.8169932667125561, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.259443759918213, + "step": 2430, + "unclipped_grad_norm": 0.7562755942344666 + }, + { + "bits_per_byte": 1.8760644917697067, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3003888130187988, + "step": 2440, + "unclipped_grad_norm": 0.7478408813476562 + }, + { + "bits_per_byte": 1.838183937055966, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2741320133209229, + "step": 2450, + "unclipped_grad_norm": 0.760445237159729 + }, + { + "bits_per_byte": 1.8195946762895068, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.261246919631958, + "step": 2460, + "unclipped_grad_norm": 0.7331485748291016 + }, + { + "bits_per_byte": 1.8163858239897086, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2590227127075195, + "step": 2470, + "unclipped_grad_norm": 0.7682369351387024 + }, + { + "bits_per_byte": 1.8266683227194942, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2661499977111816, + "step": 2480, + "unclipped_grad_norm": 0.7132375240325928 + }, + { + "bits_per_byte": 1.8042156436840528, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.250586986541748, + "step": 2490, + "unclipped_grad_norm": 0.7770085334777832 + }, + { + "bits_per_byte": 1.8313916542427904, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2694239616394043, + "step": 2500, + "unclipped_grad_norm": 0.7582318782806396 + }, + { + "bits_per_byte": 1.7788244690759694, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2329871654510498, + "step": 2510, + "unclipped_grad_norm": 0.8006963133811951 + }, + { + "bits_per_byte": 1.8448090527325032, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.278724193572998, + "step": 2520, + "unclipped_grad_norm": 0.7802237272262573 + }, + { + "bits_per_byte": 1.855150369528884, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2858922481536865, + "step": 2530, + "unclipped_grad_norm": 0.7603861689567566 + }, + { + "bits_per_byte": 1.7974587892969547, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.245903491973877, + "step": 2540, + "unclipped_grad_norm": 0.7829954624176025 + }, + { + "bits_per_byte": 1.8048358131230642, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2510168552398682, + "step": 2550, + "unclipped_grad_norm": 0.8207260966300964 + }, + { + "bits_per_byte": 1.8410618947355044, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2761268615722656, + "step": 2560, + "unclipped_grad_norm": 0.8064310550689697 + }, + { + "bits_per_byte": 1.8072459779922554, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2526874542236328, + "step": 2570, + "unclipped_grad_norm": 0.7300699353218079 + }, + { + "bits_per_byte": 1.8352736466380657, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2721147537231445, + "step": 2580, + "unclipped_grad_norm": 0.7476873993873596 + }, + { + "bits_per_byte": 1.793617556789934, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2432409524917603, + "step": 2590, + "unclipped_grad_norm": 0.7210070490837097 + }, + { + "bits_per_byte": 1.797604974550188, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2460048198699951, + "step": 2600, + "unclipped_grad_norm": 0.7853913903236389 + }, + { + "bits_per_byte": 1.7385629865436842, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2050800323486328, + "step": 2610, + "unclipped_grad_norm": 0.7092626094818115 + }, + { + "bits_per_byte": 1.8079149905041119, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2531511783599854, + "step": 2620, + "unclipped_grad_norm": 0.7522015571594238 + }, + { + "bits_per_byte": 1.8198622812942493, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.261432409286499, + "step": 2630, + "unclipped_grad_norm": 0.7675716876983643 + }, + { + "bits_per_byte": 1.680096624469618, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.164554238319397, + "step": 2640, + "unclipped_grad_norm": 0.6825466156005859 + }, + { + "bits_per_byte": 1.7638260340768748, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2225910425186157, + "step": 2650, + "unclipped_grad_norm": 0.7630164623260498 + }, + { + "bits_per_byte": 1.8201589513669878, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2616380453109741, + "step": 2660, + "unclipped_grad_norm": 0.7772946953773499 + }, + { + "bits_per_byte": 1.811244230659514, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2554588317871094, + "step": 2670, + "unclipped_grad_norm": 0.7582975625991821 + }, + { + "bits_per_byte": 1.74257860945868, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.207863450050354, + "step": 2680, + "unclipped_grad_norm": 0.7818799614906311 + }, + { + "bits_per_byte": 1.9030688636428832, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3191068172454834, + "step": 2690, + "unclipped_grad_norm": 0.7075229287147522 + }, + { + "bits_per_byte": 1.8200497423836899, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2615623474121094, + "step": 2700, + "unclipped_grad_norm": 0.8272333741188049 + }, + { + "bits_per_byte": 1.7354167359288, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2028992176055908, + "step": 2710, + "unclipped_grad_norm": 0.7310962677001953 + }, + { + "bits_per_byte": 1.728331394678554, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1979880332946777, + "step": 2720, + "unclipped_grad_norm": 0.7193791270256042 + }, + { + "bits_per_byte": 1.76859700479446, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.225898027420044, + "step": 2730, + "unclipped_grad_norm": 0.773657500743866 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2331104278564453, + "step": 2740, + "unclipped_grad_norm": 0.7759174704551697 + }, + { + "bits_per_byte": 1.7909771071512373, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.241410732269287, + "step": 2750, + "unclipped_grad_norm": 0.770866870880127 + }, + { + "bits_per_byte": 1.81886375002334, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2607402801513672, + "step": 2760, + "unclipped_grad_norm": 0.7793042659759521 + }, + { + "bits_per_byte": 1.704546881994831, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.181501865386963, + "step": 2770, + "unclipped_grad_norm": 0.7179723978042603 + }, + { + "bits_per_byte": 1.7222445846992174, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.1937689781188965, + "step": 2780, + "unclipped_grad_norm": 0.711125910282135 + }, + { + "bits_per_byte": 1.8073385046584196, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2527515888214111, + "step": 2790, + "unclipped_grad_norm": 0.7564381957054138 + }, + { + "bits_per_byte": 1.8564130661515186, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2867674827575684, + "step": 2800, + "unclipped_grad_norm": 0.7982874512672424 + }, + { + "bits_per_byte": 1.7924368958917607, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2424225807189941, + "step": 2810, + "unclipped_grad_norm": 0.827074408531189 + }, + { + "bits_per_byte": 1.781587886310033, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2349026203155518, + "step": 2820, + "unclipped_grad_norm": 0.6945767998695374 + }, + { + "bits_per_byte": 1.862601517877515, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2910569906234741, + "step": 2830, + "unclipped_grad_norm": 0.7473636269569397 + }, + { + "bits_per_byte": 1.7520161574247766, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2144050598144531, + "step": 2840, + "unclipped_grad_norm": 0.7269325852394104 + }, + { + "bits_per_byte": 1.7726036846116109, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2286752462387085, + "step": 2850, + "unclipped_grad_norm": 0.7537488341331482 + }, + { + "bits_per_byte": 1.7375121725469125, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2043516635894775, + "step": 2860, + "unclipped_grad_norm": 0.7839256525039673 + }, + { + "bits_per_byte": 1.836773851301542, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.273154616355896, + "step": 2870, + "unclipped_grad_norm": 0.7615643739700317 + }, + { + "bits_per_byte": 1.8235200083127996, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.263967752456665, + "step": 2880, + "unclipped_grad_norm": 0.7500733137130737 + }, + { + "bits_per_byte": 1.7346435019305209, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2023632526397705, + "step": 2890, + "unclipped_grad_norm": 0.7335168123245239 + }, + { + "bits_per_byte": 1.710140445731493, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1853790283203125, + "step": 2900, + "unclipped_grad_norm": 0.7237004041671753 + }, + { + "bits_per_byte": 1.7640040361205178, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2227144241333008, + "step": 2910, + "unclipped_grad_norm": 0.7653093338012695 + }, + { + "bits_per_byte": 1.7372325287566095, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.204157829284668, + "step": 2920, + "unclipped_grad_norm": 0.7592934966087341 + }, + { + "bits_per_byte": 1.7605243111456106, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2203024625778198, + "step": 2930, + "unclipped_grad_norm": 0.7871323227882385 + }, + { + "bits_per_byte": 1.7773966691085064, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2319974899291992, + "step": 2940, + "unclipped_grad_norm": 0.7994735240936279 + }, + { + "bits_per_byte": 1.761262116717812, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208138704299927, + "step": 2950, + "unclipped_grad_norm": 0.7672474384307861 + }, + { + "bits_per_byte": 1.8009139207527887, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2482984066009521, + "step": 2960, + "unclipped_grad_norm": 0.7721540927886963 + }, + { + "bits_per_byte": 1.7577087831683356, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218350887298584, + "step": 2970, + "unclipped_grad_norm": 0.7659915685653687 + }, + { + "bits_per_byte": 1.7511909846659368, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2138330936431885, + "step": 2980, + "unclipped_grad_norm": 0.7457980513572693 + }, + { + "bits_per_byte": 1.7677185174138528, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2252891063690186, + "step": 2990, + "unclipped_grad_norm": 0.7551211714744568 + }, + { + "bits_per_byte": 1.7804380103063644, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2341055870056152, + "step": 3000, + "unclipped_grad_norm": 0.7369211316108704 + }, + { + "bits_per_byte": 1.77081558099059, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.227435827255249, + "step": 3010, + "unclipped_grad_norm": 0.7526711225509644 + }, + { + "bits_per_byte": 1.726040929734363, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1964004039764404, + "step": 3020, + "unclipped_grad_norm": 0.8113403916358948 + }, + { + "bits_per_byte": 1.8045116258261877, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.250792145729065, + "step": 3030, + "unclipped_grad_norm": 0.7493385672569275 + }, + { + "bits_per_byte": 1.7229166928987893, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.194234848022461, + "step": 3040, + "unclipped_grad_norm": 0.7606903314590454 + }, + { + "bits_per_byte": 1.7403560776615798, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2063229084014893, + "step": 3050, + "unclipped_grad_norm": 0.7690193057060242 + }, + { + "bits_per_byte": 1.759874388708, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2198519706726074, + "step": 3060, + "unclipped_grad_norm": 0.7480750679969788 + }, + { + "bits_per_byte": 1.6873952242069339, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.169613242149353, + "step": 3070, + "unclipped_grad_norm": 0.7001752853393555 + }, + { + "bits_per_byte": 1.737272084766308, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2041852474212646, + "step": 3080, + "unclipped_grad_norm": 0.8322204351425171 + }, + { + "bits_per_byte": 1.8015143121869506, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.248714566230774, + "step": 3090, + "unclipped_grad_norm": 0.8183287382125854 + }, + { + "bits_per_byte": 1.7001176408045102, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.178431749343872, + "step": 3100, + "unclipped_grad_norm": 0.7750749588012695 + }, + { + "bits_per_byte": 1.7770231227908324, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.231738567352295, + "step": 3110, + "unclipped_grad_norm": 0.7082787752151489 + }, + { + "bits_per_byte": 1.749435729731231, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2126164436340332, + "step": 3120, + "unclipped_grad_norm": 0.736611545085907 + }, + { + "bits_per_byte": 1.7966217497352055, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2453233003616333, + "step": 3130, + "unclipped_grad_norm": 0.7366204857826233 + }, + { + "bits_per_byte": 1.7636881039908827, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.222495436668396, + "step": 3140, + "unclipped_grad_norm": 0.7583194375038147 + }, + { + "bits_per_byte": 1.7359698321339747, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2032825946807861, + "step": 3150, + "unclipped_grad_norm": 0.7600930333137512 + }, + { + "bits_per_byte": 1.7415977924008093, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.207183599472046, + "step": 3160, + "unclipped_grad_norm": 0.7639689445495605 + }, + { + "bits_per_byte": 1.654884311818432, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1470783948898315, + "step": 3170, + "unclipped_grad_norm": 0.7011478543281555 + }, + { + "bits_per_byte": 1.7584833930278216, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2188878059387207, + "step": 3180, + "unclipped_grad_norm": 0.7645173072814941 + }, + { + "bits_per_byte": 1.7305350083840592, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.199515461921692, + "step": 3190, + "unclipped_grad_norm": 0.7665988206863403 + }, + { + "bits_per_byte": 1.7003714871971838, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178607702255249, + "step": 3200, + "unclipped_grad_norm": 0.7379894852638245 + }, + { + "bits_per_byte": 1.6959066455981344, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755129098892212, + "step": 3210, + "unclipped_grad_norm": 0.8013631701469421 + }, + { + "bits_per_byte": 1.7098007799960389, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1851435899734497, + "step": 3220, + "unclipped_grad_norm": 0.7296364903450012 + }, + { + "bits_per_byte": 1.7987765203678656, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.246816873550415, + "step": 3230, + "unclipped_grad_norm": 0.7610434889793396 + }, + { + "bits_per_byte": 1.6862874839527267, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1688454151153564, + "step": 3240, + "unclipped_grad_norm": 0.723595917224884 + }, + { + "bits_per_byte": 1.8345631863073515, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2716223001480103, + "step": 3250, + "unclipped_grad_norm": 0.8353017568588257 + }, + { + "bits_per_byte": 1.6872093109613513, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1694843769073486, + "step": 3260, + "unclipped_grad_norm": 0.7602720260620117 + }, + { + "bits_per_byte": 1.7198954736710823, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1921406984329224, + "step": 3270, + "unclipped_grad_norm": 0.7516167163848877 + }, + { + "bits_per_byte": 1.7692164863028679, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2263274192810059, + "step": 3280, + "unclipped_grad_norm": 0.7487898468971252 + }, + { + "bits_per_byte": 1.7624494849393688, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2216368913650513, + "step": 3290, + "unclipped_grad_norm": 0.7708044052124023 + }, + { + "bits_per_byte": 1.7002387165907176, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1785156726837158, + "step": 3300, + "unclipped_grad_norm": 0.7703046202659607 + }, + { + "bits_per_byte": 1.7332599015043295, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2014042139053345, + "step": 3310, + "unclipped_grad_norm": 0.8132672905921936 + }, + { + "bits_per_byte": 1.7262779218272521, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1965646743774414, + "step": 3320, + "unclipped_grad_norm": 0.7545798420906067 + }, + { + "bits_per_byte": 1.7272033604715449, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1972061395645142, + "step": 3330, + "unclipped_grad_norm": 0.7775130867958069 + }, + { + "bits_per_byte": 1.6975052243379043, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1766209602355957, + "step": 3340, + "unclipped_grad_norm": 0.7105540633201599 + }, + { + "bits_per_byte": 1.7083408192728646, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1841316223144531, + "step": 3350, + "unclipped_grad_norm": 0.7163253426551819 + }, + { + "bits_per_byte": 1.732425613664994, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2008259296417236, + "step": 3360, + "unclipped_grad_norm": 0.7626740336418152 + }, + { + "bits_per_byte": 1.6756070173688444, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1614422798156738, + "step": 3370, + "unclipped_grad_norm": 0.8061453700065613 + }, + { + "bits_per_byte": 1.7092863798873084, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1847870349884033, + "step": 3380, + "unclipped_grad_norm": 0.7406735420227051 + }, + { + "bits_per_byte": 1.7539413312185348, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2157394886016846, + "step": 3390, + "unclipped_grad_norm": 0.7695116400718689 + }, + { + "bits_per_byte": 1.8092177590843976, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2540541887283325, + "step": 3400, + "unclipped_grad_norm": 0.7858016490936279 + }, + { + "bits_per_byte": 1.8255297975707825, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2653608322143555, + "step": 3410, + "unclipped_grad_norm": 0.7603964805603027 + }, + { + "bits_per_byte": 1.6857877023693193, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1684989929199219, + "step": 3420, + "unclipped_grad_norm": 0.7168062329292297 + }, + { + "bits_per_byte": 1.6176447364444573, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1212658882141113, + "step": 3430, + "unclipped_grad_norm": 0.7431134581565857 + }, + { + "bits_per_byte": 1.711189711884407, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1861063241958618, + "step": 3440, + "unclipped_grad_norm": 0.7753933668136597 + }, + { + "bits_per_byte": 1.77535626693867, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2305831909179688, + "step": 3450, + "unclipped_grad_norm": 0.8042635321617126 + }, + { + "bits_per_byte": 1.7083845028661837, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.184161901473999, + "step": 3460, + "unclipped_grad_norm": 0.7639697194099426 + }, + { + "bits_per_byte": 1.6937178224006033, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1739957332611084, + "step": 3470, + "unclipped_grad_norm": 0.748127281665802 + }, + { + "bits_per_byte": 1.6501064617948125, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1437666416168213, + "step": 3480, + "unclipped_grad_norm": 0.7506517767906189 + }, + { + "bits_per_byte": 1.8358463488654388, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2725117206573486, + "step": 3490, + "unclipped_grad_norm": 0.836082398891449 + }, + { + "bits_per_byte": 1.714104301868579, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.188126564025879, + "step": 3500, + "unclipped_grad_norm": 0.7544166445732117 + }, + { + "bits_per_byte": 1.744907598516664, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2094777822494507, + "step": 3510, + "unclipped_grad_norm": 0.7650617361068726 + }, + { + "bits_per_byte": 1.738600994709525, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2051063776016235, + "step": 3520, + "unclipped_grad_norm": 0.7794367671012878 + }, + { + "bits_per_byte": 1.7051804660806096, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.181941032409668, + "step": 3530, + "unclipped_grad_norm": 0.7445787787437439 + }, + { + "bits_per_byte": 1.7158172490711725, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1893138885498047, + "step": 3540, + "unclipped_grad_norm": 0.7629107236862183 + }, + { + "bits_per_byte": 1.7173026632266746, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1903434991836548, + "step": 3550, + "unclipped_grad_norm": 0.7763495445251465 + }, + { + "bits_per_byte": 1.6560372835098163, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.14787757396698, + "step": 3560, + "unclipped_grad_norm": 0.7450886964797974 + }, + { + "bits_per_byte": 1.6838996768281473, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1671903133392334, + "step": 3570, + "unclipped_grad_norm": 0.7487124800682068 + }, + { + "bits_per_byte": 1.6946102403759304, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1746143102645874, + "step": 3580, + "unclipped_grad_norm": 0.757170557975769 + }, + { + "bits_per_byte": 1.7307594457434352, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1996710300445557, + "step": 3590, + "unclipped_grad_norm": 0.7700981497764587 + }, + { + "bits_per_byte": 1.7491381997452384, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2124102115631104, + "step": 3600, + "unclipped_grad_norm": 0.7877120971679688 + }, + { + "bits_per_byte": 1.688025540622346, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1700501441955566, + "step": 3610, + "unclipped_grad_norm": 0.780867874622345 + }, + { + "bits_per_byte": 1.6436277033541595, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1392759084701538, + "step": 3620, + "unclipped_grad_norm": 0.7423334717750549 + }, + { + "bits_per_byte": 1.6950637586262558, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1749286651611328, + "step": 3630, + "unclipped_grad_norm": 0.7513464689254761 + }, + { + "bits_per_byte": 1.6758522646289749, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1616122722625732, + "step": 3640, + "unclipped_grad_norm": 0.7453336715698242 + }, + { + "bits_per_byte": 1.736063734661346, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2033476829528809, + "step": 3650, + "unclipped_grad_norm": 0.7682862877845764 + }, + { + "bits_per_byte": 1.6360068081291268, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1339935064315796, + "step": 3660, + "unclipped_grad_norm": 0.7531437277793884 + }, + { + "bits_per_byte": 1.7183529612754938, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1910715103149414, + "step": 3670, + "unclipped_grad_norm": 0.7578480243682861 + }, + { + "bits_per_byte": 1.6583936178092877, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1495108604431152, + "step": 3680, + "unclipped_grad_norm": 0.7224898934364319 + }, + { + "bits_per_byte": 1.6372521345040243, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1348567008972168, + "step": 3690, + "unclipped_grad_norm": 0.7246464490890503 + }, + { + "bits_per_byte": 1.6659130432703113, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1547229290008545, + "step": 3700, + "unclipped_grad_norm": 0.7878638505935669 + }, + { + "bits_per_byte": 1.694639821391879, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1746348142623901, + "step": 3710, + "unclipped_grad_norm": 0.7455682754516602 + }, + { + "bits_per_byte": 1.6891824679147003, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1708520650863647, + "step": 3720, + "unclipped_grad_norm": 0.7515487670898438 + }, + { + "bits_per_byte": 1.6091924770851538, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1154072284698486, + "step": 3730, + "unclipped_grad_norm": 0.7353127598762512 + }, + { + "bits_per_byte": 1.6633680440028435, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.152958869934082, + "step": 3740, + "unclipped_grad_norm": 0.7253232002258301 + }, + { + "bits_per_byte": 1.6922951819126664, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1730096340179443, + "step": 3750, + "unclipped_grad_norm": 0.7686370611190796 + }, + { + "bits_per_byte": 1.7713081393026613, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2277772426605225, + "step": 3760, + "unclipped_grad_norm": 0.7601476907730103 + }, + { + "bits_per_byte": 1.7305630415561497, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1995348930358887, + "step": 3770, + "unclipped_grad_norm": 0.7877833247184753 + }, + { + "bits_per_byte": 1.5644979697442287, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0844273567199707, + "step": 3780, + "unclipped_grad_norm": 0.7386898994445801 + }, + { + "bits_per_byte": 1.6644766441703052, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1537272930145264, + "step": 3790, + "unclipped_grad_norm": 0.7420179843902588 + }, + { + "bits_per_byte": 1.6653939996300076, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1543631553649902, + "step": 3800, + "unclipped_grad_norm": 0.7838358283042908 + }, + { + "bits_per_byte": 1.6952983429620325, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17509126663208, + "step": 3810, + "unclipped_grad_norm": 0.8048396110534668 + }, + { + "bits_per_byte": 1.6744323758634516, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.160628080368042, + "step": 3820, + "unclipped_grad_norm": 0.7709245681762695 + }, + { + "bits_per_byte": 1.720191799778519, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1923460960388184, + "step": 3830, + "unclipped_grad_norm": 0.7672807574272156 + }, + { + "bits_per_byte": 1.6467005173771254, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1414058208465576, + "step": 3840, + "unclipped_grad_norm": 0.7319741249084473 + }, + { + "bits_per_byte": 1.5964396195583748, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106567621231079, + "step": 3850, + "unclipped_grad_norm": 0.7756319642066956 + }, + { + "bits_per_byte": 1.6360811046342996, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1340450048446655, + "step": 3860, + "unclipped_grad_norm": 0.766754150390625 + }, + { + "bits_per_byte": 1.7433616464680584, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2084062099456787, + "step": 3870, + "unclipped_grad_norm": 0.7916337847709656 + }, + { + "bits_per_byte": 1.6358671582166262, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1338967084884644, + "step": 3880, + "unclipped_grad_norm": 0.7497678399085999 + }, + { + "bits_per_byte": 1.600699113872294, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1095200777053833, + "step": 3890, + "unclipped_grad_norm": 0.7555440664291382 + }, + { + "bits_per_byte": 1.6778517349279058, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1629981994628906, + "step": 3900, + "unclipped_grad_norm": 0.826478123664856 + }, + { + "bits_per_byte": 1.6805711246033486, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1648831367492676, + "step": 3910, + "unclipped_grad_norm": 0.7849931120872498 + }, + { + "bits_per_byte": 1.6375147520118918, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1350387334823608, + "step": 3920, + "unclipped_grad_norm": 0.7588120102882385 + }, + { + "bits_per_byte": 1.673005091843941, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.15963876247406, + "step": 3930, + "unclipped_grad_norm": 0.742434024810791 + }, + { + "bits_per_byte": 1.6996766772876981, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1781260967254639, + "step": 3940, + "unclipped_grad_norm": 0.7588658928871155 + }, + { + "bits_per_byte": 1.7339400928884918, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2018756866455078, + "step": 3950, + "unclipped_grad_norm": 0.7937244176864624 + }, + { + "bits_per_byte": 1.713467450112434, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1876851320266724, + "step": 3960, + "unclipped_grad_norm": 0.7884036302566528 + }, + { + "bits_per_byte": 1.6516962694193882, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1448686122894287, + "step": 3970, + "unclipped_grad_norm": 0.8004661798477173 + }, + { + "bits_per_byte": 1.6740500584305835, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1603630781173706, + "step": 3980, + "unclipped_grad_norm": 0.7692103385925293 + }, + { + "bits_per_byte": 1.691941929547794, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.172764778137207, + "step": 3990, + "unclipped_grad_norm": 0.7891059517860413 + }, + { + "bits_per_byte": 1.5813677479673616, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0961205959320068, + "step": 4000, + "unclipped_grad_norm": 0.7436221837997437 + }, + { + "bits_per_byte": 1.6793110077204767, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.164009690284729, + "step": 4010, + "unclipped_grad_norm": 0.8044730424880981 + }, + { + "bits_per_byte": 1.6584589712166156, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1495561599731445, + "step": 4020, + "unclipped_grad_norm": 0.8183668851852417 + }, + { + "bits_per_byte": 1.6850787898824628, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1680076122283936, + "step": 4030, + "unclipped_grad_norm": 0.793128252029419 + }, + { + "bits_per_byte": 1.617821878574846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1213886737823486, + "step": 4040, + "unclipped_grad_norm": 0.7582541108131409 + }, + { + "bits_per_byte": 1.6450140555027646, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1402368545532227, + "step": 4050, + "unclipped_grad_norm": 0.7559582591056824 + }, + { + "bits_per_byte": 1.6581645369183384, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1493520736694336, + "step": 4060, + "unclipped_grad_norm": 0.743245005607605 + }, + { + "bits_per_byte": 1.680131880913045, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1645786762237549, + "step": 4070, + "unclipped_grad_norm": 0.7624887824058533 + }, + { + "bits_per_byte": 1.7564567494700543, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2174830436706543, + "step": 4080, + "unclipped_grad_norm": 0.8132302761077881 + }, + { + "bits_per_byte": 1.6669108866106175, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1554145812988281, + "step": 4090, + "unclipped_grad_norm": 0.8276432752609253 + }, + { + "bits_per_byte": 1.638365894151012, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1356287002563477, + "step": 4100, + "unclipped_grad_norm": 0.7751885652542114 + }, + { + "bits_per_byte": 1.6303674970073352, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1300846338272095, + "step": 4110, + "unclipped_grad_norm": 0.7898787260055542 + }, + { + "bits_per_byte": 1.6896222275529564, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.171156883239746, + "step": 4120, + "unclipped_grad_norm": 0.8148247599601746 + }, + { + "bits_per_byte": 1.6213521664491068, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1238356828689575, + "step": 4130, + "unclipped_grad_norm": 0.8133904933929443 + }, + { + "bits_per_byte": 1.6881359534842002, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1701266765594482, + "step": 4140, + "unclipped_grad_norm": 0.8217440247535706 + }, + { + "bits_per_byte": 1.6129334437067215, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1180002689361572, + "step": 4150, + "unclipped_grad_norm": 0.7947918176651001 + }, + { + "bits_per_byte": 1.651558683298698, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.144773244857788, + "step": 4160, + "unclipped_grad_norm": 0.7860805988311768 + }, + { + "bits_per_byte": 1.6828089628563754, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1664342880249023, + "step": 4170, + "unclipped_grad_norm": 0.8207855224609375 + }, + { + "bits_per_byte": 1.7311515661874024, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1999428272247314, + "step": 4180, + "unclipped_grad_norm": 0.8373809456825256 + }, + { + "bits_per_byte": 1.6433404923272186, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.139076828956604, + "step": 4190, + "unclipped_grad_norm": 0.7621071338653564 + }, + { + "bits_per_byte": 1.682679287937625, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.166344404220581, + "step": 4200, + "unclipped_grad_norm": 0.7672834396362305 + }, + { + "bits_per_byte": 1.661693964879345, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1517984867095947, + "step": 4210, + "unclipped_grad_norm": 0.7330533862113953 + }, + { + "bits_per_byte": 1.6972419188994332, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1764384508132935, + "step": 4220, + "unclipped_grad_norm": 0.7866858243942261 + }, + { + "bits_per_byte": 1.6670219874030747, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.155491590499878, + "step": 4230, + "unclipped_grad_norm": 0.7902593612670898 + }, + { + "bits_per_byte": 1.6114956687455084, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1170036792755127, + "step": 4240, + "unclipped_grad_norm": 0.740251362323761 + }, + { + "bits_per_byte": 1.6335233786506678, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1322721242904663, + "step": 4250, + "unclipped_grad_norm": 0.7777352333068848 + }, + { + "bits_per_byte": 1.6115895712728796, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1170687675476074, + "step": 4260, + "unclipped_grad_norm": 0.774424135684967 + }, + { + "bits_per_byte": 1.619611874005026, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1226294040679932, + "step": 4270, + "unclipped_grad_norm": 0.7803477048873901 + }, + { + "bits_per_byte": 1.6299772683725275, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1298141479492188, + "step": 4280, + "unclipped_grad_norm": 0.7580373287200928 + }, + { + "bits_per_byte": 1.6795242662075467, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.164157509803772, + "step": 4290, + "unclipped_grad_norm": 0.7638103365898132 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.109092354774475, + "step": 4300, + "unclipped_grad_norm": 0.7825598120689392 + }, + { + "bits_per_byte": 1.6771565810531184, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1625163555145264, + "step": 4310, + "unclipped_grad_norm": 0.7940689921379089 + }, + { + "bits_per_byte": 1.6577068910843924, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.149034857749939, + "step": 4320, + "unclipped_grad_norm": 0.7976187467575073 + }, + { + "bits_per_byte": 1.6195226150092283, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1225675344467163, + "step": 4330, + "unclipped_grad_norm": 0.7892199158668518 + }, + { + "bits_per_byte": 1.655279183984813, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1473520994186401, + "step": 4340, + "unclipped_grad_norm": 0.8144169449806213 + }, + { + "bits_per_byte": 1.6394932404274176, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.136410117149353, + "step": 4350, + "unclipped_grad_norm": 0.8119622468948364 + }, + { + "bits_per_byte": 1.6287904160989233, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1289914846420288, + "step": 4360, + "unclipped_grad_norm": 0.7554351091384888 + }, + { + "bits_per_byte": 1.6321829458698431, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1313430070877075, + "step": 4370, + "unclipped_grad_norm": 0.808085024356842 + }, + { + "bits_per_byte": 1.6820371047193032, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1658992767333984, + "step": 4380, + "unclipped_grad_norm": 0.7708722949028015 + }, + { + "bits_per_byte": 1.6522423143358775, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1452471017837524, + "step": 4390, + "unclipped_grad_norm": 0.8123121857643127 + }, + { + "bits_per_byte": 1.6025224739367414, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1107839345932007, + "step": 4400, + "unclipped_grad_norm": 0.7717524766921997 + }, + { + "bits_per_byte": 1.6853694405624209, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1682090759277344, + "step": 4410, + "unclipped_grad_norm": 0.81525057554245 + }, + { + "bits_per_byte": 1.6142167782474597, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1188898086547852, + "step": 4420, + "unclipped_grad_norm": 0.7956154346466064 + }, + { + "bits_per_byte": 1.6755413199962148, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1613967418670654, + "step": 4430, + "unclipped_grad_norm": 0.7907513380050659 + }, + { + "bits_per_byte": 1.6011154838700328, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1098086833953857, + "step": 4440, + "unclipped_grad_norm": 0.7607184648513794 + }, + { + "bits_per_byte": 1.5990663105850023, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1083883047103882, + "step": 4450, + "unclipped_grad_norm": 0.7442869544029236 + }, + { + "bits_per_byte": 1.5859970049806358, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0993293523788452, + "step": 4460, + "unclipped_grad_norm": 0.7537500262260437 + }, + { + "bits_per_byte": 1.6451918855637568, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1403601169586182, + "step": 4470, + "unclipped_grad_norm": 0.7931647896766663 + }, + { + "bits_per_byte": 1.6505943765753102, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1441048383712769, + "step": 4480, + "unclipped_grad_norm": 0.9106901288032532 + }, + { + "bits_per_byte": 1.6301040195862135, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1299020051956177, + "step": 4490, + "unclipped_grad_norm": 0.7690432667732239 + }, + { + "bits_per_byte": 1.6520481459230536, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1451125144958496, + "step": 4500, + "unclipped_grad_norm": 0.7990603446960449 + }, + { + "bits_per_byte": 1.5968599451570833, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1068589687347412, + "step": 4510, + "unclipped_grad_norm": 0.8130239248275757 + }, + { + "bits_per_byte": 1.6430763269754933, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.138893723487854, + "step": 4520, + "unclipped_grad_norm": 0.7745712399482727 + }, + { + "bits_per_byte": 1.660709880151108, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1511163711547852, + "step": 4530, + "unclipped_grad_norm": 0.7695059776306152 + }, + { + "bits_per_byte": 1.707295852686222, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1834073066711426, + "step": 4540, + "unclipped_grad_norm": 0.8436031937599182 + }, + { + "bits_per_byte": 1.5623325361872151, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.0829263925552368, + "step": 4550, + "unclipped_grad_norm": 0.7549967169761658 + }, + { + "bits_per_byte": 1.6257084869954619, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1268552541732788, + "step": 4560, + "unclipped_grad_norm": 0.8449703454971313 + }, + { + "bits_per_byte": 1.6175644205465043, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1212102174758911, + "step": 4570, + "unclipped_grad_norm": 0.7528937458992004 + }, + { + "bits_per_byte": 1.570274179056107, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0884311199188232, + "step": 4580, + "unclipped_grad_norm": 0.790579617023468 + }, + { + "bits_per_byte": 1.609637052237634, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1157153844833374, + "step": 4590, + "unclipped_grad_norm": 0.7935193181037903 + }, + { + "bits_per_byte": 1.6088013885370918, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1151361465454102, + "step": 4600, + "unclipped_grad_norm": 0.7915687561035156 + }, + { + "bits_per_byte": 1.6615916352020816, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.151727557182312, + "step": 4610, + "unclipped_grad_norm": 0.8493216633796692 + }, + { + "bits_per_byte": 1.6675298521710726, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1558436155319214, + "step": 4620, + "unclipped_grad_norm": 0.8179065585136414 + }, + { + "bits_per_byte": 1.6286588493710132, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1289002895355225, + "step": 4630, + "unclipped_grad_norm": 0.8027536273002625 + }, + { + "bits_per_byte": 1.7031861552612044, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.180558681488037, + "step": 4640, + "unclipped_grad_norm": 0.8179929256439209 + }, + { + "bits_per_byte": 1.6059254946493637, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.113142728805542, + "step": 4650, + "unclipped_grad_norm": 0.8228299021720886 + }, + { + "bits_per_byte": 1.5739184914278899, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0909571647644043, + "step": 4660, + "unclipped_grad_norm": 0.820624589920044 + }, + { + "bits_per_byte": 1.7237126286069824, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.194786548614502, + "step": 4670, + "unclipped_grad_norm": 0.8582949638366699 + }, + { + "bits_per_byte": 1.7009104808249877, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1789813041687012, + "step": 4680, + "unclipped_grad_norm": 0.8158925175666809 + }, + { + "bits_per_byte": 1.6679322915740917, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1561225652694702, + "step": 4690, + "unclipped_grad_norm": 0.849236249923706 + }, + { + "bits_per_byte": 1.589747430648001, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.101928949356079, + "step": 4700, + "unclipped_grad_norm": 0.7403190732002258 + }, + { + "bits_per_byte": 1.6742342518496576, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1604907512664795, + "step": 4710, + "unclipped_grad_norm": 0.8582251071929932 + }, + { + "bits_per_byte": 1.607032031025015, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1139097213745117, + "step": 4720, + "unclipped_grad_norm": 0.8400667905807495 + }, + { + "bits_per_byte": 1.5176990826873464, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0519888401031494, + "step": 4730, + "unclipped_grad_norm": 0.7827285528182983 + }, + { + "bits_per_byte": 1.5868581221135059, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.099926233291626, + "step": 4740, + "unclipped_grad_norm": 0.7815871238708496 + }, + { + "bits_per_byte": 1.6377919880450826, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1352308988571167, + "step": 4750, + "unclipped_grad_norm": 0.7846289873123169 + }, + { + "bits_per_byte": 1.6125860387519786, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1177594661712646, + "step": 4760, + "unclipped_grad_norm": 0.8307337164878845 + }, + { + "bits_per_byte": 1.5936545325103024, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1046371459960938, + "step": 4770, + "unclipped_grad_norm": 0.8213374614715576 + }, + { + "bits_per_byte": 1.6121588338472355, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1174633502960205, + "step": 4780, + "unclipped_grad_norm": 0.8224621415138245 + }, + { + "bits_per_byte": 1.5705015401205475, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0885887145996094, + "step": 4790, + "unclipped_grad_norm": 0.8122685551643372 + }, + { + "bits_per_byte": 1.5722774329733569, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0898196697235107, + "step": 4800, + "unclipped_grad_norm": 0.8455460667610168 + }, + { + "bits_per_byte": 1.6480244398234671, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1423234939575195, + "step": 4810, + "unclipped_grad_norm": 0.8817458748817444 + }, + { + "bits_per_byte": 1.574972745077679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0916879177093506, + "step": 4820, + "unclipped_grad_norm": 0.7954387664794922 + }, + { + "bits_per_byte": 1.608869665649484, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1151834726333618, + "step": 4830, + "unclipped_grad_norm": 0.8687117695808411 + }, + { + "bits_per_byte": 1.5985519104762718, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1080317497253418, + "step": 4840, + "unclipped_grad_norm": 0.8505465388298035 + }, + { + "bits_per_byte": 1.6228387844831649, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1248661279678345, + "step": 4850, + "unclipped_grad_norm": 0.7912812232971191 + }, + { + "bits_per_byte": 1.5432041098103009, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0696675777435303, + "step": 4860, + "unclipped_grad_norm": 0.7933070063591003 + }, + { + "bits_per_byte": 1.635918581029234, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.13393235206604, + "step": 4870, + "unclipped_grad_norm": 0.8805916905403137 + }, + { + "bits_per_byte": 1.4766798445953597, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0235564708709717, + "step": 4880, + "unclipped_grad_norm": 0.8066675066947937 + }, + { + "bits_per_byte": 1.5344280071194227, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0635844469070435, + "step": 4890, + "unclipped_grad_norm": 0.8088182806968689 + }, + { + "bits_per_byte": 1.6101516243290155, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1160720586776733, + "step": 4900, + "unclipped_grad_norm": 0.8653903603553772 + }, + { + "bits_per_byte": 1.6779198400576476, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1630454063415527, + "step": 4910, + "unclipped_grad_norm": 0.8391038179397583 + }, + { + "bits_per_byte": 1.6284904783558185, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1287835836410522, + "step": 4920, + "unclipped_grad_norm": 0.8723531365394592 + }, + { + "bits_per_byte": 1.5831391692712484, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0973484516143799, + "step": 4930, + "unclipped_grad_norm": 0.7553638815879822 + }, + { + "bits_per_byte": 1.5755332365368409, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0920764207839966, + "step": 4940, + "unclipped_grad_norm": 0.8158645033836365 + }, + { + "bits_per_byte": 1.530142715407874, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0606141090393066, + "step": 4950, + "unclipped_grad_norm": 0.7773033380508423 + }, + { + "bits_per_byte": 1.5220387209165676, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0549968481063843, + "step": 4960, + "unclipped_grad_norm": 0.8325244784355164 + }, + { + "bits_per_byte": 1.6031579498316795, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1112244129180908, + "step": 4970, + "unclipped_grad_norm": 0.8591333627700806 + }, + { + "bits_per_byte": 1.6532000857185325, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1459109783172607, + "step": 4980, + "unclipped_grad_norm": 0.8351151347160339 + }, + { + "bits_per_byte": 1.585272098107249, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.0988268852233887, + "step": 4990, + "unclipped_grad_norm": 0.8779996037483215 + }, + { + "bits_per_byte": 1.5394700224947675, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670793056488037, + "step": 5000, + "unclipped_grad_norm": 0.7928086519241333 + }, + { + "bits_per_byte": 1.5552827953456976, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0780398845672607, + "step": 5010, + "unclipped_grad_norm": 0.8201308846473694 + }, + { + "bits_per_byte": 1.5898205232746179, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1019796133041382, + "step": 5020, + "unclipped_grad_norm": 0.7774789929389954 + }, + { + "bits_per_byte": 1.5697566832596608, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.088072419166565, + "step": 5030, + "unclipped_grad_norm": 0.7855319380760193 + }, + { + "bits_per_byte": 1.539816567536256, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0673195123672485, + "step": 5040, + "unclipped_grad_norm": 0.7920348048210144 + }, + { + "bits_per_byte": 1.5092143186070297, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0461076498031616, + "step": 5050, + "unclipped_grad_norm": 0.8045132160186768 + }, + { + "bits_per_byte": 1.5339026001210367, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0632202625274658, + "step": 5060, + "unclipped_grad_norm": 0.8044583797454834 + }, + { + "bits_per_byte": 1.6138325690054323, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1186234951019287, + "step": 5070, + "unclipped_grad_norm": 0.8723475337028503 + }, + { + "bits_per_byte": 1.574335549356232, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.091246247291565, + "step": 5080, + "unclipped_grad_norm": 0.8741123080253601 + }, + { + "bits_per_byte": 1.5594206979254568, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0809080600738525, + "step": 5090, + "unclipped_grad_norm": 0.8465037941932678 + }, + { + "bits_per_byte": 1.5640659493252613, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0841279029846191, + "step": 5100, + "unclipped_grad_norm": 0.783299446105957 + }, + { + "bits_per_byte": 1.488738924108558, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0319151878356934, + "step": 5110, + "unclipped_grad_norm": 0.8348572254180908 + }, + { + "bits_per_byte": 1.5847128105266433, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0984392166137695, + "step": 5120, + "unclipped_grad_norm": 0.8110998868942261 + }, + { + "bits_per_byte": 1.590178763136365, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1022279262542725, + "step": 5130, + "unclipped_grad_norm": 0.8208310604095459 + }, + { + "bits_per_byte": 1.5494388248693793, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0739891529083252, + "step": 5140, + "unclipped_grad_norm": 0.8678956627845764 + }, + { + "bits_per_byte": 1.6273239200350162, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1279749870300293, + "step": 5150, + "unclipped_grad_norm": 0.848126232624054 + }, + { + "bits_per_byte": 1.5085432423033631, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0456424951553345, + "step": 5160, + "unclipped_grad_norm": 0.7922812104225159 + }, + { + "bits_per_byte": 1.6700107019097687, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1575632095336914, + "step": 5170, + "unclipped_grad_norm": 0.8759046196937561 + }, + { + "bits_per_byte": 1.5869253673299932, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0999728441238403, + "step": 5180, + "unclipped_grad_norm": 0.8647724390029907 + }, + { + "bits_per_byte": 1.5237559676854329, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0561871528625488, + "step": 5190, + "unclipped_grad_norm": 0.813337504863739 + }, + { + "bits_per_byte": 1.556985939537192, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0792204141616821, + "step": 5200, + "unclipped_grad_norm": 0.8388134241104126 + }, + { + "bits_per_byte": 1.5602121620847276, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0814566612243652, + "step": 5210, + "unclipped_grad_norm": 0.8162683248519897 + }, + { + "bits_per_byte": 1.6432956048553435, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1390457153320312, + "step": 5220, + "unclipped_grad_norm": 0.8667425513267517 + }, + { + "bits_per_byte": 1.5501931407760636, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.074512004852295, + "step": 5230, + "unclipped_grad_norm": 0.8668702840805054 + }, + { + "bits_per_byte": 1.5510165937083948, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.075082778930664, + "step": 5240, + "unclipped_grad_norm": 0.8049055337905884 + }, + { + "bits_per_byte": 1.5393878147876552, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0670223236083984, + "step": 5250, + "unclipped_grad_norm": 0.8209428191184998 + }, + { + "bits_per_byte": 1.540701762240247, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0679330825805664, + "step": 5260, + "unclipped_grad_norm": 0.8524174690246582 + }, + { + "bits_per_byte": 1.5376201771020872, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0657970905303955, + "step": 5270, + "unclipped_grad_norm": 0.8064137697219849 + }, + { + "bits_per_byte": 1.5550475230793173, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0778768062591553, + "step": 5280, + "unclipped_grad_norm": 0.8169121146202087 + }, + { + "bits_per_byte": 1.5423493560355128, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.069075107574463, + "step": 5290, + "unclipped_grad_norm": 0.8263237476348877 + }, + { + "bits_per_byte": 1.5542627662434303, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.077332854270935, + "step": 5300, + "unclipped_grad_norm": 0.8492859601974487 + }, + { + "bits_per_byte": 1.5509104804128124, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0750092267990112, + "step": 5310, + "unclipped_grad_norm": 0.8885292410850525 + }, + { + "bits_per_byte": 1.5517654061702515, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0756018161773682, + "step": 5320, + "unclipped_grad_norm": 0.8326203227043152 + }, + { + "bits_per_byte": 1.5477100552629064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0727908611297607, + "step": 5330, + "unclipped_grad_norm": 0.8806997537612915 + }, + { + "bits_per_byte": 1.503639328996661, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0422433614730835, + "step": 5340, + "unclipped_grad_norm": 0.8234722018241882 + }, + { + "bits_per_byte": 1.5869049013945407, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0999586582183838, + "step": 5350, + "unclipped_grad_norm": 0.8706242442131042 + }, + { + "bits_per_byte": 1.5072631754329913, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.044755220413208, + "step": 5360, + "unclipped_grad_norm": 0.8708282709121704 + }, + { + "bits_per_byte": 1.471025742965594, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0196373462677002, + "step": 5370, + "unclipped_grad_norm": 0.8303158283233643 + }, + { + "bits_per_byte": 1.5968417149960918, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1068463325500488, + "step": 5380, + "unclipped_grad_norm": 0.883955180644989 + }, + { + "bits_per_byte": 1.531507741707772, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0615602731704712, + "step": 5390, + "unclipped_grad_norm": 0.8507125377655029 + }, + { + "bits_per_byte": 1.5483677169198058, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073246717453003, + "step": 5400, + "unclipped_grad_norm": 0.8522453904151917 + }, + { + "bits_per_byte": 1.571479089508052, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.089266300201416, + "step": 5410, + "unclipped_grad_norm": 0.8313266038894653 + }, + { + "bits_per_byte": 1.5408727129952047, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0680515766143799, + "step": 5420, + "unclipped_grad_norm": 0.8902517557144165 + }, + { + "bits_per_byte": 1.5778336764747818, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0936709642410278, + "step": 5430, + "unclipped_grad_norm": 0.8956695795059204 + }, + { + "bits_per_byte": 1.513927503153925, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0493745803833008, + "step": 5440, + "unclipped_grad_norm": 0.897573709487915 + }, + { + "bits_per_byte": 1.5078418970531444, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0451563596725464, + "step": 5450, + "unclipped_grad_norm": 0.8440007567405701 + }, + { + "bits_per_byte": 1.5030907043404085, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0418630838394165, + "step": 5460, + "unclipped_grad_norm": 0.8436890244483948 + }, + { + "bits_per_byte": 1.5464133060754008, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0718920230865479, + "step": 5470, + "unclipped_grad_norm": 0.8681005239486694 + }, + { + "bits_per_byte": 1.553998600891705, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.077149748802185, + "step": 5480, + "unclipped_grad_norm": 0.8523774147033691 + }, + { + "bits_per_byte": 1.543775436176467, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.070063591003418, + "step": 5490, + "unclipped_grad_norm": 0.8906207084655762 + }, + { + "bits_per_byte": 1.5444891641775478, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0705583095550537, + "step": 5500, + "unclipped_grad_norm": 0.8658390045166016 + }, + { + "bits_per_byte": 1.6370650173798855, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1347270011901855, + "step": 5510, + "unclipped_grad_norm": 0.9161565899848938 + }, + { + "bits_per_byte": 1.5520985365649729, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.075832724571228, + "step": 5520, + "unclipped_grad_norm": 0.845907986164093 + }, + { + "bits_per_byte": 1.5215033389244317, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0546257495880127, + "step": 5530, + "unclipped_grad_norm": 0.8585748672485352 + }, + { + "bits_per_byte": 1.5943249208833656, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1051018238067627, + "step": 5540, + "unclipped_grad_norm": 0.9360165596008301 + }, + { + "bits_per_byte": 1.5176287417831436, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0519400835037231, + "step": 5550, + "unclipped_grad_norm": 0.8308430910110474 + }, + { + "bits_per_byte": 1.5164790377621256, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0511431694030762, + "step": 5560, + "unclipped_grad_norm": 0.894058346748352 + }, + { + "bits_per_byte": 1.4905304673825959, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0331569910049438, + "step": 5570, + "unclipped_grad_norm": 0.8399901390075684 + }, + { + "bits_per_byte": 1.5655759569998369, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.085174560546875, + "step": 5580, + "unclipped_grad_norm": 0.8579773902893066 + }, + { + "bits_per_byte": 1.482571282283316, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0276401042938232, + "step": 5590, + "unclipped_grad_norm": 0.8161733746528625 + }, + { + "bits_per_byte": 1.5496305855250914, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0741220712661743, + "step": 5600, + "unclipped_grad_norm": 0.8440633416175842 + }, + { + "bits_per_byte": 1.5466313800766949, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0720431804656982, + "step": 5610, + "unclipped_grad_norm": 0.8384022116661072 + }, + { + "bits_per_byte": 1.55366271877457, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0769169330596924, + "step": 5620, + "unclipped_grad_norm": 0.8643615245819092 + }, + { + "bits_per_byte": 1.5359743031333302, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0646562576293945, + "step": 5630, + "unclipped_grad_norm": 0.8559057712554932 + }, + { + "bits_per_byte": 1.5154779267514533, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.050449252128601, + "step": 5640, + "unclipped_grad_norm": 0.8564271330833435 + }, + { + "bits_per_byte": 1.5399061704973558, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0673816204071045, + "step": 5650, + "unclipped_grad_norm": 0.899230420589447 + }, + { + "bits_per_byte": 1.6017691899259623, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1102617979049683, + "step": 5660, + "unclipped_grad_norm": 0.8796955943107605 + }, + { + "bits_per_byte": 1.5203328250026595, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.05381441116333, + "step": 5670, + "unclipped_grad_norm": 0.8509314060211182 + }, + { + "bits_per_byte": 1.5301421994599214, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.060613751411438, + "step": 5680, + "unclipped_grad_norm": 0.8624072074890137 + }, + { + "bits_per_byte": 1.6028342784827556, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1110000610351562, + "step": 5690, + "unclipped_grad_norm": 0.8914096355438232 + }, + { + "bits_per_byte": 1.526354281574668, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.057988166809082, + "step": 5700, + "unclipped_grad_norm": 0.8579604625701904 + }, + { + "bits_per_byte": 1.5347007716036911, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.063773512840271, + "step": 5710, + "unclipped_grad_norm": 0.8810272216796875 + }, + { + "bits_per_byte": 1.5491378552303696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0737805366516113, + "step": 5720, + "unclipped_grad_norm": 0.8394830226898193 + }, + { + "bits_per_byte": 1.5456280332915613, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.071347713470459, + "step": 5730, + "unclipped_grad_norm": 0.865943968296051 + }, + { + "bits_per_byte": 1.5433466834278662, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0697664022445679, + "step": 5740, + "unclipped_grad_norm": 0.886944591999054 + }, + { + "bits_per_byte": 1.5488365416260579, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0735716819763184, + "step": 5750, + "unclipped_grad_norm": 0.9145897626876831 + }, + { + "bits_per_byte": 1.5636783004302166, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0838592052459717, + "step": 5760, + "unclipped_grad_norm": 0.9009037017822266 + }, + { + "bits_per_byte": 1.511110599315443, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0474220514297485, + "step": 5770, + "unclipped_grad_norm": 0.844070315361023 + }, + { + "bits_per_byte": 1.494012772097266, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.035570740699768, + "step": 5780, + "unclipped_grad_norm": 0.8943845629692078 + }, + { + "bits_per_byte": 1.5135893852623288, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.049140214920044, + "step": 5790, + "unclipped_grad_norm": 0.8841454386711121 + }, + { + "bits_per_byte": 1.4905120652389536, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033144235610962, + "step": 5800, + "unclipped_grad_norm": 0.8580894470214844 + }, + { + "bits_per_byte": 1.5735937881830608, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0907320976257324, + "step": 5810, + "unclipped_grad_norm": 0.933175265789032 + }, + { + "bits_per_byte": 1.5042643139498963, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.042676568031311, + "step": 5820, + "unclipped_grad_norm": 0.828717052936554 + }, + { + "bits_per_byte": 1.5156406223391694, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0505620241165161, + "step": 5830, + "unclipped_grad_norm": 0.9511673450469971 + }, + { + "bits_per_byte": 1.5092205099824607, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0461119413375854, + "step": 5840, + "unclipped_grad_norm": 0.8395604491233826 + }, + { + "bits_per_byte": 1.477936693807865, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0244276523590088, + "step": 5850, + "unclipped_grad_norm": 0.9143134355545044 + }, + { + "bits_per_byte": 1.5336375748560571, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.063036561012268, + "step": 5860, + "unclipped_grad_norm": 0.9014923572540283 + }, + { + "bits_per_byte": 1.5402322496033916, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0676076412200928, + "step": 5870, + "unclipped_grad_norm": 0.8728002905845642 + }, + { + "bits_per_byte": 1.555285031120159, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.078041434288025, + "step": 5880, + "unclipped_grad_norm": 0.8936865925788879 + }, + { + "bits_per_byte": 1.564054942435606, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.084120273590088, + "step": 5890, + "unclipped_grad_norm": 0.8752698302268982 + }, + { + "bits_per_byte": 1.5458715607251832, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071516513824463, + "step": 5900, + "unclipped_grad_norm": 0.8957662582397461 + }, + { + "bits_per_byte": 1.5568008862048637, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0790921449661255, + "step": 5910, + "unclipped_grad_norm": 0.8808028101921082 + }, + { + "bits_per_byte": 1.4896764015384112, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0325649976730347, + "step": 5920, + "unclipped_grad_norm": 0.8660398721694946 + }, + { + "bits_per_byte": 1.5908711652887386, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.102707862854004, + "step": 5930, + "unclipped_grad_norm": 0.9604227542877197 + }, + { + "bits_per_byte": 1.4574508083676907, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.010227918624878, + "step": 5940, + "unclipped_grad_norm": 0.8380697965621948 + }, + { + "bits_per_byte": 1.5131755950043528, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0488533973693848, + "step": 5950, + "unclipped_grad_norm": 0.8392192125320435 + }, + { + "bits_per_byte": 1.5452194025131114, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0710644721984863, + "step": 5960, + "unclipped_grad_norm": 0.932554304599762 + }, + { + "bits_per_byte": 1.5213482105733536, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.054518222808838, + "step": 5970, + "unclipped_grad_norm": 0.927780270576477 + }, + { + "bits_per_byte": 1.5601101763727658, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0813859701156616, + "step": 5980, + "unclipped_grad_norm": 0.8920019268989563 + }, + { + "bits_per_byte": 1.5382338112003657, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0662224292755127, + "step": 5990, + "unclipped_grad_norm": 0.9537093639373779 + }, + { + "bits_per_byte": 1.4675097296513546, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.017200231552124, + "step": 6000, + "unclipped_grad_norm": 0.8773758411407471 + }, + { + "bits_per_byte": 1.506333093257125, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0441105365753174, + "step": 6010, + "unclipped_grad_norm": 0.8785988092422485 + }, + { + "bits_per_byte": 1.4794742187065786, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0254933834075928, + "step": 6020, + "unclipped_grad_norm": 0.8875186443328857 + }, + { + "bits_per_byte": 1.5245675538148544, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.056749701499939, + "step": 6030, + "unclipped_grad_norm": 0.88789963722229 + }, + { + "bits_per_byte": 1.5628784091210537, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.083304762840271, + "step": 6040, + "unclipped_grad_norm": 0.9149079322814941 + }, + { + "bits_per_byte": 1.4881958028971334, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.031538724899292, + "step": 6050, + "unclipped_grad_norm": 0.9299342632293701 + }, + { + "bits_per_byte": 1.4785307218839452, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0248394012451172, + "step": 6060, + "unclipped_grad_norm": 0.8783867359161377 + }, + { + "bits_per_byte": 1.5509557118499895, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.075040578842163, + "step": 6070, + "unclipped_grad_norm": 0.9337034821510315 + }, + { + "bits_per_byte": 1.4914187577743023, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0337727069854736, + "step": 6080, + "unclipped_grad_norm": 0.9420101046562195 + }, + { + "bits_per_byte": 1.4968499698885498, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0375373363494873, + "step": 6090, + "unclipped_grad_norm": 0.8664546012878418 + }, + { + "bits_per_byte": 1.4906561867003767, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0332441329956055, + "step": 6100, + "unclipped_grad_norm": 0.8687640428543091 + }, + { + "bits_per_byte": 1.5538661742505406, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.077057957649231, + "step": 6110, + "unclipped_grad_norm": 0.8986173868179321 + }, + { + "bits_per_byte": 1.5476213122150613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0727293491363525, + "step": 6120, + "unclipped_grad_norm": 0.9189077019691467 + }, + { + "bits_per_byte": 1.4789527673091625, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0251319408416748, + "step": 6130, + "unclipped_grad_norm": 0.8851786851882935 + }, + { + "bits_per_byte": 1.4768556108645414, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0236783027648926, + "step": 6140, + "unclipped_grad_norm": 0.9043483138084412 + }, + { + "bits_per_byte": 1.552988202817886, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0764493942260742, + "step": 6150, + "unclipped_grad_norm": 0.9005390405654907 + }, + { + "bits_per_byte": 1.3994871515820966, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.970050573348999, + "step": 6160, + "unclipped_grad_norm": 0.8554620742797852 + }, + { + "bits_per_byte": 1.5302097886417105, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0606606006622314, + "step": 6170, + "unclipped_grad_norm": 0.8954014778137207 + }, + { + "bits_per_byte": 1.4938230752333643, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0354392528533936, + "step": 6180, + "unclipped_grad_norm": 0.892278790473938 + }, + { + "bits_per_byte": 1.516749394489282, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.05133056640625, + "step": 6190, + "unclipped_grad_norm": 0.9185703992843628 + }, + { + "bits_per_byte": 1.5086576107661869, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0457217693328857, + "step": 6200, + "unclipped_grad_norm": 0.8927735090255737 + }, + { + "bits_per_byte": 1.5088584865023946, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.045861005783081, + "step": 6210, + "unclipped_grad_norm": 0.8906724452972412 + }, + { + "bits_per_byte": 1.5118191678369979, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0479131937026978, + "step": 6220, + "unclipped_grad_norm": 0.9077256917953491 + }, + { + "bits_per_byte": 1.4471287536282067, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0030732154846191, + "step": 6230, + "unclipped_grad_norm": 0.8839235305786133 + }, + { + "bits_per_byte": 1.5076786855174757, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0450432300567627, + "step": 6240, + "unclipped_grad_norm": 0.9515005946159363 + }, + { + "bits_per_byte": 1.5408861276419719, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0680608749389648, + "step": 6250, + "unclipped_grad_norm": 0.901561975479126 + }, + { + "bits_per_byte": 1.4200397662908057, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9842965602874756, + "step": 6260, + "unclipped_grad_norm": 0.889621376991272 + }, + { + "bits_per_byte": 1.5216213190229235, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0547075271606445, + "step": 6270, + "unclipped_grad_norm": 0.9411186575889587 + }, + { + "bits_per_byte": 1.5224643779774532, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0552918910980225, + "step": 6280, + "unclipped_grad_norm": 0.9406285881996155 + }, + { + "bits_per_byte": 1.452789390598705, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0069968700408936, + "step": 6290, + "unclipped_grad_norm": 0.8839805126190186 + }, + { + "bits_per_byte": 1.5074908804627336, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0449130535125732, + "step": 6300, + "unclipped_grad_norm": 0.9184122681617737 + }, + { + "bits_per_byte": 1.478450577968643, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0247838497161865, + "step": 6310, + "unclipped_grad_norm": 0.8948619365692139 + }, + { + "bits_per_byte": 1.4214500240278807, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.985274076461792, + "step": 6320, + "unclipped_grad_norm": 0.8749227523803711 + }, + { + "bits_per_byte": 1.5185149683830395, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0525543689727783, + "step": 6330, + "unclipped_grad_norm": 0.9816636443138123 + }, + { + "bits_per_byte": 1.4759477144506365, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.023048996925354, + "step": 6340, + "unclipped_grad_norm": 0.8950188159942627 + }, + { + "bits_per_byte": 1.587707716408768, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1005151271820068, + "step": 6350, + "unclipped_grad_norm": 1.01009202003479 + }, + { + "bits_per_byte": 1.4638088349874379, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0146349668502808, + "step": 6360, + "unclipped_grad_norm": 0.9816907644271851 + }, + { + "bits_per_byte": 1.4382974445064014, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9969518184661865, + "step": 6370, + "unclipped_grad_norm": 0.8970034122467041 + }, + { + "bits_per_byte": 1.4818757844432269, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0271580219268799, + "step": 6380, + "unclipped_grad_norm": 0.9058489799499512 + }, + { + "bits_per_byte": 1.527050123380059, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0584704875946045, + "step": 6390, + "unclipped_grad_norm": 0.929339587688446 + }, + { + "bits_per_byte": 1.5156593684481134, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0505750179290771, + "step": 6400, + "unclipped_grad_norm": 0.9009133577346802 + }, + { + "bits_per_byte": 1.4969940913499729, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0376372337341309, + "step": 6410, + "unclipped_grad_norm": 0.957128643989563 + }, + { + "bits_per_byte": 1.4783625228514012, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0247228145599365, + "step": 6420, + "unclipped_grad_norm": 0.9414587616920471 + }, + { + "bits_per_byte": 1.523227980947284, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.055821180343628, + "step": 6430, + "unclipped_grad_norm": 0.9234302639961243 + }, + { + "bits_per_byte": 1.4623229048839832, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.013604998588562, + "step": 6440, + "unclipped_grad_norm": 0.9177728295326233 + }, + { + "bits_per_byte": 1.5306256426914968, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0609488487243652, + "step": 6450, + "unclipped_grad_norm": 0.9453822374343872 + }, + { + "bits_per_byte": 1.5847898587542297, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0984926223754883, + "step": 6460, + "unclipped_grad_norm": 0.9647909998893738 + }, + { + "bits_per_byte": 1.5690988496201106, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0876164436340332, + "step": 6470, + "unclipped_grad_norm": 0.9592546820640564 + }, + { + "bits_per_byte": 1.4544314809491432, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0081350803375244, + "step": 6480, + "unclipped_grad_norm": 0.9334172010421753 + }, + { + "bits_per_byte": 1.5573938823850386, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0795031785964966, + "step": 6490, + "unclipped_grad_norm": 0.967822790145874 + }, + { + "bits_per_byte": 1.5280063469188563, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0591332912445068, + "step": 6500, + "unclipped_grad_norm": 0.9625112414360046 + }, + { + "bits_per_byte": 1.4687796495453256, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.018080472946167, + "step": 6510, + "unclipped_grad_norm": 0.9510544538497925 + }, + { + "bits_per_byte": 1.4284500618832987, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9901261329650879, + "step": 6520, + "unclipped_grad_norm": 0.8985839486122131 + }, + { + "bits_per_byte": 1.4546447394362132, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0082828998565674, + "step": 6530, + "unclipped_grad_norm": 0.907886803150177 + }, + { + "bits_per_byte": 1.4970680438898438, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0376884937286377, + "step": 6540, + "unclipped_grad_norm": 0.9381445050239563 + }, + { + "bits_per_byte": 1.4280130539674563, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9898232221603394, + "step": 6550, + "unclipped_grad_norm": 0.9031082391738892 + }, + { + "bits_per_byte": 1.5210671909218436, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.054323434829712, + "step": 6560, + "unclipped_grad_norm": 0.9433532357215881 + }, + { + "bits_per_byte": 1.479243074023819, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0253331661224365, + "step": 6570, + "unclipped_grad_norm": 0.9311387538909912 + }, + { + "bits_per_byte": 1.5051876888023787, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0433166027069092, + "step": 6580, + "unclipped_grad_norm": 0.9569925665855408 + }, + { + "bits_per_byte": 1.494823842278735, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0361329317092896, + "step": 6590, + "unclipped_grad_norm": 0.8958868384361267 + }, + { + "bits_per_byte": 1.4302109062541577, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9913466572761536, + "step": 6600, + "unclipped_grad_norm": 0.9136191010475159 + }, + { + "bits_per_byte": 1.436879103584736, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 0.9959686994552612, + "step": 6610, + "unclipped_grad_norm": 0.8710504770278931 + }, + { + "bits_per_byte": 1.4278971376607748, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9897428750991821, + "step": 6620, + "unclipped_grad_norm": 0.9145034551620483 + }, + { + "bits_per_byte": 1.4582168190946336, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.010758876800537, + "step": 6630, + "unclipped_grad_norm": 0.8999127149581909 + }, + { + "bits_per_byte": 1.5240701799885592, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.056404948234558, + "step": 6640, + "unclipped_grad_norm": 0.9801992774009705 + }, + { + "bits_per_byte": 1.5169175935218258, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0514471530914307, + "step": 6650, + "unclipped_grad_norm": 0.8882671594619751 + }, + { + "bits_per_byte": 1.5152221885496202, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.050271987915039, + "step": 6660, + "unclipped_grad_norm": 0.9029646515846252 + }, + { + "bits_per_byte": 1.4896082964086694, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0325177907943726, + "step": 6670, + "unclipped_grad_norm": 0.9797500371932983 + }, + { + "bits_per_byte": 1.4764526555135697, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.023398995399475, + "step": 6680, + "unclipped_grad_norm": 0.9356769919395447 + }, + { + "bits_per_byte": 1.4958203097578342, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0368236303329468, + "step": 6690, + "unclipped_grad_norm": 0.939789891242981 + }, + { + "bits_per_byte": 1.5480882451121538, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.073053002357483, + "step": 6700, + "unclipped_grad_norm": 1.0080333948135376 + }, + { + "bits_per_byte": 1.525887864625528, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0576648712158203, + "step": 6710, + "unclipped_grad_norm": 0.9197862148284912 + }, + { + "bits_per_byte": 1.534906462854123, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0639160871505737, + "step": 6720, + "unclipped_grad_norm": 0.9543269872665405 + }, + { + "bits_per_byte": 1.4820493149379474, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0272783041000366, + "step": 6730, + "unclipped_grad_norm": 0.9817740321159363 + }, + { + "bits_per_byte": 1.4384499931177168, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9970575571060181, + "step": 6740, + "unclipped_grad_norm": 0.9710240364074707 + }, + { + "bits_per_byte": 1.511069151496585, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0473933219909668, + "step": 6750, + "unclipped_grad_norm": 0.9612263441085815 + }, + { + "bits_per_byte": 1.4780942299160553, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0245368480682373, + "step": 6760, + "unclipped_grad_norm": 0.9238441586494446 + }, + { + "bits_per_byte": 1.3767874194686909, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9543163180351257, + "step": 6770, + "unclipped_grad_norm": 0.9215719699859619 + }, + { + "bits_per_byte": 1.4785716537548506, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0248677730560303, + "step": 6780, + "unclipped_grad_norm": 0.9403936266899109 + }, + { + "bits_per_byte": 1.4459957319243224, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0022878646850586, + "step": 6790, + "unclipped_grad_norm": 0.9651554822921753 + }, + { + "bits_per_byte": 1.5175859181030786, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.051910400390625, + "step": 6800, + "unclipped_grad_norm": 0.9926899075508118 + }, + { + "bits_per_byte": 1.5110634760691066, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0473893880844116, + "step": 6810, + "unclipped_grad_norm": 0.9793192148208618 + }, + { + "bits_per_byte": 1.5052492585913877, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0433592796325684, + "step": 6820, + "unclipped_grad_norm": 1.0081095695495605 + }, + { + "bits_per_byte": 1.466343515295854, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0163918733596802, + "step": 6830, + "unclipped_grad_norm": 0.960361659526825 + }, + { + "bits_per_byte": 1.4376153613130795, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9964790344238281, + "step": 6840, + "unclipped_grad_norm": 0.9702624678611755 + }, + { + "bits_per_byte": 1.4922177891702109, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0343265533447266, + "step": 6850, + "unclipped_grad_norm": 0.9893983006477356 + }, + { + "bits_per_byte": 1.4659078832412185, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.016089916229248, + "step": 6860, + "unclipped_grad_norm": 0.9693014621734619 + }, + { + "bits_per_byte": 1.413224953750367, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9795728921890259, + "step": 6870, + "unclipped_grad_norm": 0.9252057075500488 + }, + { + "bits_per_byte": 1.4200107012228098, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9842764139175415, + "step": 6880, + "unclipped_grad_norm": 0.8884434700012207 + }, + { + "bits_per_byte": 1.4366857090938405, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9958346486091614, + "step": 6890, + "unclipped_grad_norm": 0.9679457545280457 + }, + { + "bits_per_byte": 1.5416700245646047, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0686042308807373, + "step": 6900, + "unclipped_grad_norm": 0.9985603094100952 + }, + { + "bits_per_byte": 1.4893800754309745, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0323596000671387, + "step": 6910, + "unclipped_grad_norm": 0.9936915636062622 + }, + { + "bits_per_byte": 1.4499423897963222, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.00502347946167, + "step": 6920, + "unclipped_grad_norm": 0.923648476600647 + }, + { + "bits_per_byte": 1.4406846496917027, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9986065030097961, + "step": 6930, + "unclipped_grad_norm": 0.9748247861862183 + }, + { + "bits_per_byte": 1.408974058568991, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9766263961791992, + "step": 6940, + "unclipped_grad_norm": 0.9357670545578003 + }, + { + "bits_per_byte": 1.434796565665438, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9945251941680908, + "step": 6950, + "unclipped_grad_norm": 0.9624595642089844 + }, + { + "bits_per_byte": 1.49675830313564, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0374737977981567, + "step": 6960, + "unclipped_grad_norm": 0.9387544989585876 + }, + { + "bits_per_byte": 1.4929531869853003, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0348362922668457, + "step": 6970, + "unclipped_grad_norm": 0.9343914985656738 + }, + { + "bits_per_byte": 1.4660633555575984, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.016197681427002, + "step": 6980, + "unclipped_grad_norm": 0.9326976537704468 + }, + { + "bits_per_byte": 1.492015881538098, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034186601638794, + "step": 6990, + "unclipped_grad_norm": 1.0069916248321533 + }, + { + "bits_per_byte": 1.52302985693349, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0556838512420654, + "step": 7000, + "unclipped_grad_norm": 0.9719612002372742 + }, + { + "bits_per_byte": 1.512785538352196, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0485830307006836, + "step": 7010, + "unclipped_grad_norm": 0.9873834848403931 + }, + { + "bits_per_byte": 1.4473323810868282, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0032143592834473, + "step": 7020, + "unclipped_grad_norm": 0.939332127571106 + }, + { + "bits_per_byte": 1.5166639191118032, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0512713193893433, + "step": 7030, + "unclipped_grad_norm": 0.9837853312492371 + }, + { + "bits_per_byte": 1.5072451172546506, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0447427034378052, + "step": 7040, + "unclipped_grad_norm": 0.9734936952590942 + }, + { + "bits_per_byte": 1.4796651194490362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0256257057189941, + "step": 7050, + "unclipped_grad_norm": 1.005554437637329 + }, + { + "bits_per_byte": 1.4927048440374544, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0346641540527344, + "step": 7060, + "unclipped_grad_norm": 0.9903765320777893 + }, + { + "bits_per_byte": 1.4561590466770602, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.009332537651062, + "step": 7070, + "unclipped_grad_norm": 1.009922981262207 + }, + { + "bits_per_byte": 1.4353391709289103, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9949012994766235, + "step": 7080, + "unclipped_grad_norm": 0.9552373290061951 + }, + { + "bits_per_byte": 1.4780041110070032, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0244743824005127, + "step": 7090, + "unclipped_grad_norm": 1.0046122074127197 + }, + { + "bits_per_byte": 1.4728850474040718, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0209261178970337, + "step": 7100, + "unclipped_grad_norm": 0.9868813157081604 + }, + { + "bits_per_byte": 1.4862149067244954, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.030165672302246, + "step": 7110, + "unclipped_grad_norm": 0.9593554735183716 + }, + { + "bits_per_byte": 1.5328834309320238, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.062513828277588, + "step": 7120, + "unclipped_grad_norm": 1.0129024982452393 + }, + { + "bits_per_byte": 1.4811555211014134, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0266587734222412, + "step": 7130, + "unclipped_grad_norm": 1.0546875 + }, + { + "bits_per_byte": 1.4724865636020226, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0206499099731445, + "step": 7140, + "unclipped_grad_norm": 0.9776664972305298 + }, + { + "bits_per_byte": 1.4357171887955067, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.9951633214950562, + "step": 7150, + "unclipped_grad_norm": 0.9291411638259888 + }, + { + "bits_per_byte": 1.5232358921492237, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0558266639709473, + "step": 7160, + "unclipped_grad_norm": 1.0189123153686523 + }, + { + "bits_per_byte": 1.4618840051589814, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0133007764816284, + "step": 7170, + "unclipped_grad_norm": 0.9592419266700745 + }, + { + "bits_per_byte": 1.4762450724539784, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0232551097869873, + "step": 7180, + "unclipped_grad_norm": 0.9680334329605103 + }, + { + "bits_per_byte": 1.4441101141402626, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0009808540344238, + "step": 7190, + "unclipped_grad_norm": 1.0148857831954956 + }, + { + "bits_per_byte": 1.472608843266786, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0207346677780151, + "step": 7200, + "unclipped_grad_norm": 0.9935786128044128 + }, + { + "bits_per_byte": 1.5299951542934338, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.060511827468872, + "step": 7210, + "unclipped_grad_norm": 0.9725626111030579 + }, + { + "bits_per_byte": 1.5238232129019202, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0562337636947632, + "step": 7220, + "unclipped_grad_norm": 1.0141370296478271 + }, + { + "bits_per_byte": 1.4390708504873315, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9974879026412964, + "step": 7230, + "unclipped_grad_norm": 1.0297694206237793 + }, + { + "bits_per_byte": 1.4632025961431465, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0142147541046143, + "step": 7240, + "unclipped_grad_norm": 0.9345177412033081 + }, + { + "bits_per_byte": 1.5031187375124992, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0418825149536133, + "step": 7250, + "unclipped_grad_norm": 1.0408201217651367 + }, + { + "bits_per_byte": 1.3820238612308362, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9579459428787231, + "step": 7260, + "unclipped_grad_norm": 0.9544695615768433 + }, + { + "bits_per_byte": 1.4274823155068936, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9894553422927856, + "step": 7270, + "unclipped_grad_norm": 0.9940797686576843 + }, + { + "bits_per_byte": 1.4990891840027836, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0390894412994385, + "step": 7280, + "unclipped_grad_norm": 1.013985514640808 + }, + { + "bits_per_byte": 1.4370805812602216, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.99610835313797, + "step": 7290, + "unclipped_grad_norm": 0.9820091724395752 + }, + { + "bits_per_byte": 1.4451605841717328, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.001708984375, + "step": 7300, + "unclipped_grad_norm": 0.9996907114982605 + }, + { + "bits_per_byte": 1.4502488628801598, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0052359104156494, + "step": 7310, + "unclipped_grad_norm": 0.9723894000053406 + }, + { + "bits_per_byte": 1.4141343980081293, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9802032709121704, + "step": 7320, + "unclipped_grad_norm": 0.9815637469291687 + }, + { + "bits_per_byte": 1.429772436485783, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9910427331924438, + "step": 7330, + "unclipped_grad_norm": 1.0089058876037598 + }, + { + "bits_per_byte": 1.4033304478809279, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9727145433425903, + "step": 7340, + "unclipped_grad_norm": 0.9830491542816162 + }, + { + "bits_per_byte": 1.4800763299672492, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.02591073513031, + "step": 7350, + "unclipped_grad_norm": 0.9842677712440491 + }, + { + "bits_per_byte": 1.4409101189469837, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9987627863883972, + "step": 7360, + "unclipped_grad_norm": 1.0266367197036743 + }, + { + "bits_per_byte": 1.4700960047550296, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0189929008483887, + "step": 7370, + "unclipped_grad_norm": 0.9935671091079712 + }, + { + "bits_per_byte": 1.4111584102175994, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9781404733657837, + "step": 7380, + "unclipped_grad_norm": 0.9791932702064514 + }, + { + "bits_per_byte": 1.4805810990475317, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0262606143951416, + "step": 7390, + "unclipped_grad_norm": 0.9806758761405945 + }, + { + "bits_per_byte": 1.422500752033327, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9860023856163025, + "step": 7400, + "unclipped_grad_norm": 1.014562726020813 + }, + { + "bits_per_byte": 1.419007526420327, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9835810661315918, + "step": 7410, + "unclipped_grad_norm": 0.9609370231628418 + }, + { + "bits_per_byte": 1.483305992167802, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0281493663787842, + "step": 7420, + "unclipped_grad_norm": 0.9924190640449524 + }, + { + "bits_per_byte": 1.4892452410326982, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0322661399841309, + "step": 7430, + "unclipped_grad_norm": 1.0244429111480713 + }, + { + "bits_per_byte": 1.50888548777858, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0458797216415405, + "step": 7440, + "unclipped_grad_norm": 1.0394790172576904 + }, + { + "bits_per_byte": 1.4553271665948369, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0087559223175049, + "step": 7450, + "unclipped_grad_norm": 0.99248868227005 + }, + { + "bits_per_byte": 1.4425173828106221, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.999876856803894, + "step": 7460, + "unclipped_grad_norm": 1.0004469156265259 + }, + { + "bits_per_byte": 1.437790267669007, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9966002702713013, + "step": 7470, + "unclipped_grad_norm": 1.0109968185424805 + }, + { + "bits_per_byte": 1.4682612938356252, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.017721176147461, + "step": 7480, + "unclipped_grad_norm": 1.0299010276794434 + }, + { + "bits_per_byte": 1.5222762289574092, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.055161476135254, + "step": 7490, + "unclipped_grad_norm": 1.0327476263046265 + }, + { + "bits_per_byte": 1.4747042798848986, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0221871137619019, + "step": 7500, + "unclipped_grad_norm": 0.9886637926101685 + }, + { + "bits_per_byte": 1.4352613487793948, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9948473572731018, + "step": 7510, + "unclipped_grad_norm": 0.9710773825645447 + }, + { + "bits_per_byte": 1.4570311706995855, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.009937047958374, + "step": 7520, + "unclipped_grad_norm": 1.0144232511520386 + }, + { + "bits_per_byte": 1.502727820947088, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0416115522384644, + "step": 7530, + "unclipped_grad_norm": 1.0307852029800415 + }, + { + "bits_per_byte": 1.4572943041554056, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0101194381713867, + "step": 7540, + "unclipped_grad_norm": 0.9732069969177246 + }, + { + "bits_per_byte": 1.4747278415080667, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0222034454345703, + "step": 7550, + "unclipped_grad_norm": 1.0250486135482788 + }, + { + "bits_per_byte": 1.4513507557242378, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0059996843338013, + "step": 7560, + "unclipped_grad_norm": 1.0031380653381348 + }, + { + "bits_per_byte": 1.4127096077370564, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.979215681552887, + "step": 7570, + "unclipped_grad_norm": 0.9720339775085449 + }, + { + "bits_per_byte": 1.587541409185384, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1003998517990112, + "step": 7580, + "unclipped_grad_norm": 1.080464482307434 + }, + { + "bits_per_byte": 1.4421111597892842, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9995952844619751, + "step": 7590, + "unclipped_grad_norm": 0.9815735220909119 + }, + { + "bits_per_byte": 1.4267507013101233, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9889482259750366, + "step": 7600, + "unclipped_grad_norm": 1.029758334159851 + }, + { + "bits_per_byte": 1.4539179407536669, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0077791213989258, + "step": 7610, + "unclipped_grad_norm": 1.031221628189087 + }, + { + "bits_per_byte": 1.4028547438686412, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9723848104476929, + "step": 7620, + "unclipped_grad_norm": 0.9562389850616455 + }, + { + "bits_per_byte": 1.3938100042771153, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9661154747009277, + "step": 7630, + "unclipped_grad_norm": 1.023301124572754 + }, + { + "bits_per_byte": 1.4184527103886437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.983196496963501, + "step": 7640, + "unclipped_grad_norm": 0.9981763958930969 + }, + { + "bits_per_byte": 1.5608753271864544, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.081916332244873, + "step": 7650, + "unclipped_grad_norm": 1.0295648574829102 + }, + { + "bits_per_byte": 1.4665378556913289, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0165265798568726, + "step": 7660, + "unclipped_grad_norm": 1.0587892532348633 + }, + { + "bits_per_byte": 1.4931012640476933, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.034938931465149, + "step": 7670, + "unclipped_grad_norm": 1.0342050790786743 + }, + { + "bits_per_byte": 1.4465245785757257, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0026544332504272, + "step": 7680, + "unclipped_grad_norm": 1.0567405223846436 + }, + { + "bits_per_byte": 1.4541416901824393, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0079342126846313, + "step": 7690, + "unclipped_grad_norm": 1.0190099477767944 + }, + { + "bits_per_byte": 1.3949941048283057, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9669362306594849, + "step": 7700, + "unclipped_grad_norm": 1.002891182899475 + }, + { + "bits_per_byte": 1.4762163513512843, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0232352018356323, + "step": 7710, + "unclipped_grad_norm": 0.9984713792800903 + }, + { + "bits_per_byte": 1.480895139368007, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0264782905578613, + "step": 7720, + "unclipped_grad_norm": 1.0342158079147339 + }, + { + "bits_per_byte": 1.486845911070511, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.030603051185608, + "step": 7730, + "unclipped_grad_norm": 1.027107834815979 + }, + { + "bits_per_byte": 1.4387832095037634, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9972885251045227, + "step": 7740, + "unclipped_grad_norm": 1.0455139875411987 + }, + { + "bits_per_byte": 1.4381068877292456, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9968197345733643, + "step": 7750, + "unclipped_grad_norm": 1.0159157514572144 + }, + { + "bits_per_byte": 1.4538223183997872, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0077128410339355, + "step": 7760, + "unclipped_grad_norm": 0.9867706894874573 + }, + { + "bits_per_byte": 1.3986834766546148, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9694935083389282, + "step": 7770, + "unclipped_grad_norm": 0.9943742156028748 + }, + { + "bits_per_byte": 1.4760859885019304, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0231448411941528, + "step": 7780, + "unclipped_grad_norm": 1.0422483682632446 + }, + { + "bits_per_byte": 1.4162891686407895, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9816968441009521, + "step": 7790, + "unclipped_grad_norm": 0.9937946796417236 + }, + { + "bits_per_byte": 1.45477768202533, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0083750486373901, + "step": 7800, + "unclipped_grad_norm": 1.0428839921951294 + }, + { + "bits_per_byte": 1.4292601001688625, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9906876087188721, + "step": 7810, + "unclipped_grad_norm": 1.028254508972168 + }, + { + "bits_per_byte": 1.4070698666586379, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.975306510925293, + "step": 7820, + "unclipped_grad_norm": 1.0253064632415771 + }, + { + "bits_per_byte": 1.4530960356651936, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0072094202041626, + "step": 7830, + "unclipped_grad_norm": 1.0885143280029297 + }, + { + "bits_per_byte": 1.4330722676078875, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9933300018310547, + "step": 7840, + "unclipped_grad_norm": 1.0074925422668457 + }, + { + "bits_per_byte": 1.3858685333908742, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9606108665466309, + "step": 7850, + "unclipped_grad_norm": 0.9880329966545105 + }, + { + "bits_per_byte": 1.4517502714221922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0062766075134277, + "step": 7860, + "unclipped_grad_norm": 1.0082415342330933 + }, + { + "bits_per_byte": 1.4675547891058807, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0172314643859863, + "step": 7870, + "unclipped_grad_norm": 1.023923397064209 + }, + { + "bits_per_byte": 1.3867994754799946, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9612561464309692, + "step": 7880, + "unclipped_grad_norm": 1.0151392221450806 + }, + { + "bits_per_byte": 1.4568015738606837, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0097779035568237, + "step": 7890, + "unclipped_grad_norm": 1.0505098104476929 + }, + { + "bits_per_byte": 1.446776189193938, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.00282883644104, + "step": 7900, + "unclipped_grad_norm": 1.0279992818832397 + }, + { + "bits_per_byte": 1.461830518554563, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0132637023925781, + "step": 7910, + "unclipped_grad_norm": 1.0125948190689087 + }, + { + "bits_per_byte": 1.4593548282953928, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0115476846694946, + "step": 7920, + "unclipped_grad_norm": 1.0411789417266846 + }, + { + "bits_per_byte": 1.4317973602170415, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9924463033676147, + "step": 7930, + "unclipped_grad_norm": 1.0330348014831543 + }, + { + "bits_per_byte": 1.431196452834927, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9920297861099243, + "step": 7940, + "unclipped_grad_norm": 1.0142537355422974 + }, + { + "bits_per_byte": 1.49434246283897, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.035799264907837, + "step": 7950, + "unclipped_grad_norm": 1.0716009140014648 + }, + { + "bits_per_byte": 1.4186909923514142, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9833616614341736, + "step": 7960, + "unclipped_grad_norm": 1.0115852355957031 + }, + { + "bits_per_byte": 1.4420433126335188, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9995482563972473, + "step": 7970, + "unclipped_grad_norm": 1.0335451364517212 + }, + { + "bits_per_byte": 1.4872165336831205, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0308599472045898, + "step": 7980, + "unclipped_grad_norm": 1.0748283863067627 + }, + { + "bits_per_byte": 1.5234063269562288, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.055944800376892, + "step": 7990, + "unclipped_grad_norm": 1.076164722442627 + }, + { + "bits_per_byte": 1.4309937712808851, + "learning_rate": 3e-05, + "loss_nats": 0.9918892979621887, + "step": 8000, + "unclipped_grad_norm": 1.0322442054748535 + } + ] + }, + "depth-32-block-seed-2026073002": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "b4262e697e2269457fdebf31f75008383c6e8024ee1bbf96e5962fb2ba1dec18", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "population_cv": 0.39438035124793563 + }, + "activation_output_rms_by_block": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005765057983808219, + "first_to_last_ratio": 0.622587017371887, + "imbalance_abs_log_ratio": 0.4738718734697682, + "last_quartile_mean": 0.009259842918254435, + "mean": 0.007398182220640592, + "normalized": [ + 0.49149180917985336, + 0.6965082091758006, + 0.8632718983507628, + 0.9846233376628902, + 0.5069511565390686, + 0.730127144119427, + 0.8901300989627224, + 1.0709228264321597, + 0.6068057253534959, + 0.8292953231982888, + 0.9889841312125875, + 1.119997332175246, + 0.6022405569783624, + 0.8738552664669654, + 1.0668989020126547, + 1.2430080659586231, + 0.6705962885532487, + 0.9336251153230055, + 1.1108121094231989, + 1.3080235563150786, + 0.7274038039370601, + 1.01412651484229, + 1.232101109614035, + 1.4250990487866828, + 0.8294838994154143, + 1.1448566626512724, + 1.427936504016329, + 1.6627087330393202, + 0.7804514393709395, + 1.1025360304896303, + 1.4135242707919937, + 1.6516031296515918 + ], + "population_cv": 0.30590027610070303 + }, + "bits_per_byte": 8.038491115106986, + "branch_output_rms_by_sublayer": [ + 0.0023513585329055786, + 0.0027412420604377985, + 0.0023208872880786657, + 0.002683233004063368, + 0.002778682392090559, + 0.002711885841563344, + 0.002497476991266012, + 0.0027249811682850122, + 0.002628475893288851, + 0.0027012599166482687, + 0.002920303260907531, + 0.002700963756069541, + 0.002932395786046982, + 0.0026683746837079525, + 0.0032228652853518724, + 0.002738022478297353, + 0.003561305580660701, + 0.0027370047755539417, + 0.003354643704369664, + 0.002654933836311102, + 0.0034162327647209167, + 0.002663377905264497, + 0.0036349669098854065, + 0.002684312639757991, + 0.0035210708156228065, + 0.0027356327045708895, + 0.0038089773152023554, + 0.002652956871315837, + 0.003915763925760984, + 0.002679797587916255, + 0.0037244565319269896, + 0.002698024269193411, + 0.004162859171628952, + 0.0026484597474336624, + 0.004167158622294664, + 0.0026680538430809975, + 0.004440344404429197, + 0.002654345240443945, + 0.0041700867004692554, + 0.002591830212622881, + 0.004737259820103645, + 0.002620960585772991, + 0.004552462603896856, + 0.002651234855875373, + 0.004706811159849167, + 0.0026270023081451654, + 0.005186605267226696, + 0.002615687670186162, + 0.005580419674515724, + 0.00258000404573977, + 0.004905648995190859, + 0.002626793459057808, + 0.005227713380008936, + 0.002573782578110695, + 0.005349088925868273, + 0.002592637902125716, + 0.005235649179667234, + 0.002581534907221794, + 0.004925078712403774, + 0.002551236655563116, + 0.005810525268316269, + 0.002563327783718705, + 0.005564783234149218, + 0.002589041367173195 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.010012756197717748, + 0.008904953530422711, + 0.008980942528705073, + 0.007787912846174446, + 0.007907551845183929, + 0.008289745083748344, + 0.007663833460373458, + 0.007350080301903749, + 0.00709564946584838, + 0.006310668475383808, + 0.006285078227083486, + 0.006362421438950662, + 0.005604026016423943, + 0.005959901601525332, + 0.006167718108848569, + 0.005523196426059091, + 0.0054641843609823, + 0.00557989785603791, + 0.005260407000687673, + 0.004763357766419076, + 0.004970214176820314, + 0.004845025235106639, + 0.004921304265254701, + 0.004820106373346748, + 0.0054804096177630225, + 0.0042302744374162005, + 0.004588429583035324, + 0.004469681656036737, + 0.004349111730454971, + 0.003928502929307311, + 0.004468266803351117, + 0.003904642007649288 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.008362221974278682, + "first_to_last_ratio": 1.8887369414995314, + "imbalance_abs_log_ratio": 0.6359083207487641, + "last_quartile_mean": 0.004427414845626747, + "mean": 0.00600782035481319, + "normalized": [ + 1.6666204390908572, + 1.4822269982304765, + 1.4948753455169401, + 1.2962958920592769, + 1.3162097696294732, + 1.3798257261648947, + 1.2756429133626719, + 1.2234187888150154, + 1.1810688480662828, + 1.0504089840715678, + 1.0461494944748422, + 1.0590232502297414, + 0.9327885465041001, + 0.9920239370590588, + 1.0266149359654666, + 0.9193344840336578, + 0.9095119424808776, + 0.9287724210274617, + 0.875593258455752, + 0.7928595538984271, + 0.8272907449435313, + 0.8064530809788657, + 0.8191497039874, + 0.8023053434820334, + 0.9122126318860999, + 0.7041279844573081, + 0.7637428072161453, + 0.7439772483303089, + 0.7239084182952745, + 0.6538982022256998, + 0.7437417465006834, + 0.6499265585598059 + ], + "population_cv": 0.2648157248292805 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.027978340163826942, + 0.014031016267836094, + 0.0140878576785326, + 0.014151837676763535, + 0.014187770895659924, + 0.014265215955674648, + 0.014337530359625816, + 0.014401593245565891, + 0.014477464370429516, + 0.009722241200506687, + 0.009757626801729202, + 0.009786579757928848, + 0.009825476445257664, + 0.009839320555329323, + 0.00987118761986494, + 0.00996283907443285, + 0.009999874979257584, + 0.007562503684312105, + 0.007595349568873644, + 0.007635751739144325, + 0.007658981718122959, + 0.007660485804080963, + 0.007689184043556452, + 0.007726665120571852, + 0.007755559403449297, + 0.006238248664885759, + 0.00625597033649683, + 0.006290202960371971, + 0.006315375678241253, + 0.006327399518340826, + 0.00635211868211627, + 0.006382675375789404, + 0.006408242508769035, + 0.005390914157032967, + 0.005408597644418478, + 0.0054517327807843685, + 0.005475745070725679, + 0.005509276874363422, + 0.005517482291907072, + 0.0055727302096784115, + 0.005585856735706329, + 0.0048178439028561115, + 0.004830927588045597, + 0.004884926602244377, + 0.004899685736745596, + 0.004938251338899136, + 0.004948681220412254, + 0.004975184798240662, + 0.004989070352166891, + 0.004412420094013214, + 0.004419817589223385, + 0.004455919377505779, + 0.004463871940970421, + 0.004530393984168768, + 0.004542885348200798, + 0.0045819394290447235, + 0.004592607729136944, + 0.004127359017729759, + 0.004137600306421518, + 0.004185030702501535, + 0.004193917382508516, + 0.004221425391733646, + 0.004230021964758635, + 0.004281661007553339 + ], + "loss_nats": 5.571857452392578, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0023513585329055786, + 0.0036361459642648697, + 0.004408807028084993, + 0.005152894649654627, + 0.005783648230135441, + 0.006386642809957266, + 0.006751363631337881, + 0.007284422870725393, + 0.002628475893288851, + 0.0037505170330405235, + 0.004723479505628347, + 0.0054016136564314365, + 0.006024772301316261, + 0.006585344672203064, + 0.007445912808179855, + 0.007922882214188576, + 0.003561305580660701, + 0.0044892593286931515, + 0.005543973762542009, + 0.006135277915745974, + 0.006855612155050039, + 0.007316684816032648, + 0.007876631803810596, + 0.0082859443500638, + 0.0035210708156228065, + 0.0044554853811860085, + 0.0058924350887537, + 0.006464940495789051, + 0.007390674669295549, + 0.007893112488090992, + 0.008790239691734314, + 0.009196000173687935, + 0.004162859171628952, + 0.004961193539202213, + 0.006355749908834696, + 0.006907128728926182, + 0.007784162648022175, + 0.008217990398406982, + 0.00937180407345295, + 0.009676996618509293, + 0.004737259820103645, + 0.005381465889513493, + 0.006999304983764887, + 0.007502692751586437, + 0.008728929795324802, + 0.0091153085231781, + 0.010217384435236454, + 0.010543142445385456, + 0.005580419674515724, + 0.006136673036962748, + 0.008008211851119995, + 0.008469858206808567, + 0.010209310799837112, + 0.010564134456217289, + 0.012040933594107628, + 0.012301022186875343, + 0.005235649179667234, + 0.005773921962827444, + 0.007745372597128153, + 0.008156762458384037, + 0.010180563665926456, + 0.010457510128617287, + 0.011906592175364494, + 0.012218860909342766 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.95402777637355e-05, + 6.603818474104628e-05, + 6.243807729333639e-05, + 5.902352859266102e-05, + 5.3786254284204915e-05, + 5.022022014600225e-05, + 4.6831908548483625e-05, + 4.3414886022219434e-05, + 3.877186099998653e-05, + 3.6122120945947245e-05, + 3.3541862649144605e-05, + 3.1072035199031234e-05, + 2.835266786860302e-05, + 2.6703228286351077e-05, + 2.5144798200926743e-05, + 2.3762264390825294e-05, + 2.2584137695957907e-05, + 2.1571571778622456e-05, + 2.0759547624038532e-05, + 1.9997522031189874e-05, + 1.9395642084418796e-05, + 1.8848633771995082e-05, + 1.8380347682978027e-05, + 1.797073673515115e-05, + 1.755586163199041e-05, + 1.7233234757441096e-05, + 1.6949628843576647e-05, + 1.6709716874174774e-05, + 1.630509359529242e-05, + 1.611644620425068e-05, + 1.594743662280962e-05, + 1.580910975462757e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.641166717396118e-05, + "first_to_last_ratio": 3.40273807382454, + "imbalance_abs_log_ratio": 1.2245804233600555, + "last_quartile_mean": 1.6578316035520402e-05, + "mean": 3.084058886315688e-05, + "normalized": [ + 2.254829765809383, + 2.1412750915381364, + 2.0245423189025695, + 1.913826251974467, + 1.7440086673720954, + 1.6283807150646488, + 1.518515381022133, + 1.407719100788124, + 1.2571699318719751, + 1.1712526341901288, + 1.0875882687575642, + 1.0075046017085245, + 0.9193296533476374, + 0.8658469008110146, + 0.8153151132261776, + 0.770486727612793, + 0.7322862023214354, + 0.6994539525278819, + 0.6731242297658762, + 0.648415700488765, + 0.6288998621420432, + 0.6111632256967778, + 0.5959791417905044, + 0.5826975877435189, + 0.5692453444999938, + 0.5587842318415797, + 0.5495883661230995, + 0.5418092679201958, + 0.5286894380532074, + 0.5225725836741036, + 0.517092481391006, + 0.5126072600226392 + ], + "population_cv": 0.5330470009230729 + }, + "activation_output_rms_by_block": [ + 0.005864492617547512, + 0.009892112575471401, + 0.014509912580251694, + 0.019168438389897346, + 0.006793031003326178, + 0.01223041582852602, + 0.017751524224877357, + 0.023574935272336006, + 0.008267601951956749, + 0.014755276031792164, + 0.02151178941130638, + 0.02830074541270733, + 0.009282662533223629, + 0.01676790602505207, + 0.02516039088368416, + 0.0331411249935627, + 0.010437680408358574, + 0.01991143450140953, + 0.029664477333426476, + 0.04036801680922508, + 0.01357664167881012, + 0.025029508396983147, + 0.03699437528848648, + 0.05095698684453964, + 0.015438608825206757, + 0.029965175315737724, + 0.04443887993693352, + 0.060807984322309494, + 0.019359532743692398, + 0.037317242473363876, + 0.05564062297344208, + 0.07935244590044022 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.01372310781152919, + "first_to_last_ratio": 0.3207078305283738, + "imbalance_abs_log_ratio": 1.137224755592113, + "last_quartile_mean": 0.04279006156139076, + "mean": 0.02613224917149637, + "normalized": [ + 0.224415915333617, + 0.37854041994445614, + 0.5552492816453899, + 0.7335165933901026, + 0.2599481949963819, + 0.4680200218611986, + 0.6792956897196492, + 0.9021395409795135, + 0.31637544467372514, + 0.5646385787522038, + 0.8231893577216564, + 1.0829816150527234, + 0.35521865998999624, + 0.6416556766702496, + 0.9628100022529916, + 1.268207905720998, + 0.3994176061868959, + 0.7619487465750876, + 1.1351673994362057, + 1.5447586062839287, + 0.519535903309034, + 0.9578015360531602, + 1.4156598249812316, + 1.9499655965365867, + 0.5907876020884704, + 1.1466741771474498, + 1.700537892674199, + 2.3269326694097012, + 0.7408291807047638, + 1.4280149492094805, + 2.1291938021979306, + 3.0365716085010215 + ], + "population_cv": 0.6534988385002395 + }, + "bits_per_byte": 5.118322357706281, + "branch_output_rms_by_sublayer": [ + 0.003087003482505679, + 0.0049620093777775764, + 0.0026560353580862284, + 0.004914142191410065, + 0.0032712947577238083, + 0.00555405905470252, + 0.002734958194196224, + 0.005823355168104172, + 0.003094236133620143, + 0.006141432095319033, + 0.0030397549271583557, + 0.006136075593531132, + 0.0035086015705019236, + 0.006357426289469004, + 0.0038121570833027363, + 0.0064498125575482845, + 0.0047028325498104095, + 0.006525229662656784, + 0.004038033075630665, + 0.007023691199719906, + 0.004860103130340576, + 0.006682637147605419, + 0.005031581036746502, + 0.006716069765388966, + 0.005196469835937023, + 0.006977807730436325, + 0.005135253071784973, + 0.006669311318546534, + 0.006614624988287687, + 0.006923797074705362, + 0.005866772960871458, + 0.007155857048928738, + 0.006203304044902325, + 0.007240671198815107, + 0.007167549338191748, + 0.007543859537690878, + 0.007609309628605843, + 0.007281317841261625, + 0.008147990331053734, + 0.007711030542850494, + 0.00901782512664795, + 0.00788753479719162, + 0.008936969563364983, + 0.007908415980637074, + 0.009206709451973438, + 0.008078642189502716, + 0.01152731105685234, + 0.008181036449968815, + 0.010577243752777576, + 0.007894126698374748, + 0.011347576975822449, + 0.008452145382761955, + 0.011137762106955051, + 0.008351319469511509, + 0.012936098501086235, + 0.008498509414494038, + 0.013407763093709946, + 0.009082214906811714, + 0.013788831420242786, + 0.009139914996922016, + 0.015265416353940964, + 0.008129997178912163, + 0.0202538650482893, + 0.00888112559914589 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0002932670153511829, + 0.0002289733734981176, + 0.00023382191596673618, + 0.00020236837455671756, + 0.0002080571281121399, + 0.00019123637931049836, + 0.00021124835961838567, + 0.00020444780259018514, + 0.00020817260281626697, + 0.00017921191649714578, + 0.0001893830879389061, + 0.0001891854298096901, + 0.00018418310100442068, + 0.00017033524110907422, + 0.00020556753369166975, + 0.00017823357128584385, + 0.0001749817385153215, + 0.00018220334134876856, + 0.00017289789528954973, + 0.00017141475273833558, + 0.00017603112434287998, + 0.00016603952795395127, + 0.0001609386975915838, + 0.00016666055126527177, + 0.00016172141650657888, + 0.00015460675258205218, + 0.00014513294616478987, + 0.00015851211030580876, + 0.00015100080713109572, + 0.00014721653732759288, + 0.00014725738129728224, + 0.00017222312685716471 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002216775436254954, + "first_to_last_ratio": 1.4328688617518024, + "imbalance_abs_log_ratio": 0.35967863157527663, + "last_quartile_mean": 0.00015470888477154567, + "mean": 0.000183954110636719, + "normalized": [ + 1.5942400761586677, + 1.2447309424376207, + 1.2710882902122171, + 1.1001024867357483, + 1.1310273382420937, + 1.0395874201917712, + 1.1483753142954691, + 1.1114065452657267, + 1.1316550747124958, + 0.974220776458003, + 1.02951267184732, + 1.0284381749060347, + 1.0012448233252798, + 0.9259659407419281, + 1.1174935584757544, + 0.9689023565112261, + 0.9512249435995668, + 0.9904825758886794, + 0.9398968834732723, + 0.9318343153355962, + 0.956929550166532, + 0.9026138496130359, + 0.8748850299377811, + 0.9059898182672343, + 0.879139998268121, + 0.8404637006855296, + 0.7889627780669988, + 0.8616937656742324, + 0.8208612822428254, + 0.8002894679441159, + 0.8005115014151157, + 0.936228748904007 + ], + "population_cv": 0.1635293006829388 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.692298173904419, + "mean_weights": [ + 0.48010045289993286, + 0.5198994874954224 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927915811538696, + "mean_weights": [ + 0.5130036473274231, + 0.4869963526725769 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928644776344299, + "mean_weights": [ + 0.5081723928451538, + 0.4918276071548462 + ], + "sources": 2 + }, + { + "entropy_mean": 0.692646861076355, + "mean_weights": [ + 0.5149973034858704, + 0.48500269651412964 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6925361156463623, + "mean_weights": [ + 0.5161343812942505, + 0.4838656485080719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928029656410217, + "mean_weights": [ + 0.5107344388961792, + 0.4892655611038208 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924350261688232, + "mean_weights": [ + 0.5171386003494263, + 0.48286139965057373 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929639577865601, + "mean_weights": [ + 0.5072796940803528, + 0.49272027611732483 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0978727340698242, + "mean_weights": [ + 0.34929555654525757, + 0.32294148206710815, + 0.3277629613876343 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098320722579956, + "mean_weights": [ + 0.3432672619819641, + 0.3268624544143677, + 0.329870343208313 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098150610923767, + "mean_weights": [ + 0.3442404866218567, + 0.33018797636032104, + 0.32557153701782227 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984500646591187, + "mean_weights": [ + 0.34007447957992554, + 0.32975536584854126, + 0.3301700949668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982636213302612, + "mean_weights": [ + 0.3434537351131439, + 0.33091628551483154, + 0.3256300091743469 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984963178634644, + "mean_weights": [ + 0.33822712302207947, + 0.33379626274108887, + 0.32797661423683167 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0980933904647827, + "mean_weights": [ + 0.34460341930389404, + 0.33179372549057007, + 0.3236028254032135 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985530614852905, + "mean_weights": [ + 0.3358767628669739, + 0.3345070779323578, + 0.32961612939834595 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860312700271606, + "mean_weights": [ + 0.25749218463897705, + 0.24992284178733826, + 0.24571210145950317, + 0.24687285721302032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862019777297974, + "mean_weights": [ + 0.25314557552337646, + 0.25236955285072327, + 0.24678170680999756, + 0.2477031946182251 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385892391204834, + "mean_weights": [ + 0.25815317034721375, + 0.25234687328338623, + 0.2448991984128952, + 0.24460072815418243 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861613273620605, + "mean_weights": [ + 0.25378289818763733, + 0.25339171290397644, + 0.24767157435417175, + 0.24515381455421448 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3855845928192139, + "mean_weights": [ + 0.2592235505580902, + 0.2543467879295349, + 0.2462996244430542, + 0.24013003706932068 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861030340194702, + "mean_weights": [ + 0.25138330459594727, + 0.25586679577827454, + 0.24899962544441223, + 0.24375030398368835 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3856689929962158, + "mean_weights": [ + 0.2574368119239807, + 0.25204822421073914, + 0.2460038810968399, + 0.24451106786727905 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386252999305725, + "mean_weights": [ + 0.25002047419548035, + 0.25257009267807007, + 0.2502796947956085, + 0.24712975323200226 + ], + "sources": 4 + }, + { + "entropy_mean": 1.608945608139038, + "mean_weights": [ + 0.20495271682739258, + 0.2066430151462555, + 0.2000001072883606, + 0.19580814242362976, + 0.19259600341320038 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093038320541382, + "mean_weights": [ + 0.20154938101768494, + 0.20420850813388824, + 0.2010108083486557, + 0.19671520590782166, + 0.19651609659194946 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085759401321411, + "mean_weights": [ + 0.20549985766410828, + 0.209174245595932, + 0.20157773792743683, + 0.19417354464530945, + 0.18957462906837463 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093413829803467, + "mean_weights": [ + 0.20104944705963135, + 0.20404505729675293, + 0.2002137303352356, + 0.19850106537342072, + 0.1961907148361206 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6080443859100342, + "mean_weights": [ + 0.20538949966430664, + 0.21283748745918274, + 0.20341980457305908, + 0.19302619993686676, + 0.18532699346542358 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093816757202148, + "mean_weights": [ + 0.2010989934206009, + 0.20253092050552368, + 0.19950580596923828, + 0.19837552309036255, + 0.1984887421131134 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6076874732971191, + "mean_weights": [ + 0.20581099390983582, + 0.21552260220050812, + 0.20295079052448273, + 0.19259145855903625, + 0.18312416970729828 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609355092048645, + "mean_weights": [ + 0.20059309899806976, + 0.203348308801651, + 0.2011984884738922, + 0.1984248012304306, + 0.19643528759479523 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7901467084884644, + "mean_weights": [ + 0.17125868797302246, + 0.18124164640903473, + 0.17160004377365112, + 0.1620369851589203, + 0.15427924692630768, + 0.15958338975906372 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7916979789733887, + "mean_weights": [ + 0.16689208149909973, + 0.16927623748779297, + 0.1683787703514099, + 0.16569086909294128, + 0.1651553511619568, + 0.1646067053079605 + ], + "sources": 6 + }, + { + "entropy_mean": 1.789665699005127, + "mean_weights": [ + 0.1720278263092041, + 0.18269017338752747, + 0.17356479167938232, + 0.16241812705993652, + 0.15660038590431213, + 0.15269872546195984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917176485061646, + "mean_weights": [ + 0.16644492745399475, + 0.16854418814182281, + 0.16665583848953247, + 0.16677430272102356, + 0.16461026668548584, + 0.16697049140930176 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7893871068954468, + "mean_weights": [ + 0.17303556203842163, + 0.18359911441802979, + 0.17276406288146973, + 0.16385790705680847, + 0.1554735004901886, + 0.15126986801624298 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917413711547852, + "mean_weights": [ + 0.16618916392326355, + 0.16740113496780396, + 0.16675519943237305, + 0.1668717861175537, + 0.1653555929660797, + 0.16742712259292603 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7882882356643677, + "mean_weights": [ + 0.17288538813591003, + 0.18687990307807922, + 0.17607632279396057, + 0.162863090634346, + 0.15389539301395416, + 0.14739990234375 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917335033416748, + "mean_weights": [ + 0.16660982370376587, + 0.16799160838127136, + 0.16742658615112305, + 0.1664734035730362, + 0.16542598605155945, + 0.1660725623369217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.941649079322815, + "mean_weights": [ + 0.15037035942077637, + 0.16432975232601166, + 0.15334898233413696, + 0.1418340504169464, + 0.13212506473064423, + 0.12697139382362366, + 0.13102038204669952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458887577056885, + "mean_weights": [ + 0.1431310474872589, + 0.14324922859668732, + 0.14309079945087433, + 0.142366424202919, + 0.14222830533981323, + 0.14392223954200745, + 0.14201194047927856 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9409761428833008, + "mean_weights": [ + 0.1505419909954071, + 0.16567251086235046, + 0.1550290882587433, + 0.14243587851524353, + 0.13249853253364563, + 0.12776729464530945, + 0.12605470418930054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458999633789062, + "mean_weights": [ + 0.14249610900878906, + 0.14256571233272552, + 0.1435922086238861, + 0.14210085570812225, + 0.14355877041816711, + 0.1427185833454132, + 0.14296773076057434 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941575527191162, + "mean_weights": [ + 0.15046924352645874, + 0.1640411913394928, + 0.15395450592041016, + 0.14198558032512665, + 0.13400861620903015, + 0.1288735270500183, + 0.126667320728302 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458911418914795, + "mean_weights": [ + 0.14260169863700867, + 0.14355596899986267, + 0.1428615152835846, + 0.14244115352630615, + 0.14214617013931274, + 0.14205437898635864, + 0.14433911442756653 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9400205612182617, + "mean_weights": [ + 0.1507154405117035, + 0.16689133644104004, + 0.15648439526557922, + 0.1427520215511322, + 0.13353285193443298, + 0.12712189555168152, + 0.12250205874443054 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458441734313965, + "mean_weights": [ + 0.1422245353460312, + 0.14142528176307678, + 0.14198407530784607, + 0.14165860414505005, + 0.14288201928138733, + 0.14370128512382507, + 0.14612418413162231 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0741422176361084, + "mean_weights": [ + 0.13300861418247223, + 0.14654949307441711, + 0.1379268318414688, + 0.12711459398269653, + 0.1185426190495491, + 0.1134917140007019, + 0.10956315696239471, + 0.11380299925804138 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794055461883545, + "mean_weights": [ + 0.12428364902734756, + 0.1234029084444046, + 0.12436957657337189, + 0.12428068369626999, + 0.12555459141731262, + 0.12575605511665344, + 0.1263071894645691, + 0.1260453760623932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.071772813796997, + "mean_weights": [ + 0.134201318025589, + 0.1495765745639801, + 0.14145472645759583, + 0.12840476632118225, + 0.11872729659080505, + 0.11272042989730835, + 0.10793697834014893, + 0.1069779098033905 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079404592514038, + "mean_weights": [ + 0.12455718219280243, + 0.12374450266361237, + 0.12391386926174164, + 0.12458803504705429, + 0.1251734495162964, + 0.1251622885465622, + 0.1266230344772339, + 0.1262376308441162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0732176303863525, + "mean_weights": [ + 0.13281872868537903, + 0.1473538726568222, + 0.13968363404273987, + 0.12848341464996338, + 0.12016644328832626, + 0.11505335569381714, + 0.10907343029975891, + 0.1073671206831932 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0794081687927246, + "mean_weights": [ + 0.12450858950614929, + 0.12424902617931366, + 0.12409761548042297, + 0.12468484044075012, + 0.12504076957702637, + 0.1255234032869339, + 0.12684062123298645, + 0.12505511939525604 + ], + "sources": 8 + }, + { + "entropy_mean": 2.072478771209717, + "mean_weights": [ + 0.13302236795425415, + 0.14877822995185852, + 0.14031866192817688, + 0.12882456183433533, + 0.11896270513534546, + 0.11445862054824829, + 0.11000129580497742, + 0.10563355684280396 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079291820526123, + "mean_weights": [ + 0.12426379323005676, + 0.12249694019556046, + 0.12262053787708282, + 0.12315947562456131, + 0.1255876123905182, + 0.126260906457901, + 0.1273583620786667, + 0.12825237214565277 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1905503273010254, + "mean_weights": [ + 0.11892962455749512, + 0.13242119550704956, + 0.12570281326770782, + 0.11684690415859222, + 0.10851435363292694, + 0.10355254262685776, + 0.09824737906455994, + 0.0950399786233902, + 0.10074522346258163 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971964836120605, + "mean_weights": [ + 0.11044953763484955, + 0.11010265350341797, + 0.1108412891626358, + 0.11044170707464218, + 0.11131919175386429, + 0.11133328080177307, + 0.11097346246242523, + 0.11225177347660065, + 0.11228710412979126 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1902995109558105, + "mean_weights": [ + 0.11926670372486115, + 0.13237422704696655, + 0.12644407153129578, + 0.11611802130937576, + 0.11051423847675323, + 0.10466914623975754, + 0.09992008656263351, + 0.09535142779350281, + 0.09534208476543427 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197113037109375, + "mean_weights": [ + 0.11037027835845947, + 0.10917964577674866, + 0.10961420089006424, + 0.10947252064943314, + 0.11109665036201477, + 0.11174711585044861, + 0.1118166521191597, + 0.11308293044567108, + 0.11361998319625854 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1894307136535645, + "mean_weights": [ + 0.1198490560054779, + 0.13425537943840027, + 0.12682178616523743, + 0.11624304950237274, + 0.10967148095369339, + 0.10377678275108337, + 0.09994889795780182, + 0.0946216955780983, + 0.09481187164783478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.196977138519287, + "mean_weights": [ + 0.10978136211633682, + 0.1075383648276329, + 0.10839901119470596, + 0.11003260314464569, + 0.11053797602653503, + 0.11152469366788864, + 0.11391061544418335, + 0.11424341052770615, + 0.11403194814920425 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1905651092529297, + "mean_weights": [ + 0.11943811178207397, + 0.13202990591526031, + 0.12506362795829773, + 0.11625386774539948, + 0.11043960601091385, + 0.1047937273979187, + 0.1010652631521225, + 0.09624719619750977, + 0.0946686640381813 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.02856997773051262, + 0.013703026808798313, + 0.015031684190034866, + 0.014947476796805859, + 0.015670204535126686, + 0.015775443986058235, + 0.016530683264136314, + 0.016744175925850868, + 0.01773575134575367, + 0.012103146873414516, + 0.01292925700545311, + 0.013081531040370464, + 0.014021075330674648, + 0.01416850183159113, + 0.015247474424540997, + 0.015494133345782757, + 0.01669454388320446, + 0.012773395515978336, + 0.01373021025210619, + 0.013903867453336716, + 0.01499613095074892, + 0.015176740474998951, + 0.01623532734811306, + 0.016410861164331436, + 0.01758282259106636, + 0.014334342442452908, + 0.015234442427754402, + 0.015468464232981205, + 0.01637510396540165, + 0.01670125499367714, + 0.01772587187588215, + 0.017849057912826538, + 0.01901337131857872, + 0.01606663130223751, + 0.017032917588949203, + 0.0173187293112278, + 0.01834850199520588, + 0.018571525812149048, + 0.019697511568665504, + 0.019770435988903046, + 0.021141810342669487, + 0.018272221088409424, + 0.019469235092401505, + 0.01950262486934662, + 0.02084389142692089, + 0.02077464945614338, + 0.022254284471273422, + 0.0220524650067091, + 0.02407064661383629, + 0.020764723420143127, + 0.022380243986845016, + 0.02203267626464367, + 0.023947052657604218, + 0.023422541096806526, + 0.025450291112065315, + 0.024811791256070137, + 0.027382872998714447, + 0.023658527061343193, + 0.025761734694242477, + 0.025185640901327133, + 0.02756396308541298, + 0.026624467223882675, + 0.02946169301867485, + 0.02854793891310692 + ], + "loss_nats": 3.54775071144104, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1935477256774902, + "mean_weights": [ + 0.10072024166584015, + 0.09480113536119461, + 0.10205776989459991, + 0.11059839278459549, + 0.1157325804233551, + 0.11904929578304291, + 0.11986880004405975, + 0.11937490850687027, + 0.11779685318470001 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.003087003482505679, + 0.005864492617547512, + 0.006683253683149815, + 0.009892112575471401, + 0.010629965923726559, + 0.014509912580251694, + 0.01484723947942257, + 0.019168438389897346, + 0.003094236133620143, + 0.006793031003326178, + 0.007696541491895914, + 0.01223041582852602, + 0.012912816368043423, + 0.017751524224877357, + 0.018475037068128586, + 0.023574935272336006, + 0.0047028325498104095, + 0.008267601951956749, + 0.0095678074285388, + 0.014755276031792164, + 0.01631263643503189, + 0.02151178941130638, + 0.023062339052557945, + 0.02830074541270733, + 0.005196469835937023, + 0.009282662533223629, + 0.011978921480476856, + 0.01676790602505207, + 0.01994149386882782, + 0.02516039088368416, + 0.027569252997636795, + 0.0331411249935627, + 0.006203304044902325, + 0.010437680408358574, + 0.014378104358911514, + 0.01991143450140953, + 0.024205319583415985, + 0.029664477333426476, + 0.03444278985261917, + 0.04036801680922508, + 0.00901782512664795, + 0.01357664167881012, + 0.01929495297372341, + 0.025029508396983147, + 0.030904190614819527, + 0.03699437528848648, + 0.04446368291974068, + 0.05095698684453964, + 0.010577243752777576, + 0.015438608825206757, + 0.023803669959306717, + 0.029965175315737724, + 0.03807948902249336, + 0.04443887993693352, + 0.05412018671631813, + 0.060807984322309494, + 0.013407763093709946, + 0.019359532743692398, + 0.030261604115366936, + 0.037317242473363876, + 0.04958754777908325, + 0.05564062297344208, + 0.07224015146493912, + 0.07935244590044022 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.00014133188233245164, + 0.0001488620473537594, + 0.00014702342741657048, + 0.0001400294277118519, + 0.00013315585965756327, + 0.00011939816613448784, + 0.0001078194472938776, + 9.923135075950995e-05, + 8.953637006925419e-05, + 8.09328121249564e-05, + 7.468360854545608e-05, + 6.987994856899604e-05, + 7.115858170436695e-05, + 6.467441562563181e-05, + 5.974166560918093e-05, + 5.531677743420005e-05, + 5.705624425900169e-05, + 5.295562368701212e-05, + 4.8012458137236536e-05, + 4.413425631355494e-05, + 5.2346804295666516e-05, + 4.681985228671692e-05, + 4.172014814685099e-05, + 3.8159589166752994e-05, + 3.1379553547594696e-05, + 3.02761654893402e-05, + 2.9319184250198305e-05, + 2.8667578590102494e-05, + 2.2355405235430226e-05, + 2.2064739823690616e-05, + 2.201247843913734e-05, + 2.2052488930057734e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.000129606451082509, + "first_to_last_ratio": 4.9818074922726145, + "imbalance_abs_log_ratio": 1.6057927754415957, + "last_quartile_mean": 2.601594928819395e-05, + "mean": 6.850338621688934e-05, + "normalized": [ + 2.0631371693799077, + 2.173061151786652, + 2.146221311616304, + 2.04412417319785, + 1.9437850741564366, + 1.7429527608527258, + 1.5739287245233284, + 1.448561249882341, + 1.3070356812110293, + 1.181442503713818, + 1.0902177639657034, + 1.0200948073975256, + 1.038760061861336, + 0.9441053821903842, + 0.8720979926456799, + 0.8075042780959905, + 0.832896699126015, + 0.7730365841967101, + 0.7008771505867426, + 0.6442638641806842, + 0.7641491492103867, + 0.6834677068149604, + 0.6090231512754765, + 0.5570467574542299, + 0.45807302792658366, + 0.4419659692950432, + 0.4279961308389998, + 0.4184841096663127, + 0.3263401485679923, + 0.32209706763737483, + 0.3213341654300619, + 0.32191823131541414 + ], + "population_cv": 0.5858213540607353 + }, + "activation_output_rms_by_block": [ + 0.021054552868008614, + 0.038058556616306305, + 0.04883526638150215, + 0.05791458487510681, + 0.016990283504128456, + 0.027475843206048012, + 0.03445403650403023, + 0.040619369596242905, + 0.019055280834436417, + 0.029903952032327652, + 0.037492990493774414, + 0.04475866258144379, + 0.0285621490329504, + 0.03507128730416298, + 0.04228900745511055, + 0.05219251662492752, + 0.027010982856154442, + 0.04389137029647827, + 0.054182786494493484, + 0.06662839651107788, + 0.03837457299232483, + 0.041143495589494705, + 0.05716610327363014, + 0.07329611480236053, + 0.028997479006648064, + 0.053451184183359146, + 0.08026835322380066, + 0.10854664444923401, + 0.027061885222792625, + 0.05247335880994797, + 0.08323372155427933, + 0.11460508406162262 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.035675311693921685, + "first_to_last_ratio": 0.5202021080271609, + "imbalance_abs_log_ratio": 0.6535378736360024, + "last_quartile_mean": 0.06857971381396055, + "mean": 0.047658121038693935, + "normalized": [ + 0.4417831087153916, + 0.798574425236075, + 1.0246997849926245, + 1.2152091524566315, + 0.3565034276180115, + 0.5765196488493577, + 0.7229415628043072, + 0.8523074076559539, + 0.3998328179779751, + 0.6274681288431082, + 0.7867072741565622, + 0.9391612931005807, + 0.5993133680146686, + 0.7358932022453922, + 0.8873410561187638, + 1.095144237485823, + 0.5667655851187262, + 0.92096308750491, + 1.136905637771622, + 1.398049168933493, + 0.8052053281993276, + 0.8633050295056751, + 1.1995039256209161, + 1.5379564532737442, + 0.6084478022770731, + 1.121554585418124, + 1.6842534173479116, + 2.277610658655726, + 0.567833658419326, + 1.1010370880409766, + 1.746475096798326, + 2.404733580842896 + ], + "population_cv": 0.48875915937608194 + }, + "bits_per_byte": 3.059021014366605, + "branch_output_rms_by_sublayer": [ + 0.01139580737799406, + 0.01652366667985916, + 0.014748631045222282, + 0.013563322834670544, + 0.012339585460722446, + 0.012220539152622223, + 0.0120800556614995, + 0.0097816726192832, + 0.01439172774553299, + 0.008202088065445423, + 0.0171960536390543, + 0.008006753399968147, + 0.01820177212357521, + 0.008192610926926136, + 0.01519495714455843, + 0.008106541819870472, + 0.016362585127353668, + 0.008609342388808727, + 0.015944689512252808, + 0.008666405454277992, + 0.017555685713887215, + 0.009109876118600368, + 0.01772458106279373, + 0.008975221775472164, + 0.025544000789523125, + 0.009615846909582615, + 0.019797103479504585, + 0.009856111370027065, + 0.02002105489373207, + 0.009463186375796795, + 0.017730148509144783, + 0.009926749393343925, + 0.022795414552092552, + 0.010939708910882473, + 0.02134518511593342, + 0.010758070275187492, + 0.021650495007634163, + 0.011536303907632828, + 0.03255617991089821, + 0.012654256075620651, + 0.033973678946495056, + 0.014695252291858196, + 0.031387828290462494, + 0.016267452389001846, + 0.019825322553515434, + 0.01879013702273369, + 0.013387269340455532, + 0.02227928675711155, + 0.015395586378872395, + 0.021389417350292206, + 0.011897368356585503, + 0.023454682901501656, + 0.014087310060858727, + 0.024781694635748863, + 0.014466723427176476, + 0.026336129754781723, + 0.006911881268024445, + 0.026170482859015465, + 0.005224450025707483, + 0.026235800236463547, + 0.00758614856749773, + 0.030007388442754745, + 0.005234629847109318, + 0.031163375824689865 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00048079443816738296, + 0.0004352800282264352, + 0.00039416580024316875, + 0.0003828745093102654, + 0.0003643213588911513, + 0.0002989310042525185, + 0.0002673721517191021, + 0.00023887606182770654, + 0.0002716851528017292, + 0.00026407625379019375, + 0.00023115090959535868, + 0.00019271062698547964, + 0.00023181255047936123, + 0.00019176941572124543, + 0.00020689480536159778, + 0.0001935784151883457, + 0.00017612030874433946, + 0.00017372524780256995, + 0.0001906158264031438, + 0.00020699034417808569, + 0.0002469606938485226, + 0.00020806313134476826, + 0.00017307229934302834, + 0.00014129096626273445, + 0.00011634997010006574, + 0.00010725656087507846, + 0.00011122939555071074, + 0.00011004876520390145, + 7.494181919924042e-05, + 7.493373834181984e-05, + 7.712439386045853e-05, + 7.424519704760952e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00035782691907971636, + "first_to_last_ratio": 3.8366182378544447, + "imbalance_abs_log_ratio": 1.3445913113590247, + "last_quartile_mean": 9.326623002236059e-05, + "mean": 0.0002159144418958475, + "normalized": [ + 2.226782210331757, + 2.015983851772199, + 1.8255647782620577, + 1.7732695689478515, + 1.6873413176636518, + 1.384488233523149, + 1.2383245389767708, + 1.1063459198478962, + 1.258300048927641, + 1.2230597058328248, + 1.070567154127008, + 0.892532362788586, + 1.0736315201703026, + 0.8881731765481019, + 0.9582258766247762, + 0.896551492751577, + 0.8156948983954307, + 0.8046022594744787, + 0.8828303689620391, + 0.9586683612295531, + 1.1437896033265396, + 0.9636369365469933, + 0.801578152083568, + 0.6543840468572741, + 0.5388707169305075, + 0.49675491798189547, + 0.5151549594091785, + 0.5096869122677149, + 0.34709034996089216, + 0.3470529237593392, + 0.3571988662882573, + 0.34386396943018666 + ], + "population_cv": 0.48652454891086055 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6572662591934204, + "mean_weights": [ + 0.3911391496658325, + 0.6088608503341675 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6751130819320679, + "mean_weights": [ + 0.44147175550460815, + 0.5585282444953918 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6885842084884644, + "mean_weights": [ + 0.4896422326564789, + 0.5103577375411987 + ], + "sources": 2 + }, + { + "entropy_mean": 0.668085515499115, + "mean_weights": [ + 0.4332469701766968, + 0.5667530298233032 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6886032223701477, + "mean_weights": [ + 0.51185142993927, + 0.4881485104560852 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6789880990982056, + "mean_weights": [ + 0.4699174761772156, + 0.5300825834274292 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772009134292603, + "mean_weights": [ + 0.4395230710506439, + 0.5604768991470337 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6781555414199829, + "mean_weights": [ + 0.4541339576244354, + 0.5458660125732422 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0917184352874756, + "mean_weights": [ + 0.30861830711364746, + 0.355538010597229, + 0.33584368228912354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0860013961791992, + "mean_weights": [ + 0.30104559659957886, + 0.3248867988586426, + 0.37406760454177856 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0827364921569824, + "mean_weights": [ + 0.2735002934932709, + 0.34776103496551514, + 0.3787387013435364 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0724949836730957, + "mean_weights": [ + 0.2886848449707031, + 0.3024986684322357, + 0.40881651639938354 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0897927284240723, + "mean_weights": [ + 0.2973601222038269, + 0.3298739790916443, + 0.3727658987045288 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0697213411331177, + "mean_weights": [ + 0.30285951495170593, + 0.28946101665496826, + 0.4076794683933258 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0826778411865234, + "mean_weights": [ + 0.2862996757030487, + 0.3235614597797394, + 0.3901388645172119 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0579445362091064, + "mean_weights": [ + 0.28999701142311096, + 0.2830814719200134, + 0.4269214868545532 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3809051513671875, + "mean_weights": [ + 0.23795948922634125, + 0.2348281741142273, + 0.26517200469970703, + 0.26204031705856323 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3605632781982422, + "mean_weights": [ + 0.24186556041240692, + 0.22099661827087402, + 0.2607640027999878, + 0.27637380361557007 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3829561471939087, + "mean_weights": [ + 0.23731295764446259, + 0.2382979840040207, + 0.2565648853778839, + 0.2678241431713104 + ], + "sources": 4 + }, + { + "entropy_mean": 1.35977041721344, + "mean_weights": [ + 0.23858636617660522, + 0.21978360414505005, + 0.2459542453289032, + 0.29567578434944153 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3790044784545898, + "mean_weights": [ + 0.24452988803386688, + 0.2358427345752716, + 0.24750955402851105, + 0.27211782336235046 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3556556701660156, + "mean_weights": [ + 0.2410905957221985, + 0.22088873386383057, + 0.23317404091358185, + 0.3048466145992279 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3804312944412231, + "mean_weights": [ + 0.26568183302879333, + 0.22685569524765015, + 0.2384241223335266, + 0.2690383493900299 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533532619476318, + "mean_weights": [ + 0.24537262320518494, + 0.2171444594860077, + 0.23335666954517365, + 0.30412623286247253 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6053518056869507, + "mean_weights": [ + 0.20177139341831207, + 0.18913303315639496, + 0.18395328521728516, + 0.1997503787279129, + 0.22539189457893372 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585533618927002, + "mean_weights": [ + 0.19244730472564697, + 0.1780712604522705, + 0.17078758776187897, + 0.20516318082809448, + 0.25353068113327026 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6022908687591553, + "mean_weights": [ + 0.1985350400209427, + 0.1897844821214676, + 0.17792725563049316, + 0.1998080015182495, + 0.23394520580768585 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5833771228790283, + "mean_weights": [ + 0.1926397681236267, + 0.1801614761352539, + 0.17439623177051544, + 0.19375719130039215, + 0.2590453326702118 + ], + "sources": 5 + }, + { + "entropy_mean": 1.601109504699707, + "mean_weights": [ + 0.19747966527938843, + 0.19028550386428833, + 0.18151578307151794, + 0.19478440284729004, + 0.23593464493751526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5755603313446045, + "mean_weights": [ + 0.1969345360994339, + 0.17712506651878357, + 0.16844677925109863, + 0.18347150087356567, + 0.27402210235595703 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5980476140975952, + "mean_weights": [ + 0.19043894112110138, + 0.1923028826713562, + 0.1810823678970337, + 0.19125500321388245, + 0.24492081999778748 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5842485427856445, + "mean_weights": [ + 0.1964084804058075, + 0.17743000388145447, + 0.17621047794818878, + 0.18783687055110931, + 0.26211416721343994 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7856926918029785, + "mean_weights": [ + 0.15997162461280823, + 0.16431675851345062, + 0.15234553813934326, + 0.15752777457237244, + 0.18046650290489197, + 0.18537181615829468 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7751184701919556, + "mean_weights": [ + 0.16064736247062683, + 0.15155571699142456, + 0.14907033741474152, + 0.14832919836044312, + 0.18695974349975586, + 0.2034376710653305 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7829411029815674, + "mean_weights": [ + 0.14932186901569366, + 0.16724151372909546, + 0.15592646598815918, + 0.1555134654045105, + 0.17433607578277588, + 0.19766061007976532 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7661566734313965, + "mean_weights": [ + 0.165928453207016, + 0.143815815448761, + 0.1375015527009964, + 0.1444939523935318, + 0.17902101576328278, + 0.22923919558525085 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7807259559631348, + "mean_weights": [ + 0.14958028495311737, + 0.16952179372310638, + 0.15695258975028992, + 0.1499582827091217, + 0.1711740493774414, + 0.20281299948692322 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781292200088501, + "mean_weights": [ + 0.1682473123073578, + 0.15929323434829712, + 0.14874719083309174, + 0.15058940649032593, + 0.1693388819694519, + 0.20378397405147552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7818934917449951, + "mean_weights": [ + 0.14812925457954407, + 0.18386004865169525, + 0.16644306480884552, + 0.15062814950942993, + 0.15809310972690582, + 0.19284634292125702 + ], + "sources": 6 + }, + { + "entropy_mean": 1.77958083152771, + "mean_weights": [ + 0.19125157594680786, + 0.15093471109867096, + 0.14643779397010803, + 0.1490234136581421, + 0.16375312209129333, + 0.19859938323497772 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9289655685424805, + "mean_weights": [ + 0.12320417165756226, + 0.15015369653701782, + 0.13329030573368073, + 0.12274879962205887, + 0.13076716661453247, + 0.1441647708415985, + 0.19567111134529114 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9340521097183228, + "mean_weights": [ + 0.14550700783729553, + 0.14042077958583832, + 0.13418063521385193, + 0.1260203719139099, + 0.12906627357006073, + 0.1355920135974884, + 0.18921294808387756 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9230798482894897, + "mean_weights": [ + 0.12067099660634995, + 0.15888366103172302, + 0.14002522826194763, + 0.12021138519048691, + 0.12209220230579376, + 0.13571104407310486, + 0.20240548253059387 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9308109283447266, + "mean_weights": [ + 0.1450628787279129, + 0.14527732133865356, + 0.13644321262836456, + 0.12497064471244812, + 0.12634633481502533, + 0.1283716857433319, + 0.19352790713310242 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9273775815963745, + "mean_weights": [ + 0.1260491907596588, + 0.16657930612564087, + 0.14038512110710144, + 0.12195964902639389, + 0.12248437106609344, + 0.13317090272903442, + 0.18937145173549652 + ], + "sources": 7 + }, + { + "entropy_mean": 1.918284296989441, + "mean_weights": [ + 0.13994047045707703, + 0.15779416263103485, + 0.13759920001029968, + 0.11949826776981354, + 0.11786683648824692, + 0.12190048396587372, + 0.20540057122707367 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9060933589935303, + "mean_weights": [ + 0.11515206843614578, + 0.18610316514968872, + 0.15217435359954834, + 0.11993157863616943, + 0.11246243864297867, + 0.12337306141853333, + 0.19080331921577454 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9199097156524658, + "mean_weights": [ + 0.13630829751491547, + 0.1662534922361374, + 0.14684012532234192, + 0.12322510778903961, + 0.12947364151477814, + 0.1381947100162506, + 0.15970461070537567 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0490760803222656, + "mean_weights": [ + 0.10742150247097015, + 0.1722080111503601, + 0.14153704047203064, + 0.1159135103225708, + 0.10868977755308151, + 0.11180940270423889, + 0.1477845162153244, + 0.09463626146316528 + ], + "sources": 8 + }, + { + "entropy_mean": 2.051286220550537, + "mean_weights": [ + 0.1131708174943924, + 0.16089829802513123, + 0.14448881149291992, + 0.11956015229225159, + 0.10795272141695023, + 0.11352235078811646, + 0.13960374891757965, + 0.10080308467149734 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0247201919555664, + "mean_weights": [ + 0.1011081114411354, + 0.1894242763519287, + 0.1531166434288025, + 0.11718504130840302, + 0.10284577310085297, + 0.10429200530052185, + 0.14699342846870422, + 0.08503472805023193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.047537326812744, + "mean_weights": [ + 0.1111423671245575, + 0.16533835232257843, + 0.14553973078727722, + 0.12043843418359756, + 0.10876712203025818, + 0.11067016422748566, + 0.13975468277931213, + 0.09834913164377213 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0282247066497803, + "mean_weights": [ + 0.100978784263134, + 0.18681633472442627, + 0.15267831087112427, + 0.11643697321414948, + 0.10204557329416275, + 0.10493149608373642, + 0.14740025997161865, + 0.08871228992938995 + ], + "sources": 8 + }, + { + "entropy_mean": 2.053830623626709, + "mean_weights": [ + 0.11781682819128036, + 0.1550440490245819, + 0.1398259550333023, + 0.11913557350635529, + 0.10805961489677429, + 0.11164884269237518, + 0.13677509129047394, + 0.11169403791427612 + ], + "sources": 8 + }, + { + "entropy_mean": 2.017721176147461, + "mean_weights": [ + 0.09720280021429062, + 0.19308976829051971, + 0.16317345201969147, + 0.11856384575366974, + 0.10023905336856842, + 0.10130266845226288, + 0.14337101578712463, + 0.08305738866329193 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0608584880828857, + "mean_weights": [ + 0.12434182316064835, + 0.14232873916625977, + 0.13082580268383026, + 0.11684279143810272, + 0.10329097509384155, + 0.10883669555187225, + 0.1425117552280426, + 0.13102145493030548 + ], + "sources": 8 + }, + { + "entropy_mean": 2.142296314239502, + "mean_weights": [ + 0.08636107295751572, + 0.16583400964736938, + 0.14278221130371094, + 0.1055997684597969, + 0.08843953907489777, + 0.08854378759860992, + 0.12657995522022247, + 0.07546606659889221, + 0.12039357423782349 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191580057144165, + "mean_weights": [ + 0.1060912162065506, + 0.12397187203168869, + 0.11601399630308151, + 0.1091354638338089, + 0.10782144963741302, + 0.10360677540302277, + 0.12169313430786133, + 0.10359254479408264, + 0.10807354003190994 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1432788372039795, + "mean_weights": [ + 0.08927536010742188, + 0.16873395442962646, + 0.14930182695388794, + 0.112123504281044, + 0.09766747802495956, + 0.0965762585401535, + 0.1252625584602356, + 0.07491002976894379, + 0.08614903688430786 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1724743843078613, + "mean_weights": [ + 0.1105542778968811, + 0.12893787026405334, + 0.11523330211639404, + 0.10665617883205414, + 0.10366716235876083, + 0.10828493535518646, + 0.11819782853126526, + 0.10312190651893616, + 0.10534654557704926 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1452555656433105, + "mean_weights": [ + 0.09202822297811508, + 0.1701684445142746, + 0.14628148078918457, + 0.11057832837104797, + 0.09616327285766602, + 0.09830240160226822, + 0.12624149024486542, + 0.08024715632200241, + 0.07998920977115631 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1762189865112305, + "mean_weights": [ + 0.09908370673656464, + 0.13803134858608246, + 0.13144394755363464, + 0.11672185361385345, + 0.1074555367231369, + 0.10124591737985611, + 0.12084682285785675, + 0.08855879306793213, + 0.09661206603050232 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1389729976654053, + "mean_weights": [ + 0.09089216589927673, + 0.17082780599594116, + 0.14892268180847168, + 0.11308471858501434, + 0.10078474879264832, + 0.09951519966125488, + 0.12574949860572815, + 0.07681785523891449, + 0.07340531051158905 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04405451565980911, + 0.018189694732427597, + 0.0240375604480505, + 0.02713710628449917, + 0.030349314212799072, + 0.03182194381952286, + 0.034919846802949905, + 0.03608781099319458, + 0.038470037281513214, + 0.026185201480984688, + 0.02659580484032631, + 0.02762730047106743, + 0.028048591688275337, + 0.028690988197922707, + 0.029997555539011955, + 0.03034786321222782, + 0.031361404806375504, + 0.02331089787185192, + 0.0246393084526062, + 0.024178674444556236, + 0.025546060875058174, + 0.024928271770477295, + 0.02669469080865383, + 0.0256717037409544, + 0.027990806847810745, + 0.020905915647745132, + 0.02195541374385357, + 0.021578654646873474, + 0.023044798523187637, + 0.022400982677936554, + 0.02394026517868042, + 0.023724498227238655, + 0.025360919535160065, + 0.020535636693239212, + 0.02148214913904667, + 0.021988606080412865, + 0.02322523109614849, + 0.023164646700024605, + 0.023834573104977608, + 0.024238524958491325, + 0.02502058818936348, + 0.019457563757896423, + 0.01980534754693508, + 0.019637931138277054, + 0.02057415433228016, + 0.02042427472770214, + 0.02201046608388424, + 0.021937908604741096, + 0.02379545010626316, + 0.021518733352422714, + 0.022529911249876022, + 0.02267260104417801, + 0.023956971243023872, + 0.024102909490466118, + 0.02642984129488468, + 0.025153208523988724, + 0.030454812571406364, + 0.023361310362815857, + 0.02776513062417507, + 0.025513608008623123, + 0.030121317133307457, + 0.027568820863962173, + 0.03041756898164749, + 0.028944427147507668 + ], + "loss_nats": 2.120351791381836, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1635239124298096, + "mean_weights": [ + 0.1242082342505455, + 0.0880277156829834, + 0.0938715785741806, + 0.09339353442192078, + 0.09749531745910645, + 0.10569863021373749, + 0.13438579440116882, + 0.14857196807861328, + 0.11434720456600189 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01139580737799406, + 0.021054552868008614, + 0.02950318343937397, + 0.038058556616306305, + 0.04104945436120033, + 0.04883526638150215, + 0.05214058980345726, + 0.05791458487510681, + 0.01439172774553299, + 0.016990283504128456, + 0.024067245423793793, + 0.027475843206048012, + 0.030465533956885338, + 0.03445403650403023, + 0.036618731915950775, + 0.040619369596242905, + 0.016362585127353668, + 0.019055280834436417, + 0.02618422359228134, + 0.029903952032327652, + 0.033149950206279755, + 0.037492990493774414, + 0.04017872363328934, + 0.04475866258144379, + 0.025544000789523125, + 0.0285621490329504, + 0.030907167121767998, + 0.03507128730416298, + 0.03762345016002655, + 0.04228900745511055, + 0.04704070836305618, + 0.05219251662492752, + 0.022795414552092552, + 0.027010982856154442, + 0.03890290856361389, + 0.04389137029647827, + 0.04891306161880493, + 0.054182786494493484, + 0.060658931732177734, + 0.06662839651107788, + 0.033973678946495056, + 0.03837457299232483, + 0.03364448994398117, + 0.041143495589494705, + 0.04609670862555504, + 0.05716610327363014, + 0.05855504423379898, + 0.07329611480236053, + 0.015395586378872395, + 0.028997479006648064, + 0.03421308845281601, + 0.053451184183359146, + 0.058928847312927246, + 0.08026835322380066, + 0.08527950197458267, + 0.10854664444923401, + 0.006911881268024445, + 0.027061885222792625, + 0.028406694531440735, + 0.05247335880994797, + 0.05527278408408165, + 0.08323372155427933, + 0.0854114517569542, + 0.11460508406162262 + ] + }, + { + "activation_grad_rms_by_block": [ + 5.134936509421095e-05, + 5.603071258519776e-05, + 4.972576425643638e-05, + 4.7571254981448874e-05, + 4.3214979086769745e-05, + 3.9845475839683786e-05, + 3.5890228900825605e-05, + 3.406162795727141e-05, + 3.5466175177134573e-05, + 3.383069270057604e-05, + 3.1626921554561704e-05, + 3.0198520107660443e-05, + 2.883204069803469e-05, + 2.77392227872042e-05, + 2.7800235329777934e-05, + 3.112062404397875e-05, + 6.127587403170764e-05, + 6.331028998829424e-05, + 5.8612553402781487e-05, + 5.708334720111452e-05, + 0.00015888184134382755, + 0.00012872362276539207, + 0.00010434311116114259, + 8.657111175125465e-05, + 5.805831096949987e-05, + 5.078948743175715e-05, + 4.287513002054766e-05, + 3.959577952628024e-05, + 2.2001073375577107e-05, + 1.997212712012697e-05, + 1.8872458895202726e-05, + 1.86532452062238e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.4711176087730564e-05, + "first_to_last_ratio": 1.3207760209544162, + "imbalance_abs_log_ratio": 0.27821945851834845, + "last_quartile_mean": 3.385220156815194e-05, + "mean": 4.981010016535947e-05, + "normalized": [ + 1.0309026668033479, + 1.1248865671658381, + 0.9983068512481783, + 0.9550523854303026, + 0.8675947035501784, + 0.799947715571839, + 0.7205411910772573, + 0.68382974224492, + 0.7120277827066004, + 0.6791934284063869, + 0.6349499689734955, + 0.6062730250974693, + 0.5788392434931497, + 0.5568995584252106, + 0.5581244614543389, + 0.6247854138149606, + 1.2301897371875206, + 1.271033179578376, + 1.1767202476646232, + 1.146019522378179, + 3.189751492495938, + 2.584287570955602, + 2.0948183363362958, + 1.7380232415485222, + 1.165593138274326, + 1.019662423146035, + 0.8607718088943986, + 0.7949347500773989, + 0.4416990390008851, + 0.4009654076948961, + 0.3788881940118565, + 0.3744872052916737 + ], + "population_cv": 0.6175097196118516 + }, + "activation_output_rms_by_block": [ + 0.0955304205417633, + 0.19925470650196075, + 0.2359117865562439, + 0.25103330612182617, + 0.09304249286651611, + 0.11085781455039978, + 0.12910984456539154, + 0.14378803968429565, + 0.04010729491710663, + 0.06472750008106232, + 0.09987020492553711, + 0.14899125695228577, + 0.05702618882060051, + 0.10170809924602509, + 0.15923865139484406, + 0.38537684082984924, + 0.05919196084141731, + 0.08540526777505875, + 0.1145314872264862, + 0.15640178322792053, + 0.06262174993753433, + 0.08427268266677856, + 0.11030438542366028, + 0.13170360028743744, + 0.05311359465122223, + 0.08614693582057953, + 0.12623678147792816, + 0.17503507435321808, + 0.07505694776773453, + 0.13522638380527496, + 0.2187158316373825, + 0.3292153775691986 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.15731605142354965, + "first_to_last_ratio": 1.0498699791884785, + "imbalance_abs_log_ratio": 0.04866632715763954, + "last_quartile_mean": 0.14984336588531733, + "mean": 0.13496107165701687, + "normalized": [ + 0.7078368552417796, + 1.4763865169086419, + 1.7479987648273716, + 1.86004232953773, + 0.6894024456398029, + 0.8214058556983611, + 0.9566450753555414, + 1.06540380806779, + 0.2971767667867461, + 0.479601260469815, + 0.7399926786247083, + 1.1039572754054923, + 0.4225380557552472, + 0.7536106374770113, + 1.1798857955094264, + 2.8554666623367226, + 0.4385854388578413, + 0.632814090214868, + 0.8486260950679959, + 1.158865895977708, + 0.4639986121085199, + 0.624422151010662, + 0.8173051982276947, + 0.9758636225277033, + 0.39354751706627217, + 0.6383095122385285, + 0.935356987967166, + 1.2969300866107762, + 0.5561377576971261, + 1.0019658420387498, + 1.6205845800722127, + 2.439335828673987 + ], + "population_cv": 0.5772709429538598 + }, + "bits_per_byte": 2.064953725142933, + "branch_output_rms_by_sublayer": [ + 0.051234737038612366, + 0.07594770938158035, + 0.11323792487382889, + 0.04037529230117798, + 0.05108218640089035, + 0.04642988368868828, + 0.03771067038178444, + 0.02321692928671837, + 0.08815877884626389, + 0.015561277978122234, + 0.03706873208284378, + 0.01583549566566944, + 0.0438041090965271, + 0.020819898694753647, + 0.030363978818058968, + 0.027020027860999107, + 0.03348636254668236, + 0.0216005090624094, + 0.027010800316929817, + 0.024715786799788475, + 0.032826121896505356, + 0.03509491682052612, + 0.031702958047389984, + 0.05490833893418312, + 0.038218315690755844, + 0.037195492535829544, + 0.03341353312134743, + 0.042240556329488754, + 0.04180334508419037, + 0.04918703809380531, + 0.2476748526096344, + 0.04376247525215149, + 0.05078785493969917, + 0.02373618632555008, + 0.0355411171913147, + 0.01774558797478676, + 0.052597854286432266, + 0.022557877004146576, + 0.06600882858037949, + 0.03756217285990715, + 0.050489071756601334, + 0.03271714597940445, + 0.022040290758013725, + 0.03932206705212593, + 0.028325051069259644, + 0.04476166144013405, + 0.02171224355697632, + 0.049177221953868866, + 0.024430278688669205, + 0.0453680194914341, + 0.015834001824259758, + 0.0437605082988739, + 0.018936052918434143, + 0.052028607577085495, + 0.02182060107588768, + 0.060354240238666534, + 0.030554499477148056, + 0.06733348965644836, + 0.016542606055736542, + 0.07210637629032135, + 0.015565099194645882, + 0.09804103523492813, + 0.013472299091517925, + 0.1243291050195694 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00020065673277085515, + 0.0001817740596446822, + 0.0001465045892684325, + 9.755666249346124e-05, + 0.00012624719322665892, + 7.565538032235173e-05, + 8.200655559893429e-05, + 7.97391790009874e-05, + 8.09162329894288e-05, + 7.508564486922528e-05, + 9.13364471873345e-05, + 0.00011039800739152168, + 8.766456722124108e-05, + 8.422239419368561e-05, + 9.398125423258295e-05, + 0.00017161898601766447, + 0.0001497631172902175, + 0.00014237628599353527, + 0.00022365597941099996, + 0.00025566927173363617, + 0.0008281082302532628, + 0.0006115617552883831, + 0.0005788343529396405, + 0.00045039612289279527, + 0.0002758540055807746, + 0.00022394859978287228, + 0.00021964130134669458, + 0.00019854079025805258, + 0.00011545923673689007, + 0.00010570174900205617, + 9.906174415914337e-05, + 0.0001053958635417218 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00012376754404079543, + "first_to_last_ratio": 0.7369290916409895, + "imbalance_abs_log_ratio": 0.30526360357753907, + "last_quartile_mean": 0.00016795041130102567, + "mean": 0.00019904163414499135, + "normalized": [ + 1.0081143758329842, + 0.9132464191500225, + 0.7360499721465892, + 0.49013194105106844, + 0.6342753051087516, + 0.3800982676179234, + 0.4120070454164219, + 0.40061557645221907, + 0.4065291833892681, + 0.3772358742519635, + 0.4588811159016144, + 0.5546478145929135, + 0.4404333173700838, + 0.42313958361261256, + 0.4721688235543854, + 0.8622265726207266, + 0.7524210597121753, + 0.7153090689047581, + 1.123664304564935, + 1.2845014704179658, + 4.16047744890413, + 3.072531824386513, + 2.908106916555911, + 2.2628236792142964, + 1.385911077176092, + 1.1251344511155768, + 1.1034942628470255, + 0.9974837229954918, + 0.5800758079225982, + 0.5310534625386868, + 0.49769358347903286, + 0.5295166711952659 + ], + "population_cv": 0.8756839263617585 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.4580664336681366, + "mean_weights": [ + 0.81428062915802, + 0.18571940064430237 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6350213885307312, + "mean_weights": [ + 0.5189062356948853, + 0.48109376430511475 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4277062714099884, + "mean_weights": [ + 0.8402144908905029, + 0.15978553891181946 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6868079900741577, + "mean_weights": [ + 0.493515282869339, + 0.5064846873283386 + ], + "sources": 2 + }, + { + "entropy_mean": 0.38686758279800415, + "mean_weights": [ + 0.8638634085655212, + 0.13613657653331757 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6772255897521973, + "mean_weights": [ + 0.5679300427436829, + 0.43206995725631714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5690979361534119, + "mean_weights": [ + 0.7181376218795776, + 0.28186243772506714 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6592018008232117, + "mean_weights": [ + 0.5172787308692932, + 0.4827212691307068 + ], + "sources": 2 + }, + { + "entropy_mean": 1.038456916809082, + "mean_weights": [ + 0.4540913999080658, + 0.23118522763252258, + 0.3147233724594116 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0903832912445068, + "mean_weights": [ + 0.3152569830417633, + 0.30281925201416016, + 0.38192376494407654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0411981344223022, + "mean_weights": [ + 0.4020236134529114, + 0.2803056836128235, + 0.31767070293426514 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0905810594558716, + "mean_weights": [ + 0.3278552293777466, + 0.2872077226638794, + 0.384937047958374 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0150185823440552, + "mean_weights": [ + 0.48287349939346313, + 0.22136211395263672, + 0.29576438665390015 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0816482305526733, + "mean_weights": [ + 0.3306047320365906, + 0.2639211118221283, + 0.40547415614128113 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0114538669586182, + "mean_weights": [ + 0.4296638071537018, + 0.22373110055923462, + 0.346605122089386 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0918333530426025, + "mean_weights": [ + 0.36806726455688477, + 0.3041593134403229, + 0.32777342200279236 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3290088176727295, + "mean_weights": [ + 0.35130923986434937, + 0.17179939150810242, + 0.21658103168010712, + 0.2603103518486023 + ], + "sources": 4 + }, + { + "entropy_mean": 1.379033088684082, + "mean_weights": [ + 0.29040855169296265, + 0.23445811867713928, + 0.2306380271911621, + 0.24449528753757477 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3394957780838013, + "mean_weights": [ + 0.3262045383453369, + 0.18669262528419495, + 0.22525924444198608, + 0.26184359192848206 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3677098751068115, + "mean_weights": [ + 0.29889944195747375, + 0.22409284114837646, + 0.1924189329147339, + 0.2845887541770935 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2910308837890625, + "mean_weights": [ + 0.3990757167339325, + 0.18164023756980896, + 0.16051796078681946, + 0.2587660551071167 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3623619079589844, + "mean_weights": [ + 0.3024129569530487, + 0.21113446354866028, + 0.18768057227134705, + 0.29877203702926636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2281851768493652, + "mean_weights": [ + 0.46696996688842773, + 0.16045212745666504, + 0.13728894293308258, + 0.23528897762298584 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772211074829102, + "mean_weights": [ + 0.28748467564582825, + 0.24638661742210388, + 0.22340166568756104, + 0.24272704124450684 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5519046783447266, + "mean_weights": [ + 0.30400484800338745, + 0.1470816731452942, + 0.13316048681735992, + 0.1807912439107895, + 0.23496176302433014 + ], + "sources": 5 + }, + { + "entropy_mean": 1.572178602218628, + "mean_weights": [ + 0.21885336935520172, + 0.2029750645160675, + 0.14561443030834198, + 0.14960341155529022, + 0.28295373916625977 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5391805171966553, + "mean_weights": [ + 0.2898171544075012, + 0.15984714031219482, + 0.1228363886475563, + 0.16408921778202057, + 0.26341012120246887 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5521924495697021, + "mean_weights": [ + 0.210292249917984, + 0.185291588306427, + 0.14173434674739838, + 0.14211969077587128, + 0.32056212425231934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5468778610229492, + "mean_weights": [ + 0.2756267488002777, + 0.16966477036476135, + 0.1333748996257782, + 0.1590600460767746, + 0.26227355003356934 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5309922695159912, + "mean_weights": [ + 0.21606209874153137, + 0.27386677265167236, + 0.12764990329742432, + 0.10704417526721954, + 0.2753770351409912 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5832164287567139, + "mean_weights": [ + 0.22763225436210632, + 0.24636119604110718, + 0.17577087879180908, + 0.1817018985748291, + 0.1685338020324707 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5793015956878662, + "mean_weights": [ + 0.21581211686134338, + 0.21387425065040588, + 0.1464191973209381, + 0.17280855774879456, + 0.25108590722084045 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7719972133636475, + "mean_weights": [ + 0.16962555050849915, + 0.20328447222709656, + 0.1659279465675354, + 0.14008209109306335, + 0.12920910120010376, + 0.1918708235025406 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689546823501587, + "mean_weights": [ + 0.16601097583770752, + 0.1523149460554123, + 0.0875597670674324, + 0.101267509162426, + 0.1655988097190857, + 0.3272479772567749 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5251271724700928, + "mean_weights": [ + 0.13745036721229553, + 0.11702707409858704, + 0.07592719793319702, + 0.08887732028961182, + 0.11522199213504791, + 0.4654960632324219 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7189464569091797, + "mean_weights": [ + 0.1744517982006073, + 0.15982553362846375, + 0.09849169105291367, + 0.10780535638332367, + 0.16236206889152527, + 0.29706352949142456 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5877950191497803, + "mean_weights": [ + 0.14238224923610687, + 0.1503014713525772, + 0.10275131464004517, + 0.10159259289503098, + 0.08505517989397049, + 0.4179171919822693 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7369492053985596, + "mean_weights": [ + 0.17980654537677765, + 0.1706826537847519, + 0.10907262563705444, + 0.1106589287519455, + 0.15735921263694763, + 0.2724200189113617 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5987625122070312, + "mean_weights": [ + 0.15493065118789673, + 0.15333226323127747, + 0.10736657679080963, + 0.1186048835515976, + 0.07128532975912094, + 0.39448028802871704 + ], + "sources": 6 + }, + { + "entropy_mean": 1.75795578956604, + "mean_weights": [ + 0.1946164071559906, + 0.17554126679897308, + 0.11572675406932831, + 0.1155121773481369, + 0.1708213835954666, + 0.2277819961309433 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5344648361206055, + "mean_weights": [ + 0.0905577763915062, + 0.0786280632019043, + 0.06272733211517334, + 0.07359343767166138, + 0.050107162445783615, + 0.13821640610694885, + 0.5061697959899902 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7138500213623047, + "mean_weights": [ + 0.13302072882652283, + 0.10984362661838531, + 0.07243296504020691, + 0.07083481550216675, + 0.08885611593723297, + 0.11790697276592255, + 0.4071047604084015 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4565383195877075, + "mean_weights": [ + 0.09418222308158875, + 0.07457256317138672, + 0.057202234864234924, + 0.06210128590464592, + 0.04410415142774582, + 0.11846546828746796, + 0.5493720769882202 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7785513401031494, + "mean_weights": [ + 0.1471671462059021, + 0.12308044731616974, + 0.07605892419815063, + 0.07563325762748718, + 0.10904586315155029, + 0.11226382851600647, + 0.3567505478858948 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5135828256607056, + "mean_weights": [ + 0.10121430456638336, + 0.07933001220226288, + 0.061728835105895996, + 0.06817515194416046, + 0.048567138612270355, + 0.12093988060951233, + 0.5200446844100952 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7546210289001465, + "mean_weights": [ + 0.14427465200424194, + 0.12224892526865005, + 0.08193168044090271, + 0.07876674830913544, + 0.09007281810045242, + 0.10526382178068161, + 0.3774413466453552 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5284571647644043, + "mean_weights": [ + 0.10831618309020996, + 0.08754566311836243, + 0.06345342099666595, + 0.06688661873340607, + 0.05242554843425751, + 0.11085730791091919, + 0.5105152726173401 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8309729099273682, + "mean_weights": [ + 0.1468513160943985, + 0.13658970594406128, + 0.09501184523105621, + 0.0928751677274704, + 0.10057065635919571, + 0.11721575260162354, + 0.31088554859161377 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6937754154205322, + "mean_weights": [ + 0.09603364765644073, + 0.06767555326223373, + 0.045112669467926025, + 0.047895483672618866, + 0.03816477954387665, + 0.07518067210912704, + 0.3669250011444092, + 0.26301223039627075 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9070472717285156, + "mean_weights": [ + 0.10919073224067688, + 0.11743452399969101, + 0.07601121068000793, + 0.0681401938199997, + 0.0658918172121048, + 0.07979628443717957, + 0.24526360630989075, + 0.2382715940475464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.776512861251831, + "mean_weights": [ + 0.08731270581483841, + 0.08430957794189453, + 0.056895699352025986, + 0.06295455247163773, + 0.04594648256897926, + 0.09297705441713333, + 0.33455103635787964, + 0.23505286872386932 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9573358297348022, + "mean_weights": [ + 0.11900999397039413, + 0.11752337962388992, + 0.07979564368724823, + 0.07266241312026978, + 0.07412992417812347, + 0.08811671286821365, + 0.22605092823505402, + 0.2227109968662262 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7231189012527466, + "mean_weights": [ + 0.09091638028621674, + 0.07795695960521698, + 0.05235600098967552, + 0.053361356258392334, + 0.04162060469388962, + 0.07854442298412323, + 0.3417898118495941, + 0.2634544372558594 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8967883586883545, + "mean_weights": [ + 0.1089041531085968, + 0.1000017300248146, + 0.07706406712532043, + 0.07702124863862991, + 0.06017591059207916, + 0.07969298958778381, + 0.261149525642395, + 0.23599036037921906 + ], + "sources": 8 + }, + { + "entropy_mean": 1.801592230796814, + "mean_weights": [ + 0.09432460367679596, + 0.09541776776313782, + 0.06706254929304123, + 0.07247316837310791, + 0.048089414834976196, + 0.08817503601312637, + 0.3473600149154663, + 0.1870974600315094 + ], + "sources": 8 + }, + { + "entropy_mean": 1.919801950454712, + "mean_weights": [ + 0.10887958109378815, + 0.10445678234100342, + 0.0838727131485939, + 0.0852852314710617, + 0.05865198373794556, + 0.08557502925395966, + 0.27338963747024536, + 0.19988907873630524 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9332969188690186, + "mean_weights": [ + 0.07704077661037445, + 0.08837630599737167, + 0.05746292322874069, + 0.06001049652695656, + 0.04015304520726204, + 0.06584024429321289, + 0.2511133551597595, + 0.10944834351539612, + 0.25055450201034546 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0115504264831543, + "mean_weights": [ + 0.08485299348831177, + 0.08635596185922623, + 0.06204982101917267, + 0.06870189309120178, + 0.0476498119533062, + 0.08640170097351074, + 0.2653427720069885, + 0.13704031705856323, + 0.16160473227500916 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9781036376953125, + "mean_weights": [ + 0.08389372378587723, + 0.08841055631637573, + 0.060679104179143906, + 0.0658775195479393, + 0.045975662767887115, + 0.08423998951911926, + 0.29481440782546997, + 0.12572166323661804, + 0.15038736164569855 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0586161613464355, + "mean_weights": [ + 0.1038442850112915, + 0.09918913245201111, + 0.08148118108510971, + 0.0698266476392746, + 0.059573929756879807, + 0.08160553872585297, + 0.22791330516338348, + 0.13585451245307922, + 0.1407114714384079 + ], + "sources": 9 + }, + { + "entropy_mean": 2.022735118865967, + "mean_weights": [ + 0.09707076847553253, + 0.09551473706960678, + 0.06701318919658661, + 0.0673459842801094, + 0.04480960965156555, + 0.08329527080059052, + 0.24251294136047363, + 0.15929152071475983, + 0.14314597845077515 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9317231178283691, + "mean_weights": [ + 0.07746735960245132, + 0.08971680700778961, + 0.06909166276454926, + 0.08098623156547546, + 0.04421798139810562, + 0.0916394367814064, + 0.3483121693134308, + 0.10497690737247467, + 0.09359145164489746 + ], + "sources": 9 + }, + { + "entropy_mean": 2.053220272064209, + "mean_weights": [ + 0.09178322553634644, + 0.10647953301668167, + 0.07523074746131897, + 0.08394691348075867, + 0.04770101606845856, + 0.09626083076000214, + 0.26201331615448, + 0.13223087787628174, + 0.10435353964567184 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.0558863990008831, + 0.04748110845685005, + 0.061753273010253906, + 0.06085919216275215, + 0.11003358662128448, + 0.06251973658800125, + 0.1138310506939888, + 0.08963073790073395, + 0.13082429766654968, + 0.08407715708017349, + 0.10403942316770554, + 0.1004372090101242, + 0.1062953919172287, + 0.09022688120603561, + 0.10766621679067612, + 0.09703134745359421, + 0.11238158494234085, + 0.0712415874004364, + 0.08696430176496506, + 0.07720176130533218, + 0.08355452120304108, + 0.07461326569318771, + 0.08609789609909058, + 0.0729745551943779, + 0.100697360932827, + 0.06531906127929688, + 0.07805952429771423, + 0.06964895129203796, + 0.08197034895420074, + 0.08001933246850967, + 0.10499614477157593, + 0.11749708652496338, + 0.13756641745567322, + 0.09965827316045761, + 0.09882861375808716, + 0.0854005292057991, + 0.10110001266002655, + 0.08426972478628159, + 0.10269922018051147, + 0.0886540412902832, + 0.10988812148571014, + 0.04304825887084007, + 0.0575832724571228, + 0.0459144227206707, + 0.06622003018856049, + 0.05584317073225975, + 0.06460635364055634, + 0.06311307847499847, + 0.07165247201919556, + 0.05157631263136864, + 0.055589381605386734, + 0.05562245845794678, + 0.05917651578783989, + 0.059894196689128876, + 0.06219335272908211, + 0.06387592852115631, + 0.06750614196062088, + 0.050557736307382584, + 0.06024058535695076, + 0.06106090545654297, + 0.06803743541240692, + 0.06874485313892365, + 0.06952574104070663, + 0.07015988230705261 + ], + "loss_nats": 1.43131685256958, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.843801498413086, + "mean_weights": [ + 0.11151862144470215, + 0.061874739825725555, + 0.05308101326227188, + 0.040848225355148315, + 0.042743273079395294, + 0.053944505751132965, + 0.1199670359492302, + 0.3250349462032318, + 0.19098761677742004 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.051234737038612366, + 0.0955304205417633, + 0.1796528398990631, + 0.19925470650196075, + 0.21105031669139862, + 0.2359117865562439, + 0.23972436785697937, + 0.25103330612182617, + 0.08815877884626389, + 0.09304249286651611, + 0.1068187803030014, + 0.11085781455039978, + 0.12298524379730225, + 0.12910984456539154, + 0.13432784378528595, + 0.14378803968429565, + 0.03348636254668236, + 0.04010729491710663, + 0.051035381853580475, + 0.06472750008106232, + 0.07822930067777634, + 0.09987020492553711, + 0.11166217923164368, + 0.14899125695228577, + 0.038218315690755844, + 0.05702618882060051, + 0.07459890842437744, + 0.10170809924602509, + 0.12532564997673035, + 0.15923865139484406, + 0.36271119117736816, + 0.38537684082984924, + 0.05078785493969917, + 0.05919196084141731, + 0.0794382095336914, + 0.08540526777505875, + 0.10824274271726608, + 0.1145314872264862, + 0.14269457757472992, + 0.15640178322792053, + 0.050489071756601334, + 0.06262174993753433, + 0.06721649318933487, + 0.08427268266677856, + 0.09199884533882141, + 0.11030438542366028, + 0.11444488167762756, + 0.13170360028743744, + 0.024430278688669205, + 0.05311359465122223, + 0.05695623904466629, + 0.08614693582057953, + 0.09002252668142319, + 0.12623678147792816, + 0.130380317568779, + 0.17503507435321808, + 0.030554499477148056, + 0.07505694776773453, + 0.07779927551746368, + 0.13522638380527496, + 0.13734298944473267, + 0.2187158316373825, + 0.22044414281845093, + 0.3292153775691986 + ] + }, + { + "activation_grad_rms_by_block": [ + 3.57800890924409e-05, + 3.942943658330478e-05, + 3.388081677258015e-05, + 3.245583502575755e-05, + 3.3783475373638794e-05, + 2.915231198130641e-05, + 2.5950168492272496e-05, + 2.450849387969356e-05, + 2.7269536076346412e-05, + 2.568827403592877e-05, + 2.400098310317844e-05, + 2.2796592020313255e-05, + 2.010442767641507e-05, + 1.8936527339974418e-05, + 1.9039980543311685e-05, + 2.3072669137036428e-05, + 4.836148946196772e-05, + 4.9383146688342094e-05, + 4.5993543608346954e-05, + 4.280430221115239e-05, + 0.00014426946290768683, + 0.00011744296352844685, + 9.347966988570988e-05, + 7.698025729041547e-05, + 6.582303467439488e-05, + 5.6237244280055165e-05, + 4.4205091398907825e-05, + 3.852568261208944e-05, + 2.7434407456894405e-05, + 2.285257687617559e-05, + 2.028334711212665e-05, + 1.9826158677460626e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 3.186757840012433e-05, + "first_to_last_ratio": 0.8636564556008468, + "imbalance_abs_log_ratio": 0.146580210082559, + "last_quartile_mean": 3.689844288601307e-05, + "mean": 4.2179749868864747e-05, + "normalized": [ + 0.8482764645044092, + 0.9347954102594115, + 0.8032484042203742, + 0.7694648528419803, + 0.8009406285876599, + 0.6911447334785021, + 0.6152281265998635, + 0.5810488197746441, + 0.646507771172811, + 0.6090191173677569, + 0.569016724323795, + 0.5404629494292167, + 0.4766369582304063, + 0.4489483081063157, + 0.4514009827584627, + 0.5470082020108817, + 1.146557047216301, + 1.170778557053383, + 1.090417647125437, + 1.014806923801809, + 3.420348943656972, + 2.7843447126541196, + 2.216221532283493, + 1.825052484420704, + 1.5605363919661974, + 1.3332759073938238, + 1.048016916561615, + 0.913369157756135, + 0.6504165515961319, + 0.5417902416971032, + 0.48087879077488155, + 0.470039740375403 + ], + "population_cv": 0.6827037752697015 + }, + "activation_output_rms_by_block": [ + 0.15143299102783203, + 0.30262914299964905, + 0.3626292943954468, + 0.38562148809432983, + 0.13662196695804596, + 0.15442071855068207, + 0.18618692457675934, + 0.2074802666902542, + 0.04909220710396767, + 0.0881529152393341, + 0.14954733848571777, + 0.23951733112335205, + 0.08217982947826385, + 0.14409078657627106, + 0.22605396807193756, + 0.6592875719070435, + 0.06389845162630081, + 0.09458807110786438, + 0.14630787074565887, + 0.21908092498779297, + 0.07773751765489578, + 0.10736925154924393, + 0.1424422264099121, + 0.17246152460575104, + 0.06572399288415909, + 0.10123347491025925, + 0.14551502466201782, + 0.19983991980552673, + 0.09023923426866531, + 0.15600618720054626, + 0.24885882437229156, + 0.3860291540622711 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2358778491616249, + "first_to_last_ratio": 1.3542132581102162, + "imbalance_abs_log_ratio": 0.30322066438495643, + "last_quartile_mean": 0.17418072652071714, + "mean": 0.18569613725412637, + "normalized": [ + 0.8154881047449846, + 1.6297007976288655, + 1.9528101109566227, + 2.076626330501449, + 0.7357286423846173, + 0.8315774406193291, + 1.0026429592445498, + 1.1173106223869165, + 0.26436848838048077, + 0.4747159340137284, + 0.8053335987331889, + 1.2898347518967022, + 0.442550021198342, + 0.7759492938675503, + 1.2173326349948854, + 3.5503569522547695, + 0.3441022122008675, + 0.5093701598026241, + 0.7878886061342009, + 1.1797818103667892, + 0.41862754284711634, + 0.5781986267291517, + 0.7670715638795383, + 0.9287297364173834, + 0.3539330104331431, + 0.5451566004936365, + 0.7836190176798323, + 1.0761662722797756, + 0.4859510574803844, + 0.8401154138551131, + 1.3401400161152877, + 2.078821669478174 + ], + "population_cv": 0.6621635595354842 + }, + "bits_per_byte": 1.9288425276664853, + "branch_output_rms_by_sublayer": [ + 0.08832799643278122, + 0.11756844818592072, + 0.17041416466236115, + 0.05185626819729805, + 0.08745641261339188, + 0.0881219431757927, + 0.05030680447816849, + 0.03964918479323387, + 0.12814420461654663, + 0.022397184744477272, + 0.05527547001838684, + 0.022468235343694687, + 0.06746833771467209, + 0.031668391078710556, + 0.04974694177508354, + 0.045926205813884735, + 0.03680809214711189, + 0.03180550038814545, + 0.03558321297168732, + 0.03912416473031044, + 0.051462169736623764, + 0.06056958809494972, + 0.0526844747364521, + 0.10351218283176422, + 0.05210389941930771, + 0.05620818957686424, + 0.03620988130569458, + 0.0675361230969429, + 0.04957791045308113, + 0.0823286920785904, + 0.4858623445034027, + 0.07062793523073196, + 0.04707946628332138, + 0.0371064618229866, + 0.04491531476378441, + 0.023919809609651566, + 0.07858023792505264, + 0.035764746367931366, + 0.09729185700416565, + 0.07103991508483887, + 0.05887215584516525, + 0.045838613063097, + 0.026784580200910568, + 0.05455324053764343, + 0.041033755987882614, + 0.06383209675550461, + 0.032232560217380524, + 0.07182251662015915, + 0.02874751202762127, + 0.05687069892883301, + 0.022572210058569908, + 0.052836012095212936, + 0.02659173309803009, + 0.06559387594461441, + 0.025225715711712837, + 0.07689779996871948, + 0.041177187114953995, + 0.07740812748670578, + 0.027006076648831367, + 0.08842137455940247, + 0.023083338513970375, + 0.11899886280298233, + 0.016772178933024406, + 0.16321057081222534 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00017320581296885245, + 0.0001375362640366496, + 0.00014375813569971144, + 8.981648567965897e-05, + 0.00010459593200616139, + 6.156395218407552e-05, + 7.281931709149581e-05, + 7.667720183186952e-05, + 7.236475672264944e-05, + 7.890209128468536e-05, + 9.134838485444255e-05, + 0.00011250740948005864, + 7.70610699389613e-05, + 7.611216582142322e-05, + 8.146874247135892e-05, + 0.00017928549432252573, + 0.00013712162717798398, + 0.00013184310037104538, + 0.00022367504753246386, + 0.0002691717754589954, + 0.0008753648193242955, + 0.0006614972326966637, + 0.0006052862860379005, + 0.0005306195765578692, + 0.00037252876038501256, + 0.00029795485316373564, + 0.0002868510641817651, + 0.00023542884481748448, + 0.00018331496216173493, + 0.00013991664250432274, + 0.00012258431160994327, + 0.00012212563876750547 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00010749663768730933, + "first_to_last_ratio": 0.4884254111851857, + "imbalance_abs_log_ratio": 0.7165685086453754, + "last_quartile_mean": 0.00022008813469893802, + "mean": 0.0002132596174732282, + "normalized": [ + 0.812182892481255, + 0.6449240867362775, + 0.6740991914128248, + 0.42116030565859097, + 0.4904629073494984, + 0.2886807775119639, + 0.3414585375059921, + 0.3595486231306557, + 0.33932704925597945, + 0.3699813974138373, + 0.428343565166106, + 0.5275607769210333, + 0.36134862686150726, + 0.35689910130771973, + 0.3820167335786683, + 0.840691249692561, + 0.6429798046280267, + 0.6182281574597519, + 1.0488392044524941, + 1.262178834644052, + 4.104690938190328, + 3.101840097690814, + 2.838260207016858, + 2.4881390243724, + 1.746832170097925, + 1.3971461484082413, + 1.3450791461621905, + 1.1039541738230836, + 0.8595859091079332, + 0.6560859677143862, + 0.5748125831902143, + 0.5726618110568292 + ], + "population_cv": 0.9042592730039112 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2508508265018463, + "mean_weights": [ + 0.9251874089241028, + 0.07481260597705841 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5796181559562683, + "mean_weights": [ + 0.6163991689682007, + 0.38360080122947693 + ], + "sources": 2 + }, + { + "entropy_mean": 0.3817020058631897, + "mean_weights": [ + 0.8651303648948669, + 0.13486966490745544 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6884468793869019, + "mean_weights": [ + 0.4932539463043213, + 0.5067461133003235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2797870337963104, + "mean_weights": [ + 0.9142746329307556, + 0.08572537451982498 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6642253994941711, + "mean_weights": [ + 0.605044960975647, + 0.39495497941970825 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5038842558860779, + "mean_weights": [ + 0.7756115198135376, + 0.2243884801864624 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6434851884841919, + "mean_weights": [ + 0.5374104976654053, + 0.4625895023345947 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0016276836395264, + "mean_weights": [ + 0.4353652000427246, + 0.18586981296539307, + 0.3787649869918823 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0655895471572876, + "mean_weights": [ + 0.2726097106933594, + 0.2790643572807312, + 0.4483259320259094 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0300546884536743, + "mean_weights": [ + 0.40803641080856323, + 0.2817816734313965, + 0.3101818859577179 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0874333381652832, + "mean_weights": [ + 0.36608952283859253, + 0.27405625581741333, + 0.35985422134399414 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9672750234603882, + "mean_weights": [ + 0.5343947410583496, + 0.1846013069152832, + 0.2810039520263672 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0731852054595947, + "mean_weights": [ + 0.3801807165145874, + 0.24373164772987366, + 0.37608763575553894 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9721636772155762, + "mean_weights": [ + 0.49137914180755615, + 0.2109004259109497, + 0.29772043228149414 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0864832401275635, + "mean_weights": [ + 0.39626431465148926, + 0.29088544845581055, + 0.3128502070903778 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3038380146026611, + "mean_weights": [ + 0.35696184635162354, + 0.1472446322441101, + 0.19980239868164062, + 0.29599112272262573 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3611116409301758, + "mean_weights": [ + 0.3264957070350647, + 0.22676822543144226, + 0.20138126611709595, + 0.2453548014163971 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3161137104034424, + "mean_weights": [ + 0.3259783387184143, + 0.17862647771835327, + 0.20532535016536713, + 0.2900698482990265 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568220138549805, + "mean_weights": [ + 0.3223523497581482, + 0.21851874887943268, + 0.1804703176021576, + 0.27865859866142273 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2413711547851562, + "mean_weights": [ + 0.4272037148475647, + 0.16667884588241577, + 0.1365978717803955, + 0.269519567489624 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3540384769439697, + "mean_weights": [ + 0.32447606325149536, + 0.2030642032623291, + 0.18181107938289642, + 0.2906486392021179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.1544246673583984, + "mean_weights": [ + 0.5128031969070435, + 0.13654224574565887, + 0.12138728052377701, + 0.22926729917526245 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3712997436523438, + "mean_weights": [ + 0.3103735148906708, + 0.24219870567321777, + 0.2247106432914734, + 0.22271710634231567 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5205519199371338, + "mean_weights": [ + 0.3307538628578186, + 0.1283297836780548, + 0.12636876106262207, + 0.17177441716194153, + 0.2427731454372406 + ], + "sources": 5 + }, + { + "entropy_mean": 1.518624186515808, + "mean_weights": [ + 0.2150207757949829, + 0.17106646299362183, + 0.13557285070419312, + 0.11898039281368256, + 0.3593595027923584 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4772696495056152, + "mean_weights": [ + 0.30324018001556396, + 0.13238273561000824, + 0.09796822816133499, + 0.15785877406597137, + 0.30855005979537964 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4979676008224487, + "mean_weights": [ + 0.20932656526565552, + 0.16542977094650269, + 0.1272144764661789, + 0.1160244345664978, + 0.3820047676563263 + ], + "sources": 5 + }, + { + "entropy_mean": 1.49334716796875, + "mean_weights": [ + 0.28671571612358093, + 0.13947691023349762, + 0.11353637278079987, + 0.14176231622695923, + 0.31850868463516235 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5120060443878174, + "mean_weights": [ + 0.21571135520935059, + 0.28788846731185913, + 0.131511390209198, + 0.08343321830034256, + 0.2814555764198303 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5766459703445435, + "mean_weights": [ + 0.2565065026283264, + 0.22934341430664062, + 0.1593727320432663, + 0.1524350941181183, + 0.20234227180480957 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5796699523925781, + "mean_weights": [ + 0.2356114238500595, + 0.21223093569278717, + 0.1439642310142517, + 0.1658918261528015, + 0.2423015832901001 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7626371383666992, + "mean_weights": [ + 0.16389364004135132, + 0.20886825025081635, + 0.18027648329734802, + 0.1462823748588562, + 0.11240866780281067, + 0.18827059864997864 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6546597480773926, + "mean_weights": [ + 0.17577320337295532, + 0.14207448065280914, + 0.08111412078142166, + 0.0931762307882309, + 0.15011057257652283, + 0.35775139927864075 + ], + "sources": 6 + }, + { + "entropy_mean": 1.430190086364746, + "mean_weights": [ + 0.1354856789112091, + 0.09479984641075134, + 0.07003438472747803, + 0.0834486186504364, + 0.09165525436401367, + 0.5245761871337891 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7008471488952637, + "mean_weights": [ + 0.18653985857963562, + 0.149653822183609, + 0.09563066065311432, + 0.09960515052080154, + 0.15559062361717224, + 0.31297987699508667 + ], + "sources": 6 + }, + { + "entropy_mean": 1.514115333557129, + "mean_weights": [ + 0.14724519848823547, + 0.13274209201335907, + 0.096603162586689, + 0.11347334086894989, + 0.0527515783905983, + 0.4571846127510071 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7213332653045654, + "mean_weights": [ + 0.19527462124824524, + 0.16114139556884766, + 0.10406571626663208, + 0.10423287749290466, + 0.14977452158927917, + 0.2855108380317688 + ], + "sources": 6 + }, + { + "entropy_mean": 1.503009557723999, + "mean_weights": [ + 0.1677132397890091, + 0.12867671251296997, + 0.09363879263401031, + 0.12237933278083801, + 0.04204780235886574, + 0.44554415345191956 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7531253099441528, + "mean_weights": [ + 0.21246549487113953, + 0.17559614777565002, + 0.11484666168689728, + 0.11058571934700012, + 0.1656094789505005, + 0.22089649736881256 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3981997966766357, + "mean_weights": [ + 0.08501995354890823, + 0.06276540458202362, + 0.0543542318046093, + 0.06455785036087036, + 0.03336101397871971, + 0.12979380786418915, + 0.5701477527618408 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6064693927764893, + "mean_weights": [ + 0.12975823879241943, + 0.09198254346847534, + 0.06163205951452255, + 0.06021754443645477, + 0.078399658203125, + 0.10503693670034409, + 0.4729730188846588 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3297276496887207, + "mean_weights": [ + 0.09028719365596771, + 0.057085148990154266, + 0.04790058359503746, + 0.05685827136039734, + 0.031231366097927094, + 0.11244906485080719, + 0.6041883826255798 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6772459745407104, + "mean_weights": [ + 0.1455874890089035, + 0.10527922213077545, + 0.06669703871011734, + 0.06069611385464668, + 0.09951996803283691, + 0.09685218334197998, + 0.4253680109977722 + ], + "sources": 7 + }, + { + "entropy_mean": 1.3742375373840332, + "mean_weights": [ + 0.09774194657802582, + 0.06367622315883636, + 0.05300067365169525, + 0.05981069803237915, + 0.03241024166345596, + 0.10956928879022598, + 0.5837908983230591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.672610878944397, + "mean_weights": [ + 0.14811497926712036, + 0.10026681423187256, + 0.0711982250213623, + 0.06714795529842377, + 0.08556141704320908, + 0.09809904545545578, + 0.42961156368255615 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4325350522994995, + "mean_weights": [ + 0.11123399436473846, + 0.07182648032903671, + 0.05620439350605011, + 0.06410455703735352, + 0.03741488978266716, + 0.10516737401485443, + 0.5540482997894287 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7746202945709229, + "mean_weights": [ + 0.15383490920066833, + 0.11435382068157196, + 0.09083531051874161, + 0.08329388499259949, + 0.08837151527404785, + 0.11093884706497192, + 0.3583717346191406 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5557539463043213, + "mean_weights": [ + 0.09889799356460571, + 0.05227670818567276, + 0.03886467218399048, + 0.039664674550294876, + 0.02741250954568386, + 0.06309205293655396, + 0.40426212549209595, + 0.27552926540374756 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7942585945129395, + "mean_weights": [ + 0.10892648994922638, + 0.08669497072696686, + 0.060229551047086716, + 0.05406671017408371, + 0.05429235100746155, + 0.07220236957073212, + 0.24074554443359375, + 0.322842001914978 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6406831741333008, + "mean_weights": [ + 0.08106359094381332, + 0.06122073531150818, + 0.04382575303316116, + 0.0495278537273407, + 0.031025834381580353, + 0.08371539413928986, + 0.3212447166442871, + 0.3283761143684387 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8771746158599854, + "mean_weights": [ + 0.11823929846286774, + 0.09187044948339462, + 0.06966835260391235, + 0.05918115749955177, + 0.06588791310787201, + 0.07802121341228485, + 0.22871823608875275, + 0.2884133756160736 + ], + "sources": 8 + }, + { + "entropy_mean": 1.612527847290039, + "mean_weights": [ + 0.09160974621772766, + 0.05898914486169815, + 0.039636120200157166, + 0.04581688344478607, + 0.032106831669807434, + 0.06671778112649918, + 0.3060194253921509, + 0.35910406708717346 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8253357410430908, + "mean_weights": [ + 0.10355446487665176, + 0.08380813896656036, + 0.06609268486499786, + 0.07030215114355087, + 0.04850628599524498, + 0.0775601714849472, + 0.25825831294059753, + 0.2919177711009979 + ], + "sources": 8 + }, + { + "entropy_mean": 1.714170217514038, + "mean_weights": [ + 0.09696917235851288, + 0.07828262448310852, + 0.055068351328372955, + 0.0616651251912117, + 0.03598042577505112, + 0.07802210748195648, + 0.3351396918296814, + 0.25887250900268555 + ], + "sources": 8 + }, + { + "entropy_mean": 1.875918984413147, + "mean_weights": [ + 0.11156761646270752, + 0.0880264937877655, + 0.07785218954086304, + 0.07930342108011246, + 0.04990004748106003, + 0.08684437721967697, + 0.2776731252670288, + 0.22883272171020508 + ], + "sources": 8 + }, + { + "entropy_mean": 1.731968641281128, + "mean_weights": [ + 0.06617744266986847, + 0.05556279793381691, + 0.04319432005286217, + 0.045391663908958435, + 0.026374589651823044, + 0.054588086903095245, + 0.21148523688316345, + 0.10446193069219589, + 0.3927639424800873 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9649744033813477, + "mean_weights": [ + 0.07813189923763275, + 0.08751581609249115, + 0.05837181955575943, + 0.07608447968959808, + 0.03110998496413231, + 0.09630586206912994, + 0.27242881059646606, + 0.1154000535607338, + 0.18465125560760498 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8779311180114746, + "mean_weights": [ + 0.07922039180994034, + 0.064256951212883, + 0.04481552541255951, + 0.05368471518158913, + 0.03291679173707962, + 0.07845190167427063, + 0.26414477825164795, + 0.13897250592708588, + 0.24353644251823425 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9959204196929932, + "mean_weights": [ + 0.10073552280664444, + 0.0823616087436676, + 0.0686727911233902, + 0.0658436119556427, + 0.048376426100730896, + 0.07448098808526993, + 0.23861894011497498, + 0.14355987310409546, + 0.177350252866745 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9298079013824463, + "mean_weights": [ + 0.09570661187171936, + 0.07176163792610168, + 0.05499213933944702, + 0.057467732578516006, + 0.03367581591010094, + 0.06890923529863358, + 0.21427617967128754, + 0.18804281949996948, + 0.2151678204536438 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8635971546173096, + "mean_weights": [ + 0.07685244828462601, + 0.07133444398641586, + 0.05962139368057251, + 0.08040933310985565, + 0.033340416848659515, + 0.10067727416753769, + 0.36328762769699097, + 0.11175162345170975, + 0.10272547602653503 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0009379386901855, + "mean_weights": [ + 0.09139549732208252, + 0.08807491511106491, + 0.06457877159118652, + 0.07007958739995956, + 0.03462113067507744, + 0.09155915677547455, + 0.2692604660987854, + 0.15421991050243378, + 0.13621056079864502 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059407927095890045, + 0.05594591051340103, + 0.07886742800474167, + 0.07073435187339783, + 0.16177938878536224, + 0.06750936061143875, + 0.15622827410697937, + 0.10916222631931305, + 0.1895449161529541, + 0.11270010471343994, + 0.15217198431491852, + 0.14727257192134857, + 0.14916078746318817, + 0.11801524460315704, + 0.14915435016155243, + 0.13372685015201569, + 0.16036193072795868, + 0.09399056434631348, + 0.12247956544160843, + 0.10827244818210602, + 0.12010690569877625, + 0.10214843600988388, + 0.12369577586650848, + 0.09725081920623779, + 0.14881008863449097, + 0.0902416780591011, + 0.10321364551782608, + 0.09460548311471939, + 0.1168464794754982, + 0.11164536327123642, + 0.16267083585262299, + 0.19675607979297638, + 0.21974295377731323, + 0.15372900664806366, + 0.1438056081533432, + 0.10524243861436844, + 0.14797262847423553, + 0.10666082799434662, + 0.15148812532424927, + 0.11902136355638504, + 0.1692686378955841, + 0.05537491664290428, + 0.07994630187749863, + 0.06347332894802094, + 0.09600856900215149, + 0.07961282879114151, + 0.09550122171640396, + 0.09120825678110123, + 0.10489360988140106, + 0.07538540661334991, + 0.07298607379198074, + 0.07336290925741196, + 0.08083610236644745, + 0.07823792845010757, + 0.0843365490436554, + 0.08682125806808472, + 0.09205732494592667, + 0.057621169835329056, + 0.07988084107637405, + 0.0771951824426651, + 0.08835897594690323, + 0.09140052646398544, + 0.09466564655303955, + 0.09293081611394882 + ], + "loss_nats": 1.3369717597961426, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.7290174961090088, + "mean_weights": [ + 0.10267993807792664, + 0.05008202791213989, + 0.044380687177181244, + 0.0326685830950737, + 0.033161722123622894, + 0.043591417372226715, + 0.1060870811343193, + 0.3520195186138153, + 0.2353290319442749 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08832799643278122, + 0.15143299102783203, + 0.2790028154850006, + 0.30262914299964905, + 0.315830796957016, + 0.3626292943954468, + 0.3664044439792633, + 0.38562148809432983, + 0.12814420461654663, + 0.13662196695804596, + 0.14859555661678314, + 0.15442071855068207, + 0.17815031111240387, + 0.18618692457675934, + 0.19290222227573395, + 0.2074802666902542, + 0.03680809214711189, + 0.04909220710396767, + 0.06554166972637177, + 0.0881529152393341, + 0.11251214146614075, + 0.14954733848571777, + 0.16864995658397675, + 0.23951733112335205, + 0.05210389941930771, + 0.08217982947826385, + 0.09766767174005508, + 0.14409078657627106, + 0.16688434779644012, + 0.22605396807193756, + 0.6339711546897888, + 0.6592875719070435, + 0.04707946628332138, + 0.06389845162630081, + 0.084575355052948, + 0.09458807110786438, + 0.13482175767421722, + 0.14630787074565887, + 0.19051826000213623, + 0.21908092498779297, + 0.05887215584516525, + 0.07773751765489578, + 0.08281512558460236, + 0.10736925154924393, + 0.11754488199949265, + 0.1424422264099121, + 0.14818532764911652, + 0.17246152460575104, + 0.02874751202762127, + 0.06572399288415909, + 0.07124026119709015, + 0.10123347491025925, + 0.10666517913341522, + 0.14551502466201782, + 0.14956052601337433, + 0.19983991980552673, + 0.041177187114953995, + 0.09023923426866531, + 0.09537366032600403, + 0.15600618720054626, + 0.15850447118282318, + 0.24885882437229156, + 0.25068944692611694, + 0.3860291540622711 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "population_cv": 0.7299683279122143 + }, + "activation_output_rms_by_block": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.24231436662375927, + "first_to_last_ratio": 1.7382073612286646, + "imbalance_abs_log_ratio": 0.5528543299844475, + "last_quartile_mean": 0.139404752291739, + "mean": 0.17594548198394477, + "normalized": [ + 0.872176901111452, + 1.8466899321818973, + 2.1703463827261262, + 2.3075366266331874, + 0.767871429869389, + 0.870980627700565, + 1.0231460974138593, + 1.1589542538312023, + 0.24864957835709345, + 0.4666800200245053, + 0.7845885991457198, + 1.3049357734104974, + 0.46603991847126747, + 0.789568230136157, + 1.2326042132027004, + 4.183179000808161, + 0.32473565854469494, + 0.4690011715552792, + 0.7196215097851506, + 1.1150703717560293, + 0.38383209436703286, + 0.5425045292082467, + 0.7244651255474189, + 0.8882788682598759, + 0.3321309360788757, + 0.49982187502145003, + 0.6961071361611404, + 0.9322692927740504, + 0.42842289992260757, + 0.6985518108614562, + 1.0888090156129309, + 1.6624301195199809 + ], + "population_cv": 0.7619463065887785 + }, + "bits_per_byte": 1.877417995231997, + "branch_output_rms_by_sublayer": [ + 0.09253507107496262, + 0.116500623524189, + 0.19514523446559906, + 0.0468219518661499, + 0.07917427271604538, + 0.09375449270009995, + 0.04215213656425476, + 0.041188232600688934, + 0.12552575767040253, + 0.024237047880887985, + 0.05027730390429497, + 0.022429833188652992, + 0.054851483553647995, + 0.03027583472430706, + 0.036243390291929245, + 0.04546776041388512, + 0.031880609691143036, + 0.030174262821674347, + 0.035054463893175125, + 0.03663193807005882, + 0.050445348024368286, + 0.05748353153467178, + 0.05600574240088463, + 0.10596779733896255, + 0.053015924990177155, + 0.05394246056675911, + 0.032114963978528976, + 0.06537146866321564, + 0.048427484929561615, + 0.08060523867607117, + 0.5742782950401306, + 0.06690707802772522, + 0.04075472429394722, + 0.033449217677116394, + 0.03505101054906845, + 0.022565271705389023, + 0.06577442586421967, + 0.036697570234537125, + 0.08553697913885117, + 0.07367803901433945, + 0.0493597537279129, + 0.042231619358062744, + 0.023477228358387947, + 0.052135858684778214, + 0.03512806445360184, + 0.06269875913858414, + 0.030729109421372414, + 0.0728035643696785, + 0.023035654798150063, + 0.051585447043180466, + 0.019568851217627525, + 0.04862172156572342, + 0.02376176416873932, + 0.05972515419125557, + 0.02176014520227909, + 0.06787826865911484, + 0.03424545377492905, + 0.06439156830310822, + 0.021422436460852623, + 0.07232685387134552, + 0.019477492198348045, + 0.09627991914749146, + 0.017491821199655533, + 0.1271906942129135 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00028365814152869, + 0.0002107335686261075, + 0.00020559059302825965, + 0.00013034830149957224, + 0.00015811506316286034, + 8.078727112532468e-05, + 9.745502139202278e-05, + 0.00010158534732968689, + 0.00010687310625858127, + 0.00011249947044062199, + 0.00014466041390513986, + 0.00018570148639523412, + 0.0001198924920321974, + 0.00012288623237884872, + 0.00013187077826681118, + 0.000321118955435273, + 0.0002173403860671073, + 0.00022706794561596516, + 0.0003846029635623724, + 0.00045310883243459936, + 0.001499135691470673, + 0.0011394398675334944, + 0.0010162826769661553, + 0.0008650696174683238, + 0.0006964490714203023, + 0.0005559793522399823, + 0.0005128319422021078, + 0.0003792784055626199, + 0.00037680605737877213, + 0.0002696585960614849, + 0.00021598580145167561, + 0.00019464643295656736 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00015853416346156552, + "first_to_last_ratio": 0.39613292787359505, + "imbalance_abs_log_ratio": 0.9260054476059185, + "last_quartile_mean": 0.000400204457409189, + "mean": 0.0003599206213499198, + "normalized": [ + 0.7881130579981791, + 0.5855001245433764, + 0.5712109305023165, + 0.36215846986118033, + 0.439305373973387, + 0.2244585787341764, + 0.27076809610549, + 0.28224375404965807, + 0.29693521270813145, + 0.3125674489521634, + 0.4019231056074973, + 0.5159512275199496, + 0.3331081491872516, + 0.34142592863379456, + 0.36638850469921974, + 0.892193823823939, + 0.6038564427121444, + 0.6308834008018855, + 1.0685771827128963, + 1.258913231298522, + 4.165184217142125, + 3.165808792116179, + 2.8236300358520197, + 2.4035011226191765, + 1.9350074158246267, + 1.544727696220138, + 1.4248473462806275, + 1.0537834818691318, + 1.0469143334036313, + 0.7492168552335295, + 0.6000928778173319, + 0.540803781196325 + ], + "population_cv": 0.9317377718256953 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.2200722098350525, + "mean_weights": [ + 0.9384047985076904, + 0.06159520894289017 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5651201605796814, + "mean_weights": [ + 0.6557347774505615, + 0.3442652225494385 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4018177092075348, + "mean_weights": [ + 0.856421947479248, + 0.14357808232307434 + ], + "sources": 2 + }, + { + "entropy_mean": 0.687358021736145, + "mean_weights": [ + 0.46872246265411377, + 0.5312775373458862 + ], + "sources": 2 + }, + { + "entropy_mean": 0.2678314447402954, + "mean_weights": [ + 0.920531153678894, + 0.07946885377168655 + ], + "sources": 2 + }, + { + "entropy_mean": 0.657238245010376, + "mean_weights": [ + 0.6222448348999023, + 0.37775513529777527 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5044373869895935, + "mean_weights": [ + 0.7792346477508545, + 0.2207653522491455 + ], + "sources": 2 + }, + { + "entropy_mean": 0.650418758392334, + "mean_weights": [ + 0.5397791862487793, + 0.4602207541465759 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0032464265823364, + "mean_weights": [ + 0.4337552487850189, + 0.18956376612186432, + 0.37668097019195557 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0385558605194092, + "mean_weights": [ + 0.24478834867477417, + 0.2602521479129791, + 0.4949595034122467 + ], + "sources": 3 + }, + { + "entropy_mean": 1.031773328781128, + "mean_weights": [ + 0.40370821952819824, + 0.2651250958442688, + 0.33116668462753296 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0838522911071777, + "mean_weights": [ + 0.35612422227859497, + 0.2625962495803833, + 0.38127952814102173 + ], + "sources": 3 + }, + { + "entropy_mean": 0.96053147315979, + "mean_weights": [ + 0.5271762013435364, + 0.1581989824771881, + 0.3146248161792755 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0704245567321777, + "mean_weights": [ + 0.3788670599460602, + 0.23852786421775818, + 0.38260507583618164 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9674104452133179, + "mean_weights": [ + 0.48829180002212524, + 0.18356263637542725, + 0.3281455934047699 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0832805633544922, + "mean_weights": [ + 0.40388280153274536, + 0.2811351418495178, + 0.3149820566177368 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2917464971542358, + "mean_weights": [ + 0.3349481225013733, + 0.1244347095489502, + 0.20291922986507416, + 0.33769792318344116 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3533791303634644, + "mean_weights": [ + 0.32115638256073, + 0.20602071285247803, + 0.18808895349502563, + 0.28473395109176636 + ], + "sources": 4 + }, + { + "entropy_mean": 1.308367371559143, + "mean_weights": [ + 0.31139707565307617, + 0.16179001331329346, + 0.2296154499053955, + 0.29719749093055725 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3437637090682983, + "mean_weights": [ + 0.3217787742614746, + 0.1920534074306488, + 0.1763250231742859, + 0.3098427951335907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2280659675598145, + "mean_weights": [ + 0.425346314907074, + 0.14848899841308594, + 0.13232898712158203, + 0.29383569955825806 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3462648391723633, + "mean_weights": [ + 0.3221249580383301, + 0.18557843565940857, + 0.18214309215545654, + 0.3101535141468048 + ], + "sources": 4 + }, + { + "entropy_mean": 1.147221326828003, + "mean_weights": [ + 0.505380392074585, + 0.12089012563228607, + 0.11514648050069809, + 0.2585829496383667 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3729398250579834, + "mean_weights": [ + 0.30785465240478516, + 0.23079943656921387, + 0.23862706124782562, + 0.22271886467933655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5232064723968506, + "mean_weights": [ + 0.3136816620826721, + 0.11285122483968735, + 0.13196846842765808, + 0.19688263535499573, + 0.24461600184440613 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4876129627227783, + "mean_weights": [ + 0.2035508155822754, + 0.15372204780578613, + 0.13167491555213928, + 0.11437506973743439, + 0.396677166223526 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4587602615356445, + "mean_weights": [ + 0.28519001603126526, + 0.10532154142856598, + 0.09675675630569458, + 0.17811742424964905, + 0.33461427688598633 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4565696716308594, + "mean_weights": [ + 0.20232918858528137, + 0.14928825199604034, + 0.12029382586479187, + 0.1059027910232544, + 0.4221859574317932 + ], + "sources": 5 + }, + { + "entropy_mean": 1.4788098335266113, + "mean_weights": [ + 0.2697942852973938, + 0.11948063224554062, + 0.11090938746929169, + 0.15228068828582764, + 0.34753501415252686 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5018762350082397, + "mean_weights": [ + 0.208971306681633, + 0.27331358194351196, + 0.12766750156879425, + 0.07916607707738876, + 0.31088152527809143 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5627843141555786, + "mean_weights": [ + 0.27452588081359863, + 0.21448183059692383, + 0.143874391913414, + 0.14059773087501526, + 0.22652018070220947 + ], + "sources": 5 + }, + { + "entropy_mean": 1.580754280090332, + "mean_weights": [ + 0.2518634498119354, + 0.18758198618888855, + 0.13903988897800446, + 0.18044015765190125, + 0.24107450246810913 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7564777135849, + "mean_weights": [ + 0.15440437197685242, + 0.19411900639533997, + 0.17806550860404968, + 0.14696577191352844, + 0.10594475269317627, + 0.2205006331205368 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5899518728256226, + "mean_weights": [ + 0.1636897623538971, + 0.11789832264184952, + 0.07030131667852402, + 0.09238424897193909, + 0.1399868130683899, + 0.4157395362854004 + ], + "sources": 6 + }, + { + "entropy_mean": 1.2894535064697266, + "mean_weights": [ + 0.1212274506688118, + 0.06944876164197922, + 0.05726535990834236, + 0.09096578508615494, + 0.06335186958312988, + 0.5977407693862915 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6581614017486572, + "mean_weights": [ + 0.18132120370864868, + 0.12711825966835022, + 0.08560528606176376, + 0.09748884290456772, + 0.15072739124298096, + 0.35773903131484985 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3699336051940918, + "mean_weights": [ + 0.13604651391506195, + 0.10151475667953491, + 0.07715043425559998, + 0.11300846189260483, + 0.033717285841703415, + 0.538562536239624 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6926933526992798, + "mean_weights": [ + 0.19425281882286072, + 0.14001017808914185, + 0.09434761106967926, + 0.10331346094608307, + 0.14820793271064758, + 0.3198680281639099 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4148362874984741, + "mean_weights": [ + 0.16652247309684753, + 0.10329679399728775, + 0.08063912391662598, + 0.1165856122970581, + 0.033275946974754333, + 0.4996800422668457 + ], + "sources": 6 + }, + { + "entropy_mean": 1.740808129310608, + "mean_weights": [ + 0.21459883451461792, + 0.15640267729759216, + 0.10570268332958221, + 0.11097012460231781, + 0.17001523077487946, + 0.24231043457984924 + ], + "sources": 6 + }, + { + "entropy_mean": 1.3179612159729004, + "mean_weights": [ + 0.08068332821130753, + 0.049232255667448044, + 0.045073021203279495, + 0.06111945956945419, + 0.028317857533693314, + 0.13192236423492432, + 0.6036517024040222 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5319744348526, + "mean_weights": [ + 0.12406684458255768, + 0.07462044060230255, + 0.05378449708223343, + 0.05575074627995491, + 0.07357081025838852, + 0.10496053099632263, + 0.5132461786270142 + ], + "sources": 7 + }, + { + "entropy_mean": 1.2461028099060059, + "mean_weights": [ + 0.08556011319160461, + 0.0446893684566021, + 0.04084867238998413, + 0.05386578291654587, + 0.025119207799434662, + 0.11240880191326141, + 0.6375080347061157 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6323832273483276, + "mean_weights": [ + 0.1418324112892151, + 0.08991365134716034, + 0.06188817322254181, + 0.05739111453294754, + 0.09848876297473907, + 0.0972195416688919, + 0.45326635241508484 + ], + "sources": 7 + }, + { + "entropy_mean": 1.301480770111084, + "mean_weights": [ + 0.09513165056705475, + 0.05029121786355972, + 0.045042067766189575, + 0.05902485549449921, + 0.02615261822938919, + 0.1109299436211586, + 0.6134276390075684 + ], + "sources": 7 + }, + { + "entropy_mean": 1.629331111907959, + "mean_weights": [ + 0.14790503680706024, + 0.08658000826835632, + 0.06608416140079498, + 0.06427294760942459, + 0.0819322019815445, + 0.097825787961483, + 0.45539984107017517 + ], + "sources": 7 + }, + { + "entropy_mean": 1.338958501815796, + "mean_weights": [ + 0.10803166776895523, + 0.0559723861515522, + 0.047874726355075836, + 0.060293328016996384, + 0.029215168207883835, + 0.10392352938652039, + 0.5946891903877258 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7611865997314453, + "mean_weights": [ + 0.1573912799358368, + 0.10317619144916534, + 0.0853322446346283, + 0.08218386769294739, + 0.09052872657775879, + 0.11341015249490738, + 0.3679775595664978 + ], + "sources": 7 + }, + { + "entropy_mean": 1.476489782333374, + "mean_weights": [ + 0.09070298075675964, + 0.037138283252716064, + 0.030013620853424072, + 0.03533655032515526, + 0.020730752497911453, + 0.06278065592050552, + 0.3836256265640259, + 0.3396714925765991 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7213938236236572, + "mean_weights": [ + 0.10584281384944916, + 0.06887699663639069, + 0.05207539349794388, + 0.050678279250860214, + 0.04768219590187073, + 0.06809806823730469, + 0.2275032103061676, + 0.37924304604530334 + ], + "sources": 8 + }, + { + "entropy_mean": 1.5405172109603882, + "mean_weights": [ + 0.07428755611181259, + 0.04491765797138214, + 0.03515764698386192, + 0.046252522617578506, + 0.023427214473485947, + 0.07749318331480026, + 0.29666566848754883, + 0.4017985761165619 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8063910007476807, + "mean_weights": [ + 0.11433833837509155, + 0.07374332845211029, + 0.05932536721229553, + 0.05389238893985748, + 0.061260174959897995, + 0.07349519431591034, + 0.22022446990013123, + 0.3437207341194153 + ], + "sources": 8 + }, + { + "entropy_mean": 1.4859801530838013, + "mean_weights": [ + 0.08387120068073273, + 0.041406162083148956, + 0.032444946467876434, + 0.03686721250414848, + 0.025052767246961594, + 0.060124777257442474, + 0.27774184942245483, + 0.4424910843372345 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7777256965637207, + "mean_weights": [ + 0.10586117208003998, + 0.07087038457393646, + 0.05850498005747795, + 0.06360965967178345, + 0.04672512412071228, + 0.07485505938529968, + 0.2385886013507843, + 0.3409850001335144 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6318418979644775, + "mean_weights": [ + 0.09244748204946518, + 0.05762210488319397, + 0.045099519193172455, + 0.056236449629068375, + 0.029373466968536377, + 0.0723743885755539, + 0.31761035323143005, + 0.3292362093925476 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8334553241729736, + "mean_weights": [ + 0.1157165914773941, + 0.07161639630794525, + 0.06577154994010925, + 0.07425849884748459, + 0.04881399869918823, + 0.08197253942489624, + 0.27155569195747375, + 0.270294725894928 + ], + "sources": 8 + }, + { + "entropy_mean": 1.6227548122406006, + "mean_weights": [ + 0.0605507455766201, + 0.040063511580228806, + 0.032619886100292206, + 0.03868011012673378, + 0.02029048651456833, + 0.04760073497891426, + 0.19551873207092285, + 0.11294309794902802, + 0.45173269510269165 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9787883758544922, + "mean_weights": [ + 0.0788881778717041, + 0.08108963817358017, + 0.057054273784160614, + 0.08211588114500046, + 0.026040803641080856, + 0.10537256300449371, + 0.24502354860305786, + 0.12593510746955872, + 0.1984800100326538 + ], + "sources": 9 + }, + { + "entropy_mean": 1.7590525150299072, + "mean_weights": [ + 0.07131639868021011, + 0.044687941670417786, + 0.033290669322013855, + 0.043597716838121414, + 0.02658817172050476, + 0.06922741234302521, + 0.23554196953773499, + 0.14986532926559448, + 0.3258844017982483 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9736928939819336, + "mean_weights": [ + 0.09849467873573303, + 0.068942591547966, + 0.0599900484085083, + 0.06335455179214478, + 0.04671281576156616, + 0.0713595598936081, + 0.22560220956802368, + 0.1557777225971222, + 0.20976582169532776 + ], + "sources": 9 + }, + { + "entropy_mean": 1.809161901473999, + "mean_weights": [ + 0.08854075521230698, + 0.05356752127408981, + 0.04243364930152893, + 0.04702243581414223, + 0.02648795023560524, + 0.05820769816637039, + 0.18489578366279602, + 0.21069815754890442, + 0.2881460189819336 + ], + "sources": 9 + }, + { + "entropy_mean": 1.8544633388519287, + "mean_weights": [ + 0.07977832853794098, + 0.05858254432678223, + 0.053744811564683914, + 0.07626348733901978, + 0.031090255826711655, + 0.0994817242026329, + 0.3525030016899109, + 0.1274903416633606, + 0.12106549739837646 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9456019401550293, + "mean_weights": [ + 0.08823960274457932, + 0.06737187504768372, + 0.05197303742170334, + 0.0631948709487915, + 0.02739548310637474, + 0.08291902393102646, + 0.2524128556251526, + 0.186361163854599, + 0.18013209104537964 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.06279198825359344, + 0.0596221424639225, + 0.07690861076116562, + 0.07634982466697693, + 0.18058517575263977, + 0.06968431174755096, + 0.15882407128810883, + 0.11288930475711823, + 0.19892723858356476, + 0.11885104328393936, + 0.15647411346435547, + 0.1495160460472107, + 0.1533493995666504, + 0.11554427444934845, + 0.1527898609638214, + 0.13194940984249115, + 0.16355258226394653, + 0.08884616196155548, + 0.11789151281118393, + 0.10897023230791092, + 0.11456070095300674, + 0.09843935817480087, + 0.12148918211460114, + 0.09431816637516022, + 0.15094918012619019, + 0.09025302529335022, + 0.09854138642549515, + 0.0895693451166153, + 0.11213528364896774, + 0.10807885229587555, + 0.16254977881908417, + 0.2204425185918808, + 0.2323070764541626, + 0.15380074083805084, + 0.1390199512243271, + 0.08548793941736221, + 0.14812250435352325, + 0.09071414917707443, + 0.15333011746406555, + 0.10860678553581238, + 0.1781008094549179, + 0.04808543995022774, + 0.07593096792697906, + 0.05631518363952637, + 0.09691466391086578, + 0.07238366454839706, + 0.09368696808815002, + 0.08572373539209366, + 0.10724716633558273, + 0.06546958535909653, + 0.06538715958595276, + 0.06323372572660446, + 0.07412076741456985, + 0.06869969516992569, + 0.07655765861272812, + 0.07729499042034149, + 0.08420819044113159, + 0.04753003641963005, + 0.07230556756258011, + 0.06484207510948181, + 0.07723049819469452, + 0.07835857570171356, + 0.08422473073005676, + 0.08203105628490448 + ], + "loss_nats": 1.3013269901275635, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.596132755279541, + "mean_weights": [ + 0.09668220579624176, + 0.03708431497216225, + 0.034579962491989136, + 0.026012204587459564, + 0.0258483849465847, + 0.037477150559425354, + 0.0845583826303482, + 0.3670029640197754, + 0.29075440764427185 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.09253507107496262, + 0.15345558524131775, + 0.30484533309936523, + 0.3249167501926422, + 0.33420246839523315, + 0.3818626403808594, + 0.38682517409324646, + 0.40600064396858215, + 0.12552575767040253, + 0.1351035088300705, + 0.14749252796173096, + 0.15324510633945465, + 0.1726136952638626, + 0.18001793324947357, + 0.1892337054014206, + 0.20391276478767395, + 0.031880609691143036, + 0.04374876990914345, + 0.06123264133930206, + 0.08211024105548859, + 0.10403046011924744, + 0.1380448192358017, + 0.15778414905071259, + 0.2295975536108017, + 0.053015924990177155, + 0.08199761807918549, + 0.09510950744152069, + 0.13892096281051636, + 0.1597633957862854, + 0.21687114238739014, + 0.7120755910873413, + 0.7360114455223083, + 0.04075472429394722, + 0.057135771960020065, + 0.07218439877033234, + 0.08251863718032837, + 0.11449843645095825, + 0.12661415338516235, + 0.16539782285690308, + 0.19619159400463104, + 0.0493597537279129, + 0.06753352284431458, + 0.07179203629493713, + 0.095451220870018, + 0.10295005142688751, + 0.1274663656949997, + 0.1323672980070114, + 0.15628865361213684, + 0.023035654798150063, + 0.058436937630176544, + 0.06294231116771698, + 0.08794140070676804, + 0.09175154566764832, + 0.12247690558433533, + 0.1252463310956955, + 0.1640285700559616, + 0.03424545377492905, + 0.07537907361984253, + 0.07851024717092514, + 0.12290703505277634, + 0.12469635903835297, + 0.19157102704048157, + 0.19299884140491486, + 0.29249706864356995 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.034013546791773, + "cross_entropy_nats": 5.5687538385391235, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "cross_entropy_nats": 3.5575302243232727, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "cross_entropy_nats": 2.1426188945770264, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "cross_entropy_nats": 1.3501944988965988, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "cross_entropy_nats": 1.241536632180214, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "cross_entropy_nats": 1.1848131567239761, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d", + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.03141246394478, + "measured_steps": 7980, + "median_ms": 108.93342149211094, + "p95_ms": 109.53918776649516, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.018870646365954, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.558257579803467, + "step": 1, + "unclipped_grad_norm": 26.834381103515625 + }, + { + "bits_per_byte": 7.214299345307752, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.000571250915527, + "step": 10, + "unclipped_grad_norm": 12.740906715393066 + }, + { + "bits_per_byte": 6.684460697974439, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.633315086364746, + "step": 20, + "unclipped_grad_norm": 3.454355239868164 + }, + { + "bits_per_byte": 6.541223915307444, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.534030914306641, + "step": 30, + "unclipped_grad_norm": 3.060823440551758 + }, + { + "bits_per_byte": 6.411011034824987, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.443774223327637, + "step": 40, + "unclipped_grad_norm": 3.0002496242523193 + }, + { + "bits_per_byte": 6.085588902515397, + "learning_rate": 3.75e-05, + "loss_nats": 4.2182087898254395, + "step": 50, + "unclipped_grad_norm": 2.976402997970581 + }, + { + "bits_per_byte": 5.879418164452873, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.0753021240234375, + "step": 60, + "unclipped_grad_norm": 2.708517551422119 + }, + { + "bits_per_byte": 5.710178980836628, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9579944610595703, + "step": 70, + "unclipped_grad_norm": 2.559852361679077 + }, + { + "bits_per_byte": 5.512089019607543, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8206889629364014, + "step": 80, + "unclipped_grad_norm": 2.5563650131225586 + }, + { + "bits_per_byte": 5.341444737733749, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7024073600769043, + "step": 90, + "unclipped_grad_norm": 2.3861565589904785 + }, + { + "bits_per_byte": 5.169528816139475, + "learning_rate": 7.5e-05, + "loss_nats": 3.5832443237304688, + "step": 100, + "unclipped_grad_norm": 2.241894006729126 + }, + { + "bits_per_byte": 5.000991665704052, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.466423273086548, + "step": 110, + "unclipped_grad_norm": 2.1930899620056152 + }, + { + "bits_per_byte": 4.733823840860015, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2812366485595703, + "step": 120, + "unclipped_grad_norm": 1.8853988647460938 + }, + { + "bits_per_byte": 4.585053344418857, + "learning_rate": 9.75e-05, + "loss_nats": 3.178116798400879, + "step": 130, + "unclipped_grad_norm": 1.6627076864242554 + }, + { + "bits_per_byte": 4.391709416422983, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0441009998321533, + "step": 140, + "unclipped_grad_norm": 1.4586857557296753 + }, + { + "bits_per_byte": 4.260228335873064, + "learning_rate": 0.0001125, + "loss_nats": 2.952965259552002, + "step": 150, + "unclipped_grad_norm": 1.235063076019287 + }, + { + "bits_per_byte": 4.153197684831009, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.878777265548706, + "step": 160, + "unclipped_grad_norm": 1.0407519340515137 + }, + { + "bits_per_byte": 3.997309514401244, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.770723819732666, + "step": 170, + "unclipped_grad_norm": 0.9049285054206848 + }, + { + "bits_per_byte": 3.860879805055398, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.6761579513549805, + "step": 180, + "unclipped_grad_norm": 1.0858614444732666 + }, + { + "bits_per_byte": 3.870312021559318, + "learning_rate": 0.0001425, + "loss_nats": 2.6826958656311035, + "step": 190, + "unclipped_grad_norm": 0.6896197199821472 + }, + { + "bits_per_byte": 3.747811875037455, + "learning_rate": 0.00015, + "loss_nats": 2.597785234451294, + "step": 200, + "unclipped_grad_norm": 0.712465226650238 + }, + { + "bits_per_byte": 3.69472358243852, + "learning_rate": 0.0001575, + "loss_nats": 2.5609872341156006, + "step": 210, + "unclipped_grad_norm": 0.8807040452957153 + }, + { + "bits_per_byte": 3.662194783854328, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.538439989089966, + "step": 220, + "unclipped_grad_norm": 0.7631296515464783 + }, + { + "bits_per_byte": 3.6752317567203328, + "learning_rate": 0.0001725, + "loss_nats": 2.5474765300750732, + "step": 230, + "unclipped_grad_norm": 0.6869579553604126 + }, + { + "bits_per_byte": 3.6565805821995627, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.53454852104187, + "step": 240, + "unclipped_grad_norm": 0.781292736530304 + }, + { + "bits_per_byte": 3.5245367744160205, + "learning_rate": 0.0001875, + "loss_nats": 2.4430227279663086, + "step": 250, + "unclipped_grad_norm": 1.0127525329589844 + }, + { + "bits_per_byte": 3.524651658826797, + "learning_rate": 0.000195, + "loss_nats": 2.4431023597717285, + "step": 260, + "unclipped_grad_norm": 0.8615073561668396 + }, + { + "bits_per_byte": 3.485441678221887, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.415924072265625, + "step": 270, + "unclipped_grad_norm": 0.7731815576553345 + }, + { + "bits_per_byte": 3.4956767097400343, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.423018455505371, + "step": 280, + "unclipped_grad_norm": 0.9061474800109863 + }, + { + "bits_per_byte": 3.3938416545805445, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3524317741394043, + "step": 290, + "unclipped_grad_norm": 1.0149996280670166 + }, + { + "bits_per_byte": 3.483576010425327, + "learning_rate": 0.000225, + "loss_nats": 2.414630889892578, + "step": 300, + "unclipped_grad_norm": 1.1414116621017456 + }, + { + "bits_per_byte": 3.381058528107214, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.343571186065674, + "step": 310, + "unclipped_grad_norm": 1.1472890377044678 + }, + { + "bits_per_byte": 3.3707543595634197, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3364288806915283, + "step": 320, + "unclipped_grad_norm": 1.2828246355056763 + }, + { + "bits_per_byte": 3.3462052120139605, + "learning_rate": 0.0002475, + "loss_nats": 2.3194127082824707, + "step": 330, + "unclipped_grad_norm": 0.9300209879875183 + }, + { + "bits_per_byte": 3.3510090314178607, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.322742462158203, + "step": 340, + "unclipped_grad_norm": 0.7300398945808411 + }, + { + "bits_per_byte": 3.324897249502662, + "learning_rate": 0.0002625, + "loss_nats": 2.304643154144287, + "step": 350, + "unclipped_grad_norm": 0.96440589427948 + }, + { + "bits_per_byte": 3.301325651340707, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.288304567337036, + "step": 360, + "unclipped_grad_norm": 1.186806321144104 + }, + { + "bits_per_byte": 3.330423052074884, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3084733486175537, + "step": 370, + "unclipped_grad_norm": 1.40411376953125 + }, + { + "bits_per_byte": 3.351273196769586, + "learning_rate": 0.000285, + "loss_nats": 2.322925567626953, + "step": 380, + "unclipped_grad_norm": 0.8571750521659851 + }, + { + "bits_per_byte": 3.2969108566930587, + "learning_rate": 0.0002925, + "loss_nats": 2.2852444648742676, + "step": 390, + "unclipped_grad_norm": 1.172652006149292 + }, + { + "bits_per_byte": 3.2316042286462263, + "learning_rate": 0.0003, + "loss_nats": 2.2399773597717285, + "step": 400, + "unclipped_grad_norm": 1.0324530601501465 + }, + { + "bits_per_byte": 3.191360632309631, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.212082624435425, + "step": 410, + "unclipped_grad_norm": 0.9553669691085815 + }, + { + "bits_per_byte": 3.2040835648551598, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2209014892578125, + "step": 420, + "unclipped_grad_norm": 1.0212959051132202 + }, + { + "bits_per_byte": 3.1886670400318176, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2102155685424805, + "step": 430, + "unclipped_grad_norm": 0.8366522789001465 + }, + { + "bits_per_byte": 3.2100664973133752, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.225048542022705, + "step": 440, + "unclipped_grad_norm": 1.1677532196044922 + }, + { + "bits_per_byte": 3.1886219805772913, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.210184335708618, + "step": 450, + "unclipped_grad_norm": 1.3463479280471802 + }, + { + "bits_per_byte": 3.136037597145384, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.1737356185913086, + "step": 460, + "unclipped_grad_norm": 1.704139232635498 + }, + { + "bits_per_byte": 3.1589835224235, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.189640522003174, + "step": 470, + "unclipped_grad_norm": 1.1849488019943237 + }, + { + "bits_per_byte": 3.1210654754918714, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.163357734680176, + "step": 480, + "unclipped_grad_norm": 0.8924318552017212 + }, + { + "bits_per_byte": 3.0823876092087312, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1365482807159424, + "step": 490, + "unclipped_grad_norm": 1.3279240131378174 + }, + { + "bits_per_byte": 3.12389562199447, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1653194427490234, + "step": 500, + "unclipped_grad_norm": 1.3776742219924927 + }, + { + "bits_per_byte": 3.102860080002138, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1507387161254883, + "step": 510, + "unclipped_grad_norm": 1.061196208000183 + }, + { + "bits_per_byte": 2.961672642602824, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.05287504196167, + "step": 520, + "unclipped_grad_norm": 1.0864008665084839 + }, + { + "bits_per_byte": 3.0521231342057993, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.115570545196533, + "step": 530, + "unclipped_grad_norm": 0.9000556468963623 + }, + { + "bits_per_byte": 2.992112883874938, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.073974609375, + "step": 540, + "unclipped_grad_norm": 0.8655318021774292 + }, + { + "bits_per_byte": 2.9456885750316344, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0417957305908203, + "step": 550, + "unclipped_grad_norm": 0.8011268973350525 + }, + { + "bits_per_byte": 2.9526635034200264, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.046630382537842, + "step": 560, + "unclipped_grad_norm": 1.2049078941345215 + }, + { + "bits_per_byte": 2.943776471919342, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0404703617095947, + "step": 570, + "unclipped_grad_norm": 0.9911091923713684 + }, + { + "bits_per_byte": 2.836173462615215, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.9658856391906738, + "step": 580, + "unclipped_grad_norm": 0.9621384143829346 + }, + { + "bits_per_byte": 2.898327304706431, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.008967399597168, + "step": 590, + "unclipped_grad_norm": 1.0092018842697144 + }, + { + "bits_per_byte": 2.807946982025008, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9463205337524414, + "step": 600, + "unclipped_grad_norm": 1.041255235671997 + }, + { + "bits_per_byte": 2.8662821213364653, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.98675537109375, + "step": 610, + "unclipped_grad_norm": 1.0585267543792725 + }, + { + "bits_per_byte": 2.7525066546928727, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9078922271728516, + "step": 620, + "unclipped_grad_norm": 0.9587937593460083 + }, + { + "bits_per_byte": 2.7676626258001575, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9183975458145142, + "step": 630, + "unclipped_grad_norm": 0.9836012721061707 + }, + { + "bits_per_byte": 2.7750938241612886, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9235484600067139, + "step": 640, + "unclipped_grad_norm": 0.9689151644706726 + }, + { + "bits_per_byte": 2.6921524390602856, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8660578727722168, + "step": 650, + "unclipped_grad_norm": 1.0390667915344238 + }, + { + "bits_per_byte": 2.6851900654054064, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8612319231033325, + "step": 660, + "unclipped_grad_norm": 1.6513670682907104 + }, + { + "bits_per_byte": 2.6846809967588525, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8608790636062622, + "step": 670, + "unclipped_grad_norm": 0.9175130724906921 + }, + { + "bits_per_byte": 2.7078205745018398, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8769181966781616, + "step": 680, + "unclipped_grad_norm": 1.0189234018325806 + }, + { + "bits_per_byte": 2.5875252455640383, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.793535828590393, + "step": 690, + "unclipped_grad_norm": 0.9444974064826965 + }, + { + "bits_per_byte": 2.5413828162854992, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.761552333831787, + "step": 700, + "unclipped_grad_norm": 1.1904406547546387 + }, + { + "bits_per_byte": 2.623998466245768, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.818817138671875, + "step": 710, + "unclipped_grad_norm": 1.1858620643615723 + }, + { + "bits_per_byte": 2.683836561943116, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8602937459945679, + "step": 720, + "unclipped_grad_norm": 0.9852609634399414 + }, + { + "bits_per_byte": 2.4849188482154285, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.722414493560791, + "step": 730, + "unclipped_grad_norm": 1.1975579261779785 + }, + { + "bits_per_byte": 2.5244201674309004, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7497947216033936, + "step": 740, + "unclipped_grad_norm": 0.9646193385124207 + }, + { + "bits_per_byte": 2.5296962511940695, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7534518241882324, + "step": 750, + "unclipped_grad_norm": 1.191988468170166 + }, + { + "bits_per_byte": 2.4988650933565446, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7320812940597534, + "step": 760, + "unclipped_grad_norm": 1.092863917350769 + }, + { + "bits_per_byte": 2.4838112799438723, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.721646785736084, + "step": 770, + "unclipped_grad_norm": 1.0348618030548096 + }, + { + "bits_per_byte": 2.4223032327066463, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.679012656211853, + "step": 780, + "unclipped_grad_norm": 1.034103274345398 + }, + { + "bits_per_byte": 2.485370818621896, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7227277755737305, + "step": 790, + "unclipped_grad_norm": 1.060265302658081 + }, + { + "bits_per_byte": 2.4233009040643014, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.679704189300537, + "step": 800, + "unclipped_grad_norm": 1.3169445991516113 + }, + { + "bits_per_byte": 2.4599308009409677, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7050940990447998, + "step": 810, + "unclipped_grad_norm": 0.9821087121963501 + }, + { + "bits_per_byte": 2.425854846429614, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6814744472503662, + "step": 820, + "unclipped_grad_norm": 1.0924170017242432 + }, + { + "bits_per_byte": 2.4277851797028984, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6828124523162842, + "step": 830, + "unclipped_grad_norm": 1.055789828300476 + }, + { + "bits_per_byte": 2.3982261775338056, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6623237133026123, + "step": 840, + "unclipped_grad_norm": 1.230089545249939 + }, + { + "bits_per_byte": 2.339956047664374, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.621933937072754, + "step": 850, + "unclipped_grad_norm": 0.94926518201828 + }, + { + "bits_per_byte": 2.3803994878412724, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6499671936035156, + "step": 860, + "unclipped_grad_norm": 1.1705771684646606 + }, + { + "bits_per_byte": 2.3436123988217177, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6244683265686035, + "step": 870, + "unclipped_grad_norm": 1.1496235132217407 + }, + { + "bits_per_byte": 2.3796778486382517, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6494669914245605, + "step": 880, + "unclipped_grad_norm": 1.1764546632766724 + }, + { + "bits_per_byte": 2.357840007579645, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.634330153465271, + "step": 890, + "unclipped_grad_norm": 0.9926145672798157 + }, + { + "bits_per_byte": 2.2929997964819524, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.5893863439559937, + "step": 900, + "unclipped_grad_norm": 1.009945034980774 + }, + { + "bits_per_byte": 2.3411444477818364, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6227576732635498, + "step": 910, + "unclipped_grad_norm": 1.2830723524093628 + }, + { + "bits_per_byte": 2.2736926801307917, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5760036706924438, + "step": 920, + "unclipped_grad_norm": 1.0900828838348389 + }, + { + "bits_per_byte": 2.3038844064724087, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.596930980682373, + "step": 930, + "unclipped_grad_norm": 1.1090145111083984 + }, + { + "bits_per_byte": 2.2389674911124313, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.551934003829956, + "step": 940, + "unclipped_grad_norm": 0.9897664189338684 + }, + { + "bits_per_byte": 2.2434025797128814, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5550081729888916, + "step": 950, + "unclipped_grad_norm": 1.0138952732086182 + }, + { + "bits_per_byte": 2.2236732459538526, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5413328409194946, + "step": 960, + "unclipped_grad_norm": 1.0875126123428345 + }, + { + "bits_per_byte": 2.2667958318658914, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5712231397628784, + "step": 970, + "unclipped_grad_norm": 1.245374321937561 + }, + { + "bits_per_byte": 2.2344890628839638, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5488297939300537, + "step": 980, + "unclipped_grad_norm": 1.1757959127426147 + }, + { + "bits_per_byte": 2.1613678871472266, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.4981460571289062, + "step": 990, + "unclipped_grad_norm": 1.066755771636963 + }, + { + "bits_per_byte": 2.207448574654106, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5300867557525635, + "step": 1000, + "unclipped_grad_norm": 0.9658553004264832 + }, + { + "bits_per_byte": 2.213320750285166, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5341570377349854, + "step": 1010, + "unclipped_grad_norm": 1.1267611980438232 + }, + { + "bits_per_byte": 2.2186436133293697, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.537846565246582, + "step": 1020, + "unclipped_grad_norm": 0.9853240251541138 + }, + { + "bits_per_byte": 2.1280228589020402, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4750330448150635, + "step": 1030, + "unclipped_grad_norm": 0.972481369972229 + }, + { + "bits_per_byte": 2.2322469250646653, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5472756624221802, + "step": 1040, + "unclipped_grad_norm": 1.0531573295593262 + }, + { + "bits_per_byte": 2.179971250491057, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5110409259796143, + "step": 1050, + "unclipped_grad_norm": 0.9985654950141907 + }, + { + "bits_per_byte": 2.1476089311129005, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.4886090755462646, + "step": 1060, + "unclipped_grad_norm": 1.100019931793213 + }, + { + "bits_per_byte": 2.2640807417567204, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5693411827087402, + "step": 1070, + "unclipped_grad_norm": 1.0959177017211914 + }, + { + "bits_per_byte": 2.1797610876917024, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5108952522277832, + "step": 1080, + "unclipped_grad_norm": 1.088409423828125 + }, + { + "bits_per_byte": 2.1080023585151006, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.461155891418457, + "step": 1090, + "unclipped_grad_norm": 1.0188393592834473 + }, + { + "bits_per_byte": 2.1918825969071642, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5192972421646118, + "step": 1100, + "unclipped_grad_norm": 0.9501848816871643 + }, + { + "bits_per_byte": 2.205643788715952, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5288357734680176, + "step": 1110, + "unclipped_grad_norm": 0.9797388911247253 + }, + { + "bits_per_byte": 2.0930500148664373, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4507917165756226, + "step": 1120, + "unclipped_grad_norm": 0.8856902122497559 + }, + { + "bits_per_byte": 2.095190166973774, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4522751569747925, + "step": 1130, + "unclipped_grad_norm": 0.9447507262229919 + }, + { + "bits_per_byte": 2.116739593126884, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4672120809555054, + "step": 1140, + "unclipped_grad_norm": 1.0488460063934326 + }, + { + "bits_per_byte": 2.1023439573190634, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4572337865829468, + "step": 1150, + "unclipped_grad_norm": 0.9838997721672058 + }, + { + "bits_per_byte": 2.0562952585852448, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.425315260887146, + "step": 1160, + "unclipped_grad_norm": 0.9282211065292358 + }, + { + "bits_per_byte": 2.160992449020393, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.497885823249817, + "step": 1170, + "unclipped_grad_norm": 1.0253955125808716 + }, + { + "bits_per_byte": 2.0716469459492117, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4359562397003174, + "step": 1180, + "unclipped_grad_norm": 0.9352066516876221 + }, + { + "bits_per_byte": 2.0893013090255805, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4481933116912842, + "step": 1190, + "unclipped_grad_norm": 1.0826658010482788 + }, + { + "bits_per_byte": 2.0752424152481495, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.438448429107666, + "step": 1200, + "unclipped_grad_norm": 0.9942638874053955 + }, + { + "bits_per_byte": 2.2049758081000004, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5283727645874023, + "step": 1210, + "unclipped_grad_norm": 1.1482313871383667 + }, + { + "bits_per_byte": 2.0506430487646385, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4213974475860596, + "step": 1220, + "unclipped_grad_norm": 0.9142818450927734 + }, + { + "bits_per_byte": 2.00269480165204, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.3881622552871704, + "step": 1230, + "unclipped_grad_norm": 0.9387761354446411 + }, + { + "bits_per_byte": 2.0166881700394925, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.3978617191314697, + "step": 1240, + "unclipped_grad_norm": 1.0545530319213867 + }, + { + "bits_per_byte": 2.1787518934964396, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5101957321166992, + "step": 1250, + "unclipped_grad_norm": 1.1504697799682617 + }, + { + "bits_per_byte": 2.1047604855463367, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4589087963104248, + "step": 1260, + "unclipped_grad_norm": 0.9371413588523865 + }, + { + "bits_per_byte": 1.9885089846782722, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3783293962478638, + "step": 1270, + "unclipped_grad_norm": 0.9821643233299255 + }, + { + "bits_per_byte": 2.1346452228561636, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4796233177185059, + "step": 1280, + "unclipped_grad_norm": 0.9382128715515137 + }, + { + "bits_per_byte": 2.022505311222276, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4018938541412354, + "step": 1290, + "unclipped_grad_norm": 0.9164487719535828 + }, + { + "bits_per_byte": 2.050535903573151, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4213231801986694, + "step": 1300, + "unclipped_grad_norm": 0.9040083885192871 + }, + { + "bits_per_byte": 2.128253659619498, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4751930236816406, + "step": 1310, + "unclipped_grad_norm": 1.045804738998413 + }, + { + "bits_per_byte": 2.0627867437420613, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4298148155212402, + "step": 1320, + "unclipped_grad_norm": 1.0330690145492554 + }, + { + "bits_per_byte": 1.954123977343318, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3544955253601074, + "step": 1330, + "unclipped_grad_norm": 0.9727330803871155 + }, + { + "bits_per_byte": 1.9791288789175636, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3718276023864746, + "step": 1340, + "unclipped_grad_norm": 0.966886043548584 + }, + { + "bits_per_byte": 2.1481475807754027, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4889824390411377, + "step": 1350, + "unclipped_grad_norm": 0.9399362802505493 + }, + { + "bits_per_byte": 1.9898208683390537, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3792387247085571, + "step": 1360, + "unclipped_grad_norm": 0.9223746061325073 + }, + { + "bits_per_byte": 2.0666878262115826, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4325188398361206, + "step": 1370, + "unclipped_grad_norm": 0.9086402654647827 + }, + { + "bits_per_byte": 2.05495087020345, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4243834018707275, + "step": 1380, + "unclipped_grad_norm": 0.954355776309967 + }, + { + "bits_per_byte": 2.067550663170961, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4331169128417969, + "step": 1390, + "unclipped_grad_norm": 0.8977138996124268 + }, + { + "bits_per_byte": 2.0721085473841274, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4362761974334717, + "step": 1400, + "unclipped_grad_norm": 0.8833977580070496 + }, + { + "bits_per_byte": 1.9799110560136877, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.3723697662353516, + "step": 1410, + "unclipped_grad_norm": 0.9529625177383423 + }, + { + "bits_per_byte": 2.038817521691312, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4132006168365479, + "step": 1420, + "unclipped_grad_norm": 1.083083152770996 + }, + { + "bits_per_byte": 2.0292872750764, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4065947532653809, + "step": 1430, + "unclipped_grad_norm": 0.9355389475822449 + }, + { + "bits_per_byte": 1.987968787171912, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3779549598693848, + "step": 1440, + "unclipped_grad_norm": 0.8836045265197754 + }, + { + "bits_per_byte": 2.0423230440638482, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4156304597854614, + "step": 1450, + "unclipped_grad_norm": 0.924534022808075 + }, + { + "bits_per_byte": 1.9816687187055055, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3735880851745605, + "step": 1460, + "unclipped_grad_norm": 0.8437807559967041 + }, + { + "bits_per_byte": 1.9797782854072217, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3722777366638184, + "step": 1470, + "unclipped_grad_norm": 0.9143778085708618 + }, + { + "bits_per_byte": 1.8767507025466492, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3008644580841064, + "step": 1480, + "unclipped_grad_norm": 0.8930306434631348 + }, + { + "bits_per_byte": 2.001770738868954, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.387521743774414, + "step": 1490, + "unclipped_grad_norm": 0.8246955871582031 + }, + { + "bits_per_byte": 1.9125255016832254, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3256616592407227, + "step": 1500, + "unclipped_grad_norm": 0.9061107635498047 + }, + { + "bits_per_byte": 1.973825621895558, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3681516647338867, + "step": 1510, + "unclipped_grad_norm": 0.9889123439788818 + }, + { + "bits_per_byte": 1.980869687309597, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3730342388153076, + "step": 1520, + "unclipped_grad_norm": 0.9045401811599731 + }, + { + "bits_per_byte": 1.9378121108395843, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.343189001083374, + "step": 1530, + "unclipped_grad_norm": 0.8896906971931458 + }, + { + "bits_per_byte": 2.017761685746178, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3986058235168457, + "step": 1540, + "unclipped_grad_norm": 0.9600386619567871 + }, + { + "bits_per_byte": 1.9800400430018348, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3724591732025146, + "step": 1550, + "unclipped_grad_norm": 0.8903042078018188 + }, + { + "bits_per_byte": 2.037820882229562, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.412509799003601, + "step": 1560, + "unclipped_grad_norm": 0.9084417819976807 + }, + { + "bits_per_byte": 2.015308525214271, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3969054222106934, + "step": 1570, + "unclipped_grad_norm": 0.9122006893157959 + }, + { + "bits_per_byte": 1.985575476602505, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.376296043395996, + "step": 1580, + "unclipped_grad_norm": 0.8903481960296631 + }, + { + "bits_per_byte": 1.996281568601366, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3837169408798218, + "step": 1590, + "unclipped_grad_norm": 0.933029294013977 + }, + { + "bits_per_byte": 2.002402775110875, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3879598379135132, + "step": 1600, + "unclipped_grad_norm": 0.9777830243110657 + }, + { + "bits_per_byte": 2.0119596790366705, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3945841789245605, + "step": 1610, + "unclipped_grad_norm": 0.9627588391304016 + }, + { + "bits_per_byte": 2.0159194075901357, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3973288536071777, + "step": 1620, + "unclipped_grad_norm": 0.8731178641319275 + }, + { + "bits_per_byte": 1.8905791395719242, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3104496002197266, + "step": 1630, + "unclipped_grad_norm": 0.8998029828071594 + }, + { + "bits_per_byte": 2.0235616286638756, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4026260375976562, + "step": 1640, + "unclipped_grad_norm": 0.9714035391807556 + }, + { + "bits_per_byte": 1.9588359580116574, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3577616214752197, + "step": 1650, + "unclipped_grad_norm": 0.9218164682388306 + }, + { + "bits_per_byte": 1.936817019221692, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3424992561340332, + "step": 1660, + "unclipped_grad_norm": 0.8974215984344482 + }, + { + "bits_per_byte": 2.096671109580354, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4533016681671143, + "step": 1670, + "unclipped_grad_norm": 0.9378093481063843 + }, + { + "bits_per_byte": 2.0150522710644854, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3967278003692627, + "step": 1680, + "unclipped_grad_norm": 0.8720370531082153 + }, + { + "bits_per_byte": 1.8838673446393523, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3057973384857178, + "step": 1690, + "unclipped_grad_norm": 0.8577803373336792 + }, + { + "bits_per_byte": 1.927341291107104, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3359311819076538, + "step": 1700, + "unclipped_grad_norm": 0.9562329053878784 + }, + { + "bits_per_byte": 1.9962937793695772, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3837254047393799, + "step": 1710, + "unclipped_grad_norm": 0.9462636113166809 + }, + { + "bits_per_byte": 1.995081645646296, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.382885217666626, + "step": 1720, + "unclipped_grad_norm": 0.8408851027488708 + }, + { + "bits_per_byte": 1.894888164889292, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3134363889694214, + "step": 1730, + "unclipped_grad_norm": 0.8187386989593506 + }, + { + "bits_per_byte": 1.9475287866480313, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.349924087524414, + "step": 1740, + "unclipped_grad_norm": 0.9181671142578125 + }, + { + "bits_per_byte": 2.0484275682562236, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4198617935180664, + "step": 1750, + "unclipped_grad_norm": 0.9259082674980164 + }, + { + "bits_per_byte": 1.9089950418263137, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3232145309448242, + "step": 1760, + "unclipped_grad_norm": 0.9000674486160278 + }, + { + "bits_per_byte": 1.9653874651136254, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3623027801513672, + "step": 1770, + "unclipped_grad_norm": 0.8686579465866089 + }, + { + "bits_per_byte": 1.9032847018697159, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3192564249038696, + "step": 1780, + "unclipped_grad_norm": 0.8219738006591797 + }, + { + "bits_per_byte": 1.8754542973244455, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2999658584594727, + "step": 1790, + "unclipped_grad_norm": 0.8401536345481873 + }, + { + "bits_per_byte": 1.9695277754504965, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3651726245880127, + "step": 1800, + "unclipped_grad_norm": 0.9855192303657532 + }, + { + "bits_per_byte": 1.952977025044714, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3537005186080933, + "step": 1810, + "unclipped_grad_norm": 0.9152796268463135 + }, + { + "bits_per_byte": 2.0236675699768067, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4026994705200195, + "step": 1820, + "unclipped_grad_norm": 0.9160932898521423 + }, + { + "bits_per_byte": 1.863515605666851, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.2916905879974365, + "step": 1830, + "unclipped_grad_norm": 0.8749093413352966 + }, + { + "bits_per_byte": 1.9532654399502112, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.35390043258667, + "step": 1840, + "unclipped_grad_norm": 0.8691527843475342 + }, + { + "bits_per_byte": 1.8425976998077092, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.277191400527954, + "step": 1850, + "unclipped_grad_norm": 0.918012797832489 + }, + { + "bits_per_byte": 1.9936586611930571, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3818988800048828, + "step": 1860, + "unclipped_grad_norm": 0.9178117513656616 + }, + { + "bits_per_byte": 1.995530520365048, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3831963539123535, + "step": 1870, + "unclipped_grad_norm": 0.8769021034240723 + }, + { + "bits_per_byte": 1.8636951555543517, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.2918150424957275, + "step": 1880, + "unclipped_grad_norm": 0.8614026308059692 + }, + { + "bits_per_byte": 1.9426209177403593, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3465222120285034, + "step": 1890, + "unclipped_grad_norm": 0.8975904583930969 + }, + { + "bits_per_byte": 1.861964838104021, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2906156778335571, + "step": 1900, + "unclipped_grad_norm": 0.8269307613372803 + }, + { + "bits_per_byte": 1.8502344174366214, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.282484769821167, + "step": 1910, + "unclipped_grad_norm": 0.8237605690956116 + }, + { + "bits_per_byte": 1.9149965484108225, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3273744583129883, + "step": 1920, + "unclipped_grad_norm": 0.8641052842140198 + }, + { + "bits_per_byte": 1.8801597426520518, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.303227424621582, + "step": 1930, + "unclipped_grad_norm": 0.8907292485237122 + }, + { + "bits_per_byte": 1.8629336163763313, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2912871837615967, + "step": 1940, + "unclipped_grad_norm": 0.8837904930114746 + }, + { + "bits_per_byte": 1.923144226495448, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3330219984054565, + "step": 1950, + "unclipped_grad_norm": 1.0997384786605835 + }, + { + "bits_per_byte": 1.88083511852199, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.303695559501648, + "step": 1960, + "unclipped_grad_norm": 0.8385953307151794 + }, + { + "bits_per_byte": 1.8359603733629608, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2725907564163208, + "step": 1970, + "unclipped_grad_norm": 0.839948832988739 + }, + { + "bits_per_byte": 1.7988822896981462, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2468901872634888, + "step": 1980, + "unclipped_grad_norm": 0.7658324837684631 + }, + { + "bits_per_byte": 1.9198664091526536, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3307499885559082, + "step": 1990, + "unclipped_grad_norm": 0.8840731382369995 + }, + { + "bits_per_byte": 1.863330208369221, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.2915620803833008, + "step": 2000, + "unclipped_grad_norm": 0.8192090392112732 + }, + { + "bits_per_byte": 1.9331472534175815, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3399555683135986, + "step": 2010, + "unclipped_grad_norm": 0.8343153595924377 + }, + { + "bits_per_byte": 1.9117808168049895, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3251454830169678, + "step": 2020, + "unclipped_grad_norm": 0.913158118724823 + }, + { + "bits_per_byte": 1.8970498148279866, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3149347305297852, + "step": 2030, + "unclipped_grad_norm": 0.8877862691879272 + }, + { + "bits_per_byte": 1.9115138997308505, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.324960470199585, + "step": 2040, + "unclipped_grad_norm": 0.8749900460243225 + }, + { + "bits_per_byte": 1.8657040848990802, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2932075262069702, + "step": 2050, + "unclipped_grad_norm": 0.9185486435890198 + }, + { + "bits_per_byte": 1.7675556498434857, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.225176215171814, + "step": 2060, + "unclipped_grad_norm": 0.7904725670814514 + }, + { + "bits_per_byte": 1.8617049723185672, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.290435552597046, + "step": 2070, + "unclipped_grad_norm": 0.8348630666732788 + }, + { + "bits_per_byte": 1.973646415973359, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3680274486541748, + "step": 2080, + "unclipped_grad_norm": 0.8117388486862183 + }, + { + "bits_per_byte": 1.8279640400110946, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2670481204986572, + "step": 2090, + "unclipped_grad_norm": 0.7758682370185852 + }, + { + "bits_per_byte": 1.8962239541385433, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3143622875213623, + "step": 2100, + "unclipped_grad_norm": 0.8137099146842957 + }, + { + "bits_per_byte": 1.8350517890184526, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.271960973739624, + "step": 2110, + "unclipped_grad_norm": 0.9251447319984436 + }, + { + "bits_per_byte": 1.870564486595114, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2965764999389648, + "step": 2120, + "unclipped_grad_norm": 0.9138389825820923 + }, + { + "bits_per_byte": 1.8882444750864615, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3088313341140747, + "step": 2130, + "unclipped_grad_norm": 0.8262670040130615 + }, + { + "bits_per_byte": 1.7726164113277747, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2286840677261353, + "step": 2140, + "unclipped_grad_norm": 0.7359873056411743 + }, + { + "bits_per_byte": 1.8268258588276844, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2662591934204102, + "step": 2150, + "unclipped_grad_norm": 0.8173519372940063 + }, + { + "bits_per_byte": 1.8637190611428216, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.291831612586975, + "step": 2160, + "unclipped_grad_norm": 0.8174240589141846 + }, + { + "bits_per_byte": 1.8528119214251022, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2842713594436646, + "step": 2170, + "unclipped_grad_norm": 0.8957990407943726 + }, + { + "bits_per_byte": 1.782856774308099, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2357821464538574, + "step": 2180, + "unclipped_grad_norm": 0.8048707246780396 + }, + { + "bits_per_byte": 1.8923490130319534, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3116763830184937, + "step": 2190, + "unclipped_grad_norm": 0.8952655792236328 + }, + { + "bits_per_byte": 1.9442205283760343, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3476309776306152, + "step": 2200, + "unclipped_grad_norm": 0.8895230889320374 + }, + { + "bits_per_byte": 1.9876000563684622, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.377699375152588, + "step": 2210, + "unclipped_grad_norm": 0.8686981797218323 + }, + { + "bits_per_byte": 1.852814845130167, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.284273386001587, + "step": 2220, + "unclipped_grad_norm": 0.8053512573242188 + }, + { + "bits_per_byte": 1.859576858996791, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2889604568481445, + "step": 2230, + "unclipped_grad_norm": 0.8349927663803101 + }, + { + "bits_per_byte": 1.8206912376714082, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2620069980621338, + "step": 2240, + "unclipped_grad_norm": 0.8471757769584656 + }, + { + "bits_per_byte": 1.8270817690121683, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2664365768432617, + "step": 2250, + "unclipped_grad_norm": 0.8009039163589478 + }, + { + "bits_per_byte": 1.8453999851208678, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2791337966918945, + "step": 2260, + "unclipped_grad_norm": 0.8003636002540588 + }, + { + "bits_per_byte": 1.9009572606555896, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.317643165588379, + "step": 2270, + "unclipped_grad_norm": 0.8468067646026611 + }, + { + "bits_per_byte": 1.7920475271702074, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2421526908874512, + "step": 2280, + "unclipped_grad_norm": 0.9071624875068665 + }, + { + "bits_per_byte": 1.8281408381761814, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2671706676483154, + "step": 2290, + "unclipped_grad_norm": 0.822325587272644 + }, + { + "bits_per_byte": 1.8827026781277094, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3049900531768799, + "step": 2300, + "unclipped_grad_norm": 0.8915430307388306 + }, + { + "bits_per_byte": 1.831042185496237, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.269181728363037, + "step": 2310, + "unclipped_grad_norm": 0.8500391840934753 + }, + { + "bits_per_byte": 1.8431096921593277, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2775462865829468, + "step": 2320, + "unclipped_grad_norm": 0.8417274355888367 + }, + { + "bits_per_byte": 1.7906957435344257, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.241215705871582, + "step": 2330, + "unclipped_grad_norm": 0.777140736579895 + }, + { + "bits_per_byte": 1.825728265549878, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.265498399734497, + "step": 2340, + "unclipped_grad_norm": 0.8567083477973938 + }, + { + "bits_per_byte": 1.7348746466132805, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2025234699249268, + "step": 2350, + "unclipped_grad_norm": 0.7593752145767212 + }, + { + "bits_per_byte": 1.8440781264663364, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2782175540924072, + "step": 2360, + "unclipped_grad_norm": 0.8551052212715149 + }, + { + "bits_per_byte": 1.871989878805465, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2975645065307617, + "step": 2370, + "unclipped_grad_norm": 0.8494746685028076 + }, + { + "bits_per_byte": 1.7757177744707837, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2308337688446045, + "step": 2380, + "unclipped_grad_norm": 0.7841326594352722 + }, + { + "bits_per_byte": 1.7735628318554726, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2293400764465332, + "step": 2390, + "unclipped_grad_norm": 0.8951802849769592 + }, + { + "bits_per_byte": 1.8680005692360513, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2947993278503418, + "step": 2400, + "unclipped_grad_norm": 0.8339796662330627 + }, + { + "bits_per_byte": 1.8459044102358484, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.279483437538147, + "step": 2410, + "unclipped_grad_norm": 0.9356110692024231 + }, + { + "bits_per_byte": 1.931278833898608, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.338660478591919, + "step": 2420, + "unclipped_grad_norm": 0.8573734760284424 + }, + { + "bits_per_byte": 1.825236739133712, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265157699584961, + "step": 2430, + "unclipped_grad_norm": 0.8783476948738098 + }, + { + "bits_per_byte": 1.7732133631089195, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.229097843170166, + "step": 2440, + "unclipped_grad_norm": 0.8393950462341309 + }, + { + "bits_per_byte": 1.8696945983470499, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.295973539352417, + "step": 2450, + "unclipped_grad_norm": 0.8328628540039062 + }, + { + "bits_per_byte": 1.8721511985319743, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2976763248443604, + "step": 2460, + "unclipped_grad_norm": 0.8645251989364624 + }, + { + "bits_per_byte": 1.8203225068679583, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.261751413345337, + "step": 2470, + "unclipped_grad_norm": 0.8000954985618591 + }, + { + "bits_per_byte": 1.8206716316492098, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.261993408203125, + "step": 2480, + "unclipped_grad_norm": 0.7867690920829773 + }, + { + "bits_per_byte": 1.7380601092725614, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2047314643859863, + "step": 2490, + "unclipped_grad_norm": 0.7984139919281006 + }, + { + "bits_per_byte": 1.822108202731867, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2629891633987427, + "step": 2500, + "unclipped_grad_norm": 0.7798662781715393 + }, + { + "bits_per_byte": 1.7878630172920644, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2392522096633911, + "step": 2510, + "unclipped_grad_norm": 0.8921790719032288 + }, + { + "bits_per_byte": 1.7912486677569497, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2415989637374878, + "step": 2520, + "unclipped_grad_norm": 0.8250295519828796 + }, + { + "bits_per_byte": 1.7871405181757898, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2387514114379883, + "step": 2530, + "unclipped_grad_norm": 0.7922425270080566 + }, + { + "bits_per_byte": 1.7956409326773348, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2446434497833252, + "step": 2540, + "unclipped_grad_norm": 0.8244069218635559 + }, + { + "bits_per_byte": 1.833951787983534, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2711985111236572, + "step": 2550, + "unclipped_grad_norm": 0.8060020804405212 + }, + { + "bits_per_byte": 1.7851159384098327, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2373480796813965, + "step": 2560, + "unclipped_grad_norm": 0.7821792960166931 + }, + { + "bits_per_byte": 1.768823849910948, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2260552644729614, + "step": 2570, + "unclipped_grad_norm": 0.7924907207489014 + }, + { + "bits_per_byte": 1.8169237857216074, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2593955993652344, + "step": 2580, + "unclipped_grad_norm": 0.813592255115509 + }, + { + "bits_per_byte": 1.7917042497990852, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2419147491455078, + "step": 2590, + "unclipped_grad_norm": 0.8239418268203735 + }, + { + "bits_per_byte": 1.8340940176357976, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2712970972061157, + "step": 2600, + "unclipped_grad_norm": 0.7961696982383728 + }, + { + "bits_per_byte": 1.7288738279593754, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.198364019393921, + "step": 2610, + "unclipped_grad_norm": 0.7893867492675781 + }, + { + "bits_per_byte": 1.7710386424887592, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2275904417037964, + "step": 2620, + "unclipped_grad_norm": 0.7791327238082886 + }, + { + "bits_per_byte": 1.783225161146247, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2360374927520752, + "step": 2630, + "unclipped_grad_norm": 0.7614359259605408 + }, + { + "bits_per_byte": 1.8200944578729141, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2615933418273926, + "step": 2640, + "unclipped_grad_norm": 0.831463098526001 + }, + { + "bits_per_byte": 1.7566091260987187, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2175886631011963, + "step": 2650, + "unclipped_grad_norm": 0.7739477157592773 + }, + { + "bits_per_byte": 1.7470520501902724, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2109642028808594, + "step": 2660, + "unclipped_grad_norm": 0.7929992079734802 + }, + { + "bits_per_byte": 1.775358846678433, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.230584979057312, + "step": 2670, + "unclipped_grad_norm": 0.7438913583755493 + }, + { + "bits_per_byte": 1.7678870604116983, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2254059314727783, + "step": 2680, + "unclipped_grad_norm": 0.8285056352615356 + }, + { + "bits_per_byte": 1.8436208245976922, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2779005765914917, + "step": 2690, + "unclipped_grad_norm": 0.7684076428413391 + }, + { + "bits_per_byte": 1.825501592416041, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2653412818908691, + "step": 2700, + "unclipped_grad_norm": 0.8987715840339661 + }, + { + "bits_per_byte": 1.7152053347994027, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.188889741897583, + "step": 2710, + "unclipped_grad_norm": 0.8146651983261108 + }, + { + "bits_per_byte": 1.8198911743795942, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2614524364471436, + "step": 2720, + "unclipped_grad_norm": 0.8013731241226196 + }, + { + "bits_per_byte": 1.7448410412307802, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2094316482543945, + "step": 2730, + "unclipped_grad_norm": 0.7481213212013245 + }, + { + "bits_per_byte": 1.7681415947349752, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2255823612213135, + "step": 2740, + "unclipped_grad_norm": 0.8155582547187805 + }, + { + "bits_per_byte": 1.781660806953999, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2349531650543213, + "step": 2750, + "unclipped_grad_norm": 0.816902756690979 + }, + { + "bits_per_byte": 1.869134278870539, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2955851554870605, + "step": 2760, + "unclipped_grad_norm": 0.8422117233276367 + }, + { + "bits_per_byte": 1.7766440730283308, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.231475830078125, + "step": 2770, + "unclipped_grad_norm": 0.8072130084037781 + }, + { + "bits_per_byte": 1.778421169759696, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2327076196670532, + "step": 2780, + "unclipped_grad_norm": 0.8882554173469543 + }, + { + "bits_per_byte": 1.7671284449387423, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2248800992965698, + "step": 2790, + "unclipped_grad_norm": 0.8406802415847778 + }, + { + "bits_per_byte": 1.719355792112675, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.191766619682312, + "step": 2800, + "unclipped_grad_norm": 0.8179776072502136 + }, + { + "bits_per_byte": 1.7902867687906738, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2409322261810303, + "step": 2810, + "unclipped_grad_norm": 0.8046079874038696 + }, + { + "bits_per_byte": 1.7916516231079211, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2418782711029053, + "step": 2820, + "unclipped_grad_norm": 0.7889978885650635 + }, + { + "bits_per_byte": 1.717342391219024, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.190371036529541, + "step": 2830, + "unclipped_grad_norm": 0.7270697355270386 + }, + { + "bits_per_byte": 1.8021411889493455, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2491490840911865, + "step": 2840, + "unclipped_grad_norm": 0.8529788255691528 + }, + { + "bits_per_byte": 1.7264863648000979, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.196709156036377, + "step": 2850, + "unclipped_grad_norm": 0.7487845420837402 + }, + { + "bits_per_byte": 1.698717702026487, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1774613857269287, + "step": 2860, + "unclipped_grad_norm": 0.8107967972755432 + }, + { + "bits_per_byte": 1.8323354950307253, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.270078182220459, + "step": 2870, + "unclipped_grad_norm": 0.822874128818512 + }, + { + "bits_per_byte": 1.7418315168233318, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2073456048965454, + "step": 2880, + "unclipped_grad_norm": 0.7772940993309021 + }, + { + "bits_per_byte": 1.7973201712803593, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.245807409286499, + "step": 2890, + "unclipped_grad_norm": 0.7877682447433472 + }, + { + "bits_per_byte": 1.7076477291898875, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1836512088775635, + "step": 2900, + "unclipped_grad_norm": 0.8104777932167053 + }, + { + "bits_per_byte": 1.7337485042154308, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2017428874969482, + "step": 2910, + "unclipped_grad_norm": 0.8031578063964844 + }, + { + "bits_per_byte": 1.8229534974608574, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2635750770568848, + "step": 2920, + "unclipped_grad_norm": 0.7966766953468323 + }, + { + "bits_per_byte": 1.766033431400699, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.22412109375, + "step": 2930, + "unclipped_grad_norm": 0.8636581301689148 + }, + { + "bits_per_byte": 1.8135838826418518, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2570805549621582, + "step": 2940, + "unclipped_grad_norm": 0.8673250675201416 + }, + { + "bits_per_byte": 1.7613176671140407, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208523750305176, + "step": 2950, + "unclipped_grad_norm": 0.7721947431564331 + }, + { + "bits_per_byte": 1.7305283010606756, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.1995108127593994, + "step": 2960, + "unclipped_grad_norm": 0.7505601644515991 + }, + { + "bits_per_byte": 1.8081494028572378, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.253313660621643, + "step": 2970, + "unclipped_grad_norm": 0.7939547896385193 + }, + { + "bits_per_byte": 1.72656255311443, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.196761965751648, + "step": 2980, + "unclipped_grad_norm": 0.7610272765159607 + }, + { + "bits_per_byte": 1.8584056571444152, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2881486415863037, + "step": 2990, + "unclipped_grad_norm": 0.833854615688324 + }, + { + "bits_per_byte": 1.7959412143857414, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2448515892028809, + "step": 3000, + "unclipped_grad_norm": 0.7871666550636292 + }, + { + "bits_per_byte": 1.7543190051198296, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.216001272201538, + "step": 3010, + "unclipped_grad_norm": 0.7730440497398376 + }, + { + "bits_per_byte": 1.7304921847039942, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1994857788085938, + "step": 3020, + "unclipped_grad_norm": 0.8199000954627991 + }, + { + "bits_per_byte": 1.7578830015936595, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.218471646308899, + "step": 3030, + "unclipped_grad_norm": 0.9139093160629272 + }, + { + "bits_per_byte": 1.7370483353375354, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.204030156135559, + "step": 3040, + "unclipped_grad_norm": 0.7832452058792114 + }, + { + "bits_per_byte": 1.745498014957076, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2098870277404785, + "step": 3050, + "unclipped_grad_norm": 0.7990915179252625 + }, + { + "bits_per_byte": 1.7370017280391516, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2039978504180908, + "step": 3060, + "unclipped_grad_norm": 0.8140543103218079 + }, + { + "bits_per_byte": 1.7324813360438736, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.200864553451538, + "step": 3070, + "unclipped_grad_norm": 0.8290132284164429 + }, + { + "bits_per_byte": 1.73554830265671, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2029904127120972, + "step": 3080, + "unclipped_grad_norm": 0.8363073468208313 + }, + { + "bits_per_byte": 1.7064868462965634, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.1828465461730957, + "step": 3090, + "unclipped_grad_norm": 0.7568578124046326 + }, + { + "bits_per_byte": 1.8015361539836103, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2487297058105469, + "step": 3100, + "unclipped_grad_norm": 0.8538148999214172 + }, + { + "bits_per_byte": 1.7730212584879057, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2289646863937378, + "step": 3110, + "unclipped_grad_norm": 0.8123647570610046 + }, + { + "bits_per_byte": 1.664611306585931, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1538206338882446, + "step": 3120, + "unclipped_grad_norm": 0.814990758895874 + }, + { + "bits_per_byte": 1.7603762340832176, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2201998233795166, + "step": 3130, + "unclipped_grad_norm": 0.8308927416801453 + }, + { + "bits_per_byte": 1.713619138810495, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1877902746200562, + "step": 3140, + "unclipped_grad_norm": 0.7710813879966736 + }, + { + "bits_per_byte": 1.7093383186478688, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1848230361938477, + "step": 3150, + "unclipped_grad_norm": 0.7859835624694824 + }, + { + "bits_per_byte": 1.7563532159142348, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2174112796783447, + "step": 3160, + "unclipped_grad_norm": 0.811743974685669 + }, + { + "bits_per_byte": 1.8161149513145995, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.258834958076477, + "step": 3170, + "unclipped_grad_norm": 0.8695415258407593 + }, + { + "bits_per_byte": 1.6889931150161004, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1707208156585693, + "step": 3180, + "unclipped_grad_norm": 0.8201394081115723 + }, + { + "bits_per_byte": 1.7704115937437135, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2271558046340942, + "step": 3190, + "unclipped_grad_norm": 0.8031932711601257 + }, + { + "bits_per_byte": 1.6638831320421776, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1533159017562866, + "step": 3200, + "unclipped_grad_norm": 0.7531365156173706 + }, + { + "bits_per_byte": 1.7080410535124106, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1839238405227661, + "step": 3210, + "unclipped_grad_norm": 0.7989531755447388 + }, + { + "bits_per_byte": 1.7142219380017691, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1882081031799316, + "step": 3220, + "unclipped_grad_norm": 0.8516196012496948 + }, + { + "bits_per_byte": 1.687988048404458, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1700241565704346, + "step": 3230, + "unclipped_grad_norm": 0.8453141450881958 + }, + { + "bits_per_byte": 1.725164850110868, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1957931518554688, + "step": 3240, + "unclipped_grad_norm": 0.8131914734840393 + }, + { + "bits_per_byte": 1.7438406181507111, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2087382078170776, + "step": 3250, + "unclipped_grad_norm": 0.8390238285064697 + }, + { + "bits_per_byte": 1.8351064795014271, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2719988822937012, + "step": 3260, + "unclipped_grad_norm": 0.8244672417640686 + }, + { + "bits_per_byte": 1.6914478233918653, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1724222898483276, + "step": 3270, + "unclipped_grad_norm": 0.8076629042625427 + }, + { + "bits_per_byte": 1.6199020087370317, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1228305101394653, + "step": 3280, + "unclipped_grad_norm": 0.7831076979637146 + }, + { + "bits_per_byte": 1.6792207168287738, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1639471054077148, + "step": 3290, + "unclipped_grad_norm": 0.8022177219390869 + }, + { + "bits_per_byte": 1.7567814527148833, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2177081108093262, + "step": 3300, + "unclipped_grad_norm": 0.8066262602806091 + }, + { + "bits_per_byte": 1.745768887632185, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.210074782371521, + "step": 3310, + "unclipped_grad_norm": 0.8170367479324341 + }, + { + "bits_per_byte": 1.7488066171943748, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2121803760528564, + "step": 3320, + "unclipped_grad_norm": 0.8205949664115906 + }, + { + "bits_per_byte": 1.6972405430382262, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.176437497138977, + "step": 3330, + "unclipped_grad_norm": 0.7750418782234192 + }, + { + "bits_per_byte": 1.7083963696690934, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.184170126914978, + "step": 3340, + "unclipped_grad_norm": 0.7907351851463318 + }, + { + "bits_per_byte": 1.7221589373390878, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1937096118927002, + "step": 3350, + "unclipped_grad_norm": 0.7935147285461426 + }, + { + "bits_per_byte": 1.6939794080125659, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1741770505905151, + "step": 3360, + "unclipped_grad_norm": 0.7715696096420288 + }, + { + "bits_per_byte": 1.6536716621471985, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.146237850189209, + "step": 3370, + "unclipped_grad_norm": 0.7847044467926025 + }, + { + "bits_per_byte": 1.7430156173745224, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2081663608551025, + "step": 3380, + "unclipped_grad_norm": 0.9182605743408203 + }, + { + "bits_per_byte": 1.7282608817917002, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.197939157485962, + "step": 3390, + "unclipped_grad_norm": 0.8387140035629272 + }, + { + "bits_per_byte": 1.7168453613580306, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1900265216827393, + "step": 3400, + "unclipped_grad_norm": 0.827458918094635 + }, + { + "bits_per_byte": 1.744452876387783, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2091625928878784, + "step": 3410, + "unclipped_grad_norm": 0.8461803197860718 + }, + { + "bits_per_byte": 1.7169175940713928, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1900765895843506, + "step": 3420, + "unclipped_grad_norm": 0.7754979133605957 + }, + { + "bits_per_byte": 1.665198971303929, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1542279720306396, + "step": 3430, + "unclipped_grad_norm": 0.784263551235199 + }, + { + "bits_per_byte": 1.6807204055442975, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1649866104125977, + "step": 3440, + "unclipped_grad_norm": 0.768701434135437 + }, + { + "bits_per_byte": 1.674807642007634, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1608881950378418, + "step": 3450, + "unclipped_grad_norm": 0.8573304414749146 + }, + { + "bits_per_byte": 1.6607244986764313, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.151126503944397, + "step": 3460, + "unclipped_grad_norm": 0.7546505331993103 + }, + { + "bits_per_byte": 1.7168016777647113, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1899962425231934, + "step": 3470, + "unclipped_grad_norm": 0.8346678614616394 + }, + { + "bits_per_byte": 1.7794081781929978, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2333917617797852, + "step": 3480, + "unclipped_grad_norm": 0.8080501556396484 + }, + { + "bits_per_byte": 1.7098234817059528, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1851593255996704, + "step": 3490, + "unclipped_grad_norm": 0.7828541398048401 + }, + { + "bits_per_byte": 1.6618220919542377, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.15188729763031, + "step": 3500, + "unclipped_grad_norm": 0.8354499936103821 + }, + { + "bits_per_byte": 1.670962969847596, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.158223271369934, + "step": 3510, + "unclipped_grad_norm": 0.7484581470489502 + }, + { + "bits_per_byte": 1.7077951183216769, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1837533712387085, + "step": 3520, + "unclipped_grad_norm": 0.8215554356575012 + }, + { + "bits_per_byte": 1.6640239858332344, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1534135341644287, + "step": 3530, + "unclipped_grad_norm": 0.7679988741874695 + }, + { + "bits_per_byte": 1.6934656958344385, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.173820972442627, + "step": 3540, + "unclipped_grad_norm": 0.849611759185791 + }, + { + "bits_per_byte": 1.7380399873024104, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2047175168991089, + "step": 3550, + "unclipped_grad_norm": 0.8348926305770874 + }, + { + "bits_per_byte": 1.6724150193688307, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1592297554016113, + "step": 3560, + "unclipped_grad_norm": 0.82209312915802 + }, + { + "bits_per_byte": 1.707797526078789, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1837550401687622, + "step": 3570, + "unclipped_grad_norm": 0.814924955368042 + }, + { + "bits_per_byte": 1.725196322935976, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1958149671554565, + "step": 3580, + "unclipped_grad_norm": 0.8165888786315918 + }, + { + "bits_per_byte": 1.7086591591596116, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1843522787094116, + "step": 3590, + "unclipped_grad_norm": 0.8172020316123962 + }, + { + "bits_per_byte": 1.7535423314685332, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2154629230499268, + "step": 3600, + "unclipped_grad_norm": 0.8490144610404968 + }, + { + "bits_per_byte": 1.7320164669385913, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.2005423307418823, + "step": 3610, + "unclipped_grad_norm": 0.8558303117752075 + }, + { + "bits_per_byte": 1.671523289324107, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1586116552352905, + "step": 3620, + "unclipped_grad_norm": 0.7700185179710388 + }, + { + "bits_per_byte": 1.6783824733884685, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1633660793304443, + "step": 3630, + "unclipped_grad_norm": 0.8705776333808899 + }, + { + "bits_per_byte": 1.6085100499265301, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1149342060089111, + "step": 3640, + "unclipped_grad_norm": 0.7363900542259216 + }, + { + "bits_per_byte": 1.69346002040696, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1738170385360718, + "step": 3650, + "unclipped_grad_norm": 0.7923496961593628 + }, + { + "bits_per_byte": 1.6919579239343243, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1727758646011353, + "step": 3660, + "unclipped_grad_norm": 0.8016297221183777 + }, + { + "bits_per_byte": 1.7219477426438283, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1935632228851318, + "step": 3670, + "unclipped_grad_norm": 0.8946071863174438 + }, + { + "bits_per_byte": 1.6611216066172736, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1514017581939697, + "step": 3680, + "unclipped_grad_norm": 0.7832621932029724 + }, + { + "bits_per_byte": 1.7299588665036687, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1991161108016968, + "step": 3690, + "unclipped_grad_norm": 0.8317174911499023 + }, + { + "bits_per_byte": 1.6188276331170917, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1220858097076416, + "step": 3700, + "unclipped_grad_norm": 0.7848891615867615 + }, + { + "bits_per_byte": 1.69280218676741, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.17336106300354, + "step": 3710, + "unclipped_grad_norm": 0.7887690663337708 + }, + { + "bits_per_byte": 1.7133850704226707, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1876280307769775, + "step": 3720, + "unclipped_grad_norm": 0.8315525650978088 + }, + { + "bits_per_byte": 1.6847540866376338, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1677825450897217, + "step": 3730, + "unclipped_grad_norm": 0.8175208568572998 + }, + { + "bits_per_byte": 1.6646517225088835, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.153848648071289, + "step": 3740, + "unclipped_grad_norm": 0.795343816280365 + }, + { + "bits_per_byte": 1.6976003307438312, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1766868829727173, + "step": 3750, + "unclipped_grad_norm": 0.8138338327407837 + }, + { + "bits_per_byte": 1.7200370153927425, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1922388076782227, + "step": 3760, + "unclipped_grad_norm": 0.8444334268569946 + }, + { + "bits_per_byte": 1.7215936303657018, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1933177709579468, + "step": 3770, + "unclipped_grad_norm": 0.7583279609680176 + }, + { + "bits_per_byte": 1.7717356881727062, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2280735969543457, + "step": 3780, + "unclipped_grad_norm": 0.8467698693275452 + }, + { + "bits_per_byte": 1.6571842358084203, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1486725807189941, + "step": 3790, + "unclipped_grad_norm": 0.769040584564209 + }, + { + "bits_per_byte": 1.6766210270783317, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1621451377868652, + "step": 3800, + "unclipped_grad_norm": 0.7948989272117615 + }, + { + "bits_per_byte": 1.6280214816669156, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1284584999084473, + "step": 3810, + "unclipped_grad_norm": 0.8209450840950012 + }, + { + "bits_per_byte": 1.6241315780697008, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1257622241973877, + "step": 3820, + "unclipped_grad_norm": 0.81658935546875 + }, + { + "bits_per_byte": 1.6981439679032087, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1770637035369873, + "step": 3830, + "unclipped_grad_norm": 0.8478888273239136 + }, + { + "bits_per_byte": 1.649875145129402, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1436063051223755, + "step": 3840, + "unclipped_grad_norm": 0.8229350447654724 + }, + { + "bits_per_byte": 1.621423539249215, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.123885154724121, + "step": 3850, + "unclipped_grad_norm": 0.8153557181358337 + }, + { + "bits_per_byte": 1.6024271955481633, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1107178926467896, + "step": 3860, + "unclipped_grad_norm": 0.794911801815033 + }, + { + "bits_per_byte": 1.5664059452729007, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.085749864578247, + "step": 3870, + "unclipped_grad_norm": 0.75998854637146 + }, + { + "bits_per_byte": 1.6739675067581694, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1603058576583862, + "step": 3880, + "unclipped_grad_norm": 0.8571500182151794 + }, + { + "bits_per_byte": 1.6411153807903556, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.137534499168396, + "step": 3890, + "unclipped_grad_norm": 0.8254691958427429 + }, + { + "bits_per_byte": 1.6427378651185953, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.138659119606018, + "step": 3900, + "unclipped_grad_norm": 0.8312620520591736 + }, + { + "bits_per_byte": 1.6573687731927962, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1488004922866821, + "step": 3910, + "unclipped_grad_norm": 0.8051927089691162 + }, + { + "bits_per_byte": 1.6875147521492837, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1696960926055908, + "step": 3920, + "unclipped_grad_norm": 0.8500935435295105 + }, + { + "bits_per_byte": 1.685968628118027, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1686244010925293, + "step": 3930, + "unclipped_grad_norm": 0.8186569213867188 + }, + { + "bits_per_byte": 1.6142946863883005, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1189438104629517, + "step": 3940, + "unclipped_grad_norm": 0.7923241853713989 + }, + { + "bits_per_byte": 1.7195771337843353, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1919200420379639, + "step": 3950, + "unclipped_grad_norm": 0.8632136583328247 + }, + { + "bits_per_byte": 1.7081789835984027, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1840194463729858, + "step": 3960, + "unclipped_grad_norm": 0.8128528594970703 + }, + { + "bits_per_byte": 1.6418629893736563, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1380527019500732, + "step": 3970, + "unclipped_grad_norm": 0.8211596608161926 + }, + { + "bits_per_byte": 1.6416170541829225, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1378822326660156, + "step": 3980, + "unclipped_grad_norm": 0.8518010973930359 + }, + { + "bits_per_byte": 1.672920992327669, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1595804691314697, + "step": 3990, + "unclipped_grad_norm": 0.8055341243743896 + }, + { + "bits_per_byte": 1.6017095119461129, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1102204322814941, + "step": 4000, + "unclipped_grad_norm": 0.7798590064048767 + }, + { + "bits_per_byte": 1.6685314791296977, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1565378904342651, + "step": 4010, + "unclipped_grad_norm": 0.8833235502243042 + }, + { + "bits_per_byte": 1.6832612772281446, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.166747808456421, + "step": 4020, + "unclipped_grad_norm": 0.8783014416694641 + }, + { + "bits_per_byte": 1.6355704881438877, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1336910724639893, + "step": 4030, + "unclipped_grad_norm": 0.831271231174469 + }, + { + "bits_per_byte": 1.6221185211413516, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1243668794631958, + "step": 4040, + "unclipped_grad_norm": 0.828468382358551 + }, + { + "bits_per_byte": 1.6444031731269, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1398134231567383, + "step": 4050, + "unclipped_grad_norm": 0.8378818035125732 + }, + { + "bits_per_byte": 1.6213299806871455, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1238203048706055, + "step": 4060, + "unclipped_grad_norm": 0.808568000793457 + }, + { + "bits_per_byte": 1.749336667724334, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.212547779083252, + "step": 4070, + "unclipped_grad_norm": 0.8772934675216675 + }, + { + "bits_per_byte": 1.5885019322904526, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1010656356811523, + "step": 4080, + "unclipped_grad_norm": 0.8058938980102539 + }, + { + "bits_per_byte": 1.6013793052564562, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1099915504455566, + "step": 4090, + "unclipped_grad_norm": 0.8040578365325928 + }, + { + "bits_per_byte": 1.737544161319973, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.204373836517334, + "step": 4100, + "unclipped_grad_norm": 0.8918697834014893 + }, + { + "bits_per_byte": 1.6258001537483717, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1269187927246094, + "step": 4110, + "unclipped_grad_norm": 0.8254836797714233 + }, + { + "bits_per_byte": 1.6466504704257243, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1413711309432983, + "step": 4120, + "unclipped_grad_norm": 0.8797640800476074 + }, + { + "bits_per_byte": 1.7012891866221878, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.179243803024292, + "step": 4130, + "unclipped_grad_norm": 0.8626418709754944 + }, + { + "bits_per_byte": 1.6871897049391529, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1694707870483398, + "step": 4140, + "unclipped_grad_norm": 0.8501971364021301 + }, + { + "bits_per_byte": 1.629858600343432, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1297318935394287, + "step": 4150, + "unclipped_grad_norm": 0.8614740371704102 + }, + { + "bits_per_byte": 1.6422026551091102, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138288140296936, + "step": 4160, + "unclipped_grad_norm": 0.8621925115585327 + }, + { + "bits_per_byte": 1.6599294228814925, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1505753993988037, + "step": 4170, + "unclipped_grad_norm": 0.8789071440696716 + }, + { + "bits_per_byte": 1.6412854716320588, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1376523971557617, + "step": 4180, + "unclipped_grad_norm": 0.8332215547561646 + }, + { + "bits_per_byte": 1.6753517951149641, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1612653732299805, + "step": 4190, + "unclipped_grad_norm": 0.8483413457870483 + }, + { + "bits_per_byte": 1.6524258198243482, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1453742980957031, + "step": 4200, + "unclipped_grad_norm": 0.8514559268951416 + }, + { + "bits_per_byte": 1.6764456047744516, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1620235443115234, + "step": 4210, + "unclipped_grad_norm": 0.7994921803474426 + }, + { + "bits_per_byte": 1.6553684429806108, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.147413969039917, + "step": 4220, + "unclipped_grad_norm": 0.8137004375457764 + }, + { + "bits_per_byte": 1.654246428166382, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1466362476348877, + "step": 4230, + "unclipped_grad_norm": 0.8624707460403442 + }, + { + "bits_per_byte": 1.670982919835096, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.158237099647522, + "step": 4240, + "unclipped_grad_norm": 0.83209228515625 + }, + { + "bits_per_byte": 1.605450134602379, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1128132343292236, + "step": 4250, + "unclipped_grad_norm": 0.8165753483772278 + }, + { + "bits_per_byte": 1.64798711958823, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.142297625541687, + "step": 4260, + "unclipped_grad_norm": 0.8390073180198669 + }, + { + "bits_per_byte": 1.5630770490828, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.0834424495697021, + "step": 4270, + "unclipped_grad_norm": 0.7599806189537048 + }, + { + "bits_per_byte": 1.634252757072977, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1327776908874512, + "step": 4280, + "unclipped_grad_norm": 0.8349722027778625 + }, + { + "bits_per_byte": 1.622665597953746, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1247460842132568, + "step": 4290, + "unclipped_grad_norm": 0.8544122576713562 + }, + { + "bits_per_byte": 1.5775992641216556, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0935084819793701, + "step": 4300, + "unclipped_grad_norm": 0.8515903353691101 + }, + { + "bits_per_byte": 1.5682213941354084, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.0870082378387451, + "step": 4310, + "unclipped_grad_norm": 0.8175749182701111 + }, + { + "bits_per_byte": 1.709660270170284, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1850461959838867, + "step": 4320, + "unclipped_grad_norm": 0.9185000061988831 + }, + { + "bits_per_byte": 1.6398645509706304, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1366674900054932, + "step": 4330, + "unclipped_grad_norm": 0.8850803971290588 + }, + { + "bits_per_byte": 1.5869262272432476, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.099973440170288, + "step": 4340, + "unclipped_grad_norm": 0.8476384878158569 + }, + { + "bits_per_byte": 1.5594358323987327, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.080918550491333, + "step": 4350, + "unclipped_grad_norm": 0.8173791170120239 + }, + { + "bits_per_byte": 1.6383724294917448, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1356332302093506, + "step": 4360, + "unclipped_grad_norm": 0.8429195880889893 + }, + { + "bits_per_byte": 1.6201880158854165, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1230287551879883, + "step": 4370, + "unclipped_grad_norm": 0.8339970707893372 + }, + { + "bits_per_byte": 1.5560377991829855, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.0785632133483887, + "step": 4380, + "unclipped_grad_norm": 0.7866214513778687 + }, + { + "bits_per_byte": 1.7055524645544258, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1821988821029663, + "step": 4390, + "unclipped_grad_norm": 0.8178918361663818 + }, + { + "bits_per_byte": 1.6491636529027824, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.143113136291504, + "step": 4400, + "unclipped_grad_norm": 0.8444706797599792 + }, + { + "bits_per_byte": 1.6592194784987309, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.150083303451538, + "step": 4410, + "unclipped_grad_norm": 0.8914594054222107 + }, + { + "bits_per_byte": 1.676016508060549, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1617261171340942, + "step": 4420, + "unclipped_grad_norm": 0.894481897354126 + }, + { + "bits_per_byte": 1.6150391992838857, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.119459867477417, + "step": 4430, + "unclipped_grad_norm": 0.8697190880775452 + }, + { + "bits_per_byte": 1.63267068866769, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1316810846328735, + "step": 4440, + "unclipped_grad_norm": 0.8085072636604309 + }, + { + "bits_per_byte": 1.6597337066248108, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.150439739227295, + "step": 4450, + "unclipped_grad_norm": 0.8258711099624634 + }, + { + "bits_per_byte": 1.6374223973283786, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.134974718093872, + "step": 4460, + "unclipped_grad_norm": 0.8935436010360718 + }, + { + "bits_per_byte": 1.6156965169754833, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.11991548538208, + "step": 4470, + "unclipped_grad_norm": 0.8515243530273438 + }, + { + "bits_per_byte": 1.663532975365021, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1530731916427612, + "step": 4480, + "unclipped_grad_norm": 0.8462327718734741 + }, + { + "bits_per_byte": 1.6896239473794652, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1711580753326416, + "step": 4490, + "unclipped_grad_norm": 0.8624916672706604 + }, + { + "bits_per_byte": 1.6220770733224936, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.124338150024414, + "step": 4500, + "unclipped_grad_norm": 0.830740213394165 + }, + { + "bits_per_byte": 1.6091536090060587, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1153802871704102, + "step": 4510, + "unclipped_grad_norm": 0.8425999879837036 + }, + { + "bits_per_byte": 1.6507618876772505, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1442209482192993, + "step": 4520, + "unclipped_grad_norm": 0.8468135595321655 + }, + { + "bits_per_byte": 1.6238777316770272, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1255862712860107, + "step": 4530, + "unclipped_grad_norm": 0.8532826900482178 + }, + { + "bits_per_byte": 1.6469891042652731, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1416058540344238, + "step": 4540, + "unclipped_grad_norm": 0.8736191987991333 + }, + { + "bits_per_byte": 1.6559469926181134, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1478149890899658, + "step": 4550, + "unclipped_grad_norm": 0.8442186713218689 + }, + { + "bits_per_byte": 1.6446332859137545, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1399729251861572, + "step": 4560, + "unclipped_grad_norm": 0.9025750756263733 + }, + { + "bits_per_byte": 1.637351540476223, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1349256038665771, + "step": 4570, + "unclipped_grad_norm": 0.8963979482650757 + }, + { + "bits_per_byte": 1.6074537324849307, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1142020225524902, + "step": 4580, + "unclipped_grad_norm": 0.8524222373962402 + }, + { + "bits_per_byte": 1.592079171428399, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1035451889038086, + "step": 4590, + "unclipped_grad_norm": 0.8329559564590454 + }, + { + "bits_per_byte": 1.593515914493707, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1045410633087158, + "step": 4600, + "unclipped_grad_norm": 0.856568455696106 + }, + { + "bits_per_byte": 1.6099590037600493, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1159385442733765, + "step": 4610, + "unclipped_grad_norm": 0.8378997445106506 + }, + { + "bits_per_byte": 1.6641679353120065, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1535133123397827, + "step": 4620, + "unclipped_grad_norm": 0.8407614827156067 + }, + { + "bits_per_byte": 1.64286238055782, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1387454271316528, + "step": 4630, + "unclipped_grad_norm": 0.9177886247634888 + }, + { + "bits_per_byte": 1.6574955244064822, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.148888349533081, + "step": 4640, + "unclipped_grad_norm": 0.8678322434425354 + }, + { + "bits_per_byte": 1.6864685816840852, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1689709424972534, + "step": 4650, + "unclipped_grad_norm": 0.8395454287528992 + }, + { + "bits_per_byte": 1.6055636431519484, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1128919124603271, + "step": 4660, + "unclipped_grad_norm": 0.8130927681922913 + }, + { + "bits_per_byte": 1.6111651180905502, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.116774559020996, + "step": 4670, + "unclipped_grad_norm": 0.8343866467475891 + }, + { + "bits_per_byte": 1.6120890088909852, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.117414951324463, + "step": 4680, + "unclipped_grad_norm": 0.9091700911521912 + }, + { + "bits_per_byte": 1.598018764258597, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1076622009277344, + "step": 4690, + "unclipped_grad_norm": 0.8087259531021118 + }, + { + "bits_per_byte": 1.6257145063882419, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.126859426498413, + "step": 4700, + "unclipped_grad_norm": 0.8473510146141052 + }, + { + "bits_per_byte": 1.6037934257266175, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.111664891242981, + "step": 4710, + "unclipped_grad_norm": 0.8299785256385803 + }, + { + "bits_per_byte": 1.6360324335441052, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1340112686157227, + "step": 4720, + "unclipped_grad_norm": 0.8852627873420715 + }, + { + "bits_per_byte": 1.6613310814860245, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1515469551086426, + "step": 4730, + "unclipped_grad_norm": 0.8318192958831787 + }, + { + "bits_per_byte": 1.6077376758415052, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1143988370895386, + "step": 4740, + "unclipped_grad_norm": 0.9029622673988342 + }, + { + "bits_per_byte": 1.5803602735986073, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.0954222679138184, + "step": 4750, + "unclipped_grad_norm": 0.8787683248519897 + }, + { + "bits_per_byte": 1.5545300272828713, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.077518105506897, + "step": 4760, + "unclipped_grad_norm": 0.8357442021369934 + }, + { + "bits_per_byte": 1.5563697256991507, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.0787932872772217, + "step": 4770, + "unclipped_grad_norm": 0.8196295499801636 + }, + { + "bits_per_byte": 1.593790398804484, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1047313213348389, + "step": 4780, + "unclipped_grad_norm": 0.9009830951690674 + }, + { + "bits_per_byte": 1.5906680537780697, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1025670766830444, + "step": 4790, + "unclipped_grad_norm": 0.8538159728050232 + }, + { + "bits_per_byte": 1.62577985979557, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1269047260284424, + "step": 4800, + "unclipped_grad_norm": 0.9271508455276489 + }, + { + "bits_per_byte": 1.5952574108163438, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.105748176574707, + "step": 4810, + "unclipped_grad_norm": 0.8505629301071167 + }, + { + "bits_per_byte": 1.5898409892100704, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1019937992095947, + "step": 4820, + "unclipped_grad_norm": 0.8469997048377991 + }, + { + "bits_per_byte": 1.5923758415011373, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1037508249282837, + "step": 4830, + "unclipped_grad_norm": 0.8425012826919556 + }, + { + "bits_per_byte": 1.5390662072305417, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.0667994022369385, + "step": 4840, + "unclipped_grad_norm": 0.8439065217971802 + }, + { + "bits_per_byte": 1.6086853002477592, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1150556802749634, + "step": 4850, + "unclipped_grad_norm": 0.8472049236297607 + }, + { + "bits_per_byte": 1.5891265732783864, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1014986038208008, + "step": 4860, + "unclipped_grad_norm": 0.8380921483039856 + }, + { + "bits_per_byte": 1.603959560967351, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.111780047416687, + "step": 4870, + "unclipped_grad_norm": 0.8809474110603333 + }, + { + "bits_per_byte": 1.5692639529649388, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.087730884552002, + "step": 4880, + "unclipped_grad_norm": 0.7983198761940002 + }, + { + "bits_per_byte": 1.525049621185223, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0570838451385498, + "step": 4890, + "unclipped_grad_norm": 0.874304473400116 + }, + { + "bits_per_byte": 1.5072996357549742, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0447804927825928, + "step": 4900, + "unclipped_grad_norm": 0.8955963253974915 + }, + { + "bits_per_byte": 1.5786693401753242, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.094250202178955, + "step": 4910, + "unclipped_grad_norm": 0.8447177410125732 + }, + { + "bits_per_byte": 1.585648052182035, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.0990874767303467, + "step": 4920, + "unclipped_grad_norm": 0.8577683568000793 + }, + { + "bits_per_byte": 1.5737733380705616, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0908565521240234, + "step": 4930, + "unclipped_grad_norm": 0.8645251393318176 + }, + { + "bits_per_byte": 1.6512401714293001, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.14455246925354, + "step": 4940, + "unclipped_grad_norm": 0.9303934574127197 + }, + { + "bits_per_byte": 1.5141001737353912, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0494942665100098, + "step": 4950, + "unclipped_grad_norm": 0.8813614249229431 + }, + { + "bits_per_byte": 1.592528046147151, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1038563251495361, + "step": 4960, + "unclipped_grad_norm": 0.915719747543335 + }, + { + "bits_per_byte": 1.6794828183886887, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1641287803649902, + "step": 4970, + "unclipped_grad_norm": 0.9402682781219482 + }, + { + "bits_per_byte": 1.543928500735735, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.0701696872711182, + "step": 4980, + "unclipped_grad_norm": 0.9095892310142517 + }, + { + "bits_per_byte": 1.6067152389821258, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1136901378631592, + "step": 4990, + "unclipped_grad_norm": 0.934682309627533 + }, + { + "bits_per_byte": 1.6283919322968743, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1287152767181396, + "step": 5000, + "unclipped_grad_norm": 0.8894299268722534 + }, + { + "bits_per_byte": 1.5848010376265358, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.098500370979309, + "step": 5010, + "unclipped_grad_norm": 0.8762696385383606 + }, + { + "bits_per_byte": 1.5972207647585936, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1071090698242188, + "step": 5020, + "unclipped_grad_norm": 0.87384033203125 + }, + { + "bits_per_byte": 1.5759778116893213, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0923845767974854, + "step": 5030, + "unclipped_grad_norm": 0.875623881816864 + }, + { + "bits_per_byte": 1.5723771829108575, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0898888111114502, + "step": 5040, + "unclipped_grad_norm": 0.8759424686431885 + }, + { + "bits_per_byte": 1.6142312247901323, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1188998222351074, + "step": 5050, + "unclipped_grad_norm": 0.886820912361145 + }, + { + "bits_per_byte": 1.5584469321562717, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.080233097076416, + "step": 5060, + "unclipped_grad_norm": 0.880901038646698 + }, + { + "bits_per_byte": 1.6299523308881525, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1297968626022339, + "step": 5070, + "unclipped_grad_norm": 0.9343949556350708 + }, + { + "bits_per_byte": 1.651922254622622, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1450252532958984, + "step": 5080, + "unclipped_grad_norm": 0.9880313277244568 + }, + { + "bits_per_byte": 1.6199948793684975, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1228948831558228, + "step": 5090, + "unclipped_grad_norm": 1.0230180025100708 + }, + { + "bits_per_byte": 1.5480533326340287, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.073028802871704, + "step": 5100, + "unclipped_grad_norm": 0.9155668020248413 + }, + { + "bits_per_byte": 1.5785750936826513, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0941848754882812, + "step": 5110, + "unclipped_grad_norm": 0.8749560713768005 + }, + { + "bits_per_byte": 1.553970911684916, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0771305561065674, + "step": 5120, + "unclipped_grad_norm": 0.9114576578140259 + }, + { + "bits_per_byte": 1.6072428817549729, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.114055871963501, + "step": 5130, + "unclipped_grad_norm": 0.89686119556427 + }, + { + "bits_per_byte": 1.5469013928385495, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.072230339050293, + "step": 5140, + "unclipped_grad_norm": 0.9089201092720032 + }, + { + "bits_per_byte": 1.5509646549478342, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0750467777252197, + "step": 5150, + "unclipped_grad_norm": 0.9353929162025452 + }, + { + "bits_per_byte": 1.5786652125917033, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0942473411560059, + "step": 5160, + "unclipped_grad_norm": 0.9100342988967896 + }, + { + "bits_per_byte": 1.5893422395225683, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1016480922698975, + "step": 5170, + "unclipped_grad_norm": 0.8825995922088623 + }, + { + "bits_per_byte": 1.611850984902191, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1172499656677246, + "step": 5180, + "unclipped_grad_norm": 0.8858869671821594 + }, + { + "bits_per_byte": 1.5765690880429875, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.092794418334961, + "step": 5190, + "unclipped_grad_norm": 0.893822431564331 + }, + { + "bits_per_byte": 1.5180597303062058, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0522388219833374, + "step": 5200, + "unclipped_grad_norm": 0.9365838766098022 + }, + { + "bits_per_byte": 1.6136103674205176, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.118469476699829, + "step": 5210, + "unclipped_grad_norm": 0.9187290668487549 + }, + { + "bits_per_byte": 1.578151672396227, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0938913822174072, + "step": 5220, + "unclipped_grad_norm": 0.9324937462806702 + }, + { + "bits_per_byte": 1.6149896682804372, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1194255352020264, + "step": 5230, + "unclipped_grad_norm": 0.9387596249580383 + }, + { + "bits_per_byte": 1.5729148006774545, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.090261459350586, + "step": 5240, + "unclipped_grad_norm": 0.8688122630119324 + }, + { + "bits_per_byte": 1.5356076361216906, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0644021034240723, + "step": 5250, + "unclipped_grad_norm": 0.890831708908081 + }, + { + "bits_per_byte": 1.5141386978491844, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0495209693908691, + "step": 5260, + "unclipped_grad_norm": 0.8596166968345642 + }, + { + "bits_per_byte": 1.6116535488190005, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1171131134033203, + "step": 5270, + "unclipped_grad_norm": 0.92216956615448 + }, + { + "bits_per_byte": 1.5994467362087108, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1086519956588745, + "step": 5280, + "unclipped_grad_norm": 0.9085080623626709 + }, + { + "bits_per_byte": 1.5769201046333985, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.093037724494934, + "step": 5290, + "unclipped_grad_norm": 0.8931360244750977 + }, + { + "bits_per_byte": 1.537008606795619, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.065373182296753, + "step": 5300, + "unclipped_grad_norm": 0.8922452330589294 + }, + { + "bits_per_byte": 1.5906943671236518, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1025853157043457, + "step": 5310, + "unclipped_grad_norm": 0.887484073638916 + }, + { + "bits_per_byte": 1.5623235930893702, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0829201936721802, + "step": 5320, + "unclipped_grad_norm": 0.9026566743850708 + }, + { + "bits_per_byte": 1.6297041599229574, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.129624843597412, + "step": 5330, + "unclipped_grad_norm": 0.9325461387634277 + }, + { + "bits_per_byte": 1.5387120949524151, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0665539503097534, + "step": 5340, + "unclipped_grad_norm": 0.8923580050468445 + }, + { + "bits_per_byte": 1.5882112816104945, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1008641719818115, + "step": 5350, + "unclipped_grad_norm": 0.9464812278747559 + }, + { + "bits_per_byte": 1.4963152758270173, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.037166714668274, + "step": 5360, + "unclipped_grad_norm": 0.8723886609077454 + }, + { + "bits_per_byte": 1.5320489711100373, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0619354248046875, + "step": 5370, + "unclipped_grad_norm": 0.9227972626686096 + }, + { + "bits_per_byte": 1.5933907111238788, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1044542789459229, + "step": 5380, + "unclipped_grad_norm": 0.9233055710792542 + }, + { + "bits_per_byte": 1.5140970780476757, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0494921207427979, + "step": 5390, + "unclipped_grad_norm": 0.8707529902458191 + }, + { + "bits_per_byte": 1.5792117734561455, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0946261882781982, + "step": 5400, + "unclipped_grad_norm": 0.9207195043563843 + }, + { + "bits_per_byte": 1.5488073045754112, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0735514163970947, + "step": 5410, + "unclipped_grad_norm": 0.8852476477622986 + }, + { + "bits_per_byte": 1.5115954184082252, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0477581024169922, + "step": 5420, + "unclipped_grad_norm": 0.9259113073348999 + }, + { + "bits_per_byte": 1.5742989170515984, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0912208557128906, + "step": 5430, + "unclipped_grad_norm": 0.9123548865318298 + }, + { + "bits_per_byte": 1.5198012266288425, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0534459352493286, + "step": 5440, + "unclipped_grad_norm": 0.8873674273490906 + }, + { + "bits_per_byte": 1.5575636292614403, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0796208381652832, + "step": 5450, + "unclipped_grad_norm": 0.851129949092865 + }, + { + "bits_per_byte": 1.5371788696199733, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0654911994934082, + "step": 5460, + "unclipped_grad_norm": 0.8905500173568726 + }, + { + "bits_per_byte": 1.5449696837040585, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0708913803100586, + "step": 5470, + "unclipped_grad_norm": 0.8742530345916748 + }, + { + "bits_per_byte": 1.5184021477640735, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.052476167678833, + "step": 5480, + "unclipped_grad_norm": 0.9149489998817444 + }, + { + "bits_per_byte": 1.6054506505503316, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1128135919570923, + "step": 5490, + "unclipped_grad_norm": 1.0173310041427612 + }, + { + "bits_per_byte": 1.5652256283400292, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.08493173122406, + "step": 5500, + "unclipped_grad_norm": 0.9233388304710388 + }, + { + "bits_per_byte": 1.5242242764437322, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0565117597579956, + "step": 5510, + "unclipped_grad_norm": 0.8623188734054565 + }, + { + "bits_per_byte": 1.4956853033769069, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0367300510406494, + "step": 5520, + "unclipped_grad_norm": 0.8464505672454834 + }, + { + "bits_per_byte": 1.501370017918526, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.040670394897461, + "step": 5530, + "unclipped_grad_norm": 0.9552764296531677 + }, + { + "bits_per_byte": 1.6303358521995766, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1300626993179321, + "step": 5540, + "unclipped_grad_norm": 0.930858314037323 + }, + { + "bits_per_byte": 1.5750513411491232, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0917423963546753, + "step": 5550, + "unclipped_grad_norm": 0.9512235522270203 + }, + { + "bits_per_byte": 1.6036597952068972, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.111572265625, + "step": 5560, + "unclipped_grad_norm": 0.9797965884208679 + }, + { + "bits_per_byte": 1.5364850916063928, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0650103092193604, + "step": 5570, + "unclipped_grad_norm": 0.8889045715332031 + }, + { + "bits_per_byte": 1.5818145588943031, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0964303016662598, + "step": 5580, + "unclipped_grad_norm": 0.91837477684021 + }, + { + "bits_per_byte": 1.571553042047923, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0893175601959229, + "step": 5590, + "unclipped_grad_norm": 0.9484079480171204 + }, + { + "bits_per_byte": 1.5589711352761015, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0805964469909668, + "step": 5600, + "unclipped_grad_norm": 0.9200659394264221 + }, + { + "bits_per_byte": 1.4898029807694462, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.032652735710144, + "step": 5610, + "unclipped_grad_norm": 0.898281455039978 + }, + { + "bits_per_byte": 1.5617056594248202, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0824918746948242, + "step": 5620, + "unclipped_grad_norm": 0.9515117406845093 + }, + { + "bits_per_byte": 1.481016903084818, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0265626907348633, + "step": 5630, + "unclipped_grad_norm": 0.8537728786468506 + }, + { + "bits_per_byte": 1.471526900410208, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0199847221374512, + "step": 5640, + "unclipped_grad_norm": 0.851723313331604 + }, + { + "bits_per_byte": 1.5525594500692852, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0761522054672241, + "step": 5650, + "unclipped_grad_norm": 0.9591006636619568 + }, + { + "bits_per_byte": 1.4925528113740916, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0345587730407715, + "step": 5660, + "unclipped_grad_norm": 0.9846746325492859 + }, + { + "bits_per_byte": 1.5646704683430441, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0845469236373901, + "step": 5670, + "unclipped_grad_norm": 0.9337268471717834 + }, + { + "bits_per_byte": 1.5783896963850212, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0940563678741455, + "step": 5680, + "unclipped_grad_norm": 1.0177373886108398 + }, + { + "bits_per_byte": 1.5898387534356093, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1019922494888306, + "step": 5690, + "unclipped_grad_norm": 0.9374083876609802 + }, + { + "bits_per_byte": 1.5923674143512452, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1037449836730957, + "step": 5700, + "unclipped_grad_norm": 0.9426813721656799 + }, + { + "bits_per_byte": 1.5582474322812707, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.080094814300537, + "step": 5710, + "unclipped_grad_norm": 0.968785285949707 + }, + { + "bits_per_byte": 1.5819518010496918, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0965254306793213, + "step": 5720, + "unclipped_grad_norm": 0.9166642427444458 + }, + { + "bits_per_byte": 1.5197234904706525, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0533920526504517, + "step": 5730, + "unclipped_grad_norm": 0.9203759431838989 + }, + { + "bits_per_byte": 1.6015581672133536, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1101155281066895, + "step": 5740, + "unclipped_grad_norm": 0.934572160243988 + }, + { + "bits_per_byte": 1.4943809869527633, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0358259677886963, + "step": 5750, + "unclipped_grad_norm": 0.8915490508079529 + }, + { + "bits_per_byte": 1.50218005620409, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0412318706512451, + "step": 5760, + "unclipped_grad_norm": 0.9438080787658691 + }, + { + "bits_per_byte": 1.5178872317073904, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.052119255065918, + "step": 5770, + "unclipped_grad_norm": 0.9708113670349121 + }, + { + "bits_per_byte": 1.6077987296825615, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.114441156387329, + "step": 5780, + "unclipped_grad_norm": 1.0216169357299805 + }, + { + "bits_per_byte": 1.454466909375221, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0081596374511719, + "step": 5790, + "unclipped_grad_norm": 0.8829967975616455 + }, + { + "bits_per_byte": 1.5674770532224742, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0864923000335693, + "step": 5800, + "unclipped_grad_norm": 0.9498817920684814 + }, + { + "bits_per_byte": 1.4688064788388602, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.018099069595337, + "step": 5810, + "unclipped_grad_norm": 0.9071933031082153 + }, + { + "bits_per_byte": 1.4812057400354652, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.02669358253479, + "step": 5820, + "unclipped_grad_norm": 0.9959194660186768 + }, + { + "bits_per_byte": 1.5459386339590195, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0715630054473877, + "step": 5830, + "unclipped_grad_norm": 0.9805806875228882 + }, + { + "bits_per_byte": 1.5386066695874363, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0664808750152588, + "step": 5840, + "unclipped_grad_norm": 0.9813235998153687 + }, + { + "bits_per_byte": 1.4904921152514534, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.033130407333374, + "step": 5850, + "unclipped_grad_norm": 0.8884730339050293 + }, + { + "bits_per_byte": 1.505957139182339, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0438499450683594, + "step": 5860, + "unclipped_grad_norm": 0.9569032192230225 + }, + { + "bits_per_byte": 1.5707541826346652, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0887638330459595, + "step": 5870, + "unclipped_grad_norm": 0.9548988342285156 + }, + { + "bits_per_byte": 1.5056532458382643, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0436393022537231, + "step": 5880, + "unclipped_grad_norm": 0.9100088477134705 + }, + { + "bits_per_byte": 1.5621944341185723, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0828306674957275, + "step": 5890, + "unclipped_grad_norm": 1.0087047815322876 + }, + { + "bits_per_byte": 1.564086587243365, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0841422080993652, + "step": 5900, + "unclipped_grad_norm": 0.9735300540924072 + }, + { + "bits_per_byte": 1.5205851235514751, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.053989291191101, + "step": 5910, + "unclipped_grad_norm": 0.928986668586731 + }, + { + "bits_per_byte": 1.4827504882055151, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0277643203735352, + "step": 5920, + "unclipped_grad_norm": 0.9661484956741333 + }, + { + "bits_per_byte": 1.5936927126587939, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.104663610458374, + "step": 5930, + "unclipped_grad_norm": 0.9932063817977905 + }, + { + "bits_per_byte": 1.5284586612906255, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0594468116760254, + "step": 5940, + "unclipped_grad_norm": 1.0194250345230103 + }, + { + "bits_per_byte": 1.5388254315193337, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0666325092315674, + "step": 5950, + "unclipped_grad_norm": 0.9446147084236145 + }, + { + "bits_per_byte": 1.4754302186541906, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0226902961730957, + "step": 5960, + "unclipped_grad_norm": 0.9241304993629456 + }, + { + "bits_per_byte": 1.5277755462013984, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0589733123779297, + "step": 5970, + "unclipped_grad_norm": 0.9580891728401184 + }, + { + "bits_per_byte": 1.5012076662961116, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.040557861328125, + "step": 5980, + "unclipped_grad_norm": 0.954113245010376 + }, + { + "bits_per_byte": 1.4896813890352862, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0325684547424316, + "step": 5990, + "unclipped_grad_norm": 0.8945097923278809 + }, + { + "bits_per_byte": 1.5812009247960246, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0960049629211426, + "step": 6000, + "unclipped_grad_norm": 1.0032626390457153 + }, + { + "bits_per_byte": 1.556720054358958, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0790361166000366, + "step": 6010, + "unclipped_grad_norm": 0.9715147614479065 + }, + { + "bits_per_byte": 1.5090167105411882, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0459706783294678, + "step": 6020, + "unclipped_grad_norm": 0.9192096590995789 + }, + { + "bits_per_byte": 1.4969094758857484, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0375785827636719, + "step": 6030, + "unclipped_grad_norm": 0.9740291833877563 + }, + { + "bits_per_byte": 1.5263979651679873, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.058018445968628, + "step": 6040, + "unclipped_grad_norm": 0.9185873866081238 + }, + { + "bits_per_byte": 1.5301623214300724, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0606276988983154, + "step": 6050, + "unclipped_grad_norm": 0.9192250370979309 + }, + { + "bits_per_byte": 1.5237477125181913, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0561814308166504, + "step": 6060, + "unclipped_grad_norm": 0.9670439958572388 + }, + { + "bits_per_byte": 1.4068365721927425, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 0.9751448035240173, + "step": 6070, + "unclipped_grad_norm": 0.917002260684967 + }, + { + "bits_per_byte": 1.4967151354902735, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0374438762664795, + "step": 6080, + "unclipped_grad_norm": 0.9365084171295166 + }, + { + "bits_per_byte": 1.4969400887976019, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.037599802017212, + "step": 6090, + "unclipped_grad_norm": 0.9915803074836731 + }, + { + "bits_per_byte": 1.5907201645212812, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.1026031970977783, + "step": 6100, + "unclipped_grad_norm": 1.0093804597854614 + }, + { + "bits_per_byte": 1.5313995646203795, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0614852905273438, + "step": 6110, + "unclipped_grad_norm": 1.0098049640655518 + }, + { + "bits_per_byte": 1.4730523865233613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0210421085357666, + "step": 6120, + "unclipped_grad_norm": 0.9365134239196777 + }, + { + "bits_per_byte": 1.5512489422697104, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0752438306808472, + "step": 6130, + "unclipped_grad_norm": 0.9237281680107117 + }, + { + "bits_per_byte": 1.497698360305256, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0381253957748413, + "step": 6140, + "unclipped_grad_norm": 0.9138990044593811 + }, + { + "bits_per_byte": 1.4838755987074594, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0285441875457764, + "step": 6150, + "unclipped_grad_norm": 0.9429982900619507 + }, + { + "bits_per_byte": 1.551023816979731, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0750877857208252, + "step": 6160, + "unclipped_grad_norm": 1.0046567916870117 + }, + { + "bits_per_byte": 1.4928049379402566, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.034733533859253, + "step": 6170, + "unclipped_grad_norm": 0.964278519153595 + }, + { + "bits_per_byte": 1.4789379768011883, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0251216888427734, + "step": 6180, + "unclipped_grad_norm": 0.9606515169143677 + }, + { + "bits_per_byte": 1.501445346319604, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0407226085662842, + "step": 6190, + "unclipped_grad_norm": 1.0252128839492798 + }, + { + "bits_per_byte": 1.4915814533620186, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0338854789733887, + "step": 6200, + "unclipped_grad_norm": 0.8806238770484924 + }, + { + "bits_per_byte": 1.5186611536362729, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0526556968688965, + "step": 6210, + "unclipped_grad_norm": 0.989686906337738 + }, + { + "bits_per_byte": 1.539449556559315, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0670651197433472, + "step": 6220, + "unclipped_grad_norm": 0.9960981011390686 + }, + { + "bits_per_byte": 1.4808485320696232, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.026445984840393, + "step": 6230, + "unclipped_grad_norm": 0.9705929160118103 + }, + { + "bits_per_byte": 1.552468127281677, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0760889053344727, + "step": 6240, + "unclipped_grad_norm": 1.0799938440322876 + }, + { + "bits_per_byte": 1.4757860507588256, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0229369401931763, + "step": 6250, + "unclipped_grad_norm": 0.9792200922966003 + }, + { + "bits_per_byte": 1.445374530589406, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0018572807312012, + "step": 6260, + "unclipped_grad_norm": 0.9422674179077148 + }, + { + "bits_per_byte": 1.5013631386124917, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.040665626525879, + "step": 6270, + "unclipped_grad_norm": 0.9675420522689819 + }, + { + "bits_per_byte": 1.5119997496204038, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.048038363456726, + "step": 6280, + "unclipped_grad_norm": 0.9240654110908508 + }, + { + "bits_per_byte": 1.5072342823476461, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0447351932525635, + "step": 6290, + "unclipped_grad_norm": 0.9724128842353821 + }, + { + "bits_per_byte": 1.4760084243263911, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0230910778045654, + "step": 6300, + "unclipped_grad_norm": 0.9724760055541992 + }, + { + "bits_per_byte": 1.539534687971492, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0671241283416748, + "step": 6310, + "unclipped_grad_norm": 1.0172570943832397 + }, + { + "bits_per_byte": 1.498671266161187, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.03879976272583, + "step": 6320, + "unclipped_grad_norm": 0.9810369610786438 + }, + { + "bits_per_byte": 1.5103010769778318, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.046860933303833, + "step": 6330, + "unclipped_grad_norm": 0.9735086560249329 + }, + { + "bits_per_byte": 1.4197871237766881, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 0.9841214418411255, + "step": 6340, + "unclipped_grad_norm": 0.9049187302589417 + }, + { + "bits_per_byte": 1.5345103868091858, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0636415481567383, + "step": 6350, + "unclipped_grad_norm": 0.9579975008964539 + }, + { + "bits_per_byte": 1.5509206273892133, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0750162601470947, + "step": 6360, + "unclipped_grad_norm": 0.989890456199646 + }, + { + "bits_per_byte": 1.4729708667468522, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0209856033325195, + "step": 6370, + "unclipped_grad_norm": 1.021422266960144 + }, + { + "bits_per_byte": 1.4625091620948676, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0137341022491455, + "step": 6380, + "unclipped_grad_norm": 1.0111291408538818 + }, + { + "bits_per_byte": 1.576993369242666, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0930885076522827, + "step": 6390, + "unclipped_grad_norm": 1.0084502696990967 + }, + { + "bits_per_byte": 1.5426105976821733, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0692561864852905, + "step": 6400, + "unclipped_grad_norm": 1.060890793800354 + }, + { + "bits_per_byte": 1.542634847235945, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0692729949951172, + "step": 6410, + "unclipped_grad_norm": 1.0786292552947998 + }, + { + "bits_per_byte": 1.4677484415707522, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0173656940460205, + "step": 6420, + "unclipped_grad_norm": 1.0185354948043823 + }, + { + "bits_per_byte": 1.5249625979638863, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.057023525238037, + "step": 6430, + "unclipped_grad_norm": 0.9899823069572449 + }, + { + "bits_per_byte": 1.447091949340922, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0030477046966553, + "step": 6440, + "unclipped_grad_norm": 0.9853183031082153 + }, + { + "bits_per_byte": 1.4600725118974434, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.01204514503479, + "step": 6450, + "unclipped_grad_norm": 0.9329874515533447 + }, + { + "bits_per_byte": 1.4881399085356029, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.031499981880188, + "step": 6460, + "unclipped_grad_norm": 0.9553210735321045 + }, + { + "bits_per_byte": 1.5093300629310604, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0461878776550293, + "step": 6470, + "unclipped_grad_norm": 1.0880736112594604 + }, + { + "bits_per_byte": 1.4620568477230984, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.013420581817627, + "step": 6480, + "unclipped_grad_norm": 0.978254497051239 + }, + { + "bits_per_byte": 1.5301041912940807, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0605874061584473, + "step": 6490, + "unclipped_grad_norm": 1.0214275121688843 + }, + { + "bits_per_byte": 1.5703515712489953, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.088484764099121, + "step": 6500, + "unclipped_grad_norm": 1.024044156074524 + }, + { + "bits_per_byte": 1.5418132861127734, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.068703532218933, + "step": 6510, + "unclipped_grad_norm": 1.0472792387008667 + }, + { + "bits_per_byte": 1.5326505664227554, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0623524188995361, + "step": 6520, + "unclipped_grad_norm": 0.9873340129852295 + }, + { + "bits_per_byte": 1.493115194642413, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0349485874176025, + "step": 6530, + "unclipped_grad_norm": 0.9528921246528625 + }, + { + "bits_per_byte": 1.4717057623671055, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.020108699798584, + "step": 6540, + "unclipped_grad_norm": 0.9596993327140808 + }, + { + "bits_per_byte": 1.532975785615537, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0625778436660767, + "step": 6550, + "unclipped_grad_norm": 0.9932572245597839 + }, + { + "bits_per_byte": 1.5382425823155597, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0662285089492798, + "step": 6560, + "unclipped_grad_norm": 1.0132641792297363 + }, + { + "bits_per_byte": 1.5160394501065202, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0508384704589844, + "step": 6570, + "unclipped_grad_norm": 0.9644629955291748 + }, + { + "bits_per_byte": 1.503854823258192, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0423927307128906, + "step": 6580, + "unclipped_grad_norm": 0.9727261662483215 + }, + { + "bits_per_byte": 1.5231672710715294, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0557790994644165, + "step": 6590, + "unclipped_grad_norm": 1.029261589050293 + }, + { + "bits_per_byte": 1.415425385776831, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9810981154441833, + "step": 6600, + "unclipped_grad_norm": 0.9430753588676453 + }, + { + "bits_per_byte": 1.4673490978554489, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0170888900756836, + "step": 6610, + "unclipped_grad_norm": 0.9699459671974182 + }, + { + "bits_per_byte": 1.5770974187464382, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.093160629272461, + "step": 6620, + "unclipped_grad_norm": 1.053364872932434 + }, + { + "bits_per_byte": 1.520643081704816, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0540294647216797, + "step": 6630, + "unclipped_grad_norm": 1.0116336345672607 + }, + { + "bits_per_byte": 1.4843385760035823, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.028865098953247, + "step": 6640, + "unclipped_grad_norm": 0.9483272433280945 + }, + { + "bits_per_byte": 1.481005552229861, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.026554822921753, + "step": 6650, + "unclipped_grad_norm": 0.9847526550292969 + }, + { + "bits_per_byte": 1.5181830418668742, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0523242950439453, + "step": 6660, + "unclipped_grad_norm": 1.0222764015197754 + }, + { + "bits_per_byte": 1.483456133022005, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.028253436088562, + "step": 6670, + "unclipped_grad_norm": 0.9609436392784119 + }, + { + "bits_per_byte": 1.5070292790278177, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.044593095779419, + "step": 6680, + "unclipped_grad_norm": 1.0481340885162354 + }, + { + "bits_per_byte": 1.4999456576040804, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0396831035614014, + "step": 6690, + "unclipped_grad_norm": 1.0560411214828491 + }, + { + "bits_per_byte": 1.5028304945896531, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0416827201843262, + "step": 6700, + "unclipped_grad_norm": 1.0250314474105835 + }, + { + "bits_per_byte": 1.5313854620430087, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0614755153656006, + "step": 6710, + "unclipped_grad_norm": 0.9873788356781006 + }, + { + "bits_per_byte": 1.5848861690387128, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0985593795776367, + "step": 6720, + "unclipped_grad_norm": 1.065596103668213 + }, + { + "bits_per_byte": 1.4535899698384716, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0075517892837524, + "step": 6730, + "unclipped_grad_norm": 0.9767746329307556 + }, + { + "bits_per_byte": 1.4828168735087481, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0278103351593018, + "step": 6740, + "unclipped_grad_norm": 0.9589064717292786 + }, + { + "bits_per_byte": 1.4599156637198565, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0119364261627197, + "step": 6750, + "unclipped_grad_norm": 0.9927419424057007 + }, + { + "bits_per_byte": 1.4484303183299363, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0039753913879395, + "step": 6760, + "unclipped_grad_norm": 0.9825301766395569 + }, + { + "bits_per_byte": 1.469467236183475, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.018557071685791, + "step": 6770, + "unclipped_grad_norm": 1.0425026416778564 + }, + { + "bits_per_byte": 1.4605650702095145, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0123865604400635, + "step": 6780, + "unclipped_grad_norm": 1.0209197998046875 + }, + { + "bits_per_byte": 1.494792713418929, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0361113548278809, + "step": 6790, + "unclipped_grad_norm": 0.9975507855415344 + }, + { + "bits_per_byte": 1.4806505800384802, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0263087749481201, + "step": 6800, + "unclipped_grad_norm": 1.0428087711334229 + }, + { + "bits_per_byte": 1.4777557680591573, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0243022441864014, + "step": 6810, + "unclipped_grad_norm": 0.9905012249946594 + }, + { + "bits_per_byte": 1.4914748241184836, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0338115692138672, + "step": 6820, + "unclipped_grad_norm": 1.0114696025848389 + }, + { + "bits_per_byte": 1.4751165222990168, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.022472858428955, + "step": 6830, + "unclipped_grad_norm": 0.9909161329269409 + }, + { + "bits_per_byte": 1.462379831141419, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0136444568634033, + "step": 6840, + "unclipped_grad_norm": 1.0066962242126465 + }, + { + "bits_per_byte": 1.542950607382929, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0694918632507324, + "step": 6850, + "unclipped_grad_norm": 1.0904512405395508 + }, + { + "bits_per_byte": 1.5094851912821385, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.046295404434204, + "step": 6860, + "unclipped_grad_norm": 1.0577117204666138 + }, + { + "bits_per_byte": 1.4431473552607326, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0003135204315186, + "step": 6870, + "unclipped_grad_norm": 0.931652843952179 + }, + { + "bits_per_byte": 1.4721159409894133, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0203930139541626, + "step": 6880, + "unclipped_grad_norm": 1.0439471006393433 + }, + { + "bits_per_byte": 1.4601951355275085, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0121301412582397, + "step": 6890, + "unclipped_grad_norm": 0.9945755004882812 + }, + { + "bits_per_byte": 1.5078214311176918, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0451421737670898, + "step": 6900, + "unclipped_grad_norm": 1.0577389001846313 + }, + { + "bits_per_byte": 1.51248938422741, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0483777523040771, + "step": 6910, + "unclipped_grad_norm": 1.042457938194275 + }, + { + "bits_per_byte": 1.4546720846777004, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.008301854133606, + "step": 6920, + "unclipped_grad_norm": 1.0035841464996338 + }, + { + "bits_per_byte": 1.4388924184870613, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9973642230033875, + "step": 6930, + "unclipped_grad_norm": 1.0526496171951294 + }, + { + "bits_per_byte": 1.5114688391771902, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0476703643798828, + "step": 6940, + "unclipped_grad_norm": 1.0117003917694092 + }, + { + "bits_per_byte": 1.4256293744264978, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9881709814071655, + "step": 6950, + "unclipped_grad_norm": 1.020249843597412 + }, + { + "bits_per_byte": 1.4392803253560824, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 0.9976330995559692, + "step": 6960, + "unclipped_grad_norm": 1.007938265800476 + }, + { + "bits_per_byte": 1.425250496646647, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 0.9879083633422852, + "step": 6970, + "unclipped_grad_norm": 1.0789695978164673 + }, + { + "bits_per_byte": 1.5310960152416067, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0612748861312866, + "step": 6980, + "unclipped_grad_norm": 1.0816211700439453 + }, + { + "bits_per_byte": 1.4834984407541174, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0282827615737915, + "step": 6990, + "unclipped_grad_norm": 1.0542293787002563 + }, + { + "bits_per_byte": 1.494992041311279, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0362495183944702, + "step": 7000, + "unclipped_grad_norm": 1.040958046913147 + }, + { + "bits_per_byte": 1.4039343649594325, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9731331467628479, + "step": 7010, + "unclipped_grad_norm": 1.0020650625228882 + }, + { + "bits_per_byte": 1.4906596263533938, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0332465171813965, + "step": 7020, + "unclipped_grad_norm": 1.0739529132843018 + }, + { + "bits_per_byte": 1.4642052549976767, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0149097442626953, + "step": 7030, + "unclipped_grad_norm": 1.0350828170776367 + }, + { + "bits_per_byte": 1.4574465088014192, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0102249383926392, + "step": 7040, + "unclipped_grad_norm": 1.0089728832244873 + }, + { + "bits_per_byte": 1.4534154074478458, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0074307918548584, + "step": 7050, + "unclipped_grad_norm": 1.0608088970184326 + }, + { + "bits_per_byte": 1.4416395833606184, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 0.9992684125900269, + "step": 7060, + "unclipped_grad_norm": 0.9864199757575989 + }, + { + "bits_per_byte": 1.4906718371216052, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0332549810409546, + "step": 7070, + "unclipped_grad_norm": 1.0150113105773926 + }, + { + "bits_per_byte": 1.4931912109740944, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.035001277923584, + "step": 7080, + "unclipped_grad_norm": 1.0090219974517822 + }, + { + "bits_per_byte": 1.4509221749582877, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0057026147842407, + "step": 7090, + "unclipped_grad_norm": 1.0420514345169067 + }, + { + "bits_per_byte": 1.4389093587781714, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9973759651184082, + "step": 7100, + "unclipped_grad_norm": 1.0253499746322632 + }, + { + "bits_per_byte": 1.41304419998431, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 0.979447603225708, + "step": 7110, + "unclipped_grad_norm": 0.9803867936134338 + }, + { + "bits_per_byte": 1.4722433801337027, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0204813480377197, + "step": 7120, + "unclipped_grad_norm": 1.0725315809249878 + }, + { + "bits_per_byte": 1.5197811046586915, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0534319877624512, + "step": 7130, + "unclipped_grad_norm": 1.0701161623001099 + }, + { + "bits_per_byte": 1.465326925846604, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0156872272491455, + "step": 7140, + "unclipped_grad_norm": 1.0326207876205444 + }, + { + "bits_per_byte": 1.5100518741167317, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.046688199043274, + "step": 7150, + "unclipped_grad_norm": 1.0673091411590576 + }, + { + "bits_per_byte": 1.5022746466620645, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.041297435760498, + "step": 7160, + "unclipped_grad_norm": 1.059691071510315 + }, + { + "bits_per_byte": 1.514605802728928, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.049844741821289, + "step": 7170, + "unclipped_grad_norm": 1.085069179534912 + }, + { + "bits_per_byte": 1.4489440305080636, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0043314695358276, + "step": 7180, + "unclipped_grad_norm": 0.9880333542823792 + }, + { + "bits_per_byte": 1.5073161460894569, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0447919368743896, + "step": 7190, + "unclipped_grad_norm": 1.0678606033325195 + }, + { + "bits_per_byte": 1.4434393818018978, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0005159378051758, + "step": 7200, + "unclipped_grad_norm": 1.0179466009140015 + }, + { + "bits_per_byte": 1.4303239848470999, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 0.9914250373840332, + "step": 7210, + "unclipped_grad_norm": 1.005952000617981 + }, + { + "bits_per_byte": 1.4493132772594661, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0045874118804932, + "step": 7220, + "unclipped_grad_norm": 1.0151865482330322 + }, + { + "bits_per_byte": 1.475039818036732, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0224196910858154, + "step": 7230, + "unclipped_grad_norm": 1.0841153860092163 + }, + { + "bits_per_byte": 1.5119313005253603, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0479909181594849, + "step": 7240, + "unclipped_grad_norm": 1.09213387966156 + }, + { + "bits_per_byte": 1.494972951237033, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.03623628616333, + "step": 7250, + "unclipped_grad_norm": 1.0454415082931519 + }, + { + "bits_per_byte": 1.4613747645297765, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0129477977752686, + "step": 7260, + "unclipped_grad_norm": 1.0101300477981567 + }, + { + "bits_per_byte": 1.4686286487778681, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0179758071899414, + "step": 7270, + "unclipped_grad_norm": 1.0528877973556519 + }, + { + "bits_per_byte": 1.4749450555961066, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.022354006767273, + "step": 7280, + "unclipped_grad_norm": 1.0265198945999146 + }, + { + "bits_per_byte": 1.473522931056122, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0213682651519775, + "step": 7290, + "unclipped_grad_norm": 1.066664457321167 + }, + { + "bits_per_byte": 1.4559291058728565, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0091731548309326, + "step": 7300, + "unclipped_grad_norm": 1.0080223083496094 + }, + { + "bits_per_byte": 1.445694074354709, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0020787715911865, + "step": 7310, + "unclipped_grad_norm": 1.0805282592773438 + }, + { + "bits_per_byte": 1.4250067972303744, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9877394437789917, + "step": 7320, + "unclipped_grad_norm": 1.0912657976150513 + }, + { + "bits_per_byte": 1.4783119599520478, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0246877670288086, + "step": 7330, + "unclipped_grad_norm": 1.0849530696868896 + }, + { + "bits_per_byte": 1.4380544330207323, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9967833757400513, + "step": 7340, + "unclipped_grad_norm": 1.042580485343933 + }, + { + "bits_per_byte": 1.4567071553853599, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0097124576568604, + "step": 7350, + "unclipped_grad_norm": 1.0958631038665771 + }, + { + "bits_per_byte": 1.4481006275882324, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0037468671798706, + "step": 7360, + "unclipped_grad_norm": 1.0337251424789429 + }, + { + "bits_per_byte": 1.4027950658887918, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 0.9723434448242188, + "step": 7370, + "unclipped_grad_norm": 1.0347329378128052 + }, + { + "bits_per_byte": 1.4160821015291507, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.981553316116333, + "step": 7380, + "unclipped_grad_norm": 1.0372161865234375 + }, + { + "bits_per_byte": 1.421894943145663, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 0.9855824708938599, + "step": 7390, + "unclipped_grad_norm": 1.038377046585083 + }, + { + "bits_per_byte": 1.4461825050831596, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0024173259735107, + "step": 7400, + "unclipped_grad_norm": 1.057859182357788 + }, + { + "bits_per_byte": 1.475963364871865, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0230598449707031, + "step": 7410, + "unclipped_grad_norm": 1.0708297491073608 + }, + { + "bits_per_byte": 1.469500256852441, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0185799598693848, + "step": 7420, + "unclipped_grad_norm": 1.062455654144287 + }, + { + "bits_per_byte": 1.41574011402791, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9813162684440613, + "step": 7430, + "unclipped_grad_norm": 1.0484251976013184 + }, + { + "bits_per_byte": 1.3821044351027654, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9580017924308777, + "step": 7440, + "unclipped_grad_norm": 1.030645728111267 + }, + { + "bits_per_byte": 1.4987479704234719, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0388529300689697, + "step": 7450, + "unclipped_grad_norm": 1.0465614795684814 + }, + { + "bits_per_byte": 1.4339140366925356, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.993913471698761, + "step": 7460, + "unclipped_grad_norm": 1.0371458530426025 + }, + { + "bits_per_byte": 1.4012647642614147, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9712827205657959, + "step": 7470, + "unclipped_grad_norm": 1.0733681917190552 + }, + { + "bits_per_byte": 1.5010874504231586, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.040474534034729, + "step": 7480, + "unclipped_grad_norm": 1.0303142070770264 + }, + { + "bits_per_byte": 1.4650142613873354, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0154705047607422, + "step": 7490, + "unclipped_grad_norm": 1.0710126161575317 + }, + { + "bits_per_byte": 1.472811954777455, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0208754539489746, + "step": 7500, + "unclipped_grad_norm": 1.0901119709014893 + }, + { + "bits_per_byte": 1.4645168875610401, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0151257514953613, + "step": 7510, + "unclipped_grad_norm": 1.024397611618042 + }, + { + "bits_per_byte": 1.4743771688829574, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0219603776931763, + "step": 7520, + "unclipped_grad_norm": 1.0934079885482788 + }, + { + "bits_per_byte": 1.46691879726299, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0167906284332275, + "step": 7530, + "unclipped_grad_norm": 1.081713318824768 + }, + { + "bits_per_byte": 1.4640198577000465, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0147812366485596, + "step": 7540, + "unclipped_grad_norm": 1.0780924558639526 + }, + { + "bits_per_byte": 1.4520784143200383, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0065040588378906, + "step": 7550, + "unclipped_grad_norm": 1.04312264919281 + }, + { + "bits_per_byte": 1.3812490793886991, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 0.9574089050292969, + "step": 7560, + "unclipped_grad_norm": 1.0638409852981567 + }, + { + "bits_per_byte": 1.4099118799641457, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9772764444351196, + "step": 7570, + "unclipped_grad_norm": 1.0123975276947021 + }, + { + "bits_per_byte": 1.3981073347742243, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9690941572189331, + "step": 7580, + "unclipped_grad_norm": 1.0492113828659058 + }, + { + "bits_per_byte": 1.472455950690169, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0206286907196045, + "step": 7590, + "unclipped_grad_norm": 1.0436540842056274 + }, + { + "bits_per_byte": 1.5055485084038889, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0435667037963867, + "step": 7600, + "unclipped_grad_norm": 1.140743374824524 + }, + { + "bits_per_byte": 1.4701950667619266, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.01906156539917, + "step": 7610, + "unclipped_grad_norm": 1.082565426826477 + }, + { + "bits_per_byte": 1.458967179400348, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0112789869308472, + "step": 7620, + "unclipped_grad_norm": 1.0694279670715332 + }, + { + "bits_per_byte": 1.4490995028244436, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0044392347335815, + "step": 7630, + "unclipped_grad_norm": 1.0964275598526 + }, + { + "bits_per_byte": 1.4314776444690875, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9922246932983398, + "step": 7640, + "unclipped_grad_norm": 1.15219247341156 + }, + { + "bits_per_byte": 1.4066524647649938, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 0.9750171899795532, + "step": 7650, + "unclipped_grad_norm": 1.064915418624878 + }, + { + "bits_per_byte": 1.416584462852321, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9819015264511108, + "step": 7660, + "unclipped_grad_norm": 1.141569972038269 + }, + { + "bits_per_byte": 1.4608237321164121, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0125658512115479, + "step": 7670, + "unclipped_grad_norm": 1.0876517295837402 + }, + { + "bits_per_byte": 1.4539619683122877, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0078096389770508, + "step": 7680, + "unclipped_grad_norm": 1.0938481092453003 + }, + { + "bits_per_byte": 1.433180616677931, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9934051036834717, + "step": 7690, + "unclipped_grad_norm": 1.070209264755249 + }, + { + "bits_per_byte": 1.4952543148538446, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0364313125610352, + "step": 7700, + "unclipped_grad_norm": 1.0864635705947876 + }, + { + "bits_per_byte": 1.412102165014209, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9787946343421936, + "step": 7710, + "unclipped_grad_norm": 1.040504813194275 + }, + { + "bits_per_byte": 1.4180753804526507, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9829349517822266, + "step": 7720, + "unclipped_grad_norm": 1.0190634727478027 + }, + { + "bits_per_byte": 1.4255394275000965, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9881086349487305, + "step": 7730, + "unclipped_grad_norm": 1.0293750762939453 + }, + { + "bits_per_byte": 1.468768986620972, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0180730819702148, + "step": 7740, + "unclipped_grad_norm": 1.0531129837036133 + }, + { + "bits_per_byte": 1.4037740771288283, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9730220437049866, + "step": 7750, + "unclipped_grad_norm": 1.1033527851104736 + }, + { + "bits_per_byte": 1.4111542826339787, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9781376123428345, + "step": 7760, + "unclipped_grad_norm": 1.0707266330718994 + }, + { + "bits_per_byte": 1.4583370349675868, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.010842204093933, + "step": 7770, + "unclipped_grad_norm": 1.0769577026367188 + }, + { + "bits_per_byte": 1.4516912813729461, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0062357187271118, + "step": 7780, + "unclipped_grad_norm": 1.1653907299041748 + }, + { + "bits_per_byte": 1.4586383485718983, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.011051058769226, + "step": 7790, + "unclipped_grad_norm": 1.0739812850952148 + }, + { + "bits_per_byte": 1.4426457678594913, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 0.9999658465385437, + "step": 7800, + "unclipped_grad_norm": 1.0972295999526978 + }, + { + "bits_per_byte": 1.4198621082124645, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9841734170913696, + "step": 7810, + "unclipped_grad_norm": 1.075659990310669 + }, + { + "bits_per_byte": 1.4512271001982673, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0059139728546143, + "step": 7820, + "unclipped_grad_norm": 1.1029958724975586 + }, + { + "bits_per_byte": 1.4583721194283628, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0108665227890015, + "step": 7830, + "unclipped_grad_norm": 1.081362009048462 + }, + { + "bits_per_byte": 1.4242661539444337, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9872260689735413, + "step": 7840, + "unclipped_grad_norm": 1.0629743337631226 + }, + { + "bits_per_byte": 1.423362471105475, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9865996837615967, + "step": 7850, + "unclipped_grad_norm": 1.0624163150787354 + }, + { + "bits_per_byte": 1.4807702799634808, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0263917446136475, + "step": 7860, + "unclipped_grad_norm": 1.0853880643844604 + }, + { + "bits_per_byte": 1.4488509878939468, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0042669773101807, + "step": 7870, + "unclipped_grad_norm": 1.1049087047576904 + }, + { + "bits_per_byte": 1.4220910033676464, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9857183694839478, + "step": 7880, + "unclipped_grad_norm": 1.0985115766525269 + }, + { + "bits_per_byte": 1.4594506226319235, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0116140842437744, + "step": 7890, + "unclipped_grad_norm": 1.0736068487167358 + }, + { + "bits_per_byte": 1.4105420243969071, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 0.9777132272720337, + "step": 7900, + "unclipped_grad_norm": 1.1535842418670654 + }, + { + "bits_per_byte": 1.463574250651661, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0144723653793335, + "step": 7910, + "unclipped_grad_norm": 1.1884382963180542 + }, + { + "bits_per_byte": 1.4830432026772835, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0279672145843506, + "step": 7920, + "unclipped_grad_norm": 1.1013721227645874 + }, + { + "bits_per_byte": 1.435883066062264, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9952782988548279, + "step": 7930, + "unclipped_grad_norm": 1.0824542045593262 + }, + { + "bits_per_byte": 1.4594231054077853, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0115950107574463, + "step": 7940, + "unclipped_grad_norm": 1.1213319301605225 + }, + { + "bits_per_byte": 1.408353201199376, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9761960506439209, + "step": 7950, + "unclipped_grad_norm": 1.0871061086654663 + }, + { + "bits_per_byte": 1.4735662706841393, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0213983058929443, + "step": 7960, + "unclipped_grad_norm": 1.1091856956481934 + }, + { + "bits_per_byte": 1.4267374586460069, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9889390468597412, + "step": 7970, + "unclipped_grad_norm": 1.0478838682174683 + }, + { + "bits_per_byte": 1.472857186214632, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0209068059921265, + "step": 7980, + "unclipped_grad_norm": 1.2172881364822388 + }, + { + "bits_per_byte": 1.4136101948882995, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9798399209976196, + "step": 7990, + "unclipped_grad_norm": 1.1755717992782593 + }, + { + "bits_per_byte": 1.46345644253582, + "learning_rate": 3e-05, + "loss_nats": 1.0143907070159912, + "step": 8000, + "unclipped_grad_norm": 1.0856302976608276 + } + ] + }, + "depth-32-block-seed-2026073003": { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "513b85666fcdbf75424596e968cd73733314ec188345e453998de08b18dcef65", + "depth": 32, + "diagnostics": [ + { + "activation_grad_rms_by_block": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "population_cv": 0.2498932502086342 + }, + "activation_output_rms_by_block": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.005511203169589862, + "first_to_last_ratio": 0.6356203088456116, + "imbalance_abs_log_ratio": 0.45315389258709987, + "last_quartile_mean": 0.008670590119436383, + "mean": 0.007157071326219011, + "normalized": [ + 0.4829581625738996, + 0.6886426787339398, + 0.8458438602600115, + 0.9725589429115095, + 0.5050274053272774, + 0.7207530129760953, + 0.8901040788372985, + 1.0544005221525679, + 0.5528397766850007, + 0.7916459680002588, + 0.9438571997073384, + 1.1102770425361632, + 0.6125217730505899, + 0.9295491721602742, + 1.1083022471557418, + 1.3061042379491232, + 0.6819634955048963, + 0.9650194302374437, + 1.1362396940587196, + 1.3715945478756304, + 0.7619466772614072, + 1.0650300118366156, + 1.3201940438400879, + 1.490851576861243, + 0.7782743937317399, + 1.0812722353862223, + 1.3287334463206348, + 1.5181249897087195, + 0.8365475786723071, + 1.138987569259587, + 1.3797450031110483, + 1.6300892253166077 + ], + "population_cv": 0.3025375276561286 + }, + "bits_per_byte": 8.06433666787865, + "branch_output_rms_by_sublayer": [ + 0.0021227982360869646, + 0.0027448106557130814, + 0.00224129855632782, + 0.0027243648655712605, + 0.0024903982412070036, + 0.0027103275060653687, + 0.0023722455371171236, + 0.0027234479784965515, + 0.0023577234242111444, + 0.0027595730498433113, + 0.0023993616923689842, + 0.0027272943407297134, + 0.002688874490559101, + 0.002728298306465149, + 0.0029664887115359306, + 0.0027007071767002344, + 0.0029547461308538914, + 0.002684502862393856, + 0.003052416956052184, + 0.002683063969016075, + 0.0028214550111442804, + 0.0027139110025018454, + 0.0032544767018407583, + 0.002666700165718794, + 0.00348988501355052, + 0.002679243916645646, + 0.003914494998753071, + 0.0026584581937640905, + 0.0035071265883743763, + 0.002684473292902112, + 0.004211232997477055, + 0.0026407993864268064, + 0.004072706215083599, + 0.002663099905475974, + 0.004050548188388348, + 0.0026283918414264917, + 0.003929397091269493, + 0.002597011858597398, + 0.004410331137478352, + 0.0026434636674821377, + 0.004824656527489424, + 0.0025708195753395557, + 0.00433384021744132, + 0.002691218862310052, + 0.004475373309105635, + 0.0026598076801747084, + 0.004755842965096235, + 0.0026596153620630503, + 0.004843301139771938, + 0.0026765079237520695, + 0.0050001065246760845, + 0.002582729561254382, + 0.005346097983419895, + 0.002614464145153761, + 0.004958339501172304, + 0.0025760054122656584, + 0.0054801348596811295, + 0.002634295728057623, + 0.005455824080854654, + 0.00259007909335196, + 0.005286100786179304, + 0.00257018837146461, + 0.005402065813541412, + 0.0025643527042120695 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.006159982634052032, + 0.005514663551817784, + 0.005191681274696802, + 0.00489831575692433, + 0.004666415075367917, + 0.004399813239791133, + 0.004483878567781538, + 0.004868281936291656, + 0.00443828064080646, + 0.004266304440803418, + 0.0039471188591477255, + 0.004494388758593749, + 0.0042334459038140095, + 0.00446434979681851, + 0.004322339302329385, + 0.004638471384339534, + 0.004165330760400444, + 0.004499937764754925, + 0.003981478974932938, + 0.004372386299982334, + 0.00458495381977442, + 0.00429501466057645, + 0.003963900484885007, + 0.004271473918508079, + 0.004041799650627023, + 0.004206875397956982, + 0.003955591472942362, + 0.003916464042690177, + 0.003989587235588314, + 0.003922309599766117, + 0.003679536678174494, + 0.003763244111925236 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.005022879004590398, + "first_to_last_ratio": 1.2766484804448084, + "imbalance_abs_log_ratio": 0.24423826934058726, + "last_quartile_mean": 0.003934426023708838, + "mean": 0.0043936754999019155, + "normalized": [ + 1.4020112851278042, + 1.255136741878387, + 1.1816260155791434, + 1.1148560600421402, + 1.0620754936207943, + 1.0013969488391563, + 1.0205302070855338, + 1.108020366183241, + 1.0101521245493756, + 0.9710103627130996, + 0.8983637638318627, + 1.0229223252136126, + 0.9635317637610054, + 1.0160854612313022, + 0.9837638902613262, + 1.0557155130011497, + 0.9480287655502622, + 1.0241852783291305, + 0.9061841219320409, + 0.995154580733134, + 1.0435349219296635, + 0.9775448051801577, + 0.9021832597727114, + 0.9721869351988865, + 0.9199131002545027, + 0.9574843199164109, + 0.9002921296829196, + 0.891386731400079, + 0.9080295610536958, + 0.892717179467095, + 0.8374620925593245, + 0.8565138941210488 + ], + "population_cv": 0.11556590700625036 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6931471824645996, + "mean_weights": [ + 0.5, + 0.5 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0986123085021973, + "mean_weights": [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3862943649291992, + "mean_weights": [ + 0.25, + 0.25, + 0.25, + 0.25 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6094379425048828, + "mean_weights": [ + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343, + 0.20000001788139343 + ], + "sources": 5 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.791759729385376, + "mean_weights": [ + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592, + 0.1666666567325592 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9459102153778076, + "mean_weights": [ + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548, + 0.1428571492433548 + ], + "sources": 7 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079441547393799, + "mean_weights": [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028097106143832207, + 0.014079716056585312, + 0.014136734418570995, + 0.014167090877890587, + 0.014224153943359852, + 0.014255538582801819, + 0.014337832108139992, + 0.014355218037962914, + 0.014408928342163563, + 0.009655896574258804, + 0.009704872034490108, + 0.009734091348946095, + 0.009773381985723972, + 0.009828771464526653, + 0.00987160298973322, + 0.009915035218000412, + 0.00994801800698042, + 0.007479202933609486, + 0.007515520788729191, + 0.007559895049780607, + 0.007602147292345762, + 0.0076358383521437645, + 0.007658426649868488, + 0.007706487085670233, + 0.007734050042927265, + 0.0062490664422512054, + 0.0062688966281712055, + 0.006315518636256456, + 0.006336068734526634, + 0.0063744354993104935, + 0.006399397272616625, + 0.006439714692533016, + 0.006462901830673218, + 0.0054116579703986645, + 0.005432929843664169, + 0.005481995176523924, + 0.005504137370735407, + 0.005541813559830189, + 0.005560056772083044, + 0.005614723544567823, + 0.005631812382489443, + 0.004862982779741287, + 0.004873365629464388, + 0.004923288244754076, + 0.004937449004501104, + 0.0049925679340958595, + 0.005005750339478254, + 0.005004179198294878, + 0.005022916477173567, + 0.004448203835636377, + 0.0044684382155537605, + 0.004490054678171873, + 0.004500201437622309, + 0.00455933902412653, + 0.004573327023535967, + 0.004625064320862293, + 0.00463611725717783, + 0.004160413984209299, + 0.004166214261204004, + 0.004242690280079842, + 0.0042496914975345135, + 0.004270957782864571, + 0.004283483140170574, + 0.004303553607314825 + ], + "loss_nats": 5.5897722244262695, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1972246170043945, + "mean_weights": [ + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519, + 0.11111113429069519 + ], + "sources": 9 + }, + "step": 0, + "stream_state_rms_by_sublayer": [ + 0.0021227982360869646, + 0.0034565660171210766, + 0.00413061399012804, + 0.004928664769977331, + 0.005416445434093475, + 0.0060537648387253284, + 0.006388718727976084, + 0.006960673723369837, + 0.0023577234242111444, + 0.0036145171616226435, + 0.004388594534248114, + 0.0051584807224571705, + 0.00574506726115942, + 0.006370538379997015, + 0.007066869642585516, + 0.007546419743448496, + 0.0029547461308538914, + 0.00395671371370554, + 0.004991721361875534, + 0.005665866658091545, + 0.006190058775246143, + 0.006755253300070763, + 0.0074937604367733, + 0.007946331985294819, + 0.00348988501355052, + 0.004383862018585205, + 0.006057556718587875, + 0.006652849726378918, + 0.0074848332442343235, + 0.007932198233902454, + 0.00897275935858488, + 0.009347881190478802, + 0.004072706215083599, + 0.004880861379206181, + 0.0063449000008404255, + 0.006906712893396616, + 0.007735534105449915, + 0.008132148534059525, + 0.00946879480034113, + 0.009816600009799004, + 0.004824656527489424, + 0.005453306715935469, + 0.007074959110468626, + 0.007622495759278536, + 0.009095527231693268, + 0.009448722936213017, + 0.010346688330173492, + 0.010670131072402, + 0.004843301139771938, + 0.0055701653473079205, + 0.007334613241255283, + 0.007738742511719465, + 0.009146629832684994, + 0.009509840048849583, + 0.010587968863546848, + 0.010865328833460808, + 0.0054801348596811295, + 0.005987230688333511, + 0.0077637480571866035, + 0.00815181527286768, + 0.009536821395158768, + 0.009874933399260044, + 0.011367839761078358, + 0.011666664853692055 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.886429764563218e-05, + 6.52737362543121e-05, + 6.186874816194177e-05, + 5.901425174670294e-05, + 5.411080928752199e-05, + 5.1034174248343334e-05, + 4.815354259335436e-05, + 4.550318044493906e-05, + 4.120725498069078e-05, + 3.8442405639216304e-05, + 3.604330777307041e-05, + 3.37670098815579e-05, + 3.108748569502495e-05, + 2.9203254598542117e-05, + 2.7565200070966966e-05, + 2.5956482204492204e-05, + 2.4566294086980633e-05, + 2.329334893147461e-05, + 2.2260202968027443e-05, + 2.14147730730474e-05, + 2.0902438336634077e-05, + 2.0279332602513023e-05, + 1.9777973648160696e-05, + 1.9328235794091597e-05, + 1.9035323930438608e-05, + 1.8655482563190162e-05, + 1.832095585996285e-05, + 1.8063024981529452e-05, + 1.7733544154907577e-05, + 1.752249227138236e-05, + 1.7320473489235155e-05, + 1.7174717868329026e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.6727842547843466e-05, + "first_to_last_ratio": 3.155359202626439, + "imbalance_abs_log_ratio": 1.1491023413751247, + "last_quartile_mean": 1.79782518898719e-05, + "mean": 3.227324236831919e-05, + "normalized": [ + 2.133789250541258, + 2.0225341944070556, + 1.9170292050567193, + 1.8285814320482991, + 1.67664620337743, + 1.581315371598383, + 1.4920577871848402, + 1.409935200362978, + 1.2768241415105412, + 1.1911541208190792, + 1.1168170635514478, + 1.0462850151897058, + 0.9632588303411923, + 0.904875136661487, + 0.8541193275958588, + 0.8042725273235084, + 0.7611969633115008, + 0.7217542218299198, + 0.6897417592562383, + 0.6635457580819046, + 0.647670850610065, + 0.628363657145899, + 0.6128288389014056, + 0.5988935221787641, + 0.5898175247840765, + 0.5780479801280578, + 0.5676825294116558, + 0.559690432569022, + 0.5494813304632692, + 0.5429417990112825, + 0.5366821620079203, + 0.5321658627392354 + ], + "population_cv": 0.49221598973047537 + }, + "activation_output_rms_by_block": [ + 0.0055193523876369, + 0.009546536952257156, + 0.013977172784507275, + 0.018744563683867455, + 0.00660830270498991, + 0.011765030212700367, + 0.017156247049570084, + 0.022267481312155724, + 0.007449587807059288, + 0.013175037689507008, + 0.019073888659477234, + 0.025712018832564354, + 0.008972279727458954, + 0.01624389924108982, + 0.023743286728858948, + 0.03214457631111145, + 0.00991375558078289, + 0.019281653687357903, + 0.02868141047656536, + 0.03889070823788643, + 0.0127727584913373, + 0.022841427475214005, + 0.034929387271404266, + 0.04634999856352806, + 0.014845813624560833, + 0.029589947313070297, + 0.046351123601198196, + 0.06254568696022034, + 0.01947367936372757, + 0.0370744951069355, + 0.05507569760084152, + 0.07403328269720078 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.013198085885960609, + "first_to_last_ratio": 0.31146869331458016, + "imbalance_abs_log_ratio": 1.1664564488083473, + "last_quartile_mean": 0.04237371578346938, + "mean": 0.0251484402542701, + "normalized": [ + 0.21947096248642048, + 0.3796075166385794, + 0.5557868656341027, + 0.7453569085933552, + 0.26277187126417706, + 0.46782345520226504, + 0.6821992487847045, + 0.8854418439877121, + 0.2962246458125521, + 0.5238908479530829, + 0.7584521531604159, + 1.0224100808080359, + 0.35677281122575777, + 0.6459207440641045, + 0.9441256192748351, + 1.2781936369057099, + 0.39420956053525336, + 0.7667137004285568, + 1.140484665711838, + 1.5464461352144077, + 0.5078946591595691, + 0.9082641803733982, + 1.3889285744261377, + 1.8430565909811456, + 0.5903274109431131, + 1.1766116313335198, + 1.8431013268637195, + 2.487060283971303, + 0.7743493897368454, + 1.4742264224772512, + 2.190024392923927, + 2.9438518631242045 + ], + "population_cv": 0.6624204772463126 + }, + "bits_per_byte": 5.18286091123436, + "branch_output_rms_by_sublayer": [ + 0.002723898272961378, + 0.004827107302844524, + 0.0025367254856973886, + 0.004943604115396738, + 0.0027546093333512545, + 0.00549353240057826, + 0.002648916095495224, + 0.005891259294003248, + 0.0027809101156890392, + 0.0059933834709227085, + 0.0025255572982132435, + 0.006044833920896053, + 0.003397067543119192, + 0.0061737713403999805, + 0.0033452848438173532, + 0.0059272525832057, + 0.003559867851436138, + 0.0063678608275949955, + 0.0038245979230850935, + 0.0061998614110052586, + 0.003386941971257329, + 0.006479958072304726, + 0.00461281044408679, + 0.006899545900523663, + 0.004887574352324009, + 0.007097913417965174, + 0.005150536075234413, + 0.006852242164313793, + 0.0054018739610910416, + 0.006816149223595858, + 0.006125702988356352, + 0.007103090640157461, + 0.005625482182949781, + 0.007109561935067177, + 0.006412550341337919, + 0.007626448292285204, + 0.0072620403952896595, + 0.007445307914167643, + 0.008774910122156143, + 0.007442969363182783, + 0.008495661430060863, + 0.007471293676644564, + 0.007638709619641304, + 0.007774693425744772, + 0.009623398073017597, + 0.007449399214237928, + 0.008996927179396152, + 0.007351893465965986, + 0.00961991585791111, + 0.008440978825092316, + 0.011907265521585941, + 0.008385274559259415, + 0.014190610498189926, + 0.008216616697609425, + 0.013635334558784962, + 0.007803319953382015, + 0.014384749345481396, + 0.007909612730145454, + 0.01446358859539032, + 0.007761957123875618, + 0.013604369945824146, + 0.008686874993145466, + 0.015084062702953815, + 0.008338849991559982 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00023786681750428726, + 0.00020576573809540593, + 0.00019740566304489066, + 0.0001813902134209802, + 0.0001696969276866905, + 0.00015923444695871368, + 0.00017314685145041394, + 0.00015953319119459735, + 0.0001633125663305534, + 0.0001601758879769712, + 0.00014581270072009644, + 0.00016837778263142155, + 0.0001603117902727929, + 0.00016340123818181664, + 0.00015138579649161358, + 0.00015756725288262655, + 0.00013845240957374987, + 0.0001515626583813202, + 0.00014303549429743115, + 0.00016115390114098118, + 0.00015144047315466305, + 0.00013355245347473064, + 0.00014934284038847835, + 0.00014161248575260865, + 0.00014194691162247068, + 0.00015146794722523853, + 0.00015520640136475585, + 0.0001429086966261467, + 0.00014288517283478702, + 0.00013975759356060232, + 0.00013469418893153685, + 0.00013332122361590492 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00018550498116949745, + "first_to_last_ratio": 1.2992954513055368, + "imbalance_abs_log_ratio": 0.26182215701959494, + "last_quartile_mean": 0.00014277351697268036, + "mean": 0.00015833517864966492, + "normalized": [ + 1.5022992333914333, + 1.2995579368416077, + 1.2467580782011416, + 1.1456090489045851, + 1.0717575786626974, + 1.0056795231275704, + 1.0935463169149642, + 1.0075663068381233, + 1.0314357663491922, + 1.0116253971038178, + 0.9209115874541316, + 1.0634262333071103, + 1.0124837173898131, + 1.0319957925671146, + 0.9561096768430238, + 0.9951499990489321, + 0.874426079880151, + 0.9572266862859976, + 0.9033715407863586, + 1.0178022503612605, + 0.9564549991113648, + 0.8434793494011273, + 0.9432069465681839, + 0.894384223141863, + 0.8964963619142705, + 0.9566285175348117, + 0.9802395316594056, + 0.9025707227220104, + 0.9024221531396832, + 0.8826692510944287, + 0.8506902261408595, + 0.8420189673129665 + ], + "population_cv": 0.13848454603612886 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6920576691627502, + "mean_weights": [ + 0.5229578018188477, + 0.47704219818115234 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6922386288642883, + "mean_weights": [ + 0.5206724405288696, + 0.479327529668808 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6930604577064514, + "mean_weights": [ + 0.5023424625396729, + 0.4976575970649719 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6924410462379456, + "mean_weights": [ + 0.5176653861999512, + 0.48233455419540405 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929901838302612, + "mean_weights": [ + 0.5013687014579773, + 0.4986313283443451 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6929076313972473, + "mean_weights": [ + 0.5098068714141846, + 0.49019312858581543 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6927430629730225, + "mean_weights": [ + 0.5127849578857422, + 0.4872150421142578 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6928876638412476, + "mean_weights": [ + 0.5100651383399963, + 0.48993486166000366 + ], + "sources": 2 + }, + { + "entropy_mean": 1.098104476928711, + "mean_weights": [ + 0.33572155237197876, + 0.32083213329315186, + 0.34344637393951416 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098517656326294, + "mean_weights": [ + 0.3379058539867401, + 0.3309776186943054, + 0.33111652731895447 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984238386154175, + "mean_weights": [ + 0.34072375297546387, + 0.3312225341796875, + 0.3280537724494934 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0985376834869385, + "mean_weights": [ + 0.3372243344783783, + 0.33110499382019043, + 0.3316706418991089 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0983879566192627, + "mean_weights": [ + 0.34103795886039734, + 0.33302855491638184, + 0.3259335160255432 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0984759330749512, + "mean_weights": [ + 0.3378061056137085, + 0.3331226110458374, + 0.3290712833404541 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0982115268707275, + "mean_weights": [ + 0.3427826464176178, + 0.33246177434921265, + 0.32475557923316956 + ], + "sources": 3 + }, + { + "entropy_mean": 1.098510980606079, + "mean_weights": [ + 0.336602121591568, + 0.33530712127685547, + 0.32809075713157654 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3860652446746826, + "mean_weights": [ + 0.25563088059425354, + 0.25276094675064087, + 0.24706901609897614, + 0.24453918635845184 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861669301986694, + "mean_weights": [ + 0.25406360626220703, + 0.25270599126815796, + 0.24658669531345367, + 0.24664369225502014 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386000394821167, + "mean_weights": [ + 0.2551618814468384, + 0.2541123330593109, + 0.2483193278312683, + 0.24240648746490479 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386232852935791, + "mean_weights": [ + 0.25183945894241333, + 0.25246676802635193, + 0.24847325682640076, + 0.24722051620483398 + ], + "sources": 4 + }, + { + "entropy_mean": 1.385960340499878, + "mean_weights": [ + 0.25454509258270264, + 0.25490614771842957, + 0.24972698092460632, + 0.24082177877426147 + ], + "sources": 4 + }, + { + "entropy_mean": 1.386134147644043, + "mean_weights": [ + 0.25200384855270386, + 0.25437670946121216, + 0.24930089712142944, + 0.24431857466697693 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3859115839004517, + "mean_weights": [ + 0.2541099786758423, + 0.2563633620738983, + 0.2499358206987381, + 0.2395908385515213 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3861732482910156, + "mean_weights": [ + 0.2492598295211792, + 0.25510817766189575, + 0.2504415512084961, + 0.24519044160842896 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6091704368591309, + "mean_weights": [ + 0.20294126868247986, + 0.20561827719211578, + 0.20076946914196014, + 0.19495637714862823, + 0.19571462273597717 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093817949295044, + "mean_weights": [ + 0.1999536156654358, + 0.20290037989616394, + 0.19905918836593628, + 0.1980593204498291, + 0.2000274658203125 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6089024543762207, + "mean_weights": [ + 0.20434537529945374, + 0.20689502358436584, + 0.20309439301490784, + 0.19485914707183838, + 0.1908060759305954 + ], + "sources": 5 + }, + { + "entropy_mean": 1.609398365020752, + "mean_weights": [ + 0.20015040040016174, + 0.2025333195924759, + 0.2003234326839447, + 0.19931097328662872, + 0.19768190383911133 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085304021835327, + "mean_weights": [ + 0.20286625623703003, + 0.21047379076480865, + 0.20388861000537872, + 0.19463706016540527, + 0.18813428282737732 + ], + "sources": 5 + }, + { + "entropy_mean": 1.60928213596344, + "mean_weights": [ + 0.20026028156280518, + 0.20464842021465302, + 0.20178082585334778, + 0.1985689103603363, + 0.19474159181118011 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6085617542266846, + "mean_weights": [ + 0.20295967161655426, + 0.2110326737165451, + 0.20332126319408417, + 0.194858580827713, + 0.18782781064510345 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6093013286590576, + "mean_weights": [ + 0.20157885551452637, + 0.20404376089572906, + 0.20028473436832428, + 0.19877153635025024, + 0.19532111287117004 + ], + "sources": 5 + }, + { + "entropy_mean": 1.79081130027771, + "mean_weights": [ + 0.1695082187652588, + 0.17698298394680023, + 0.17063108086585999, + 0.16265572607517242, + 0.15738935768604279, + 0.16283263266086578 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917133569717407, + "mean_weights": [ + 0.16738522052764893, + 0.1689736694097519, + 0.16775797307491302, + 0.16576024889945984, + 0.16475272178649902, + 0.16537019610404968 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7902145385742188, + "mean_weights": [ + 0.17057637870311737, + 0.18038460612297058, + 0.17244961857795715, + 0.164292573928833, + 0.15751495957374573, + 0.15478186309337616 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917354106903076, + "mean_weights": [ + 0.16660897433757782, + 0.16850554943084717, + 0.1666441559791565, + 0.16635674238204956, + 0.16625551879405975, + 0.1656290590763092 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7898752689361572, + "mean_weights": [ + 0.17184747755527496, + 0.180228590965271, + 0.17470473051071167, + 0.16429948806762695, + 0.15632927417755127, + 0.15259043872356415 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917178869247437, + "mean_weights": [ + 0.1659121811389923, + 0.16851314902305603, + 0.16693055629730225, + 0.16679996252059937, + 0.16575691103935242, + 0.16608723998069763 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7883799076080322, + "mean_weights": [ + 0.1728566288948059, + 0.18512636423110962, + 0.17585304379463196, + 0.16398008167743683, + 0.1537502557039261, + 0.1484336107969284 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7917424440383911, + "mean_weights": [ + 0.1667865514755249, + 0.1680906116962433, + 0.16619735956192017, + 0.16709285974502563, + 0.16617436707019806, + 0.16565825045108795 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9426109790802002, + "mean_weights": [ + 0.14892546832561493, + 0.16000479459762573, + 0.15354622900485992, + 0.14328595995903015, + 0.13467344641685486, + 0.12974688410758972, + 0.12981724739074707 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1419999748468399, + 0.1434222161769867, + 0.14279979467391968, + 0.14312142133712769, + 0.14279043674468994, + 0.1428302675485611, + 0.143035888671875 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9415357112884521, + "mean_weights": [ + 0.1496729701757431, + 0.16313126683235168, + 0.15414540469646454, + 0.1440294086933136, + 0.1343868225812912, + 0.12868303060531616, + 0.1259511113166809 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458801746368408, + "mean_weights": [ + 0.1429852843284607, + 0.1429750919342041, + 0.14214137196540833, + 0.1414642632007599, + 0.1421438455581665, + 0.14385518431663513, + 0.14443498849868774 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9424511194229126, + "mean_weights": [ + 0.14928653836250305, + 0.16045087575912476, + 0.15314346551895142, + 0.1440504789352417, + 0.1356543004512787, + 0.13001246750354767, + 0.1274019032716751 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458882808685303, + "mean_weights": [ + 0.14319148659706116, + 0.14391475915908813, + 0.14284171164035797, + 0.1420421153306961, + 0.14134715497493744, + 0.14342516660690308, + 0.1432376205921173 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9395697116851807, + "mean_weights": [ + 0.15016257762908936, + 0.16719450056552887, + 0.15599417686462402, + 0.14574657380580902, + 0.13377557694911957, + 0.12622776627540588, + 0.12089881300926208 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9458978176116943, + "mean_weights": [ + 0.1428529918193817, + 0.14342239499092102, + 0.14335417747497559, + 0.14274317026138306, + 0.1419316530227661, + 0.14220726490020752, + 0.14348836243152618 + ], + "sources": 7 + }, + { + "entropy_mean": 2.074862480163574, + "mean_weights": [ + 0.13159659504890442, + 0.14486080408096313, + 0.13573890924453735, + 0.12842535972595215, + 0.12030278891324997, + 0.11495959758758545, + 0.10915482044219971, + 0.11496111750602722 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079418659210205, + "mean_weights": [ + 0.12459750473499298, + 0.12504151463508606, + 0.12489251047372818, + 0.125042125582695, + 0.12461285293102264, + 0.12601442635059357, + 0.1261361837387085, + 0.12366288155317307 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0741164684295654, + "mean_weights": [ + 0.1323997974395752, + 0.14626389741897583, + 0.1369163542985916, + 0.12836164236068726, + 0.12002508342266083, + 0.11502262949943542, + 0.10908686369657516, + 0.11192375421524048 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0793213844299316, + "mean_weights": [ + 0.12429114431142807, + 0.12243181467056274, + 0.12357296049594879, + 0.12375843524932861, + 0.12469147145748138, + 0.12560917437076569, + 0.12766826152801514, + 0.12797673046588898 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0734808444976807, + "mean_weights": [ + 0.1330580711364746, + 0.14781729876995087, + 0.1374434530735016, + 0.12885303795337677, + 0.11949820816516876, + 0.11495119333267212, + 0.10982058197259903, + 0.10855817794799805 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07940673828125, + "mean_weights": [ + 0.12457560002803802, + 0.12431882321834564, + 0.12479400634765625, + 0.12437158077955246, + 0.12397614866495132, + 0.12488298863172531, + 0.12619814276695251, + 0.12688270211219788 + ], + "sources": 8 + }, + { + "entropy_mean": 2.07299542427063, + "mean_weights": [ + 0.13274571299552917, + 0.14826728403568268, + 0.13701492547988892, + 0.12918588519096375, + 0.12125477194786072, + 0.1169566735625267, + 0.10889489948749542, + 0.10567984730005264 + ], + "sources": 8 + }, + { + "entropy_mean": 2.079298734664917, + "mean_weights": [ + 0.12391229718923569, + 0.12222036719322205, + 0.12298265099525452, + 0.12416766583919525, + 0.125657856464386, + 0.12553520500659943, + 0.126623272895813, + 0.1289007067680359 + ], + "sources": 8 + }, + { + "entropy_mean": 2.189734935760498, + "mean_weights": [ + 0.1203557699918747, + 0.13466420769691467, + 0.12461012601852417, + 0.11655421555042267, + 0.10905998200178146, + 0.10487955063581467, + 0.09900732338428497, + 0.0950026735663414, + 0.0958661437034607 + ], + "sources": 9 + }, + { + "entropy_mean": 2.197068691253662, + "mean_weights": [ + 0.1099812239408493, + 0.10853179544210434, + 0.10949578881263733, + 0.11001666635274887, + 0.1107073426246643, + 0.11118210852146149, + 0.11218173801898956, + 0.11490923166275024, + 0.11299411952495575 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1907877922058105, + "mean_weights": [ + 0.11948458850383759, + 0.13228623569011688, + 0.12348943203687668, + 0.11669500172138214, + 0.11027230322360992, + 0.1054498627781868, + 0.10031206905841827, + 0.09697622060775757, + 0.09503427147865295 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1971726417541504, + "mean_weights": [ + 0.11100216209888458, + 0.11012113094329834, + 0.1099993959069252, + 0.11000204086303711, + 0.10997181385755539, + 0.11260797083377838, + 0.11264219880104065, + 0.11185945570468903, + 0.11179381608963013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.190375804901123, + "mean_weights": [ + 0.11983220279216766, + 0.13351142406463623, + 0.12386792153120041, + 0.11615028977394104, + 0.10967587679624557, + 0.10584976524114609, + 0.10062196850776672, + 0.09596225619316101, + 0.09452830255031586 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1969356536865234, + "mean_weights": [ + 0.10960094630718231, + 0.10724944621324539, + 0.10892992466688156, + 0.1096237450838089, + 0.11032412201166153, + 0.11112386733293533, + 0.11386257410049438, + 0.11429048329591751, + 0.11499489098787308 + ], + "sources": 9 + }, + { + "entropy_mean": 2.188795566558838, + "mean_weights": [ + 0.12045153975486755, + 0.13515985012054443, + 0.12505559623241425, + 0.11722971498966217, + 0.11039333790540695, + 0.10568177700042725, + 0.09951679408550262, + 0.09492656588554382, + 0.09158482402563095 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.028617795556783676, + 0.014952295459806919, + 0.015273182652890682, + 0.014778315089643002, + 0.015790818259119987, + 0.01543855108320713, + 0.01658804342150688, + 0.016689632087945938, + 0.017823126167058945, + 0.01178102195262909, + 0.012884888797998428, + 0.012982882559299469, + 0.013985250145196915, + 0.014135614968836308, + 0.015267650596797466, + 0.015412751585245132, + 0.016450626775622368, + 0.012501097284257412, + 0.013461960479617119, + 0.013585822656750679, + 0.014477741904556751, + 0.014657454565167427, + 0.01562186423689127, + 0.01575874350965023, + 0.016906939446926117, + 0.013761760666966438, + 0.014646334573626518, + 0.014892088249325752, + 0.015772713348269463, + 0.016054842621088028, + 0.016963548958301544, + 0.017220331355929375, + 0.018231164664030075, + 0.015453077852725983, + 0.016293151304125786, + 0.016655873507261276, + 0.017603756859898567, + 0.017766626551747322, + 0.018859725445508957, + 0.018939858302474022, + 0.020210443064570427, + 0.017597980797290802, + 0.018590020015835762, + 0.018625281751155853, + 0.019812360405921936, + 0.01988193951547146, + 0.021170100197196007, + 0.0208931565284729, + 0.022594628855586052, + 0.019704710692167282, + 0.021153371781110764, + 0.021027294918894768, + 0.022732628509402275, + 0.02250491827726364, + 0.024445844814181328, + 0.023921677842736244, + 0.02632913924753666, + 0.022959763184189796, + 0.02505173534154892, + 0.02448389306664467, + 0.026562821120023727, + 0.025734128430485725, + 0.028511708602309227, + 0.027005694806575775 + ], + "loss_nats": 3.5924854278564453, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1922647953033447, + "mean_weights": [ + 0.09991011023521423, + 0.0929517149925232, + 0.10029369592666626, + 0.10801893472671509, + 0.11530699580907822, + 0.11817377060651779, + 0.12114627659320831, + 0.12246126681566238, + 0.12173722684383392 + ], + "sources": 9 + }, + "step": 100, + "stream_state_rms_by_sublayer": [ + 0.002723898272961378, + 0.0055193523876369, + 0.006246842443943024, + 0.009546536952257156, + 0.009986688382923603, + 0.013977172784507275, + 0.014312135986983776, + 0.018744563683867455, + 0.0027809101156890392, + 0.00660830270498991, + 0.007167908828705549, + 0.011765030212700367, + 0.012327533215284348, + 0.017156247049570084, + 0.017696699127554893, + 0.022267481312155724, + 0.003559867851436138, + 0.007449587807059288, + 0.008642485365271568, + 0.013175037689507008, + 0.014099182561039925, + 0.019073888659477234, + 0.020258057862520218, + 0.025712018832564354, + 0.004887574352324009, + 0.008972279727458954, + 0.011225050315260887, + 0.01624389924108982, + 0.01857108250260353, + 0.023743286728858948, + 0.026577936485409737, + 0.03214457631111145, + 0.005625482182949781, + 0.00991375558078289, + 0.013686468824744225, + 0.019281653687357903, + 0.023031942546367645, + 0.02868141047656536, + 0.03310495615005493, + 0.03889070823788643, + 0.008495661430060863, + 0.0127727584913373, + 0.017283262684941292, + 0.022841427475214005, + 0.029314789921045303, + 0.034929387271404266, + 0.04071469604969025, + 0.04634999856352806, + 0.00961991585791111, + 0.014845813624560833, + 0.023407738655805588, + 0.029589947313070297, + 0.0400763601064682, + 0.046351123601198196, + 0.05648425966501236, + 0.06254568696022034, + 0.014384749345481396, + 0.01947367936372757, + 0.03149676322937012, + 0.0370744951069355, + 0.04818879812955856, + 0.05507569760084152, + 0.06738673150539398, + 0.07403328269720078 + ] + }, + { + "activation_grad_rms_by_block": [ + 0.0001494410098530352, + 0.00015486938355024904, + 0.00014897517394274473, + 0.0001431136333849281, + 0.0001401594199705869, + 0.00012453079398255795, + 0.00011206007184227929, + 0.00010208225285168737, + 9.998067253036425e-05, + 9.082697215490043e-05, + 8.336143946507946e-05, + 7.669178739888594e-05, + 7.405834912788123e-05, + 6.847811891930178e-05, + 6.22319639660418e-05, + 5.739524931414053e-05, + 5.947135286987759e-05, + 5.3890027629677206e-05, + 4.890635682386346e-05, + 4.4437034375732765e-05, + 4.31341104558669e-05, + 3.886142803821713e-05, + 3.5836677852785215e-05, + 3.340865805512294e-05, + 3.223997919121757e-05, + 3.0436871384154074e-05, + 2.9292581530171447e-05, + 2.8587397537194192e-05, + 2.2913789507583715e-05, + 2.2530648493557237e-05, + 2.244284223706927e-05, + 2.2519136109622195e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 0.00013440396742225857, + "first_to_last_ratio": 5.096772825661455, + "imbalance_abs_log_ratio": 1.6286075601589056, + "last_quartile_mean": 2.6370405748821213e-05, + "mean": 7.053641201082428e-05, + "normalized": [ + 2.1186363977530145, + 2.195594858531836, + 2.112032207136982, + 2.0289327073081966, + 1.9870506023056362, + 1.7654824002594274, + 1.5886840377574483, + 1.4472277500593902, + 1.4174334883240383, + 1.287660792002894, + 1.1818213754945048, + 1.0872652182409996, + 1.0499307664886113, + 0.9708194245660434, + 0.8822672176250175, + 0.8136967514782698, + 0.8431298271983457, + 0.7640029609304118, + 0.6933490863748281, + 0.6299871670381194, + 0.6115155169679979, + 0.5509413780821966, + 0.5080592679889337, + 0.47363704932987183, + 0.4570686014801849, + 0.43150580695093105, + 0.4152831239229507, + 0.4052856775987878, + 0.32485051130850606, + 0.319418692435047, + 0.31817385655546626, + 0.3192554805051111 + ], + "population_cv": 0.606636113722199 + }, + "activation_output_rms_by_block": [ + 0.021243074908852577, + 0.03624089062213898, + 0.04951142147183418, + 0.05851924046874046, + 0.017167888581752777, + 0.02674153819680214, + 0.03352004662156105, + 0.03946679085493088, + 0.020600929856300354, + 0.027865095064044, + 0.035815127193927765, + 0.04293550178408623, + 0.021564343944191933, + 0.029616132378578186, + 0.035480253398418427, + 0.04259813204407692, + 0.021436138078570366, + 0.03586751967668533, + 0.04331304505467415, + 0.051551222801208496, + 0.025608548894524574, + 0.04091007262468338, + 0.05466190353035927, + 0.07478146255016327, + 0.02793189324438572, + 0.05498877540230751, + 0.08125203847885132, + 0.10916247963905334, + 0.026290444657206535, + 0.05761713907122612, + 0.08842677623033524, + 0.12436132878065109 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.03530136146582663, + "first_to_last_ratio": 0.495430868506741, + "imbalance_abs_log_ratio": 0.7023274536001348, + "last_quartile_mean": 0.07125385943800211, + "mean": 0.04553272487828508, + "normalized": [ + 0.4665452148018396, + 0.7959306349228079, + 1.0873810342821493, + 1.2852127920121195, + 0.37704505117242126, + 0.5873037088881845, + 0.7361748437231577, + 0.8667785853016872, + 0.452442279950724, + 0.6119795188741951, + 0.7865799222354093, + 0.9429591991003927, + 0.4736009980038805, + 0.6504361963345259, + 0.7792253482140983, + 0.9355498085815704, + 0.47078531179216626, + 0.7877305777891374, + 0.9512508897821419, + 1.1321796123340224, + 0.5624207416309823, + 0.8984762658953829, + 1.2004970859195814, + 1.6423673907077576, + 0.6134465556157976, + 1.2076759198861848, + 1.784475642431869, + 2.397451062523908, + 0.5773966905667149, + 1.2654006371295428, + 1.9420488553389141, + 2.7312516242567333 + ], + "population_cv": 0.5563959277170655 + }, + "bits_per_byte": 3.0745070201808953, + "branch_output_rms_by_sublayer": [ + 0.01049753837287426, + 0.01706349104642868, + 0.013886291533708572, + 0.012710319831967354, + 0.01245343592017889, + 0.013922877609729767, + 0.01395952794700861, + 0.011416541412472725, + 0.014544809237122536, + 0.008325628004968166, + 0.014345151372253895, + 0.00801781378686428, + 0.015135552734136581, + 0.007913725450634956, + 0.014776745811104774, + 0.008828459307551384, + 0.01795734278857708, + 0.008378393016755581, + 0.013709688559174538, + 0.008136541582643986, + 0.016144096851348877, + 0.008199729025363922, + 0.018440017476677895, + 0.009040302596986294, + 0.018456056714057922, + 0.00936646107584238, + 0.012218938209116459, + 0.009016151539981365, + 0.01701919361948967, + 0.009497544728219509, + 0.01305371057242155, + 0.009566655382514, + 0.017418840900063515, + 0.011108046397566795, + 0.026864025741815567, + 0.0114391278475523, + 0.035265542566776276, + 0.012216469272971153, + 0.027374446392059326, + 0.01392423827201128, + 0.01969025284051895, + 0.015176027081906796, + 0.018602686002850533, + 0.017532862722873688, + 0.0133047541603446, + 0.018514130264520645, + 0.015662170946598053, + 0.022848375141620636, + 0.012774706818163395, + 0.0230264812707901, + 0.016374029219150543, + 0.02426965720951557, + 0.010044055990874767, + 0.025484448298811913, + 0.006929588038474321, + 0.027965376153588295, + 0.005331806372851133, + 0.02452327311038971, + 0.01223068032413721, + 0.02779858186841011, + 0.00525518087670207, + 0.03124845214188099, + 0.010387555696070194, + 0.03395458683371544 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0004975869912568777, + 0.0004115293940014644, + 0.00043817390286628833, + 0.00039988528646951293, + 0.0003968514715139995, + 0.00031053290415162903, + 0.00024154094413850278, + 0.00022869177850848996, + 0.0002476899080691786, + 0.00021028907646624228, + 0.0002010354933691209, + 0.0001845208571901889, + 0.0002273114134138899, + 0.0001715437435350543, + 0.00015668460984383524, + 0.00015894955615086492, + 0.00016556491904516564, + 0.0001408361881569501, + 0.00014198814555526783, + 0.00012254378127881199, + 0.00011450446233020633, + 0.0001100879077446283, + 0.00010549256620556044, + 9.975908855450905e-05, + 0.00010071115476761671, + 9.794352566848755e-05, + 9.525064309078406e-05, + 8.226123370955787e-05, + 6.340888944803813e-05, + 6.739220497758678e-05, + 6.650164980556211e-05, + 7.228417264196778e-05 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0003655990841133456, + "first_to_last_ratio": 4.5292712934136725, + "imbalance_abs_log_ratio": 1.5105610641741143, + "last_quartile_mean": 8.071918426370012e-05, + "mean": 0.0001915421207476825, + "normalized": [ + 2.5977941004023166, + 2.148505991241321, + 2.28761121134026, + 2.0877146233348847, + 2.071875731379053, + 1.621225153712757, + 1.2610330469123472, + 1.1939503312158923, + 1.293135458155751, + 1.0978738026152224, + 1.0495628459389565, + 0.963343501000792, + 1.186743743499249, + 0.8955927963282188, + 0.8180164719499744, + 0.8298412669255573, + 0.8643786463201141, + 0.7352752888356756, + 0.7412894093529857, + 0.6397745874404216, + 0.5978030413531993, + 0.574745164744451, + 0.5507538801062211, + 0.5208206328983963, + 0.5257911647552604, + 0.5113419716048152, + 0.4972830138821386, + 0.4294681158820424, + 0.33104410246958865, + 0.3518401316353921, + 0.3471907356250086, + 0.3773800371417385 + ], + "population_cv": 0.6210714087578444 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6506600379943848, + "mean_weights": [ + 0.3747701942920685, + 0.6252298355102539 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6806463003158569, + "mean_weights": [ + 0.45339685678482056, + 0.5466030836105347 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6869370937347412, + "mean_weights": [ + 0.4668571949005127, + 0.5331428050994873 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6791321635246277, + "mean_weights": [ + 0.4727538824081421, + 0.5272461175918579 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6804035902023315, + "mean_weights": [ + 0.4487338662147522, + 0.5512661933898926 + ], + "sources": 2 + }, + { + "entropy_mean": 0.682766318321228, + "mean_weights": [ + 0.4646179676055908, + 0.5353820323944092 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6873732805252075, + "mean_weights": [ + 0.4883750379085541, + 0.5116249322891235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6835889220237732, + "mean_weights": [ + 0.46665582060813904, + 0.5333442091941833 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0856965780258179, + "mean_weights": [ + 0.27463093400001526, + 0.3524433374404907, + 0.372925728559494 + ], + "sources": 3 + }, + { + "entropy_mean": 1.077350378036499, + "mean_weights": [ + 0.2904614210128784, + 0.3131614923477173, + 0.3963770568370819 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0850789546966553, + "mean_weights": [ + 0.27783387899398804, + 0.34068456292152405, + 0.3814815878868103 + ], + "sources": 3 + }, + { + "entropy_mean": 1.062077522277832, + "mean_weights": [ + 0.28390786051750183, + 0.29115602374076843, + 0.42493611574172974 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0936429500579834, + "mean_weights": [ + 0.30466002225875854, + 0.3333379328250885, + 0.36200207471847534 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0627415180206299, + "mean_weights": [ + 0.2850727438926697, + 0.29934102296829224, + 0.4155862331390381 + ], + "sources": 3 + }, + { + "entropy_mean": 1.087097406387329, + "mean_weights": [ + 0.28865373134613037, + 0.3320052921772003, + 0.3793409764766693 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0500720739364624, + "mean_weights": [ + 0.2745887339115143, + 0.286259263753891, + 0.4391520023345947 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3743348121643066, + "mean_weights": [ + 0.2166871875524521, + 0.23293715715408325, + 0.2495419979095459, + 0.30083364248275757 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3609715700149536, + "mean_weights": [ + 0.23144879937171936, + 0.21839870512485504, + 0.24986028671264648, + 0.3002921938896179 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3810515403747559, + "mean_weights": [ + 0.2385442852973938, + 0.23454180359840393, + 0.24874086678028107, + 0.27817302942276 + ], + "sources": 4 + }, + { + "entropy_mean": 1.348030686378479, + "mean_weights": [ + 0.23874622583389282, + 0.21300309896469116, + 0.22721335291862488, + 0.32103732228279114 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3825427293777466, + "mean_weights": [ + 0.24534732103347778, + 0.2375141680240631, + 0.25072187185287476, + 0.26641663908958435 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3499886989593506, + "mean_weights": [ + 0.24629786610603333, + 0.22150886058807373, + 0.2235339879989624, + 0.30865928530693054 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3807358741760254, + "mean_weights": [ + 0.24433431029319763, + 0.23110222816467285, + 0.23992134630680084, + 0.2846421003341675 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3399771451950073, + "mean_weights": [ + 0.24593281745910645, + 0.21928201615810394, + 0.21277041733264923, + 0.3220147490501404 + ], + "sources": 4 + }, + { + "entropy_mean": 1.6075894832611084, + "mean_weights": [ + 0.19650746881961823, + 0.19567455351352692, + 0.18970930576324463, + 0.21128275990486145, + 0.20682592689990997 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5706448554992676, + "mean_weights": [ + 0.1975565403699875, + 0.1772933304309845, + 0.16386017203330994, + 0.21728253364562988, + 0.2440074384212494 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6042816638946533, + "mean_weights": [ + 0.21478912234306335, + 0.18552109599113464, + 0.17930489778518677, + 0.20310312509536743, + 0.2172817587852478 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5764570236206055, + "mean_weights": [ + 0.20157726109027863, + 0.17631688714027405, + 0.1703747808933258, + 0.19828173518180847, + 0.25344932079315186 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6011263132095337, + "mean_weights": [ + 0.20813238620758057, + 0.18490281701087952, + 0.17333751916885376, + 0.20533671975135803, + 0.22829055786132812 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5817476511001587, + "mean_weights": [ + 0.20727968215942383, + 0.18177060782909393, + 0.17236655950546265, + 0.18925222754478455, + 0.24933092296123505 + ], + "sources": 5 + }, + { + "entropy_mean": 1.603710412979126, + "mean_weights": [ + 0.2016504406929016, + 0.1979413628578186, + 0.17889465391635895, + 0.19759604334831238, + 0.22391751408576965 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5885388851165771, + "mean_weights": [ + 0.20988406240940094, + 0.18461555242538452, + 0.17252585291862488, + 0.19482356309890747, + 0.2381509691476822 + ], + "sources": 5 + }, + { + "entropy_mean": 1.783994197845459, + "mean_weights": [ + 0.16140900552272797, + 0.16211673617362976, + 0.15125815570354462, + 0.1585075855255127, + 0.17266258597373962, + 0.19404590129852295 + ], + "sources": 6 + }, + { + "entropy_mean": 1.765566349029541, + "mean_weights": [ + 0.1630604863166809, + 0.1509958952665329, + 0.14303800463676453, + 0.14991191029548645, + 0.16761469841003418, + 0.22537899017333984 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7848252058029175, + "mean_weights": [ + 0.16126418113708496, + 0.1664336919784546, + 0.15361233055591583, + 0.1546509563922882, + 0.16693615913391113, + 0.19710269570350647 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7731435298919678, + "mean_weights": [ + 0.16123875975608826, + 0.15981441736221313, + 0.1509810984134674, + 0.14913737773895264, + 0.16297483444213867, + 0.2158534824848175 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7815005779266357, + "mean_weights": [ + 0.15527218580245972, + 0.17254948616027832, + 0.15193141996860504, + 0.15522462129592896, + 0.1650869995355606, + 0.19993525743484497 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7801456451416016, + "mean_weights": [ + 0.1661754697561264, + 0.16477231681346893, + 0.15300315618515015, + 0.15292593836784363, + 0.1582789272069931, + 0.2048441767692566 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7743862867355347, + "mean_weights": [ + 0.15205824375152588, + 0.1771804541349411, + 0.15399661660194397, + 0.15122173726558685, + 0.15813913941383362, + 0.20740382373332977 + ], + "sources": 6 + }, + { + "entropy_mean": 1.781898856163025, + "mean_weights": [ + 0.17058727145195007, + 0.1644459068775177, + 0.15141303837299347, + 0.15486885607242584, + 0.15909278392791748, + 0.19959214329719543 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9389331340789795, + "mean_weights": [ + 0.12895603477954865, + 0.15703952312469482, + 0.13832643628120422, + 0.13419194519519806, + 0.1312965601682663, + 0.15331263840198517, + 0.15687686204910278 + ], + "sources": 7 + }, + { + "entropy_mean": 1.941878080368042, + "mean_weights": [ + 0.14007402956485748, + 0.15248695015907288, + 0.14026451110839844, + 0.13737857341766357, + 0.1305495798587799, + 0.14631187915802002, + 0.1529344916343689 + ], + "sources": 7 + }, + { + "entropy_mean": 1.936349630355835, + "mean_weights": [ + 0.12599323689937592, + 0.16029544174671173, + 0.1399340182542801, + 0.13338115811347961, + 0.12978437542915344, + 0.14873966574668884, + 0.16187208890914917 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9385986328125, + "mean_weights": [ + 0.1397915780544281, + 0.15300613641738892, + 0.14345945417881012, + 0.13018865883350372, + 0.1257915198802948, + 0.14249205589294434, + 0.1652706116437912 + ], + "sources": 7 + }, + { + "entropy_mean": 1.939380407333374, + "mean_weights": [ + 0.12855647504329681, + 0.15473899245262146, + 0.1372847855091095, + 0.13797816634178162, + 0.13512183725833893, + 0.15090687572956085, + 0.15541282296180725 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9388402700424194, + "mean_weights": [ + 0.14282521605491638, + 0.15391701459884644, + 0.1462559849023819, + 0.13105326890945435, + 0.12675246596336365, + 0.13615548610687256, + 0.16304057836532593 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9313902854919434, + "mean_weights": [ + 0.11694864928722382, + 0.17515113949775696, + 0.15024805068969727, + 0.1394161432981491, + 0.13194763660430908, + 0.1451682150363922, + 0.14112016558647156 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9366493225097656, + "mean_weights": [ + 0.14114168286323547, + 0.16076743602752686, + 0.14226865768432617, + 0.12737974524497986, + 0.12711969017982483, + 0.1412842869758606, + 0.16003848612308502 + ], + "sources": 7 + }, + { + "entropy_mean": 2.064785957336426, + "mean_weights": [ + 0.106104776263237, + 0.15484434366226196, + 0.13487765192985535, + 0.12357068061828613, + 0.12023378908634186, + 0.12704278528690338, + 0.11963208019733429, + 0.11369390785694122 + ], + "sources": 8 + }, + { + "entropy_mean": 2.062044143676758, + "mean_weights": [ + 0.11719764769077301, + 0.14862585067749023, + 0.12573552131652832, + 0.12179906666278839, + 0.11081404983997345, + 0.1260838657617569, + 0.12823458015918732, + 0.12150939553976059 + ], + "sources": 8 + }, + { + "entropy_mean": 2.054962158203125, + "mean_weights": [ + 0.10514847934246063, + 0.16456037759780884, + 0.14191412925720215, + 0.1262327879667282, + 0.1181878000497818, + 0.12522782385349274, + 0.1187765821814537, + 0.09995201230049133 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0610406398773193, + "mean_weights": [ + 0.11198848485946655, + 0.15103809535503387, + 0.13588255643844604, + 0.12388211488723755, + 0.11551118642091751, + 0.12874530255794525, + 0.12429477274417877, + 0.10865749418735504 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0338308811187744, + "mean_weights": [ + 0.0961676836013794, + 0.17635446786880493, + 0.14848342537879944, + 0.13749772310256958, + 0.12027012556791306, + 0.12292921543121338, + 0.10873906314373016, + 0.08955828845500946 + ], + "sources": 8 + }, + { + "entropy_mean": 2.068699598312378, + "mean_weights": [ + 0.11418519914150238, + 0.14292436838150024, + 0.1359846293926239, + 0.13056153059005737, + 0.12100572884082794, + 0.11985942721366882, + 0.12221725285053253, + 0.1132618635892868 + ], + "sources": 8 + }, + { + "entropy_mean": 2.036879539489746, + "mean_weights": [ + 0.09746269136667252, + 0.17786666750907898, + 0.15074899792671204, + 0.13200224936008453, + 0.11842608451843262, + 0.12164927273988724, + 0.1129462867975235, + 0.08889776468276978 + ], + "sources": 8 + }, + { + "entropy_mean": 2.06752347946167, + "mean_weights": [ + 0.11505836993455887, + 0.14705273509025574, + 0.1377231776714325, + 0.12483140081167221, + 0.11869609355926514, + 0.1215740442276001, + 0.12386028468608856, + 0.1112038791179657 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1517844200134277, + "mean_weights": [ + 0.08786984533071518, + 0.16287443041801453, + 0.13886377215385437, + 0.12039706110954285, + 0.10935483872890472, + 0.1119438111782074, + 0.10067413747310638, + 0.07631799578666687, + 0.0917041003704071 + ], + "sources": 9 + }, + { + "entropy_mean": 2.177518129348755, + "mean_weights": [ + 0.10418819636106491, + 0.13366681337356567, + 0.12014994025230408, + 0.11398349702358246, + 0.10732515156269073, + 0.12029910087585449, + 0.11035183072090149, + 0.09321349114179611, + 0.09682197868824005 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1535391807556152, + "mean_weights": [ + 0.08976081013679504, + 0.163266122341156, + 0.13892531394958496, + 0.12143255770206451, + 0.10986581444740295, + 0.11029069125652313, + 0.10418520867824554, + 0.08120252937078476, + 0.0810709297657013 + ], + "sources": 9 + }, + { + "entropy_mean": 2.191593647003174, + "mean_weights": [ + 0.10739251971244812, + 0.12338980287313461, + 0.11759916692972183, + 0.1066875234246254, + 0.1060015857219696, + 0.1124890148639679, + 0.11740898340940475, + 0.10738331079483032, + 0.10164809972047806 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1465134620666504, + "mean_weights": [ + 0.08895281702280045, + 0.1655423492193222, + 0.14156755805015564, + 0.12290289252996445, + 0.11169517785310745, + 0.11284246295690536, + 0.1037452220916748, + 0.0791342705488205, + 0.07361726462841034 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1857833862304688, + "mean_weights": [ + 0.10998444259166718, + 0.12038750946521759, + 0.11211743205785751, + 0.11047163605690002, + 0.10957512259483337, + 0.12139561027288437, + 0.11392621695995331, + 0.10294321179389954, + 0.09919881820678711 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1400556564331055, + "mean_weights": [ + 0.08778931945562363, + 0.16802933812141418, + 0.1466498225927353, + 0.12668846547603607, + 0.11206841468811035, + 0.11208955943584442, + 0.10068599879741669, + 0.07686632871627808, + 0.0691327303647995 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.04377923160791397, + 0.017109708860516548, + 0.023813940584659576, + 0.02606838569045067, + 0.029579024761915207, + 0.030872320756316185, + 0.034570932388305664, + 0.034764163196086884, + 0.037983279675245285, + 0.025188947096467018, + 0.02627667225897312, + 0.026965361088514328, + 0.02788778953254223, + 0.028485212475061417, + 0.03023744560778141, + 0.030349206179380417, + 0.031768180429935455, + 0.022602196782827377, + 0.02424551732838154, + 0.023387571796774864, + 0.025113960728049278, + 0.02450813725590706, + 0.026638666167855263, + 0.025169003754854202, + 0.027698444202542305, + 0.020798001438379288, + 0.022435806691646576, + 0.02118327096104622, + 0.022681118920445442, + 0.021654823794960976, + 0.023235918954014778, + 0.022377775982022285, + 0.023750189691781998, + 0.018887808546423912, + 0.019677624106407166, + 0.01950881816446781, + 0.020757248625159264, + 0.020426612347364426, + 0.02137788012623787, + 0.021234560757875443, + 0.02207154594361782, + 0.01860968954861164, + 0.019271831959486008, + 0.019469313323497772, + 0.020465917885303497, + 0.020844420418143272, + 0.021911408752202988, + 0.02213398739695549, + 0.024193286895751953, + 0.0209572184830904, + 0.02259150519967079, + 0.022487536072731018, + 0.024306390434503555, + 0.023487187922000885, + 0.02611992135643959, + 0.024889562278985977, + 0.02851196378469467, + 0.023824406787753105, + 0.027104467153549194, + 0.025868752971291542, + 0.03068944811820984, + 0.0271493811160326, + 0.03286270797252655, + 0.028410611674189568 + ], + "loss_nats": 2.1310858726501465, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 2.1613364219665527, + "mean_weights": [ + 0.1255333125591278, + 0.08712290227413177, + 0.09080196917057037, + 0.08838342130184174, + 0.09530800580978394, + 0.09807588160037994, + 0.13817133009433746, + 0.15537576377391815, + 0.12122740596532822 + ], + "sources": 9 + }, + "step": 500, + "stream_state_rms_by_sublayer": [ + 0.01049753837287426, + 0.021243074908852577, + 0.028129518032073975, + 0.03624089062213898, + 0.040982723236083984, + 0.04951142147183418, + 0.05135732889175415, + 0.05851924046874046, + 0.014544809237122536, + 0.017167888581752777, + 0.023477500304579735, + 0.02674153819680214, + 0.029818039387464523, + 0.03352004662156105, + 0.03540094941854477, + 0.03946679085493088, + 0.01795734278857708, + 0.020600929856300354, + 0.024303091689944267, + 0.027865095064044, + 0.031844362616539, + 0.035815127193927765, + 0.03825921565294266, + 0.04293550178408623, + 0.018456056714057922, + 0.021564343944191933, + 0.02560909278690815, + 0.029616132378578186, + 0.0305727981030941, + 0.035480253398418427, + 0.037264734506607056, + 0.04259813204407692, + 0.017418840900063515, + 0.021436138078570366, + 0.02998829260468483, + 0.03586751967668533, + 0.03708711639046669, + 0.04331304505467415, + 0.04352690279483795, + 0.051551222801208496, + 0.01969025284051895, + 0.025608548894524574, + 0.030230700969696045, + 0.04091007262468338, + 0.041474223136901855, + 0.05466190353035927, + 0.056570712476968765, + 0.07478146255016327, + 0.012774706818163395, + 0.02793189324438572, + 0.03524070605635643, + 0.05498877540230751, + 0.05920406058430672, + 0.08125203847885132, + 0.08430205285549164, + 0.10916247963905334, + 0.005331806372851133, + 0.026290444657206535, + 0.032364487648010254, + 0.05761713907122612, + 0.059356335550546646, + 0.08842677623033524, + 0.09272631257772446, + 0.12436132878065109 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.294298509601504e-05, + 6.491570093203336e-05, + 6.380436389008537e-05, + 6.04347005719319e-05, + 7.516580808442086e-05, + 7.751413795631379e-05, + 6.965622014831752e-05, + 6.359309190884233e-05, + 6.543538620462641e-05, + 6.0226317145861685e-05, + 5.608194260275923e-05, + 5.3055387979839e-05, + 4.572631951305084e-05, + 4.3727712181862444e-05, + 4.314268517191522e-05, + 4.164052370470017e-05, + 7.694400119362399e-05, + 7.326623017434031e-05, + 6.749018211849034e-05, + 6.173704605316743e-05, + 0.00010233377543045208, + 8.934178913477808e-05, + 7.613583147758618e-05, + 6.664563989033923e-05, + 5.163387322681956e-05, + 4.2875533836195245e-05, + 3.8466609112219885e-05, + 3.59200312232133e-05, + 2.308324837940745e-05, + 2.0679148292401806e-05, + 1.9936938770115376e-05, + 1.9895105651812628e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 6.725337607349502e-05, + "first_to_last_ratio": 2.130880302861834, + "imbalance_abs_log_ratio": 0.7565351820700446, + "last_quartile_mean": 3.1561311061523156e-05, + "mean": 5.6670258345548064e-05, + "normalized": [ + 1.1106881622493918, + 1.145498588275503, + 1.1258880010928651, + 1.0664271230850946, + 1.3263713679598177, + 1.367809856868302, + 1.2291495077292092, + 1.122159908308202, + 1.154668923611267, + 1.062750001572791, + 0.9896186154790125, + 0.9362122130506743, + 0.8068839078557516, + 0.7716166020495587, + 0.7612932503122151, + 0.7347861986228512, + 1.357749257546284, + 1.2928515294142118, + 1.190927729797305, + 1.089408233799192, + 1.8057756999530474, + 1.5765198843812338, + 1.3434883429213678, + 1.176024987992221, + 0.9111282484717285, + 0.7565791137700626, + 0.678779490957478, + 0.6338427293588494, + 0.4073256246452393, + 0.3649030178459797, + 0.3518060328673549, + 0.35106784815593767 + ], + "population_cv": 0.34914284904214865 + }, + "activation_output_rms_by_block": [ + 0.10241243988275528, + 0.1819411665201187, + 0.24157075583934784, + 0.26444777846336365, + 0.10636615753173828, + 0.13034582138061523, + 0.14029134809970856, + 0.15042904019355774, + 0.04812335968017578, + 0.07954440265893936, + 0.10673297941684723, + 0.14089107513427734, + 0.06885477155447006, + 0.10946234315633774, + 0.13876035809516907, + 0.1687854677438736, + 0.05059739947319031, + 0.07094386219978333, + 0.09166282415390015, + 0.12053482979536057, + 0.048727843910455704, + 0.07109228521585464, + 0.09690473228693008, + 0.11853098124265671, + 0.05259029567241669, + 0.08889466524124146, + 0.12455403059720993, + 0.1728687882423401, + 0.06489895284175873, + 0.14679722487926483, + 0.23576539754867554, + 0.35208994150161743 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.16472556348890066, + "first_to_last_ratio": 1.0640676779683806, + "imbalance_abs_log_ratio": 0.0620989960097457, + "last_quartile_mean": 0.15480741206556559, + "mean": 0.127700416254811, + "normalized": [ + 0.8019742056157706, + 1.4247499879489567, + 1.8916990478505735, + 2.0708450780276983, + 0.8329350886335192, + 1.020715712629516, + 1.098597422108428, + 1.177983945591802, + 0.3768457542389787, + 0.6228985385624584, + 0.8358076077361742, + 1.1032937813855357, + 0.5391898755997675, + 0.8571807858317286, + 1.086608502655851, + 1.3217299731199175, + 0.396219534415841, + 0.5555491851978249, + 0.7177958133746243, + 0.9438875246486875, + 0.3815793662976406, + 0.5567114603134765, + 0.7588442945524017, + 0.9281957312194052, + 0.411825561848436, + 0.6961188369493074, + 0.9753611949758815, + 1.3537057537651325, + 0.5082125395132666, + 1.1495438244018596, + 1.8462382853806345, + 2.7571557856089037 + ], + "population_cv": 0.532159231456062 + }, + "bits_per_byte": 2.0533380169036586, + "branch_output_rms_by_sublayer": [ + 0.05045261234045029, + 0.0860176682472229, + 0.09517527371644974, + 0.031032679602503777, + 0.08140470087528229, + 0.011454286053776741, + 0.035610511898994446, + 0.03601690009236336, + 0.1042611226439476, + 0.01171922218054533, + 0.050615254789590836, + 0.008982792496681213, + 0.04264022409915924, + 0.01206296682357788, + 0.03225839138031006, + 0.026524480432271957, + 0.040291935205459595, + 0.024220434948801994, + 0.036216676235198975, + 0.026388581842184067, + 0.02871432714164257, + 0.03031591884791851, + 0.03194425627589226, + 0.032942064106464386, + 0.05663999170064926, + 0.029961559921503067, + 0.03894488140940666, + 0.02937687374651432, + 0.03595346212387085, + 0.031781863421201706, + 0.03414257988333702, + 0.035546254366636276, + 0.03680795058608055, + 0.030760737136006355, + 0.022694969549775124, + 0.03042135201394558, + 0.022243596613407135, + 0.03508196026086807, + 0.02950950525701046, + 0.04756028205156326, + 0.035728659480810165, + 0.0295625701546669, + 0.024367405101656914, + 0.03350908309221268, + 0.0301151555031538, + 0.038334235548973083, + 0.022455131635069847, + 0.043223608285188675, + 0.025910304859280586, + 0.04338310658931732, + 0.025968369096517563, + 0.046161677688360214, + 0.01597527042031288, + 0.050398919731378555, + 0.015805471688508987, + 0.062427736818790436, + 0.013562481850385666, + 0.0627857968211174, + 0.02184087596833706, + 0.08928166329860687, + 0.012465676292777061, + 0.09919292479753494, + 0.01876053772866726, + 0.13455425202846527 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00034319970120439075, + 0.0002207216513437987, + 0.00019628105547348242, + 0.00016658832628724645, + 0.00021517433675904697, + 0.00019244240293185932, + 0.0001402257608174488, + 0.00015224450647613049, + 0.00018347185792801386, + 0.00016427150761043105, + 0.00015319316983403424, + 0.00015483500137268905, + 0.00013970482137555046, + 0.00012521709576159806, + 0.00014171058058500498, + 0.00014304258084243227, + 0.00024403444477986432, + 0.0002443248488541129, + 0.0002486173108247806, + 0.0002822915928542698, + 0.00039666535280576766, + 0.0003607137256748422, + 0.0003593407909791253, + 0.00030791631238161, + 0.0002432000733486066, + 0.00020051667741228585, + 0.0001766506293272091, + 0.00017705575210645458, + 0.00010859706957547057, + 0.0001081942080930384, + 0.00010514694748959741, + 0.00011321633644894032 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.0002033597176616755, + "first_to_last_ratio": 1.3198987369921267, + "imbalance_abs_log_ratio": 0.2775550192556754, + "last_quartile_mean": 0.00015407221172520036, + "mean": 0.00020340020092372293, + "normalized": [ + 1.6873124984428802, + 1.085159455799021, + 0.9649993194799731, + 0.8190175109498476, + 1.0578865496781857, + 0.9461269067478802, + 0.6894081725614166, + 0.7484973258862397, + 0.9020239758603661, + 0.8076270665633928, + 0.7531613495872759, + 0.7612332764152662, + 0.6868470175599342, + 0.6156193317063423, + 0.6967081642075066, + 0.703256831570558, + 1.1997748461978148, + 1.2012025934317396, + 1.2223061220967748, + 1.3878629006867693, + 1.9501718828415566, + 1.7734187283822467, + 1.7666688102922847, + 1.5138446814862372, + 1.1956727291523621, + 0.985823398903532, + 0.8684879785023163, + 0.8704797306117324, + 0.5339083692262209, + 0.5319277345926137, + 0.516946133839015, + 0.5566186067406962 + ], + "population_cv": 0.38898494230064556 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.39672940969467163, + "mean_weights": [ + 0.854300856590271, + 0.1456991732120514 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6258258819580078, + "mean_weights": [ + 0.5948886871337891, + 0.40511131286621094 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5053832530975342, + "mean_weights": [ + 0.778947651386261, + 0.221052348613739 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6737473011016846, + "mean_weights": [ + 0.5406605005264282, + 0.4593394994735718 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6529731750488281, + "mean_weights": [ + 0.5619670748710632, + 0.43803298473358154 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6891955733299255, + "mean_weights": [ + 0.5099825263023376, + 0.49001747369766235 + ], + "sources": 2 + }, + { + "entropy_mean": 0.48565131425857544, + "mean_weights": [ + 0.7992967367172241, + 0.20070326328277588 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6875144243240356, + "mean_weights": [ + 0.5191382169723511, + 0.4808618128299713 + ], + "sources": 2 + }, + { + "entropy_mean": 1.048142671585083, + "mean_weights": [ + 0.3059995770454407, + 0.2586204409599304, + 0.4353799819946289 + ], + "sources": 3 + }, + { + "entropy_mean": 1.078475832939148, + "mean_weights": [ + 0.2919672429561615, + 0.28997206687927246, + 0.4180607199668884 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0040717124938965, + "mean_weights": [ + 0.27878478169441223, + 0.21860124170780182, + 0.5026139616966248 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0727062225341797, + "mean_weights": [ + 0.3306647837162018, + 0.2443503737449646, + 0.42498481273651123 + ], + "sources": 3 + }, + { + "entropy_mean": 1.036304235458374, + "mean_weights": [ + 0.3365095853805542, + 0.20819777250289917, + 0.455292671918869 + ], + "sources": 3 + }, + { + "entropy_mean": 1.071068525314331, + "mean_weights": [ + 0.3291535973548889, + 0.24366843700408936, + 0.42717796564102173 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0272725820541382, + "mean_weights": [ + 0.29237720370292664, + 0.2188395857810974, + 0.48878318071365356 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0856674909591675, + "mean_weights": [ + 0.3748946189880371, + 0.26831528544425964, + 0.35679009556770325 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3326265811920166, + "mean_weights": [ + 0.26391589641571045, + 0.14978256821632385, + 0.2545396089553833, + 0.3317619264125824 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3771977424621582, + "mean_weights": [ + 0.27525436878204346, + 0.2067744880914688, + 0.23979949951171875, + 0.2781716585159302 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3576545715332031, + "mean_weights": [ + 0.2848266065120697, + 0.17341844737529755, + 0.23106196522712708, + 0.3106929659843445 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3772028684616089, + "mean_weights": [ + 0.2702856659889221, + 0.21318203210830688, + 0.23082655668258667, + 0.2857057452201843 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3277652263641357, + "mean_weights": [ + 0.3119839131832123, + 0.16646917164325714, + 0.1912841796875, + 0.3302627205848694 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3750126361846924, + "mean_weights": [ + 0.2621780037879944, + 0.21051618456840515, + 0.23754313588142395, + 0.2897626757621765 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568847179412842, + "mean_weights": [ + 0.2628105878829956, + 0.1954234540462494, + 0.21842113137245178, + 0.3233448266983032 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3733241558074951, + "mean_weights": [ + 0.25842776894569397, + 0.22964225709438324, + 0.21348324418067932, + 0.2984467148780823 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5855276584625244, + "mean_weights": [ + 0.18060272932052612, + 0.1563272476196289, + 0.18859915435314178, + 0.24991817772388458, + 0.2245527058839798 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5906484127044678, + "mean_weights": [ + 0.19773373007774353, + 0.18790484964847565, + 0.15935488045215607, + 0.18813443183898926, + 0.2668721079826355 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5916568040847778, + "mean_weights": [ + 0.2148856818675995, + 0.14808836579322815, + 0.18413493037223816, + 0.22754491865634918, + 0.22534608840942383 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5474352836608887, + "mean_weights": [ + 0.19825294613838196, + 0.1729484647512436, + 0.11353601515293121, + 0.19613605737686157, + 0.3191264867782593 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5919160842895508, + "mean_weights": [ + 0.21136541664600372, + 0.1572093367576599, + 0.18091553449630737, + 0.2257765829563141, + 0.2247331142425537 + ], + "sources": 5 + }, + { + "entropy_mean": 1.542572021484375, + "mean_weights": [ + 0.2053474485874176, + 0.19789552688598633, + 0.10896679759025574, + 0.1649853140115738, + 0.32280486822128296 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5774884223937988, + "mean_weights": [ + 0.1874420940876007, + 0.14777487516403198, + 0.18392466008663177, + 0.23523849248886108, + 0.24561989307403564 + ], + "sources": 5 + }, + { + "entropy_mean": 1.585869312286377, + "mean_weights": [ + 0.2123720645904541, + 0.19153764843940735, + 0.14032581448554993, + 0.1918477714061737, + 0.26391667127609253 + ], + "sources": 5 + }, + { + "entropy_mean": 1.747511625289917, + "mean_weights": [ + 0.13209694623947144, + 0.11971477419137955, + 0.14703533053398132, + 0.17972922325134277, + 0.15732964873313904, + 0.2640940546989441 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6791677474975586, + "mean_weights": [ + 0.1429731547832489, + 0.128341943025589, + 0.08682706952095032, + 0.12974600493907928, + 0.16802877187728882, + 0.3440830111503601 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7079570293426514, + "mean_weights": [ + 0.12207645177841187, + 0.11454501748085022, + 0.14774824678897858, + 0.1675407588481903, + 0.1342848241329193, + 0.3138047158718109 + ], + "sources": 6 + }, + { + "entropy_mean": 1.689455270767212, + "mean_weights": [ + 0.14791087806224823, + 0.15263086557388306, + 0.0942135751247406, + 0.12123478949069977, + 0.14674627780914307, + 0.3372636139392853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6673545837402344, + "mean_weights": [ + 0.119766965508461, + 0.10584045946598053, + 0.1391725242137909, + 0.15867865085601807, + 0.1261359602212906, + 0.3504054546356201 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7276721000671387, + "mean_weights": [ + 0.16197484731674194, + 0.16130125522613525, + 0.10335841774940491, + 0.12991490960121155, + 0.15279719233512878, + 0.2906533479690552 + ], + "sources": 6 + }, + { + "entropy_mean": 1.655259609222412, + "mean_weights": [ + 0.12291313707828522, + 0.10535868257284164, + 0.14431551098823547, + 0.1644165962934494, + 0.11736815422773361, + 0.34562790393829346 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7456457614898682, + "mean_weights": [ + 0.17144063115119934, + 0.17272350192070007, + 0.09372985363006592, + 0.13482090830802917, + 0.1837429702281952, + 0.2435421198606491 + ], + "sources": 6 + }, + { + "entropy_mean": 1.8196223974227905, + "mean_weights": [ + 0.097611203789711, + 0.08662202209234238, + 0.10460364818572998, + 0.12231925129890442, + 0.09176328778266907, + 0.22007866203784943, + 0.27700191736221313 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8201665878295898, + "mean_weights": [ + 0.12032124400138855, + 0.12977322936058044, + 0.07435770332813263, + 0.09230788052082062, + 0.1133352667093277, + 0.15118572115898132, + 0.31871896982192993 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7483060359954834, + "mean_weights": [ + 0.09602338075637817, + 0.07951871305704117, + 0.09892410784959793, + 0.11023538559675217, + 0.08284324407577515, + 0.17450374364852905, + 0.35795143246650696 + ], + "sources": 7 + }, + { + "entropy_mean": 1.851926565170288, + "mean_weights": [ + 0.14006076753139496, + 0.14811867475509644, + 0.08932580053806305, + 0.09198777377605438, + 0.11175547540187836, + 0.12843111157417297, + 0.29032039642333984 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7523608207702637, + "mean_weights": [ + 0.10785886645317078, + 0.08607444912195206, + 0.09595336019992828, + 0.10649852454662323, + 0.0838143527507782, + 0.14955362677574158, + 0.3702468276023865 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8693451881408691, + "mean_weights": [ + 0.1378144919872284, + 0.1244172528386116, + 0.09968579560518265, + 0.10974414646625519, + 0.11655184626579285, + 0.130243182182312, + 0.2815432548522949 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7898914813995361, + "mean_weights": [ + 0.10115306824445724, + 0.09615937620401382, + 0.11218561232089996, + 0.1163020133972168, + 0.09238721430301666, + 0.14564841985702515, + 0.33616429567337036 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9094455242156982, + "mean_weights": [ + 0.14588291943073273, + 0.14179158210754395, + 0.1023377850651741, + 0.11511939018964767, + 0.13270339369773865, + 0.13753530383110046, + 0.22462961077690125 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9082129001617432, + "mean_weights": [ + 0.09145341068506241, + 0.07698438316583633, + 0.08525429666042328, + 0.08438773453235626, + 0.06885559856891632, + 0.11482041329145432, + 0.2519546449184418, + 0.2262895107269287 + ], + "sources": 8 + }, + { + "entropy_mean": 2.025789499282837, + "mean_weights": [ + 0.11047068238258362, + 0.12069687247276306, + 0.09073689579963684, + 0.09690164029598236, + 0.10987945646047592, + 0.11353722959756851, + 0.15697816014289856, + 0.20079906284809113 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9302079677581787, + "mean_weights": [ + 0.0971018448472023, + 0.08240298926830292, + 0.0830029845237732, + 0.08926018327474594, + 0.07480385154485703, + 0.10335960984230042, + 0.22211775183677673, + 0.24795079231262207 + ], + "sources": 8 + }, + { + "entropy_mean": 2.010532855987549, + "mean_weights": [ + 0.10700424760580063, + 0.09611141681671143, + 0.09769981354475021, + 0.10596007108688354, + 0.09634587168693542, + 0.11705268919467926, + 0.17551329731941223, + 0.20431260764598846 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9408204555511475, + "mean_weights": [ + 0.08590779453516006, + 0.09530991315841675, + 0.09476494789123535, + 0.1030702069401741, + 0.08048167079687119, + 0.12287819385528564, + 0.22555866837501526, + 0.19202858209609985 + ], + "sources": 8 + }, + { + "entropy_mean": 2.0024678707122803, + "mean_weights": [ + 0.10720442980527878, + 0.09982497990131378, + 0.10896851122379303, + 0.1345594823360443, + 0.09260416775941849, + 0.1016254797577858, + 0.1643495261669159, + 0.19086343050003052 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9389225244522095, + "mean_weights": [ + 0.08632777631282806, + 0.09041796624660492, + 0.09941554069519043, + 0.10805264860391617, + 0.07865813374519348, + 0.11747924983501434, + 0.2661469578742981, + 0.1535017192363739 + ], + "sources": 8 + }, + { + "entropy_mean": 2.008256673812866, + "mean_weights": [ + 0.1016545370221138, + 0.10481878370046616, + 0.10823093354701996, + 0.1270347535610199, + 0.0919865146279335, + 0.14368976652622223, + 0.20522482693195343, + 0.11735989153385162 + ], + "sources": 8 + }, + { + "entropy_mean": 2.094825267791748, + "mean_weights": [ + 0.08519881963729858, + 0.09903685003519058, + 0.08900875598192215, + 0.09697237610816956, + 0.071251779794693, + 0.09201781451702118, + 0.19253811240196228, + 0.1091965064406395, + 0.16477900743484497 + ], + "sources": 9 + }, + { + "entropy_mean": 2.140371322631836, + "mean_weights": [ + 0.10001853853464127, + 0.10924500226974487, + 0.10232416540384293, + 0.11669869720935822, + 0.10123330354690552, + 0.1287832260131836, + 0.14476822316646576, + 0.09069216251373291, + 0.10623672604560852 + ], + "sources": 9 + }, + { + "entropy_mean": 2.117004632949829, + "mean_weights": [ + 0.09184888750314713, + 0.09620077908039093, + 0.09620209038257599, + 0.10222946852445602, + 0.06924223154783249, + 0.0821654349565506, + 0.17978665232658386, + 0.14873988926410675, + 0.13358455896377563 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1423683166503906, + "mean_weights": [ + 0.09596475213766098, + 0.07681103050708771, + 0.11225531250238419, + 0.11592596024274826, + 0.0810609757900238, + 0.11778091639280319, + 0.1649385243654251, + 0.12366649508476257, + 0.11159604042768478 + ], + "sources": 9 + }, + { + "entropy_mean": 2.093705415725708, + "mean_weights": [ + 0.08838655799627304, + 0.09557235240936279, + 0.09992480278015137, + 0.10721968859434128, + 0.07486072182655334, + 0.09290309250354767, + 0.21980023384094238, + 0.12490785866975784, + 0.09642469882965088 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1627297401428223, + "mean_weights": [ + 0.0994885265827179, + 0.09216944873332977, + 0.11588040739297867, + 0.12231601774692535, + 0.09873615205287933, + 0.13436099886894226, + 0.13960200548171997, + 0.1002899631857872, + 0.09715649485588074 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1300058364868164, + "mean_weights": [ + 0.089942067861557, + 0.1065598577260971, + 0.11013364791870117, + 0.12377548217773438, + 0.08726659417152405, + 0.11411209404468536, + 0.1931532621383667, + 0.10132836550474167, + 0.07372863590717316 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.05571148172020912, + 0.04887383058667183, + 0.061161335557699203, + 0.06501953303813934, + 0.09595561027526855, + 0.11691275984048843, + 0.12880638241767883, + 0.07614059001207352, + 0.13726800680160522, + 0.09499973803758621, + 0.10176635533571243, + 0.09968593716621399, + 0.09925490617752075, + 0.09547581523656845, + 0.10055620223283768, + 0.09975490719079971, + 0.0990053340792656, + 0.06202486902475357, + 0.07309409230947495, + 0.06455004960298538, + 0.07328634709119797, + 0.062386661767959595, + 0.07434380799531937, + 0.07036563754081726, + 0.07861478626728058, + 0.05747222527861595, + 0.06236656382679939, + 0.057312749326229095, + 0.06600815057754517, + 0.06149744614958763, + 0.07462523877620697, + 0.06584152579307556, + 0.07498960196971893, + 0.04952141270041466, + 0.05233807489275932, + 0.04880477488040924, + 0.05626988410949707, + 0.0513896569609642, + 0.061378348618745804, + 0.057161398231983185, + 0.06856362521648407, + 0.04507989436388016, + 0.04801172763109207, + 0.04335150495171547, + 0.0518198125064373, + 0.047289349138736725, + 0.05105668306350708, + 0.05114771053195, + 0.054886870086193085, + 0.042588189244270325, + 0.04603559896349907, + 0.044108107686042786, + 0.04718422889709473, + 0.05026337504386902, + 0.05210597813129425, + 0.05349528789520264, + 0.05379685014486313, + 0.04645879194140434, + 0.05137190222740173, + 0.05323651432991028, + 0.05757243186235428, + 0.05820684880018234, + 0.06228902190923691, + 0.058245208114385605 + ], + "loss_nats": 1.4232654571533203, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.9258099794387817, + "mean_weights": [ + 0.11242504417896271, + 0.0684729665517807, + 0.07014584541320801, + 0.059657223522663116, + 0.05741160362958908, + 0.05127572640776634, + 0.12547241151332855, + 0.2706507742404938, + 0.18448840081691742 + ], + "sources": 9 + }, + "step": 2000, + "stream_state_rms_by_sublayer": [ + 0.05045261234045029, + 0.10241243988275528, + 0.16565440595149994, + 0.1819411665201187, + 0.23719695210456848, + 0.24157075583934784, + 0.24811406433582306, + 0.26444777846336365, + 0.1042611226439476, + 0.10636615753173828, + 0.12873972952365875, + 0.13034582138061523, + 0.1382850706577301, + 0.14029134809970856, + 0.14404986798763275, + 0.15042904019355774, + 0.040291935205459595, + 0.04812335968017578, + 0.06678888201713562, + 0.07954440265893936, + 0.09121427685022354, + 0.10673297941684723, + 0.12122578918933868, + 0.14089107513427734, + 0.05663999170064926, + 0.06885477155447006, + 0.09547740966081619, + 0.10946234315633774, + 0.12204843014478683, + 0.13876035809516907, + 0.14941293001174927, + 0.1687854677438736, + 0.03680795058608055, + 0.05059739947319031, + 0.055156365036964417, + 0.07094386219978333, + 0.07373107224702835, + 0.09166282415390015, + 0.09760652482509613, + 0.12053482979536057, + 0.035728659480810165, + 0.048727843910455704, + 0.05559300258755684, + 0.07109228521585464, + 0.08069392293691635, + 0.09690473228693008, + 0.10087756812572479, + 0.11853098124265671, + 0.025910304859280586, + 0.05259029567241669, + 0.060516636818647385, + 0.08889466524124146, + 0.09090615063905716, + 0.12455403059720993, + 0.1273101419210434, + 0.1728687882423401, + 0.013562481850385666, + 0.06489895284175873, + 0.07113923132419586, + 0.14679722487926483, + 0.14901472628116608, + 0.23576539754867554, + 0.23766301572322845, + 0.35208994150161743 + ] + }, + { + "activation_grad_rms_by_block": [ + 4.705962055595592e-05, + 4.8540012357989326e-05, + 4.8112680815393105e-05, + 4.5881148253101856e-05, + 6.218662019819021e-05, + 6.19510465185158e-05, + 5.5873108067316934e-05, + 5.11586076754611e-05, + 5.377902198233642e-05, + 4.923442247672938e-05, + 4.591418110067025e-05, + 4.340316809248179e-05, + 3.853498856187798e-05, + 3.637185727711767e-05, + 3.548145105014555e-05, + 3.440591899561696e-05, + 7.517015910707414e-05, + 7.005458610365167e-05, + 6.350460171233863e-05, + 5.674128988175653e-05, + 0.00010789759107865393, + 9.314908675150946e-05, + 7.818391168257222e-05, + 6.73785325489007e-05, + 5.735654485761188e-05, + 4.582547626341693e-05, + 3.993835343862884e-05, + 3.59240839316044e-05, + 2.40640074480325e-05, + 2.1051846488262527e-05, + 1.9940822312491946e-05, + 1.993459591176361e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 5.259535555524053e-05, + "first_to_last_ratio": 1.59358297228639, + "imbalance_abs_log_ratio": 0.46598492272985975, + "last_quartile_mean": 3.300446633147658e-05, + "mean": 5.106260448428657e-05, + "normalized": [ + 0.921606350307445, + 0.9505980521014451, + 0.9422292752458163, + 0.8985273805847644, + 1.2178505351666271, + 1.2132371065714032, + 1.0942079557361923, + 1.0018801076079868, + 1.0531977858451338, + 0.9641972432463806, + 0.8991742893725557, + 0.8499991046449304, + 0.7546616344988103, + 0.712299296999438, + 0.6948617566317875, + 0.6737987484795189, + 1.4721176067352022, + 1.3719352314903852, + 1.2436616258358075, + 1.111210257581308, + 2.1130451955668876, + 1.8242133884920444, + 1.531138344238427, + 1.319527925169485, + 1.1232592899812257, + 0.8974371112918603, + 0.7821448561426008, + 0.7035301918972675, + 0.4712647874324093, + 0.41227521981846393, + 0.3905171409466258, + 0.3903952043397641 + ], + "population_cv": 0.3859968358313239 + }, + "activation_output_rms_by_block": [ + 0.16142085194587708, + 0.2773628532886505, + 0.3768865168094635, + 0.40868353843688965, + 0.14338424801826477, + 0.16827327013015747, + 0.18227216601371765, + 0.19825012981891632, + 0.06985136866569519, + 0.1086883395910263, + 0.14860449731349945, + 0.21192558109760284, + 0.08672910183668137, + 0.13778631389141083, + 0.18114051222801208, + 0.23228809237480164, + 0.062035780400037766, + 0.0909910649061203, + 0.12021689862012863, + 0.1606491208076477, + 0.061457011848688126, + 0.092374786734581, + 0.12395651638507843, + 0.15162576735019684, + 0.06645520776510239, + 0.10725662857294083, + 0.14660172164440155, + 0.20186874270439148, + 0.07922796159982681, + 0.17203646898269653, + 0.2743971645832062, + 0.41855114698410034 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.23956669680774212, + "first_to_last_ratio": 1.306969485354029, + "imbalance_abs_log_ratio": 0.26771108728004156, + "last_quartile_mean": 0.18329938035458326, + "mean": 0.1694765428546816, + "normalized": [ + 0.9524672218755847, + 1.6365855039093904, + 2.223827029163714, + 2.4114460417529116, + 0.846041851002414, + 0.9929000633110867, + 1.0755008506988941, + 1.1697791710850909, + 0.41215950885657104, + 0.6413178942658843, + 0.8768440446706599, + 1.250471466598887, + 0.5117469378110197, + 0.8130111198312341, + 1.068823503104685, + 1.370620904003087, + 0.36604346156170187, + 0.5368947429151949, + 0.7093424057110318, + 0.9479136056332998, + 0.36262842522923483, + 0.5450594234376621, + 0.7314081010689808, + 0.8946711137494054, + 0.392120388141766, + 0.6328700524939813, + 0.8650266235965518, + 1.1911308745392848, + 0.46748629789882584, + 1.0151048993854561, + 1.6190863936756676, + 2.4696700790208403 + ], + "population_cv": 0.5508327874312803 + }, + "bits_per_byte": 1.9147359946947653, + "branch_output_rms_by_sublayer": [ + 0.08130212873220444, + 0.13576549291610718, + 0.1432749330997467, + 0.041234951466321945, + 0.12958019971847534, + 0.023600688204169273, + 0.04409155994653702, + 0.0554339736700058, + 0.14100632071495056, + 0.01666521653532982, + 0.0481393076479435, + 0.008341255597770214, + 0.04208913818001747, + 0.01377889234572649, + 0.04236427694559097, + 0.03695881366729736, + 0.05848926678299904, + 0.033706482499837875, + 0.04504085332155228, + 0.0386466309428215, + 0.04397343844175339, + 0.04499480128288269, + 0.05665678158402443, + 0.057638294994831085, + 0.06478945910930634, + 0.04709118604660034, + 0.04678606986999512, + 0.044345736503601074, + 0.04817425087094307, + 0.04982399195432663, + 0.049439266324043274, + 0.060589101165533066, + 0.04376237466931343, + 0.040811967104673386, + 0.02850259095430374, + 0.04198655113577843, + 0.02922668866813183, + 0.05064959079027176, + 0.04021619260311127, + 0.07217346876859665, + 0.04409762844443321, + 0.039372652769088745, + 0.030135832726955414, + 0.04702961444854736, + 0.03803896903991699, + 0.054020222276449203, + 0.030100898817181587, + 0.06186771020293236, + 0.03295852243900299, + 0.054687224328517914, + 0.0306381918489933, + 0.05866995081305504, + 0.02206242084503174, + 0.06297128647565842, + 0.02074088715016842, + 0.07938925921916962, + 0.021799447014927864, + 0.07573848962783813, + 0.026608876883983612, + 0.1105475053191185, + 0.01902451552450657, + 0.12047550827264786, + 0.02519184537231922, + 0.16982334852218628 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.00023209204329478853, + 0.00014705811075659293, + 0.0001443553349415038, + 0.0001409677954299832, + 0.0001547598070491459, + 0.00010679063100428474, + 0.00010173131038502872, + 0.00012885752112210694, + 0.00018269174364617834, + 0.00015061935227176197, + 0.00015101151039961197, + 0.0001756105532961918, + 0.00014068815971577485, + 0.0001326364304799064, + 0.00015121016067746042, + 0.00016411832907359437, + 0.0002779928432077, + 0.00028472923376674154, + 0.0003007215693300081, + 0.0003372879732191828, + 0.0005019822813973469, + 0.0004655562424498461, + 0.00043909856222474334, + 0.00038874448014086744, + 0.0003187708239967095, + 0.00026126643200496464, + 0.0002213996139509037, + 0.00021927053392099255, + 0.00013957865743180746, + 0.00013181726813679266, + 0.00012444014246083002, + 0.00012714374135915476 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00014457656924792933, + "first_to_last_ratio": 0.74925318033778, + "imbalance_abs_log_ratio": 0.28867832809930477, + "last_quartile_mean": 0.0001929609016577694, + "mean": 0.00021703122476695333, + "normalized": [ + 1.069394708268395, + 0.6775896459806783, + 0.6651362498484334, + 0.6495277146472977, + 0.7130762276963877, + 0.49205192072687143, + 0.46874043352179906, + 0.5937280281234798, + 0.8417763104933472, + 0.6939985360792947, + 0.6958054563889003, + 0.8091487917683791, + 0.6482392562030871, + 0.6111398515228881, + 0.6967207637510633, + 0.7561968525488604, + 1.2808886993390323, + 1.311927507539442, + 1.3856143034391522, + 1.554098833388423, + 2.3129495856477433, + 2.145111805684907, + 2.0232045536131698, + 1.7911914774397033, + 1.468778625467388, + 1.2038195530873992, + 1.0201279294656385, + 1.0103179123485286, + 0.6431270780584042, + 0.6073654529588411, + 0.573374373178113, + 0.5858315617749513 + ], + "population_cv": 0.5013169471957857 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.22879935801029205, + "mean_weights": [ + 0.9326952695846558, + 0.06730473041534424 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5610548257827759, + "mean_weights": [ + 0.6759846210479736, + 0.3240153193473816 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43457120656967163, + "mean_weights": [ + 0.8295697569847107, + 0.1704302579164505 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6696063280105591, + "mean_weights": [ + 0.545992374420166, + 0.45400765538215637 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6690812110900879, + "mean_weights": [ + 0.5202739238739014, + 0.47972607612609863 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865538358688354, + "mean_weights": [ + 0.5361902713775635, + 0.4638097584247589 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4152516722679138, + "mean_weights": [ + 0.8452726602554321, + 0.15472736954689026 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6853918433189392, + "mean_weights": [ + 0.5220736265182495, + 0.4779263138771057 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0265861749649048, + "mean_weights": [ + 0.3228098750114441, + 0.23137259483337402, + 0.4458175599575043 + ], + "sources": 3 + }, + { + "entropy_mean": 1.066486120223999, + "mean_weights": [ + 0.2909930944442749, + 0.26484501361846924, + 0.44416186213493347 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8915231227874756, + "mean_weights": [ + 0.21873247623443604, + 0.16816270351409912, + 0.6131048202514648 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0651111602783203, + "mean_weights": [ + 0.3639928698539734, + 0.22285741567611694, + 0.41314971446990967 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0005524158477783, + "mean_weights": [ + 0.3166183531284332, + 0.18387392163276672, + 0.49950775504112244 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0648009777069092, + "mean_weights": [ + 0.35599708557128906, + 0.22703218460083008, + 0.41697072982788086 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0006463527679443, + "mean_weights": [ + 0.28650256991386414, + 0.19727063179016113, + 0.5162267684936523 + ], + "sources": 3 + }, + { + "entropy_mean": 1.072684407234192, + "mean_weights": [ + 0.41332390904426575, + 0.2456190586090088, + 0.34105703234672546 + ], + "sources": 3 + }, + { + "entropy_mean": 1.304518699645996, + "mean_weights": [ + 0.2721603810787201, + 0.12663133442401886, + 0.2481587529182434, + 0.35304954648017883 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3646502494812012, + "mean_weights": [ + 0.29890745878219604, + 0.18950369954109192, + 0.2219487726688385, + 0.2896400988101959 + ], + "sources": 4 + }, + { + "entropy_mean": 1.331923484802246, + "mean_weights": [ + 0.31035515666007996, + 0.14580613374710083, + 0.22454026341438293, + 0.31929847598075867 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3639233112335205, + "mean_weights": [ + 0.2952677309513092, + 0.19548878073692322, + 0.21280673146247864, + 0.29643672704696655 + ], + "sources": 4 + }, + { + "entropy_mean": 1.2764989137649536, + "mean_weights": [ + 0.3656603693962097, + 0.14041098952293396, + 0.17602922022342682, + 0.3178994059562683 + ], + "sources": 4 + }, + { + "entropy_mean": 1.364103078842163, + "mean_weights": [ + 0.27470695972442627, + 0.18989428877830505, + 0.23374854028224945, + 0.30165016651153564 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3429455757141113, + "mean_weights": [ + 0.2830207943916321, + 0.17677831649780273, + 0.23265977203845978, + 0.3075411319732666 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3732160329818726, + "mean_weights": [ + 0.2642192244529724, + 0.22012627124786377, + 0.22637531161308289, + 0.28927919268608093 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5599325895309448, + "mean_weights": [ + 0.18314200639724731, + 0.13039523363113403, + 0.18218961358070374, + 0.26679670810699463, + 0.2374764084815979 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5772080421447754, + "mean_weights": [ + 0.19838765263557434, + 0.18937477469444275, + 0.15325719118118286, + 0.16343195736408234, + 0.2955484390258789 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5662866830825806, + "mean_weights": [ + 0.22973918914794922, + 0.12454323470592499, + 0.16623356938362122, + 0.21617932617664337, + 0.2633046507835388 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5421115159988403, + "mean_weights": [ + 0.2021060585975647, + 0.1677371859550476, + 0.11891372501850128, + 0.17452098429203033, + 0.3367220163345337 + ], + "sources": 5 + }, + { + "entropy_mean": 1.564535140991211, + "mean_weights": [ + 0.22419840097427368, + 0.12987476587295532, + 0.1717071682214737, + 0.21590615808963776, + 0.25831347703933716 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5267750024795532, + "mean_weights": [ + 0.215010404586792, + 0.19819971919059753, + 0.10917140543460846, + 0.13804513216018677, + 0.33957335352897644 + ], + "sources": 5 + }, + { + "entropy_mean": 1.556459903717041, + "mean_weights": [ + 0.2037975788116455, + 0.12750469148159027, + 0.1812792420387268, + 0.22193393111228943, + 0.2654845714569092 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5851528644561768, + "mean_weights": [ + 0.21840542554855347, + 0.1923592984676361, + 0.1442098617553711, + 0.1790001392364502, + 0.26602524518966675 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7158764600753784, + "mean_weights": [ + 0.13390135765075684, + 0.10310791432857513, + 0.14190614223480225, + 0.16341811418533325, + 0.15331929922103882, + 0.3043471872806549 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6410574913024902, + "mean_weights": [ + 0.14665581285953522, + 0.12231245636940002, + 0.08100371062755585, + 0.11105543375015259, + 0.16015371680259705, + 0.3788188695907593 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6537573337554932, + "mean_weights": [ + 0.12217442691326141, + 0.09171081334352493, + 0.1348741054534912, + 0.16064059734344482, + 0.1268058568239212, + 0.363794207572937 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6571812629699707, + "mean_weights": [ + 0.15173450112342834, + 0.14253422617912292, + 0.09320533275604248, + 0.10436245799064636, + 0.14149092137813568, + 0.3666725754737854 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6046829223632812, + "mean_weights": [ + 0.12253664433956146, + 0.08343374729156494, + 0.1302124410867691, + 0.14439499378204346, + 0.11938183009624481, + 0.4000403583049774 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7041404247283936, + "mean_weights": [ + 0.1659325659275055, + 0.15505465865135193, + 0.10219550132751465, + 0.11347830295562744, + 0.1465306580066681, + 0.3168083429336548 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5808616876602173, + "mean_weights": [ + 0.12713032960891724, + 0.08526259660720825, + 0.1294248104095459, + 0.1489948332309723, + 0.10688205063343048, + 0.40230536460876465 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7305450439453125, + "mean_weights": [ + 0.17891491949558258, + 0.17673785984516144, + 0.09075747430324554, + 0.11627361923456192, + 0.17774534225463867, + 0.25957077741622925 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7513642311096191, + "mean_weights": [ + 0.09819091856479645, + 0.06520214676856995, + 0.091708242893219, + 0.11103156208992004, + 0.08388154953718185, + 0.22660189867019653, + 0.3233836889266968 + ], + "sources": 7 + }, + { + "entropy_mean": 1.766061782836914, + "mean_weights": [ + 0.12103402614593506, + 0.1159343495965004, + 0.06956179440021515, + 0.07680170238018036, + 0.10650800168514252, + 0.1487376093864441, + 0.36142250895500183 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6586570739746094, + "mean_weights": [ + 0.09605097770690918, + 0.06012085825204849, + 0.08562932163476944, + 0.10102558881044388, + 0.07161714881658554, + 0.17364460229873657, + 0.4119114875793457 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7821578979492188, + "mean_weights": [ + 0.13399386405944824, + 0.12986589968204498, + 0.0802837461233139, + 0.07629825174808502, + 0.10134232044219971, + 0.12680892646312714, + 0.351406991481781 + ], + "sources": 7 + }, + { + "entropy_mean": 1.655133843421936, + "mean_weights": [ + 0.10968073457479477, + 0.0657510757446289, + 0.08727138489484787, + 0.09466101229190826, + 0.0703636109828949, + 0.14186590909957886, + 0.4304063022136688 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8160101175308228, + "mean_weights": [ + 0.13769462704658508, + 0.11569900065660477, + 0.08787183463573456, + 0.0929793268442154, + 0.10649852454662323, + 0.1315194070339203, + 0.3277372717857361 + ], + "sources": 7 + }, + { + "entropy_mean": 1.6999847888946533, + "mean_weights": [ + 0.10220035165548325, + 0.07546016573905945, + 0.1028895378112793, + 0.10639216750860214, + 0.07665040343999863, + 0.13976991176605225, + 0.3966374397277832 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8706753253936768, + "mean_weights": [ + 0.14694823324680328, + 0.12422746419906616, + 0.09348422288894653, + 0.1027352511882782, + 0.12104703485965729, + 0.14027650654315948, + 0.27128130197525024 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7911043167114258, + "mean_weights": [ + 0.09078209102153778, + 0.05932023376226425, + 0.06744872033596039, + 0.0717800185084343, + 0.05460817739367485, + 0.10924961417913437, + 0.271850049495697, + 0.27496111392974854 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9753689765930176, + "mean_weights": [ + 0.10981250554323196, + 0.10515635460615158, + 0.07847881317138672, + 0.0806870236992836, + 0.10027817636728287, + 0.1099884957075119, + 0.1757785975933075, + 0.23982004821300507 + ], + "sources": 8 + }, + { + "entropy_mean": 1.81705904006958, + "mean_weights": [ + 0.0980612188577652, + 0.060419049113988876, + 0.06808654963970184, + 0.07581339031457901, + 0.06074263155460358, + 0.09064903855323792, + 0.23993313312530518, + 0.3062949776649475 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9814136028289795, + "mean_weights": [ + 0.11045937240123749, + 0.08757394552230835, + 0.08714327216148376, + 0.09700852632522583, + 0.09435248374938965, + 0.114869624376297, + 0.1777447760105133, + 0.23084799945354462 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8373757600784302, + "mean_weights": [ + 0.08649781346321106, + 0.07379771769046783, + 0.0769633948802948, + 0.0880836620926857, + 0.066677525639534, + 0.1187550276517868, + 0.23661984503269196, + 0.25260502099990845 + ], + "sources": 8 + }, + { + "entropy_mean": 1.968585729598999, + "mean_weights": [ + 0.10324696451425552, + 0.0860632136464119, + 0.09797552973031998, + 0.1271316111087799, + 0.08284065127372742, + 0.10640716552734375, + 0.17243653535842896, + 0.22389833629131317 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8529754877090454, + "mean_weights": [ + 0.08385773003101349, + 0.07188580930233002, + 0.08174896240234375, + 0.09558320790529251, + 0.06307300925254822, + 0.108722984790802, + 0.2897399365901947, + 0.20538833737373352 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9637342691421509, + "mean_weights": [ + 0.10303820669651031, + 0.08717727661132812, + 0.09220535308122635, + 0.11336663365364075, + 0.08865514397621155, + 0.14618384838104248, + 0.2349606156349182, + 0.13441291451454163 + ], + "sources": 8 + }, + { + "entropy_mean": 1.962258219718933, + "mean_weights": [ + 0.07841141521930695, + 0.07437640428543091, + 0.07400830090045929, + 0.07925958186388016, + 0.05560165271162987, + 0.08937598764896393, + 0.1956467628479004, + 0.1118645966053009, + 0.24145527184009552 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0993216037750244, + "mean_weights": [ + 0.1043272316455841, + 0.09323035180568695, + 0.08778752386569977, + 0.10311444103717804, + 0.09694567322731018, + 0.13869863748550415, + 0.1680138111114502, + 0.0874539315700531, + 0.12042839825153351 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0489909648895264, + "mean_weights": [ + 0.0919051468372345, + 0.07878251373767853, + 0.07574650645256042, + 0.09311886876821518, + 0.05773407965898514, + 0.07325181365013123, + 0.17863741517066956, + 0.18801960349082947, + 0.1628040373325348 + ], + "sources": 9 + }, + { + "entropy_mean": 2.126072406768799, + "mean_weights": [ + 0.09415344893932343, + 0.07414553314447403, + 0.10936354845762253, + 0.11765885353088379, + 0.07923045754432678, + 0.12059831619262695, + 0.17183929681777954, + 0.12057285010814667, + 0.11243770271539688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.028747081756592, + "mean_weights": [ + 0.08796507865190506, + 0.0757753849029541, + 0.0847562700510025, + 0.09340266138315201, + 0.06450396031141281, + 0.08635804057121277, + 0.24226921796798706, + 0.14922890067100525, + 0.11574049293994904 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1297550201416016, + "mean_weights": [ + 0.10085826367139816, + 0.08009681105613708, + 0.0982624888420105, + 0.11403533071279526, + 0.09301646053791046, + 0.12771156430244446, + 0.17309343814849854, + 0.10872717201709747, + 0.10419845581054688 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1051812171936035, + "mean_weights": [ + 0.09203394502401352, + 0.08722756803035736, + 0.10295312106609344, + 0.11847955733537674, + 0.07401065528392792, + 0.11382973194122314, + 0.21416787803173065, + 0.11479790508747101, + 0.08249962329864502 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.059127259999513626, + 0.055979277938604355, + 0.0761958509683609, + 0.07538121938705444, + 0.13674014806747437, + 0.18601374328136444, + 0.18700072169303894, + 0.08912894874811172, + 0.2049511820077896, + 0.12973569333553314, + 0.14231130480766296, + 0.13784728944301605, + 0.13109618425369263, + 0.1306019276380539, + 0.13412337005138397, + 0.13534265756607056, + 0.13132649660110474, + 0.08038423955440521, + 0.09715579450130463, + 0.0852823480963707, + 0.09982985258102417, + 0.08248664438724518, + 0.10234102606773376, + 0.09743349254131317, + 0.11231652647256851, + 0.07765212655067444, + 0.08827291429042816, + 0.07436380535364151, + 0.08991213887929916, + 0.08191535621881485, + 0.1065397635102272, + 0.09093168377876282, + 0.10877490043640137, + 0.06605636328458786, + 0.07123783230781555, + 0.06527329236268997, + 0.07851143926382065, + 0.0699063166975975, + 0.08676032721996307, + 0.08022409677505493, + 0.10016339272260666, + 0.05980068817734718, + 0.06517573446035385, + 0.05788872763514519, + 0.07041674107313156, + 0.0643942728638649, + 0.07159053534269333, + 0.07148924469947815, + 0.07631570100784302, + 0.056988880038261414, + 0.06180137023329735, + 0.05852367728948593, + 0.06370948255062103, + 0.06811124831438065, + 0.07006713002920151, + 0.07283915579319, + 0.0730312243103981, + 0.057786133140325546, + 0.06770572811365128, + 0.07090020179748535, + 0.07336394488811493, + 0.07763758301734924, + 0.08089767396450043, + 0.07564623653888702 + ], + "loss_nats": 1.3271938562393188, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.8321013450622559, + "mean_weights": [ + 0.09867185354232788, + 0.05898427963256836, + 0.057430200278759, + 0.05067210644483566, + 0.04944504797458649, + 0.047245003283023834, + 0.1162109300494194, + 0.30178403854370117, + 0.219556525349617 + ], + "sources": 9 + }, + "step": 4000, + "stream_state_rms_by_sublayer": [ + 0.08130212873220444, + 0.16142085194587708, + 0.2561342120170593, + 0.2773628532886505, + 0.3685380816459656, + 0.3768865168094635, + 0.38441672921180725, + 0.40868353843688965, + 0.14100632071495056, + 0.14338424801826477, + 0.16686734557151794, + 0.16827327013015747, + 0.18023288249969482, + 0.18227216601371765, + 0.18965557217597961, + 0.19825012981891632, + 0.05848926678299904, + 0.06985136866569519, + 0.09072534739971161, + 0.1086883395910263, + 0.12624414265155792, + 0.14860449731349945, + 0.17554737627506256, + 0.21192558109760284, + 0.06478945910930634, + 0.08672910183668137, + 0.11512516438961029, + 0.13778631389141083, + 0.15369470417499542, + 0.18114051222801208, + 0.19735252857208252, + 0.23228809237480164, + 0.04376237466931343, + 0.062035780400037766, + 0.07002050429582596, + 0.0909910649061203, + 0.09636416286230087, + 0.12021689862012863, + 0.12901709973812103, + 0.1606491208076477, + 0.04409762844443321, + 0.061457011848688126, + 0.07137759029865265, + 0.092374786734581, + 0.10297419875860214, + 0.12395651638507843, + 0.1298016905784607, + 0.15162576735019684, + 0.03295852243900299, + 0.06645520776510239, + 0.07529281079769135, + 0.10725662857294083, + 0.11068660020828247, + 0.14660172164440155, + 0.1495535671710968, + 0.20186874270439148, + 0.021799447014927864, + 0.07922796159982681, + 0.08539005368947983, + 0.17203646898269653, + 0.17447474598884583, + 0.2743971645832062, + 0.27695441246032715, + 0.41855114698410034 + ] + }, + { + "activation_grad_rms_by_block": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ], + "activation_grad_statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "population_cv": 0.42684708327825066 + }, + "activation_output_rms_by_block": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "activation_output_statistics": { + "first_quartile_mean": 0.2482638694345951, + "first_to_last_ratio": 1.6504997944655762, + "imbalance_abs_log_ratio": 0.5010781477825214, + "last_quartile_mean": 0.150417388882488, + "mean": 0.1577255482552573, + "normalized": [ + 1.0413313772202522, + 1.8759762186094133, + 2.6842751238870792, + 2.86288304745295, + 0.8807369558111258, + 0.9863695480754758, + 1.0723915340184837, + 1.1882317995492147, + 0.3938410687519575, + 0.6240548714989927, + 0.8686355260833801, + 1.2755653854399962, + 0.532526584341831, + 0.7695868670150526, + 1.006314124418815, + 1.3133324349770026, + 0.3684251942759779, + 0.5299682889456079, + 0.6940544684956076, + 0.9337557124120812, + 0.3540539965769534, + 0.5274131112327584, + 0.7125108234042923, + 0.874443203356988, + 0.3801289540418877, + 0.5954785163594047, + 0.7985322424957776, + 1.083277634082024, + 0.4248333923179985, + 0.8857998846594436, + 1.4024216545081702, + 2.0588504556840044 + ], + "population_cv": 0.6067899489376947 + }, + "bits_per_byte": 1.8565786854442996, + "branch_output_rms_by_sublayer": [ + 0.08626964688301086, + 0.13547764718532562, + 0.16337208449840546, + 0.039667773991823196, + 0.15584443509578705, + 0.026703152805566788, + 0.04305505007505417, + 0.05304057523608208, + 0.1360139101743698, + 0.016798952594399452, + 0.03393339365720749, + 0.00753376679494977, + 0.03730965405702591, + 0.01294450368732214, + 0.037494905292987823, + 0.03526800498366356, + 0.05234000086784363, + 0.030294200405478477, + 0.04223497211933136, + 0.034149833023548126, + 0.041619230061769485, + 0.04367009550333023, + 0.05722156912088394, + 0.05930330231785774, + 0.062187470495700836, + 0.04527241364121437, + 0.0369485504925251, + 0.04281846433877945, + 0.04293901473283768, + 0.04812675341963768, + 0.045369699597358704, + 0.06094835698604584, + 0.04149825870990753, + 0.038031257688999176, + 0.02440357394516468, + 0.040106989443302155, + 0.026009807363152504, + 0.04961257055401802, + 0.03634416684508324, + 0.07085621356964111, + 0.038355328142642975, + 0.03753058984875679, + 0.02631118707358837, + 0.04544757679104805, + 0.033921729773283005, + 0.05368982255458832, + 0.02731483057141304, + 0.06174425780773163, + 0.027557097375392914, + 0.05051252990961075, + 0.024315863847732544, + 0.054937634617090225, + 0.02051021158695221, + 0.05777662619948387, + 0.018631720915436745, + 0.07133159041404724, + 0.019497457891702652, + 0.0636468306183815, + 0.021715763956308365, + 0.09261580556631088, + 0.017114246264100075, + 0.1020733118057251, + 0.020576251670718193, + 0.12878532707691193 + ], + "capture": { + "all_gradients_finite": true, + "all_gradients_present": true, + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset", + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "core_parameter_grad_rms_by_block": [ + 0.0003637111411518744, + 0.0002328437345071155, + 0.00026754486121072013, + 0.00020961054010746675, + 0.00023324217449373482, + 0.00013226952769200274, + 0.0001490970084679736, + 0.0001891597343986376, + 0.0002918746014559555, + 0.00023307226647352922, + 0.00024024223572494834, + 0.00029360747277183056, + 0.00025626387819923106, + 0.00024114885933974834, + 0.0002716298155375107, + 0.0003086967264992385, + 0.0005417594428612147, + 0.0005350465304351142, + 0.0005646380221663597, + 0.0006164043223392351, + 0.0009244274084097353, + 0.0008484364345428017, + 0.0007892136159088089, + 0.0006791712867767833, + 0.0005903268079560613, + 0.00047055631494833115, + 0.0003773406829883918, + 0.0003589161190616393, + 0.00026391753197443733, + 0.00022775345188039026, + 0.00019334088037647755, + 0.0001778215115139631 + ], + "core_parameter_grad_statistics": { + "first_quartile_mean": 0.00022218484025369068, + "first_to_last_ratio": 0.6682317907333766, + "imbalance_abs_log_ratio": 0.4031201735174996, + "last_quartile_mean": 0.00033249666258746146, + "mean": 0.00037728390444285195, + "normalized": [ + 0.9640250667172751, + 0.6171578796900011, + 0.7091340448403511, + 0.5555777430182339, + 0.6182139543911144, + 0.3505835423520907, + 0.3951851820664077, + 0.5013723940277184, + 0.7736206024664017, + 0.6177636091253748, + 0.6367677838780912, + 0.7782136192780627, + 0.6792335299266601, + 0.6391708114234587, + 0.7199613138509996, + 0.81820804668331, + 1.4359463430099129, + 1.4181536082893218, + 1.496586563903859, + 1.633794378929312, + 2.4502169255666204, + 2.2488010343184843, + 2.0918295390158974, + 1.8001597173347186, + 1.5646753042057733, + 1.2472207518187604, + 1.000150492891086, + 0.9513157461399342, + 0.6995197220622846, + 0.6036659640085139, + 0.5124546213069721, + 0.471320163462998 + ], + "population_cv": 0.5587385684613161 + }, + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.21598131954669952, + "mean_weights": [ + 0.9387255907058716, + 0.06127442046999931 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5377621054649353, + "mean_weights": [ + 0.7082071900367737, + 0.2917928099632263 + ], + "sources": 2 + }, + { + "entropy_mean": 0.43774232268333435, + "mean_weights": [ + 0.829714834690094, + 0.1702851504087448 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6749181151390076, + "mean_weights": [ + 0.5407299399375916, + 0.45927006006240845 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6663151383399963, + "mean_weights": [ + 0.5011438131332397, + 0.49885618686676025 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6865031719207764, + "mean_weights": [ + 0.5308656692504883, + 0.4691343307495117 + ], + "sources": 2 + }, + { + "entropy_mean": 0.4133813679218292, + "mean_weights": [ + 0.8479139804840088, + 0.15208598971366882 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6850745677947998, + "mean_weights": [ + 0.5398513674736023, + 0.4601486027240753 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0074119567871094, + "mean_weights": [ + 0.32744842767715454, + 0.20981493592262268, + 0.4627366065979004 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0489801168441772, + "mean_weights": [ + 0.28095993399620056, + 0.24264614284038544, + 0.4763939380645752 + ], + "sources": 3 + }, + { + "entropy_mean": 0.8527575731277466, + "mean_weights": [ + 0.20522087812423706, + 0.1493145227432251, + 0.6454645991325378 + ], + "sources": 3 + }, + { + "entropy_mean": 1.049942135810852, + "mean_weights": [ + 0.35788866877555847, + 0.20253834128379822, + 0.4395729899406433 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9791408181190491, + "mean_weights": [ + 0.31465935707092285, + 0.1638563871383667, + 0.5214842557907104 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0485806465148926, + "mean_weights": [ + 0.346283882856369, + 0.20800183713436127, + 0.4457142949104309 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9801348447799683, + "mean_weights": [ + 0.2748759686946869, + 0.1817527413368225, + 0.5433712601661682 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0688585042953491, + "mean_weights": [ + 0.4051227569580078, + 0.23231396079063416, + 0.36256328225135803 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2915732860565186, + "mean_weights": [ + 0.26621368527412415, + 0.1108347550034523, + 0.2563389539718628, + 0.36661261320114136 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3568191528320312, + "mean_weights": [ + 0.294538676738739, + 0.1710844486951828, + 0.2252884805202484, + 0.30908840894699097 + ], + "sources": 4 + }, + { + "entropy_mean": 1.320021152496338, + "mean_weights": [ + 0.304654598236084, + 0.12911874055862427, + 0.23493197560310364, + 0.3312946557998657 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3578683137893677, + "mean_weights": [ + 0.29558902978897095, + 0.17812995612621307, + 0.21904321014881134, + 0.30723777413368225 + ], + "sources": 4 + }, + { + "entropy_mean": 1.258887767791748, + "mean_weights": [ + 0.35744374990463257, + 0.12067858874797821, + 0.17883820831775665, + 0.34303945302963257 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3539760112762451, + "mean_weights": [ + 0.2736307978630066, + 0.1722734123468399, + 0.2365075945854187, + 0.3175881803035736 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3203014135360718, + "mean_weights": [ + 0.28207293152809143, + 0.15420743823051453, + 0.22419029474258423, + 0.3395293354988098 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3734662532806396, + "mean_weights": [ + 0.2766023576259613, + 0.21176086366176605, + 0.22839435935020447, + 0.283242404460907 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5325813293457031, + "mean_weights": [ + 0.17451074719429016, + 0.10643941164016724, + 0.1691833734512329, + 0.27394938468933105, + 0.27591708302497864 + ], + "sources": 5 + }, + { + "entropy_mean": 1.560360074043274, + "mean_weights": [ + 0.19377154111862183, + 0.17211592197418213, + 0.14605137705802917, + 0.16254930198192596, + 0.3255118727684021 + ], + "sources": 5 + }, + { + "entropy_mean": 1.540909767150879, + "mean_weights": [ + 0.21921303868293762, + 0.10205383598804474, + 0.16381604969501495, + 0.215368390083313, + 0.2995486855506897 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5142419338226318, + "mean_weights": [ + 0.2000826597213745, + 0.15480534732341766, + 0.10827909409999847, + 0.16766759753227234, + 0.3691653311252594 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5333527326583862, + "mean_weights": [ + 0.21770668029785156, + 0.10525082051753998, + 0.15970060229301453, + 0.217601016163826, + 0.29974091053009033 + ], + "sources": 5 + }, + { + "entropy_mean": 1.512608289718628, + "mean_weights": [ + 0.21392285823822021, + 0.1860550343990326, + 0.10291537642478943, + 0.1385909616947174, + 0.35851576924324036 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5262645483016968, + "mean_weights": [ + 0.20246261358261108, + 0.10530520975589752, + 0.16650918126106262, + 0.21146778762340546, + 0.3142552077770233 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5794764757156372, + "mean_weights": [ + 0.2210189402103424, + 0.18839940428733826, + 0.14016607403755188, + 0.17440536618232727, + 0.2760102152824402 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6602650880813599, + "mean_weights": [ + 0.12695768475532532, + 0.08036886155605316, + 0.12672865390777588, + 0.15391480922698975, + 0.1544383019208908, + 0.3575916886329651 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5925545692443848, + "mean_weights": [ + 0.14075040817260742, + 0.11028306186199188, + 0.07537610828876495, + 0.10244421660900116, + 0.1520608514547348, + 0.41908538341522217 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5918848514556885, + "mean_weights": [ + 0.11773697286844254, + 0.07344591617584229, + 0.11976177245378494, + 0.14735516905784607, + 0.1265447735786438, + 0.4151553511619568 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6208878755569458, + "mean_weights": [ + 0.14785674214363098, + 0.12492769956588745, + 0.08781817555427551, + 0.10019385814666748, + 0.14005999267101288, + 0.3991435170173645 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5470075607299805, + "mean_weights": [ + 0.12043031305074692, + 0.06855648756027222, + 0.11753211170434952, + 0.13396470248699188, + 0.1162077933549881, + 0.44330859184265137 + ], + "sources": 6 + }, + { + "entropy_mean": 1.683492660522461, + "mean_weights": [ + 0.1658952832221985, + 0.1435280442237854, + 0.09461991488933563, + 0.10784181207418442, + 0.15157341957092285, + 0.3365415334701538 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5540567636489868, + "mean_weights": [ + 0.12987136840820312, + 0.07178180664777756, + 0.12132491171360016, + 0.1441602110862732, + 0.11020088195800781, + 0.42266082763671875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7208337783813477, + "mean_weights": [ + 0.18048501014709473, + 0.1685236394405365, + 0.08726631104946136, + 0.11160479485988617, + 0.18141084909439087, + 0.27070939540863037 + ], + "sources": 6 + }, + { + "entropy_mean": 1.6729483604431152, + "mean_weights": [ + 0.09229261428117752, + 0.051420144736766815, + 0.08079522103071213, + 0.09550285339355469, + 0.07590493559837341, + 0.22372649610042572, + 0.3803577423095703 + ], + "sources": 7 + }, + { + "entropy_mean": 1.710235595703125, + "mean_weights": [ + 0.11657393723726273, + 0.10617929697036743, + 0.061112720519304276, + 0.06796726584434509, + 0.10333786904811859, + 0.14365722239017487, + 0.4011717140674591 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5640671253204346, + "mean_weights": [ + 0.0904226303100586, + 0.04561381787061691, + 0.07565715163946152, + 0.08878682553768158, + 0.06555216014385223, + 0.16504976153373718, + 0.4689176678657532 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7556228637695312, + "mean_weights": [ + 0.13281463086605072, + 0.12175846099853516, + 0.07235272228717804, + 0.0707377940416336, + 0.10551950335502625, + 0.12620802223682404, + 0.3706088662147522 + ], + "sources": 7 + }, + { + "entropy_mean": 1.594892978668213, + "mean_weights": [ + 0.10721130669116974, + 0.053925663232803345, + 0.07673212885856628, + 0.08865121006965637, + 0.06888366490602493, + 0.14094024896621704, + 0.4636557698249817 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7775511741638184, + "mean_weights": [ + 0.13637228310108185, + 0.10478455573320389, + 0.07710614800453186, + 0.08548048138618469, + 0.11000139266252518, + 0.12931162118911743, + 0.3569435477256775 + ], + "sources": 7 + }, + { + "entropy_mean": 1.654036045074463, + "mean_weights": [ + 0.10232305526733398, + 0.061968062072992325, + 0.08900615572929382, + 0.1005433052778244, + 0.07864195853471756, + 0.14099234342575073, + 0.4265251159667969 + ], + "sources": 7 + }, + { + "entropy_mean": 1.84956693649292, + "mean_weights": [ + 0.14851760864257812, + 0.11630287766456604, + 0.08512166142463684, + 0.09215595573186874, + 0.1263951063156128, + 0.14375528693199158, + 0.28775155544281006 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7161309719085693, + "mean_weights": [ + 0.0837959572672844, + 0.04434927552938461, + 0.0567089319229126, + 0.06495873630046844, + 0.04885464161634445, + 0.09997115284204483, + 0.2760006785392761, + 0.32536062598228455 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9285151958465576, + "mean_weights": [ + 0.10564712435007095, + 0.09173665940761566, + 0.07135222852230072, + 0.07045470178127289, + 0.09785081446170807, + 0.10425601899623871, + 0.1782122403383255, + 0.2804902195930481 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7219383716583252, + "mean_weights": [ + 0.0923302173614502, + 0.04646994173526764, + 0.05750556290149689, + 0.06549686193466187, + 0.054404959082603455, + 0.08427833020687103, + 0.2333412766456604, + 0.3661728501319885 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9470824003219604, + "mean_weights": [ + 0.10804469138383865, + 0.07850410044193268, + 0.07298290729522705, + 0.08511960506439209, + 0.09968686103820801, + 0.11502514779567719, + 0.18013443052768707, + 0.26050227880477905 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7855088710784912, + "mean_weights": [ + 0.08493785560131073, + 0.05972691997885704, + 0.07017223536968231, + 0.07963316142559052, + 0.0645163506269455, + 0.11586237698793411, + 0.23710276186466217, + 0.28804832696914673 + ], + "sources": 8 + }, + { + "entropy_mean": 1.973182201385498, + "mean_weights": [ + 0.1049819216132164, + 0.0840599462389946, + 0.09310521930456161, + 0.11566175520420074, + 0.0963074192404747, + 0.1096896231174469, + 0.16087493300437927, + 0.23531918227672577 + ], + "sources": 8 + }, + { + "entropy_mean": 1.802741289138794, + "mean_weights": [ + 0.08190082013607025, + 0.05632811039686203, + 0.07102417945861816, + 0.08421063423156738, + 0.06341388821601868, + 0.10201875865459442, + 0.2913346588611603, + 0.24976897239685059 + ], + "sources": 8 + }, + { + "entropy_mean": 1.945815086364746, + "mean_weights": [ + 0.1064736619591713, + 0.07925166189670563, + 0.07823627442121506, + 0.09447507560253143, + 0.09724453091621399, + 0.1459730863571167, + 0.2451319396495819, + 0.153213769197464 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8717057704925537, + "mean_weights": [ + 0.07336360216140747, + 0.05720400810241699, + 0.057025715708732605, + 0.06655597686767578, + 0.05032068490982056, + 0.08329878002405167, + 0.1901058554649353, + 0.11727118492126465, + 0.3048541843891144 + ], + "sources": 9 + }, + { + "entropy_mean": 2.09096622467041, + "mean_weights": [ + 0.10326676070690155, + 0.08232800662517548, + 0.07164021581411362, + 0.08822821080684662, + 0.10803373903036118, + 0.13578416407108307, + 0.17320571839809418, + 0.09955950826406479, + 0.13795366883277893 + ], + "sources": 9 + }, + { + "entropy_mean": 1.96881103515625, + "mean_weights": [ + 0.08843819797039032, + 0.06052382290363312, + 0.0642135739326477, + 0.07744920998811722, + 0.05161681026220322, + 0.06631232053041458, + 0.16417551040649414, + 0.2115624099969864, + 0.21570813655853271 + ], + "sources": 9 + }, + { + "entropy_mean": 2.124063014984131, + "mean_weights": [ + 0.09220977127552032, + 0.06839151680469513, + 0.10039852559566498, + 0.10902463644742966, + 0.0849665105342865, + 0.12137697637081146, + 0.17017139494419098, + 0.12761826813220978, + 0.1258423924446106 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9633058309555054, + "mean_weights": [ + 0.0846441313624382, + 0.05872901901602745, + 0.07210009545087814, + 0.07792744785547256, + 0.05918677896261215, + 0.07445863634347916, + 0.23511825501918793, + 0.18020221590995789, + 0.15763342380523682 + ], + "sources": 9 + }, + { + "entropy_mean": 2.1238560676574707, + "mean_weights": [ + 0.10231245309114456, + 0.07575537264347076, + 0.08897507935762405, + 0.0967075377702713, + 0.09801244735717773, + 0.12438583374023438, + 0.1766587197780609, + 0.12203006446361542, + 0.11516247689723969 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0835556983947754, + "mean_weights": [ + 0.09133295714855194, + 0.07054752856492996, + 0.09000501036643982, + 0.10956106334924698, + 0.07114207744598389, + 0.11565886437892914, + 0.22511571645736694, + 0.13003571331501007, + 0.09660108387470245 + ], + "sources": 9 + } + ], + "layer_input_rms_by_sublayer": [ + 0.062413111329078674, + 0.05931045487523079, + 0.07552354782819748, + 0.07934419065713882, + 0.14653873443603516, + 0.214645653963089, + 0.2105521410703659, + 0.09514089673757553, + 0.2186809629201889, + 0.12821291387081146, + 0.14272794127464294, + 0.1321839988231659, + 0.1287408471107483, + 0.12568901479244232, + 0.13253600895404816, + 0.1335759311914444, + 0.13371841609477997, + 0.07788944244384766, + 0.09585829079151154, + 0.08352623134851456, + 0.09883332997560501, + 0.07934392243623734, + 0.10020580142736435, + 0.09343641996383667, + 0.11237040907144547, + 0.07063088566064835, + 0.08538161963224411, + 0.06870647519826889, + 0.08663177490234375, + 0.07622843235731125, + 0.10358677804470062, + 0.08666495233774185, + 0.10874549299478531, + 0.057347409427165985, + 0.06467291712760925, + 0.05777324363589287, + 0.07278503477573395, + 0.06351501494646072, + 0.08346924930810928, + 0.07469023764133453, + 0.09857872873544693, + 0.051319729536771774, + 0.06093711405992508, + 0.05102822557091713, + 0.06834813207387924, + 0.05902284011244774, + 0.0680762529373169, + 0.06656526774168015, + 0.07369214296340942, + 0.050236333161592484, + 0.05633483827114105, + 0.05184732377529144, + 0.05850831791758537, + 0.061248794198036194, + 0.06479237228631973, + 0.06608261168003082, + 0.06705968081951141, + 0.048912134021520615, + 0.061747051775455475, + 0.062179163098335266, + 0.0654391348361969, + 0.07131517678499222, + 0.07205834239721298, + 0.06802479177713394 + ], + "loss_nats": 1.2868822813034058, + "loss_scale": 1.0, + "output_weights": { + "entropy_mean": 1.679484486579895, + "mean_weights": [ + 0.08993907272815704, + 0.043493133038282394, + 0.04922422021627426, + 0.037924446165561676, + 0.03939434140920639, + 0.03898489475250244, + 0.09112586826086044, + 0.32717907428741455, + 0.2827349305152893 + ], + "sources": 9 + }, + "step": 8000, + "stream_state_rms_by_sublayer": [ + 0.08626964688301086, + 0.16424456238746643, + 0.27676042914390564, + 0.29588937759399414, + 0.41335058212280273, + 0.42337876558303833, + 0.43042075634002686, + 0.45154979825019836, + 0.1360139101743698, + 0.13891471922397614, + 0.15445689857006073, + 0.1555756777524948, + 0.1670466512441635, + 0.16914354264736176, + 0.17834730446338654, + 0.1874145120382309, + 0.05234000086784363, + 0.06211879849433899, + 0.08269349485635757, + 0.09842939674854279, + 0.11548293381929398, + 0.137006014585495, + 0.16410532593727112, + 0.20118924975395203, + 0.062187470495700836, + 0.08399304747581482, + 0.0988159254193306, + 0.12138351052999496, + 0.13199056684970856, + 0.1587214469909668, + 0.17199495434761047, + 0.2071460783481598, + 0.04149825870990753, + 0.0581100657582283, + 0.06434677541255951, + 0.08358953893184662, + 0.08728200942277908, + 0.10947012156248093, + 0.11647984385490417, + 0.1472771316766739, + 0.038355328142642975, + 0.05584336072206497, + 0.06353354454040527, + 0.08318652212619781, + 0.09196189045906067, + 0.11238116025924683, + 0.1169828325510025, + 0.1379220336675644, + 0.027557097375392914, + 0.05995604768395424, + 0.06579430401325226, + 0.09392217546701431, + 0.09665275365114212, + 0.1259489357471466, + 0.12799666821956635, + 0.17086055874824524, + 0.019497457891702652, + 0.06700707972049713, + 0.07176875323057175, + 0.13971327245235443, + 0.14131556451320648, + 0.2211977243423462, + 0.22250843048095703, + 0.32473331689834595 + ] + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.061194200882085, + "cross_entropy_nats": 5.587594032287598, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "cross_entropy_nats": 3.598757028579712, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "cross_entropy_nats": 2.1405738294124603, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "cross_entropy_nats": 1.3509249985218048, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "cross_entropy_nats": 1.2388448417186737, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "cross_entropy_nats": 1.1804966628551483, + "step": 8000 + } + ], + "gradient_gate": null, + "hashes": { + "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a", + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "path": "experiments/k3/attnres_gradient/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "attnres_aggregation_groups": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + }, + "sublayers": 64, + "sublayers_per_attnres_group": 8, + "transformer_blocks_per_attnres_group": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 109.96596761305909, + "measured_steps": 7980, + "median_ms": 109.86474549281411, + "p95_ms": 110.51533734425902, + "peak_allocated_bytes": 12815843328, + "peak_reserved_bytes": 13337886720, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.082576459898554, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.602415084838867, + "step": 1, + "unclipped_grad_norm": 19.083253860473633 + }, + { + "bits_per_byte": 7.337116283872504, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.0857014656066895, + "step": 10, + "unclipped_grad_norm": 11.73559856414795 + }, + { + "bits_per_byte": 6.794594947933964, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.709654331207275, + "step": 20, + "unclipped_grad_norm": 3.3705036640167236 + }, + { + "bits_per_byte": 6.669727976170927, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.623103141784668, + "step": 30, + "unclipped_grad_norm": 3.037728786468506 + }, + { + "bits_per_byte": 6.488515360401613, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.497496128082275, + "step": 40, + "unclipped_grad_norm": 3.071707010269165 + }, + { + "bits_per_byte": 6.170372909737745, + "learning_rate": 3.75e-05, + "loss_nats": 4.276976585388184, + "step": 50, + "unclipped_grad_norm": 2.8251144886016846 + }, + { + "bits_per_byte": 5.9458358004242795, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.121339321136475, + "step": 60, + "unclipped_grad_norm": 2.6051926612854004 + }, + { + "bits_per_byte": 5.763860269615738, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9952034950256348, + "step": 70, + "unclipped_grad_norm": 2.513765573501587 + }, + { + "bits_per_byte": 5.595572150058764, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8785550594329834, + "step": 80, + "unclipped_grad_norm": 2.595860719680786 + }, + { + "bits_per_byte": 5.4051244099034195, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.746546745300293, + "step": 90, + "unclipped_grad_norm": 2.317351818084717 + }, + { + "bits_per_byte": 5.1779332643218385, + "learning_rate": 7.5e-05, + "loss_nats": 3.5890698432922363, + "step": 100, + "unclipped_grad_norm": 2.214414358139038 + }, + { + "bits_per_byte": 4.995566300999934, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.462662696838379, + "step": 110, + "unclipped_grad_norm": 2.0333311557769775 + }, + { + "bits_per_byte": 4.799066491360726, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3264594078063965, + "step": 120, + "unclipped_grad_norm": 1.8019887208938599 + }, + { + "bits_per_byte": 4.633962458601826, + "learning_rate": 9.75e-05, + "loss_nats": 3.2120180130004883, + "step": 130, + "unclipped_grad_norm": 1.6018544435501099 + }, + { + "bits_per_byte": 4.476944362155892, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.1031813621520996, + "step": 140, + "unclipped_grad_norm": 1.3790134191513062 + }, + { + "bits_per_byte": 4.275779695094682, + "learning_rate": 0.0001125, + "loss_nats": 2.963744640350342, + "step": 150, + "unclipped_grad_norm": 1.2061023712158203 + }, + { + "bits_per_byte": 4.128819144071206, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.861879348754883, + "step": 160, + "unclipped_grad_norm": 1.0463906526565552 + }, + { + "bits_per_byte": 4.021535678532382, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7875161170959473, + "step": 170, + "unclipped_grad_norm": 0.9023508429527283 + }, + { + "bits_per_byte": 3.919193618518149, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7165780067443848, + "step": 180, + "unclipped_grad_norm": 0.9033810496330261 + }, + { + "bits_per_byte": 3.8348498868820102, + "learning_rate": 0.0001425, + "loss_nats": 2.6581153869628906, + "step": 190, + "unclipped_grad_norm": 0.8499347567558289 + }, + { + "bits_per_byte": 3.7665400977858114, + "learning_rate": 0.00015, + "loss_nats": 2.610766649246216, + "step": 200, + "unclipped_grad_norm": 0.8223105072975159 + }, + { + "bits_per_byte": 3.6822345462981643, + "learning_rate": 0.0001575, + "loss_nats": 2.552330493927002, + "step": 210, + "unclipped_grad_norm": 0.8248683214187622 + }, + { + "bits_per_byte": 3.6564082555833983, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5344290733337402, + "step": 220, + "unclipped_grad_norm": 1.0457000732421875 + }, + { + "bits_per_byte": 3.6121584953776043, + "learning_rate": 0.0001725, + "loss_nats": 2.5037574768066406, + "step": 230, + "unclipped_grad_norm": 1.2628414630889893 + }, + { + "bits_per_byte": 3.5855720413460244, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4853291511535645, + "step": 240, + "unclipped_grad_norm": 0.9464337229728699 + }, + { + "bits_per_byte": 3.5024318443006237, + "learning_rate": 0.0001875, + "loss_nats": 2.4277007579803467, + "step": 250, + "unclipped_grad_norm": 0.8258991837501526 + }, + { + "bits_per_byte": 3.520142273721174, + "learning_rate": 0.000195, + "loss_nats": 2.439976692199707, + "step": 260, + "unclipped_grad_norm": 1.1166419982910156 + }, + { + "bits_per_byte": 3.497664141253405, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.42439603805542, + "step": 270, + "unclipped_grad_norm": 0.9347699880599976 + }, + { + "bits_per_byte": 3.454170760746106, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.3942487239837646, + "step": 280, + "unclipped_grad_norm": 1.2552868127822876 + }, + { + "bits_per_byte": 3.4220065653817433, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3719542026519775, + "step": 290, + "unclipped_grad_norm": 0.8032089471817017 + }, + { + "bits_per_byte": 3.435558454304431, + "learning_rate": 0.000225, + "loss_nats": 2.38134765625, + "step": 300, + "unclipped_grad_norm": 1.544488787651062 + }, + { + "bits_per_byte": 3.330305071976392, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.308391571044922, + "step": 310, + "unclipped_grad_norm": 0.8757423162460327 + }, + { + "bits_per_byte": 3.390824390953808, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3503403663635254, + "step": 320, + "unclipped_grad_norm": 0.8823365569114685 + }, + { + "bits_per_byte": 3.3979087003081485, + "learning_rate": 0.0002475, + "loss_nats": 2.355250835418701, + "step": 330, + "unclipped_grad_norm": 0.898759663105011 + }, + { + "bits_per_byte": 3.325636774901372, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3051557540893555, + "step": 340, + "unclipped_grad_norm": 0.9536095261573792 + }, + { + "bits_per_byte": 3.3170438337336625, + "learning_rate": 0.0002625, + "loss_nats": 2.2991995811462402, + "step": 350, + "unclipped_grad_norm": 1.2654696702957153 + }, + { + "bits_per_byte": 3.313786482326321, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2969417572021484, + "step": 360, + "unclipped_grad_norm": 1.3773932456970215 + }, + { + "bits_per_byte": 3.3236555347634327, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3037824630737305, + "step": 370, + "unclipped_grad_norm": 1.2408338785171509 + }, + { + "bits_per_byte": 3.208911117864879, + "learning_rate": 0.000285, + "loss_nats": 2.224247694015503, + "step": 380, + "unclipped_grad_norm": 1.0960922241210938 + }, + { + "bits_per_byte": 3.2804789462990227, + "learning_rate": 0.0002925, + "loss_nats": 2.2738547325134277, + "step": 390, + "unclipped_grad_norm": 0.9866538643836975 + }, + { + "bits_per_byte": 3.27038631641579, + "learning_rate": 0.0003, + "loss_nats": 2.2668590545654297, + "step": 400, + "unclipped_grad_norm": 1.2531954050064087 + }, + { + "bits_per_byte": 3.2889521875417316, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2797279357910156, + "step": 410, + "unclipped_grad_norm": 1.448500633239746 + }, + { + "bits_per_byte": 3.205672684549132, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2220029830932617, + "step": 420, + "unclipped_grad_norm": 0.9654834866523743 + }, + { + "bits_per_byte": 3.218386673996816, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2308156490325928, + "step": 430, + "unclipped_grad_norm": 0.974204421043396 + }, + { + "bits_per_byte": 3.1822768526563587, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2057862281799316, + "step": 440, + "unclipped_grad_norm": 1.4095590114593506 + }, + { + "bits_per_byte": 3.237966210866943, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.244387149810791, + "step": 450, + "unclipped_grad_norm": 0.9198341369628906 + }, + { + "bits_per_byte": 3.1857660366770637, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.208204746246338, + "step": 460, + "unclipped_grad_norm": 1.4268778562545776 + }, + { + "bits_per_byte": 3.1576792059993566, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.1887364387512207, + "step": 470, + "unclipped_grad_norm": 1.2876354455947876 + }, + { + "bits_per_byte": 3.131057323541699, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.170283555984497, + "step": 480, + "unclipped_grad_norm": 1.0718865394592285 + }, + { + "bits_per_byte": 3.1147361699748184, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.158970594406128, + "step": 490, + "unclipped_grad_norm": 0.8680974841117859 + }, + { + "bits_per_byte": 3.059730958749366, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1208438873291016, + "step": 500, + "unclipped_grad_norm": 1.296195387840271 + }, + { + "bits_per_byte": 3.12068642572937, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.163094997406006, + "step": 510, + "unclipped_grad_norm": 1.0469720363616943 + }, + { + "bits_per_byte": 3.1397613655018657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.176316738128662, + "step": 520, + "unclipped_grad_norm": 1.2089059352874756 + }, + { + "bits_per_byte": 2.9844827015867583, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.06868577003479, + "step": 530, + "unclipped_grad_norm": 1.1353814601898193 + }, + { + "bits_per_byte": 3.0015299659055814, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0805020332336426, + "step": 540, + "unclipped_grad_norm": 0.919367253780365 + }, + { + "bits_per_byte": 2.987599371185694, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0708460807800293, + "step": 550, + "unclipped_grad_norm": 0.8667266368865967 + }, + { + "bits_per_byte": 2.995183462123442, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0761029720306396, + "step": 560, + "unclipped_grad_norm": 0.9804732203483582 + }, + { + "bits_per_byte": 2.962294187903042, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0533058643341064, + "step": 570, + "unclipped_grad_norm": 0.9414609670639038 + }, + { + "bits_per_byte": 2.956716790535561, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0494399070739746, + "step": 580, + "unclipped_grad_norm": 0.8727462291717529 + }, + { + "bits_per_byte": 2.9113505189603663, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0179944038391113, + "step": 590, + "unclipped_grad_norm": 1.1642946004867554 + }, + { + "bits_per_byte": 2.8665478345320485, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.986939549446106, + "step": 600, + "unclipped_grad_norm": 0.9213871359825134 + }, + { + "bits_per_byte": 2.875505550902238, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9931485652923584, + "step": 610, + "unclipped_grad_norm": 0.7122263312339783 + }, + { + "bits_per_byte": 2.878308696128651, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9950915575027466, + "step": 620, + "unclipped_grad_norm": 1.4622896909713745 + }, + { + "bits_per_byte": 2.790202156074285, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.934020757675171, + "step": 630, + "unclipped_grad_norm": 1.1472660303115845 + }, + { + "bits_per_byte": 2.7890029210498186, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9331895112991333, + "step": 640, + "unclipped_grad_norm": 1.1472764015197754 + }, + { + "bits_per_byte": 2.7734051265124666, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.9223779439926147, + "step": 650, + "unclipped_grad_norm": 0.9182148575782776 + }, + { + "bits_per_byte": 2.795219749913207, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.937498688697815, + "step": 660, + "unclipped_grad_norm": 1.083983302116394 + }, + { + "bits_per_byte": 2.7702638633944576, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.9202005863189697, + "step": 670, + "unclipped_grad_norm": 1.1721765995025635 + }, + { + "bits_per_byte": 2.755497089026075, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.9099650382995605, + "step": 680, + "unclipped_grad_norm": 1.280714750289917 + }, + { + "bits_per_byte": 2.7128121989604823, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8803781270980835, + "step": 690, + "unclipped_grad_norm": 1.0380759239196777 + }, + { + "bits_per_byte": 2.7091302223881604, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8778259754180908, + "step": 700, + "unclipped_grad_norm": 1.1401211023330688 + }, + { + "bits_per_byte": 2.6870259802033667, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.862504482269287, + "step": 710, + "unclipped_grad_norm": 1.6450653076171875 + }, + { + "bits_per_byte": 2.688485424978589, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.863516092300415, + "step": 720, + "unclipped_grad_norm": 1.0721391439437866 + }, + { + "bits_per_byte": 2.65192277331841, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8381727933883667, + "step": 730, + "unclipped_grad_norm": 1.0645538568496704 + }, + { + "bits_per_byte": 2.6143492076364594, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.8121287822723389, + "step": 740, + "unclipped_grad_norm": 1.1877458095550537 + }, + { + "bits_per_byte": 2.6277774410331767, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.8214365243911743, + "step": 750, + "unclipped_grad_norm": 1.1091431379318237 + }, + { + "bits_per_byte": 2.540277655771055, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7607862949371338, + "step": 760, + "unclipped_grad_norm": 1.222191333770752 + }, + { + "bits_per_byte": 2.5790566478529025, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.787665843963623, + "step": 770, + "unclipped_grad_norm": 1.254854679107666 + }, + { + "bits_per_byte": 2.5138998166949715, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7425025701522827, + "step": 780, + "unclipped_grad_norm": 0.9573470950126648 + }, + { + "bits_per_byte": 2.5258618979930834, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7507940530776978, + "step": 790, + "unclipped_grad_norm": 1.108724594116211 + }, + { + "bits_per_byte": 2.430956195819507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6850104331970215, + "step": 800, + "unclipped_grad_norm": 0.9832104444503784 + }, + { + "bits_per_byte": 2.468076415233783, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7107402086257935, + "step": 810, + "unclipped_grad_norm": 0.9492170214653015 + }, + { + "bits_per_byte": 2.4602348662676934, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7053048610687256, + "step": 820, + "unclipped_grad_norm": 1.192505121231079 + }, + { + "bits_per_byte": 2.4583616312344954, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7040064334869385, + "step": 830, + "unclipped_grad_norm": 0.9113606214523315 + }, + { + "bits_per_byte": 2.4751598646748696, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7156500816345215, + "step": 840, + "unclipped_grad_norm": 1.3671480417251587 + }, + { + "bits_per_byte": 2.386899400147981, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6544725894927979, + "step": 850, + "unclipped_grad_norm": 1.0324512720108032 + }, + { + "bits_per_byte": 2.472668523994471, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.7139232158660889, + "step": 860, + "unclipped_grad_norm": 1.503003478050232 + }, + { + "bits_per_byte": 2.4348980381772827, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6877427101135254, + "step": 870, + "unclipped_grad_norm": 1.1144746541976929 + }, + { + "bits_per_byte": 2.3709108610278693, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.64339017868042, + "step": 880, + "unclipped_grad_norm": 1.0134795904159546 + }, + { + "bits_per_byte": 2.3377405671559597, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6203982830047607, + "step": 890, + "unclipped_grad_norm": 1.0625330209732056 + }, + { + "bits_per_byte": 2.389408283058768, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6562116146087646, + "step": 900, + "unclipped_grad_norm": 0.9719302654266357 + }, + { + "bits_per_byte": 2.3782030974071033, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6484447717666626, + "step": 910, + "unclipped_grad_norm": 1.0672563314437866 + }, + { + "bits_per_byte": 2.3530695528100125, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6310235261917114, + "step": 920, + "unclipped_grad_norm": 1.1197935342788696 + }, + { + "bits_per_byte": 2.297453115243394, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5924731492996216, + "step": 930, + "unclipped_grad_norm": 1.0027272701263428 + }, + { + "bits_per_byte": 2.344061961471073, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6247799396514893, + "step": 940, + "unclipped_grad_norm": 1.0651428699493408 + }, + { + "bits_per_byte": 2.367935217202641, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6413276195526123, + "step": 950, + "unclipped_grad_norm": 0.9822106957435608 + }, + { + "bits_per_byte": 2.309206925551311, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6006202697753906, + "step": 960, + "unclipped_grad_norm": 0.8860561847686768 + }, + { + "bits_per_byte": 2.2419830349126597, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5540242195129395, + "step": 970, + "unclipped_grad_norm": 1.1816905736923218 + }, + { + "bits_per_byte": 2.3851068249780383, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.65323007106781, + "step": 980, + "unclipped_grad_norm": 0.9257189035415649 + }, + { + "bits_per_byte": 2.232579711494085, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.547506332397461, + "step": 990, + "unclipped_grad_norm": 0.9634203910827637 + }, + { + "bits_per_byte": 2.2247954327507324, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5421106815338135, + "step": 1000, + "unclipped_grad_norm": 0.809658944606781 + }, + { + "bits_per_byte": 2.2354428786656486, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5494909286499023, + "step": 1010, + "unclipped_grad_norm": 1.3530596494674683 + }, + { + "bits_per_byte": 2.2067197121797495, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5295815467834473, + "step": 1020, + "unclipped_grad_norm": 1.0303069353103638 + }, + { + "bits_per_byte": 2.1954008459958647, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5217359066009521, + "step": 1030, + "unclipped_grad_norm": 0.9071640968322754 + }, + { + "bits_per_byte": 2.291673982226451, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5884673595428467, + "step": 1040, + "unclipped_grad_norm": 1.3239798545837402 + }, + { + "bits_per_byte": 2.196653911590051, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5226044654846191, + "step": 1050, + "unclipped_grad_norm": 0.9085557460784912 + }, + { + "bits_per_byte": 2.225028985190604, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5422725677490234, + "step": 1060, + "unclipped_grad_norm": 0.9488101005554199 + }, + { + "bits_per_byte": 2.275690258620563, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5773882865905762, + "step": 1070, + "unclipped_grad_norm": 1.047879695892334 + }, + { + "bits_per_byte": 2.235951603346901, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5498435497283936, + "step": 1080, + "unclipped_grad_norm": 0.8919407725334167 + }, + { + "bits_per_byte": 2.1733449309359636, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5064479112625122, + "step": 1090, + "unclipped_grad_norm": 1.0036922693252563 + }, + { + "bits_per_byte": 2.169043472855234, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5034663677215576, + "step": 1100, + "unclipped_grad_norm": 0.9904995560646057 + }, + { + "bits_per_byte": 2.124962083664635, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4729114770889282, + "step": 1110, + "unclipped_grad_norm": 0.9176430702209473 + }, + { + "bits_per_byte": 2.176794214981668, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5088387727737427, + "step": 1120, + "unclipped_grad_norm": 0.9229135513305664 + }, + { + "bits_per_byte": 2.210271497885368, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.53204345703125, + "step": 1130, + "unclipped_grad_norm": 0.871993899345398 + }, + { + "bits_per_byte": 2.123456375556331, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4718677997589111, + "step": 1140, + "unclipped_grad_norm": 0.8673478960990906 + }, + { + "bits_per_byte": 2.1911007637763418, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.518755316734314, + "step": 1150, + "unclipped_grad_norm": 0.8959671258926392 + }, + { + "bits_per_byte": 2.0953000638876755, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4523513317108154, + "step": 1160, + "unclipped_grad_norm": 0.9113589525222778 + }, + { + "bits_per_byte": 2.1570418355474237, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.495147466659546, + "step": 1170, + "unclipped_grad_norm": 0.8980047702789307 + }, + { + "bits_per_byte": 2.1555593450969863, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4941198825836182, + "step": 1180, + "unclipped_grad_norm": 0.948503315448761 + }, + { + "bits_per_byte": 2.1334348089593913, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4787843227386475, + "step": 1190, + "unclipped_grad_norm": 1.1094794273376465 + }, + { + "bits_per_byte": 2.1317098229712372, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4775886535644531, + "step": 1200, + "unclipped_grad_norm": 0.8304017782211304 + }, + { + "bits_per_byte": 2.1385674591917407, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.482342004776001, + "step": 1210, + "unclipped_grad_norm": 0.8620252013206482 + }, + { + "bits_per_byte": 2.0980762078379027, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4542756080627441, + "step": 1220, + "unclipped_grad_norm": 0.88591068983078 + }, + { + "bits_per_byte": 2.108927281211441, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4617969989776611, + "step": 1230, + "unclipped_grad_norm": 1.0479243993759155 + }, + { + "bits_per_byte": 2.081558994049039, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.442826747894287, + "step": 1240, + "unclipped_grad_norm": 0.9365729689598083 + }, + { + "bits_per_byte": 2.0172010223043655, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.3982172012329102, + "step": 1250, + "unclipped_grad_norm": 0.8966301083564758 + }, + { + "bits_per_byte": 2.0690183631134245, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4341342449188232, + "step": 1260, + "unclipped_grad_norm": 0.8974369168281555 + }, + { + "bits_per_byte": 2.101840220134686, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4568846225738525, + "step": 1270, + "unclipped_grad_norm": 0.9220007061958313 + }, + { + "bits_per_byte": 2.1256708241688407, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.473402738571167, + "step": 1280, + "unclipped_grad_norm": 0.9313441514968872 + }, + { + "bits_per_byte": 2.076867651298803, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.439574956893921, + "step": 1290, + "unclipped_grad_norm": 0.8451681137084961 + }, + { + "bits_per_byte": 2.032476865319302, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4088056087493896, + "step": 1300, + "unclipped_grad_norm": 0.9314314126968384 + }, + { + "bits_per_byte": 2.058903719450881, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4271233081817627, + "step": 1310, + "unclipped_grad_norm": 0.8796194195747375 + }, + { + "bits_per_byte": 2.0275063947267156, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4053603410720825, + "step": 1320, + "unclipped_grad_norm": 0.9140937924385071 + }, + { + "bits_per_byte": 2.037489299678698, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4122799634933472, + "step": 1330, + "unclipped_grad_norm": 0.821657657623291 + }, + { + "bits_per_byte": 2.0199990080512524, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4001566171646118, + "step": 1340, + "unclipped_grad_norm": 0.8321346044540405 + }, + { + "bits_per_byte": 2.048903616233812, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.420191764831543, + "step": 1350, + "unclipped_grad_norm": 0.9421875476837158 + }, + { + "bits_per_byte": 2.00944598061166, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3928418159484863, + "step": 1360, + "unclipped_grad_norm": 0.8682821393013 + }, + { + "bits_per_byte": 2.0435301902902543, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4164671897888184, + "step": 1370, + "unclipped_grad_norm": 0.8776147961616516 + }, + { + "bits_per_byte": 2.0135448431296727, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.39568293094635, + "step": 1380, + "unclipped_grad_norm": 0.87440025806427 + }, + { + "bits_per_byte": 2.0533221944997795, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4232544898986816, + "step": 1390, + "unclipped_grad_norm": 0.7801422476768494 + }, + { + "bits_per_byte": 2.1564656936670334, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4947481155395508, + "step": 1400, + "unclipped_grad_norm": 0.8561526536941528 + }, + { + "bits_per_byte": 2.0373921294809607, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.412212610244751, + "step": 1410, + "unclipped_grad_norm": 0.8376067876815796 + }, + { + "bits_per_byte": 2.0325960492963495, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4088882207870483, + "step": 1420, + "unclipped_grad_norm": 0.751315712928772 + }, + { + "bits_per_byte": 1.9860154082234123, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.376600980758667, + "step": 1430, + "unclipped_grad_norm": 0.9057046175003052 + }, + { + "bits_per_byte": 2.015529006972677, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3970582485198975, + "step": 1440, + "unclipped_grad_norm": 0.8342951536178589 + }, + { + "bits_per_byte": 1.9818307263626185, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3737003803253174, + "step": 1450, + "unclipped_grad_norm": 0.8142686486244202 + }, + { + "bits_per_byte": 2.1060192265680016, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.459781289100647, + "step": 1460, + "unclipped_grad_norm": 0.8357279896736145 + }, + { + "bits_per_byte": 2.079546797033944, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.441431999206543, + "step": 1470, + "unclipped_grad_norm": 0.7720605731010437 + }, + { + "bits_per_byte": 2.025708316111945, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.404114007949829, + "step": 1480, + "unclipped_grad_norm": 0.8328851461410522 + }, + { + "bits_per_byte": 2.0422327531721454, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4155678749084473, + "step": 1490, + "unclipped_grad_norm": 0.9698525071144104 + }, + { + "bits_per_byte": 1.9217496191796013, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3320553302764893, + "step": 1500, + "unclipped_grad_norm": 0.7396577000617981 + }, + { + "bits_per_byte": 1.9436662282923034, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.347246766090393, + "step": 1510, + "unclipped_grad_norm": 0.809975266456604 + }, + { + "bits_per_byte": 1.9993110429963143, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3858168125152588, + "step": 1520, + "unclipped_grad_norm": 0.8165730834007263 + }, + { + "bits_per_byte": 2.03089187320895, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4077069759368896, + "step": 1530, + "unclipped_grad_norm": 0.7620540261268616 + }, + { + "bits_per_byte": 2.002577509484152, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3880809545516968, + "step": 1540, + "unclipped_grad_norm": 0.8928612470626831 + }, + { + "bits_per_byte": 2.0629018001354886, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4298945665359497, + "step": 1550, + "unclipped_grad_norm": 0.7963082194328308 + }, + { + "bits_per_byte": 1.9429003895480113, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3467159271240234, + "step": 1560, + "unclipped_grad_norm": 0.7710158228874207 + }, + { + "bits_per_byte": 1.9502758655302628, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3518282175064087, + "step": 1570, + "unclipped_grad_norm": 0.8057935833930969 + }, + { + "bits_per_byte": 1.9693647358974786, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3650596141815186, + "step": 1580, + "unclipped_grad_norm": 0.8588574528694153 + }, + { + "bits_per_byte": 1.972771368245769, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3674209117889404, + "step": 1590, + "unclipped_grad_norm": 0.8608299493789673 + }, + { + "bits_per_byte": 1.9610373359427014, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3592875003814697, + "step": 1600, + "unclipped_grad_norm": 0.8587453365325928 + }, + { + "bits_per_byte": 1.9257736692444893, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3348445892333984, + "step": 1610, + "unclipped_grad_norm": 0.8028180003166199 + }, + { + "bits_per_byte": 1.929578785394829, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3374820947647095, + "step": 1620, + "unclipped_grad_norm": 0.7991352677345276 + }, + { + "bits_per_byte": 1.9716548568763677, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3666470050811768, + "step": 1630, + "unclipped_grad_norm": 0.7573540806770325 + }, + { + "bits_per_byte": 1.9236465878186182, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3333702087402344, + "step": 1640, + "unclipped_grad_norm": 0.8524133563041687 + }, + { + "bits_per_byte": 1.9335809936630575, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3402562141418457, + "step": 1650, + "unclipped_grad_norm": 0.7908568978309631 + }, + { + "bits_per_byte": 1.9524230689262851, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3533165454864502, + "step": 1660, + "unclipped_grad_norm": 0.8385109901428223 + }, + { + "bits_per_byte": 1.8864807930018634, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3076088428497314, + "step": 1670, + "unclipped_grad_norm": 0.8104477524757385 + }, + { + "bits_per_byte": 1.918672677573015, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3299225568771362, + "step": 1680, + "unclipped_grad_norm": 0.7970330119132996 + }, + { + "bits_per_byte": 1.9517956762159374, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.352881669998169, + "step": 1690, + "unclipped_grad_norm": 0.7619603276252747 + }, + { + "bits_per_byte": 1.8990867773448057, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3163466453552246, + "step": 1700, + "unclipped_grad_norm": 0.8769286274909973 + }, + { + "bits_per_byte": 1.938959579086141, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3439843654632568, + "step": 1710, + "unclipped_grad_norm": 0.7246651649475098 + }, + { + "bits_per_byte": 1.847509524316351, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.2805960178375244, + "step": 1720, + "unclipped_grad_norm": 0.7415109276771545 + }, + { + "bits_per_byte": 1.944612132872049, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3479024171829224, + "step": 1730, + "unclipped_grad_norm": 0.8057509660720825 + }, + { + "bits_per_byte": 1.956270664791388, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3559834957122803, + "step": 1740, + "unclipped_grad_norm": 0.8188740611076355 + }, + { + "bits_per_byte": 1.908994697861012, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3232142925262451, + "step": 1750, + "unclipped_grad_norm": 0.7595267295837402 + }, + { + "bits_per_byte": 1.9438418225788343, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3473684787750244, + "step": 1760, + "unclipped_grad_norm": 0.7832275629043579 + }, + { + "bits_per_byte": 1.9277638524802738, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.33622407913208, + "step": 1770, + "unclipped_grad_norm": 0.7276267409324646 + }, + { + "bits_per_byte": 1.987644083927083, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.377729892730713, + "step": 1780, + "unclipped_grad_norm": 0.7699368000030518 + }, + { + "bits_per_byte": 1.8704368754681737, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2964880466461182, + "step": 1790, + "unclipped_grad_norm": 0.7781511545181274 + }, + { + "bits_per_byte": 2.006532250540742, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.390822172164917, + "step": 1800, + "unclipped_grad_norm": 0.8113785982131958 + }, + { + "bits_per_byte": 1.8832052114335305, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3053383827209473, + "step": 1810, + "unclipped_grad_norm": 0.8088275790214539 + }, + { + "bits_per_byte": 1.9263209180395349, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.335223913192749, + "step": 1820, + "unclipped_grad_norm": 0.7881855964660645 + }, + { + "bits_per_byte": 1.968589438107389, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3645222187042236, + "step": 1830, + "unclipped_grad_norm": 0.7948311567306519 + }, + { + "bits_per_byte": 1.9398172565659937, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3445788621902466, + "step": 1840, + "unclipped_grad_norm": 0.8343047499656677 + }, + { + "bits_per_byte": 1.8747907882574166, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2995059490203857, + "step": 1850, + "unclipped_grad_norm": 0.7942258715629578 + }, + { + "bits_per_byte": 1.9135037390013332, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3263397216796875, + "step": 1860, + "unclipped_grad_norm": 0.769598126411438 + }, + { + "bits_per_byte": 1.987024774401326, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3773006200790405, + "step": 1870, + "unclipped_grad_norm": 0.7647593021392822 + }, + { + "bits_per_byte": 1.8769319722606586, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.300990104675293, + "step": 1880, + "unclipped_grad_norm": 0.8446611166000366 + }, + { + "bits_per_byte": 1.8953207012562119, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3137362003326416, + "step": 1890, + "unclipped_grad_norm": 0.7048609852790833 + }, + { + "bits_per_byte": 1.915848206497895, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3279647827148438, + "step": 1900, + "unclipped_grad_norm": 0.7416327595710754 + }, + { + "bits_per_byte": 1.9362319342434569, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3420937061309814, + "step": 1910, + "unclipped_grad_norm": 0.8084686398506165 + }, + { + "bits_per_byte": 1.8860243510464736, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3072924613952637, + "step": 1920, + "unclipped_grad_norm": 0.8338462710380554 + }, + { + "bits_per_byte": 1.930916638435891, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.338409423828125, + "step": 1930, + "unclipped_grad_norm": 0.6878981590270996 + }, + { + "bits_per_byte": 1.9257588787365152, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.334834337234497, + "step": 1940, + "unclipped_grad_norm": 0.6981199383735657 + }, + { + "bits_per_byte": 1.8699291826828268, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.2961361408233643, + "step": 1950, + "unclipped_grad_norm": 0.76193767786026 + }, + { + "bits_per_byte": 1.859464038377825, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.2888822555541992, + "step": 1960, + "unclipped_grad_norm": 0.7367894053459167 + }, + { + "bits_per_byte": 1.8691335909399354, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2955846786499023, + "step": 1970, + "unclipped_grad_norm": 0.8052698373794556 + }, + { + "bits_per_byte": 1.8850715671606937, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3066320419311523, + "step": 1980, + "unclipped_grad_norm": 0.7624203562736511 + }, + { + "bits_per_byte": 1.8715028239382214, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.297226905822754, + "step": 1990, + "unclipped_grad_norm": 0.779162585735321 + }, + { + "bits_per_byte": 1.9325518494802945, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3395428657531738, + "step": 2000, + "unclipped_grad_norm": 0.7498615384101868 + }, + { + "bits_per_byte": 1.9335421255839624, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3402292728424072, + "step": 2010, + "unclipped_grad_norm": 0.7343811392784119 + }, + { + "bits_per_byte": 1.838237939608337, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.2741694450378418, + "step": 2020, + "unclipped_grad_norm": 0.7684847116470337 + }, + { + "bits_per_byte": 1.8631659649376469, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.2914482355117798, + "step": 2030, + "unclipped_grad_norm": 0.6878870129585266 + }, + { + "bits_per_byte": 1.8841837927169398, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3060166835784912, + "step": 2040, + "unclipped_grad_norm": 0.853155255317688 + }, + { + "bits_per_byte": 1.8339112000779305, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2711703777313232, + "step": 2050, + "unclipped_grad_norm": 0.8042898774147034 + }, + { + "bits_per_byte": 1.8136427007084468, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2571213245391846, + "step": 2060, + "unclipped_grad_norm": 0.7331950664520264 + }, + { + "bits_per_byte": 1.9306452498128293, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3382213115692139, + "step": 2070, + "unclipped_grad_norm": 0.7544640898704529 + }, + { + "bits_per_byte": 1.9159075405124426, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3280059099197388, + "step": 2080, + "unclipped_grad_norm": 0.7521310448646545 + }, + { + "bits_per_byte": 1.8394872215842044, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2750353813171387, + "step": 2090, + "unclipped_grad_norm": 0.7430872321128845 + }, + { + "bits_per_byte": 1.7890550290451945, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2400784492492676, + "step": 2100, + "unclipped_grad_norm": 0.7162277698516846 + }, + { + "bits_per_byte": 1.8155914361253733, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2584720849990845, + "step": 2110, + "unclipped_grad_norm": 0.7056337594985962 + }, + { + "bits_per_byte": 1.834290249840432, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2714331150054932, + "step": 2120, + "unclipped_grad_norm": 0.8666519522666931 + }, + { + "bits_per_byte": 1.8865812308699674, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.307678461074829, + "step": 2130, + "unclipped_grad_norm": 0.762921154499054 + }, + { + "bits_per_byte": 1.9030090136803828, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3190653324127197, + "step": 2140, + "unclipped_grad_norm": 0.7508745789527893 + }, + { + "bits_per_byte": 1.8090877402003454, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2539640665054321, + "step": 2150, + "unclipped_grad_norm": 0.7472740411758423 + }, + { + "bits_per_byte": 1.898260228724759, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3157737255096436, + "step": 2160, + "unclipped_grad_norm": 0.7288866639137268 + }, + { + "bits_per_byte": 1.8098697453138184, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2545061111450195, + "step": 2170, + "unclipped_grad_norm": 0.7270416617393494 + }, + { + "bits_per_byte": 1.896411071262682, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3144919872283936, + "step": 2180, + "unclipped_grad_norm": 0.7397056221961975 + }, + { + "bits_per_byte": 1.8517294626205718, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.283521056175232, + "step": 2190, + "unclipped_grad_norm": 0.7186560034751892 + }, + { + "bits_per_byte": 1.8299633383273748, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.268433928489685, + "step": 2200, + "unclipped_grad_norm": 0.6916610598564148 + }, + { + "bits_per_byte": 1.8274437924922347, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2666875123977661, + "step": 2210, + "unclipped_grad_norm": 0.7494430541992188 + }, + { + "bits_per_byte": 1.8262679471082857, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2658724784851074, + "step": 2220, + "unclipped_grad_norm": 0.782909095287323 + }, + { + "bits_per_byte": 1.782249847533204, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2353614568710327, + "step": 2230, + "unclipped_grad_norm": 0.7370295524597168 + }, + { + "bits_per_byte": 1.8783687153259665, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3019859790802002, + "step": 2240, + "unclipped_grad_norm": 0.7077252268791199 + }, + { + "bits_per_byte": 1.844123529886164, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2782490253448486, + "step": 2250, + "unclipped_grad_norm": 0.7420696020126343 + }, + { + "bits_per_byte": 1.897104505310961, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3149726390838623, + "step": 2260, + "unclipped_grad_norm": 0.7444695234298706 + }, + { + "bits_per_byte": 1.7929542197055561, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.242781162261963, + "step": 2270, + "unclipped_grad_norm": 0.7164454460144043 + }, + { + "bits_per_byte": 1.8643507534194408, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2922694683074951, + "step": 2280, + "unclipped_grad_norm": 0.7152382135391235 + }, + { + "bits_per_byte": 1.8035947863144381, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2501566410064697, + "step": 2290, + "unclipped_grad_norm": 0.7178454399108887 + }, + { + "bits_per_byte": 1.7935200426268947, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.243173360824585, + "step": 2300, + "unclipped_grad_norm": 0.7068437337875366 + }, + { + "bits_per_byte": 1.8020768674379228, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2491044998168945, + "step": 2310, + "unclipped_grad_norm": 0.7017168998718262 + }, + { + "bits_per_byte": 1.8564474626816911, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2867913246154785, + "step": 2320, + "unclipped_grad_norm": 0.6947462558746338 + }, + { + "bits_per_byte": 1.7936961528613782, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.243295431137085, + "step": 2330, + "unclipped_grad_norm": 0.6949398517608643 + }, + { + "bits_per_byte": 1.8129625093242843, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2566498517990112, + "step": 2340, + "unclipped_grad_norm": 0.7047421336174011 + }, + { + "bits_per_byte": 1.871945851246844, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2975339889526367, + "step": 2350, + "unclipped_grad_norm": 0.7291849851608276 + }, + { + "bits_per_byte": 1.7987307729827362, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2467851638793945, + "step": 2360, + "unclipped_grad_norm": 0.7353813052177429 + }, + { + "bits_per_byte": 1.7311453748119714, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.1999385356903076, + "step": 2370, + "unclipped_grad_norm": 0.6746760010719299 + }, + { + "bits_per_byte": 1.834468595849377, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2715567350387573, + "step": 2380, + "unclipped_grad_norm": 0.7166123390197754 + }, + { + "bits_per_byte": 1.8260691351638882, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2657346725463867, + "step": 2390, + "unclipped_grad_norm": 0.6613916158676147 + }, + { + "bits_per_byte": 1.8641103216735346, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2921028137207031, + "step": 2400, + "unclipped_grad_norm": 0.824955403804779 + }, + { + "bits_per_byte": 1.793840446305452, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.243395447731018, + "step": 2410, + "unclipped_grad_norm": 0.7929511070251465 + }, + { + "bits_per_byte": 1.8233349549804712, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2638394832611084, + "step": 2420, + "unclipped_grad_norm": 0.725745677947998 + }, + { + "bits_per_byte": 1.825773496987055, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265529751777649, + "step": 2430, + "unclipped_grad_norm": 0.7284960746765137 + }, + { + "bits_per_byte": 1.7725216488871491, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2286183834075928, + "step": 2440, + "unclipped_grad_norm": 0.7458904981613159 + }, + { + "bits_per_byte": 1.9001444706476118, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.317079782485962, + "step": 2450, + "unclipped_grad_norm": 0.8321223258972168 + }, + { + "bits_per_byte": 1.7654523020234334, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.223718285560608, + "step": 2460, + "unclipped_grad_norm": 0.7338651418685913 + }, + { + "bits_per_byte": 1.7862824967306352, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2381566762924194, + "step": 2470, + "unclipped_grad_norm": 0.7279908061027527 + }, + { + "bits_per_byte": 1.8666427662074894, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2938581705093384, + "step": 2480, + "unclipped_grad_norm": 0.7701565027236938 + }, + { + "bits_per_byte": 1.850673661126925, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2827892303466797, + "step": 2490, + "unclipped_grad_norm": 0.6776601672172546 + }, + { + "bits_per_byte": 1.8000481600883451, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2476983070373535, + "step": 2500, + "unclipped_grad_norm": 0.7444605231285095 + }, + { + "bits_per_byte": 1.7603095048146828, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.220153570175171, + "step": 2510, + "unclipped_grad_norm": 0.7512273788452148 + }, + { + "bits_per_byte": 1.7805241736144466, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2341653108596802, + "step": 2520, + "unclipped_grad_norm": 0.6968159079551697 + }, + { + "bits_per_byte": 1.803689032807111, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2502219676971436, + "step": 2530, + "unclipped_grad_norm": 0.7150991559028625 + }, + { + "bits_per_byte": 1.8049077018711248, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2510666847229004, + "step": 2540, + "unclipped_grad_norm": 0.7343016862869263 + }, + { + "bits_per_byte": 1.742795479581418, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2080137729644775, + "step": 2550, + "unclipped_grad_norm": 0.7049802541732788 + }, + { + "bits_per_byte": 1.7179068382791558, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.1907622814178467, + "step": 2560, + "unclipped_grad_norm": 0.7055768370628357 + }, + { + "bits_per_byte": 1.7731971967397384, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2290866374969482, + "step": 2570, + "unclipped_grad_norm": 0.6996883749961853 + }, + { + "bits_per_byte": 1.7458676056737803, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2101432085037231, + "step": 2580, + "unclipped_grad_norm": 0.6933735609054565 + }, + { + "bits_per_byte": 1.7458146350173145, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2101064920425415, + "step": 2590, + "unclipped_grad_norm": 0.7094681859016418 + }, + { + "bits_per_byte": 1.7637799427264436, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2225590944290161, + "step": 2600, + "unclipped_grad_norm": 0.7199745178222656 + }, + { + "bits_per_byte": 1.786311561798631, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2381768226623535, + "step": 2610, + "unclipped_grad_norm": 0.7421150207519531 + }, + { + "bits_per_byte": 1.7252271078304804, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.1958363056182861, + "step": 2620, + "unclipped_grad_norm": 0.7062536478042603 + }, + { + "bits_per_byte": 1.7243496523457782, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.195228099822998, + "step": 2630, + "unclipped_grad_norm": 0.6937817931175232 + }, + { + "bits_per_byte": 1.7124126805146924, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1869540214538574, + "step": 2640, + "unclipped_grad_norm": 0.7252171039581299 + }, + { + "bits_per_byte": 1.781446344588373, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2348045110702515, + "step": 2650, + "unclipped_grad_norm": 0.6726356744766235 + }, + { + "bits_per_byte": 1.6915262474806587, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1724766492843628, + "step": 2660, + "unclipped_grad_norm": 0.6618354320526123 + }, + { + "bits_per_byte": 1.7563157236963467, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2173852920532227, + "step": 2670, + "unclipped_grad_norm": 0.7113842368125916 + }, + { + "bits_per_byte": 1.858153874543552, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2879741191864014, + "step": 2680, + "unclipped_grad_norm": 0.7770810127258301 + }, + { + "bits_per_byte": 1.784997614346039, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2372660636901855, + "step": 2690, + "unclipped_grad_norm": 0.7249491810798645 + }, + { + "bits_per_byte": 1.7001362149308035, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.1784446239471436, + "step": 2700, + "unclipped_grad_norm": 0.6978740096092224 + }, + { + "bits_per_byte": 1.7671420315681605, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2248895168304443, + "step": 2710, + "unclipped_grad_norm": 0.6759880185127258 + }, + { + "bits_per_byte": 1.8404900524213854, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2757304906845093, + "step": 2720, + "unclipped_grad_norm": 0.7600791454315186 + }, + { + "bits_per_byte": 1.708491820040322, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.1842362880706787, + "step": 2730, + "unclipped_grad_norm": 0.6715761423110962 + }, + { + "bits_per_byte": 1.7849604660934528, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2372403144836426, + "step": 2740, + "unclipped_grad_norm": 0.7194302082061768 + }, + { + "bits_per_byte": 1.7485555225241152, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2120063304901123, + "step": 2750, + "unclipped_grad_norm": 0.6991336941719055 + }, + { + "bits_per_byte": 1.8629487508496072, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2912976741790771, + "step": 2760, + "unclipped_grad_norm": 0.8377771973609924 + }, + { + "bits_per_byte": 1.7856594035865592, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.237724781036377, + "step": 2770, + "unclipped_grad_norm": 0.7150954604148865 + }, + { + "bits_per_byte": 1.7901013714930438, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2408037185668945, + "step": 2780, + "unclipped_grad_norm": 0.6968044638633728 + }, + { + "bits_per_byte": 1.752669863480706, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2148581743240356, + "step": 2790, + "unclipped_grad_norm": 0.7223197817802429 + }, + { + "bits_per_byte": 1.7567675221201633, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2176984548568726, + "step": 2800, + "unclipped_grad_norm": 0.7307369709014893 + }, + { + "bits_per_byte": 1.7124591158304254, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.1869862079620361, + "step": 2810, + "unclipped_grad_norm": 0.7025272250175476 + }, + { + "bits_per_byte": 1.7803284573577647, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2340296506881714, + "step": 2820, + "unclipped_grad_norm": 0.7275472283363342 + }, + { + "bits_per_byte": 1.7225665362216327, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.1939921379089355, + "step": 2830, + "unclipped_grad_norm": 0.6614095568656921 + }, + { + "bits_per_byte": 1.7602923065495966, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2201416492462158, + "step": 2840, + "unclipped_grad_norm": 0.7144313454627991 + }, + { + "bits_per_byte": 1.7021394688480533, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.179833173751831, + "step": 2850, + "unclipped_grad_norm": 0.7220690846443176 + }, + { + "bits_per_byte": 1.744293620453084, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2090522050857544, + "step": 2860, + "unclipped_grad_norm": 0.7174125909805298 + }, + { + "bits_per_byte": 1.7433115995166573, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2083715200424194, + "step": 2870, + "unclipped_grad_norm": 0.730305552482605 + }, + { + "bits_per_byte": 1.7421880368585705, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2075927257537842, + "step": 2880, + "unclipped_grad_norm": 0.7075303792953491 + }, + { + "bits_per_byte": 1.7085926018737276, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.1843061447143555, + "step": 2890, + "unclipped_grad_norm": 0.6744402050971985 + }, + { + "bits_per_byte": 1.7378320602775172, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.204573392868042, + "step": 2900, + "unclipped_grad_norm": 0.6889722347259521 + }, + { + "bits_per_byte": 1.7031551983840492, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.180537223815918, + "step": 2910, + "unclipped_grad_norm": 0.7552688717842102 + }, + { + "bits_per_byte": 1.7508614659068837, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2136046886444092, + "step": 2920, + "unclipped_grad_norm": 0.6863148808479309 + }, + { + "bits_per_byte": 1.701510700276499, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.1793973445892334, + "step": 2930, + "unclipped_grad_norm": 0.6827563047409058 + }, + { + "bits_per_byte": 1.7106488264474435, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.1857314109802246, + "step": 2940, + "unclipped_grad_norm": 0.6807736158370972 + }, + { + "bits_per_byte": 1.7703209588867088, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.227092981338501, + "step": 2950, + "unclipped_grad_norm": 0.6866815686225891 + }, + { + "bits_per_byte": 1.7651767858167513, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2235273122787476, + "step": 2960, + "unclipped_grad_norm": 0.741482138633728 + }, + { + "bits_per_byte": 1.7465264712092357, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2105998992919922, + "step": 2970, + "unclipped_grad_norm": 0.6754798293113708 + }, + { + "bits_per_byte": 1.7663952828981142, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2243719100952148, + "step": 2980, + "unclipped_grad_norm": 0.688591480255127 + }, + { + "bits_per_byte": 1.7069632382394535, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.1831767559051514, + "step": 2990, + "unclipped_grad_norm": 0.7298949956893921 + }, + { + "bits_per_byte": 1.79113911480835, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.241523027420044, + "step": 3000, + "unclipped_grad_norm": 0.713433027267456 + }, + { + "bits_per_byte": 1.6766162115641075, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1621417999267578, + "step": 3010, + "unclipped_grad_norm": 0.7228698134422302 + }, + { + "bits_per_byte": 1.7365501015979858, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2036848068237305, + "step": 3020, + "unclipped_grad_norm": 0.6969108581542969 + }, + { + "bits_per_byte": 1.6833617150962485, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.1668174266815186, + "step": 3030, + "unclipped_grad_norm": 0.669261634349823 + }, + { + "bits_per_byte": 1.6671776317021056, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1555994749069214, + "step": 3040, + "unclipped_grad_norm": 0.6839757561683655 + }, + { + "bits_per_byte": 1.8241205717296125, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2643840312957764, + "step": 3050, + "unclipped_grad_norm": 0.7963703870773315 + }, + { + "bits_per_byte": 1.7604708245411922, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2202653884887695, + "step": 3060, + "unclipped_grad_norm": 0.7280165553092957 + }, + { + "bits_per_byte": 1.6594571585222233, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.1502480506896973, + "step": 3070, + "unclipped_grad_norm": 0.7014188766479492 + }, + { + "bits_per_byte": 1.7706030104341237, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2272884845733643, + "step": 3080, + "unclipped_grad_norm": 0.7385008335113525 + }, + { + "bits_per_byte": 1.7493225651469633, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2125380039215088, + "step": 3090, + "unclipped_grad_norm": 0.6976860165596008 + }, + { + "bits_per_byte": 1.706454857523503, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1828243732452393, + "step": 3100, + "unclipped_grad_norm": 0.7350029349327087 + }, + { + "bits_per_byte": 1.8454660264587992, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.279179573059082, + "step": 3110, + "unclipped_grad_norm": 0.8262335658073425 + }, + { + "bits_per_byte": 1.7237298268720687, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.194798469543457, + "step": 3120, + "unclipped_grad_norm": 0.7770306468009949 + }, + { + "bits_per_byte": 1.6658043502349662, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1546475887298584, + "step": 3130, + "unclipped_grad_norm": 0.6939164996147156 + }, + { + "bits_per_byte": 1.7147224075157799, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1885550022125244, + "step": 3140, + "unclipped_grad_norm": 0.744745135307312 + }, + { + "bits_per_byte": 1.7012960659282221, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.179248571395874, + "step": 3150, + "unclipped_grad_norm": 0.6927894353866577 + }, + { + "bits_per_byte": 1.700822425707746, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.1789202690124512, + "step": 3160, + "unclipped_grad_norm": 0.6962236166000366 + }, + { + "bits_per_byte": 1.7730090477196945, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2289562225341797, + "step": 3170, + "unclipped_grad_norm": 0.7581587433815002 + }, + { + "bits_per_byte": 1.7445914944043783, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2092586755752563, + "step": 3180, + "unclipped_grad_norm": 0.7753618359565735 + }, + { + "bits_per_byte": 1.7320080397886992, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2005364894866943, + "step": 3190, + "unclipped_grad_norm": 0.7117310166358948 + }, + { + "bits_per_byte": 1.7215322325593436, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1932752132415771, + "step": 3200, + "unclipped_grad_norm": 0.712298572063446 + }, + { + "bits_per_byte": 1.6960516269728116, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1756134033203125, + "step": 3210, + "unclipped_grad_norm": 0.7590813636779785 + }, + { + "bits_per_byte": 1.6593746068498092, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.150190830230713, + "step": 3220, + "unclipped_grad_norm": 0.6987801194190979 + }, + { + "bits_per_byte": 1.7324070395387008, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2008130550384521, + "step": 3230, + "unclipped_grad_norm": 0.6938228607177734 + }, + { + "bits_per_byte": 1.7474627447605329, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2112488746643066, + "step": 3240, + "unclipped_grad_norm": 0.78226238489151 + }, + { + "bits_per_byte": 1.7110863503112386, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.1860346794128418, + "step": 3250, + "unclipped_grad_norm": 0.6999595165252686 + }, + { + "bits_per_byte": 1.7142417160066183, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.18822181224823, + "step": 3260, + "unclipped_grad_norm": 0.700433075428009 + }, + { + "bits_per_byte": 1.748484321706658, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2119569778442383, + "step": 3270, + "unclipped_grad_norm": 0.7274234294891357 + }, + { + "bits_per_byte": 1.738793787261142, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.20524001121521, + "step": 3280, + "unclipped_grad_norm": 0.7183425426483154 + }, + { + "bits_per_byte": 1.6962500949519075, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.175750970840454, + "step": 3290, + "unclipped_grad_norm": 0.7457373738288879 + }, + { + "bits_per_byte": 1.7210296992535226, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1929268836975098, + "step": 3300, + "unclipped_grad_norm": 0.7834483981132507 + }, + { + "bits_per_byte": 1.6835460804979734, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.166945219039917, + "step": 3310, + "unclipped_grad_norm": 0.6523665189743042 + }, + { + "bits_per_byte": 1.6871995079502522, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1694775819778442, + "step": 3320, + "unclipped_grad_norm": 0.7348048686981201 + }, + { + "bits_per_byte": 1.7008401399207849, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.178932547569275, + "step": 3330, + "unclipped_grad_norm": 0.7228167653083801 + }, + { + "bits_per_byte": 1.643402750046831, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1391199827194214, + "step": 3340, + "unclipped_grad_norm": 0.7094708681106567 + }, + { + "bits_per_byte": 1.686364876145615, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1688990592956543, + "step": 3350, + "unclipped_grad_norm": 0.7186631560325623 + }, + { + "bits_per_byte": 1.7065993229502279, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.182924509048462, + "step": 3360, + "unclipped_grad_norm": 0.6950210332870483 + }, + { + "bits_per_byte": 1.6548356407282379, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1470446586608887, + "step": 3370, + "unclipped_grad_norm": 0.6926314830780029 + }, + { + "bits_per_byte": 1.6926787032240904, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1732754707336426, + "step": 3380, + "unclipped_grad_norm": 0.7107762694358826 + }, + { + "bits_per_byte": 1.7003921251152874, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1786220073699951, + "step": 3390, + "unclipped_grad_norm": 0.700124979019165 + }, + { + "bits_per_byte": 1.6889117672222422, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1706644296646118, + "step": 3400, + "unclipped_grad_norm": 0.7049819827079773 + }, + { + "bits_per_byte": 1.6802605239358903, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.1646678447723389, + "step": 3410, + "unclipped_grad_norm": 0.6922183036804199 + }, + { + "bits_per_byte": 1.664921907253389, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1540359258651733, + "step": 3420, + "unclipped_grad_norm": 0.7144856452941895 + }, + { + "bits_per_byte": 1.698013089105902, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1769729852676392, + "step": 3430, + "unclipped_grad_norm": 0.6953245401382446 + }, + { + "bits_per_byte": 1.7072490734051875, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1833748817443848, + "step": 3440, + "unclipped_grad_norm": 0.727461040019989 + }, + { + "bits_per_byte": 1.7747906159999822, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2301911115646362, + "step": 3450, + "unclipped_grad_norm": 0.7409155368804932 + }, + { + "bits_per_byte": 1.7588486841782542, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2191410064697266, + "step": 3460, + "unclipped_grad_norm": 0.748551070690155 + }, + { + "bits_per_byte": 1.7339454243506687, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2018793821334839, + "step": 3470, + "unclipped_grad_norm": 0.7151429653167725 + }, + { + "bits_per_byte": 1.7067943512763062, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1830596923828125, + "step": 3480, + "unclipped_grad_norm": 0.7189637422561646 + }, + { + "bits_per_byte": 1.6200879219826143, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1229593753814697, + "step": 3490, + "unclipped_grad_norm": 0.6848708391189575 + }, + { + "bits_per_byte": 1.6373195517031627, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1349034309387207, + "step": 3500, + "unclipped_grad_norm": 0.6704034209251404 + }, + { + "bits_per_byte": 1.6849599498707166, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.167925238609314, + "step": 3510, + "unclipped_grad_norm": 0.7942876219749451 + }, + { + "bits_per_byte": 1.6514716600773613, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1447129249572754, + "step": 3520, + "unclipped_grad_norm": 0.7149106860160828 + }, + { + "bits_per_byte": 1.6705435041621417, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1579325199127197, + "step": 3530, + "unclipped_grad_norm": 0.7720932364463806 + }, + { + "bits_per_byte": 1.6828739722984016, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1664793491363525, + "step": 3540, + "unclipped_grad_norm": 0.6750161647796631 + }, + { + "bits_per_byte": 1.7101762181228726, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.185403823852539, + "step": 3550, + "unclipped_grad_norm": 0.6993052363395691 + }, + { + "bits_per_byte": 1.6591575647444203, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1500403881072998, + "step": 3560, + "unclipped_grad_norm": 0.6886851787567139 + }, + { + "bits_per_byte": 1.7182958630354075, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1910319328308105, + "step": 3570, + "unclipped_grad_norm": 0.749891459941864 + }, + { + "bits_per_byte": 1.7607525321233055, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2204606533050537, + "step": 3580, + "unclipped_grad_norm": 0.8161401152610779 + }, + { + "bits_per_byte": 1.690903842267186, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1720452308654785, + "step": 3590, + "unclipped_grad_norm": 0.6925192475318909 + }, + { + "bits_per_byte": 1.6772946831217612, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1626120805740356, + "step": 3600, + "unclipped_grad_norm": 0.7061766982078552 + }, + { + "bits_per_byte": 1.6673109182565242, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1556918621063232, + "step": 3610, + "unclipped_grad_norm": 0.7162109017372131 + }, + { + "bits_per_byte": 1.6598764522250269, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.150538682937622, + "step": 3620, + "unclipped_grad_norm": 0.6953511238098145 + }, + { + "bits_per_byte": 1.6688818077895051, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.15678071975708, + "step": 3630, + "unclipped_grad_norm": 0.7265485525131226 + }, + { + "bits_per_byte": 1.6378186453559664, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.135249376296997, + "step": 3640, + "unclipped_grad_norm": 0.68751460313797 + }, + { + "bits_per_byte": 1.6996694540163617, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1781210899353027, + "step": 3650, + "unclipped_grad_norm": 0.7145417928695679 + }, + { + "bits_per_byte": 1.6073453834148872, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1141269207000732, + "step": 3660, + "unclipped_grad_norm": 0.71860671043396 + }, + { + "bits_per_byte": 1.62879970316207, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1289979219436646, + "step": 3670, + "unclipped_grad_norm": 0.6769548654556274 + }, + { + "bits_per_byte": 1.702227523965295, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.179894208908081, + "step": 3680, + "unclipped_grad_norm": 0.7466121315956116 + }, + { + "bits_per_byte": 1.653425382991163, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1460671424865723, + "step": 3690, + "unclipped_grad_norm": 0.6637522578239441 + }, + { + "bits_per_byte": 1.5989233930021354, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1082892417907715, + "step": 3700, + "unclipped_grad_norm": 0.7335078120231628 + }, + { + "bits_per_byte": 1.6701862961962997, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1576849222183228, + "step": 3710, + "unclipped_grad_norm": 0.7247957587242126 + }, + { + "bits_per_byte": 1.6538059805975223, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1463309526443481, + "step": 3720, + "unclipped_grad_norm": 0.7534144520759583 + }, + { + "bits_per_byte": 1.7119505631318241, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1866337060928345, + "step": 3730, + "unclipped_grad_norm": 0.6951875686645508 + }, + { + "bits_per_byte": 1.6437530787066386, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1393628120422363, + "step": 3740, + "unclipped_grad_norm": 0.7089563608169556 + }, + { + "bits_per_byte": 1.6267918057132467, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1276061534881592, + "step": 3750, + "unclipped_grad_norm": 0.6922760009765625 + }, + { + "bits_per_byte": 1.6201740852906965, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1230190992355347, + "step": 3760, + "unclipped_grad_norm": 0.6971311569213867 + }, + { + "bits_per_byte": 1.6730486034546093, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1596689224243164, + "step": 3770, + "unclipped_grad_norm": 0.6925700902938843 + }, + { + "bits_per_byte": 1.6258194158052683, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.126932144165039, + "step": 3780, + "unclipped_grad_norm": 0.71840900182724 + }, + { + "bits_per_byte": 1.6611288298886098, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1514067649841309, + "step": 3790, + "unclipped_grad_norm": 0.6731990575790405 + }, + { + "bits_per_byte": 1.622545554063444, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1246628761291504, + "step": 3800, + "unclipped_grad_norm": 0.7068047523498535 + }, + { + "bits_per_byte": 1.613386446009094, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.118314266204834, + "step": 3810, + "unclipped_grad_norm": 0.679490327835083 + }, + { + "bits_per_byte": 1.723293162921528, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1944957971572876, + "step": 3820, + "unclipped_grad_norm": 0.7167435884475708 + }, + { + "bits_per_byte": 1.6410942269242994, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1375198364257812, + "step": 3830, + "unclipped_grad_norm": 0.749890148639679 + }, + { + "bits_per_byte": 1.6271811744348, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1278760433197021, + "step": 3840, + "unclipped_grad_norm": 0.6869876980781555 + }, + { + "bits_per_byte": 1.6520392028252087, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.145106315612793, + "step": 3850, + "unclipped_grad_norm": 0.6890377998352051 + }, + { + "bits_per_byte": 1.6079827511189846, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1145687103271484, + "step": 3860, + "unclipped_grad_norm": 0.717820942401886 + }, + { + "bits_per_byte": 1.6751354409401786, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1611154079437256, + "step": 3870, + "unclipped_grad_norm": 0.7368612289428711 + }, + { + "bits_per_byte": 1.6338718155013159, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1325136423110962, + "step": 3880, + "unclipped_grad_norm": 0.7661339640617371 + }, + { + "bits_per_byte": 1.656585564200767, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1482576131820679, + "step": 3890, + "unclipped_grad_norm": 0.7343658804893494 + }, + { + "bits_per_byte": 1.6647341021986468, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1539057493209839, + "step": 3900, + "unclipped_grad_norm": 0.7533658146858215 + }, + { + "bits_per_byte": 1.6762937440937398, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.16191828250885, + "step": 3910, + "unclipped_grad_norm": 0.7147406935691833 + }, + { + "bits_per_byte": 1.6322013480134854, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1313557624816895, + "step": 3920, + "unclipped_grad_norm": 0.6887187361717224 + }, + { + "bits_per_byte": 1.6221845624792828, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1244126558303833, + "step": 3930, + "unclipped_grad_norm": 0.7910217642784119 + }, + { + "bits_per_byte": 1.63472175376188, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1331027746200562, + "step": 3940, + "unclipped_grad_norm": 0.730076789855957 + }, + { + "bits_per_byte": 1.6972579132859635, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1764495372772217, + "step": 3950, + "unclipped_grad_norm": 0.7732967138290405 + }, + { + "bits_per_byte": 1.7135035664691152, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.187710165977478, + "step": 3960, + "unclipped_grad_norm": 0.7405394911766052 + }, + { + "bits_per_byte": 1.685746770498414, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1684706211090088, + "step": 3970, + "unclipped_grad_norm": 0.7160924673080444 + }, + { + "bits_per_byte": 1.6665311489175123, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1551513671875, + "step": 3980, + "unclipped_grad_norm": 0.7374541759490967 + }, + { + "bits_per_byte": 1.7040839046987084, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1811809539794922, + "step": 3990, + "unclipped_grad_norm": 0.7841057181358337 + }, + { + "bits_per_byte": 1.693502844087025, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.17384672164917, + "step": 4000, + "unclipped_grad_norm": 0.7609017491340637 + }, + { + "bits_per_byte": 1.699465998540391, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1779800653457642, + "step": 4010, + "unclipped_grad_norm": 0.7629905343055725 + }, + { + "bits_per_byte": 1.5838037102341824, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.097809076309204, + "step": 4020, + "unclipped_grad_norm": 0.6868616342544556 + }, + { + "bits_per_byte": 1.617556165379263, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1212044954299927, + "step": 4030, + "unclipped_grad_norm": 0.749467670917511 + }, + { + "bits_per_byte": 1.6189757101794846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1221884489059448, + "step": 4040, + "unclipped_grad_norm": 0.7323976755142212 + }, + { + "bits_per_byte": 1.6111234982890412, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1167457103729248, + "step": 4050, + "unclipped_grad_norm": 0.7304336428642273 + }, + { + "bits_per_byte": 1.6428754512392856, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1387544870376587, + "step": 4060, + "unclipped_grad_norm": 0.745829164981842 + }, + { + "bits_per_byte": 1.6097484969953932, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1157926321029663, + "step": 4070, + "unclipped_grad_norm": 0.7523763179779053 + }, + { + "bits_per_byte": 1.6445053308215125, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1398842334747314, + "step": 4080, + "unclipped_grad_norm": 0.7511109113693237 + }, + { + "bits_per_byte": 1.6034616711931031, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1114349365234375, + "step": 4090, + "unclipped_grad_norm": 0.7438745498657227 + }, + { + "bits_per_byte": 1.561935084281071, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.082650899887085, + "step": 4100, + "unclipped_grad_norm": 0.6872318387031555 + }, + { + "bits_per_byte": 1.6907129415247284, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1719129085540771, + "step": 4110, + "unclipped_grad_norm": 0.7602059841156006 + }, + { + "bits_per_byte": 1.6125045189754696, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1177029609680176, + "step": 4120, + "unclipped_grad_norm": 0.7136654257774353 + }, + { + "bits_per_byte": 1.5891117827704122, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1014883518218994, + "step": 4130, + "unclipped_grad_norm": 0.7062129378318787 + }, + { + "bits_per_byte": 1.6146775197691212, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1192091703414917, + "step": 4140, + "unclipped_grad_norm": 0.7150430679321289 + }, + { + "bits_per_byte": 1.6618133208390438, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.151881217956543, + "step": 4150, + "unclipped_grad_norm": 0.7448095679283142 + }, + { + "bits_per_byte": 1.6228898633304711, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.124901533126831, + "step": 4160, + "unclipped_grad_norm": 0.7362097501754761 + }, + { + "bits_per_byte": 1.5905947891688021, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1025162935256958, + "step": 4170, + "unclipped_grad_norm": 0.7651577591896057 + }, + { + "bits_per_byte": 1.5608787668394717, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.081918716430664, + "step": 4180, + "unclipped_grad_norm": 0.7135135531425476 + }, + { + "bits_per_byte": 1.713072749928704, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1874115467071533, + "step": 4190, + "unclipped_grad_norm": 0.7463747262954712 + }, + { + "bits_per_byte": 1.5972510337051453, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1071300506591797, + "step": 4200, + "unclipped_grad_norm": 0.7313898801803589 + }, + { + "bits_per_byte": 1.6169509584308768, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1207849979400635, + "step": 4210, + "unclipped_grad_norm": 0.7539675831794739 + }, + { + "bits_per_byte": 1.6536405332873922, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1462162733078003, + "step": 4220, + "unclipped_grad_norm": 0.7351350784301758 + }, + { + "bits_per_byte": 1.6357479742395782, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1338140964508057, + "step": 4230, + "unclipped_grad_norm": 0.7141795754432678 + }, + { + "bits_per_byte": 1.6650933739562992, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1541547775268555, + "step": 4240, + "unclipped_grad_norm": 0.7625938653945923 + }, + { + "bits_per_byte": 1.6824343846427963, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1661746501922607, + "step": 4250, + "unclipped_grad_norm": 0.702212393283844 + }, + { + "bits_per_byte": 1.6672134040934852, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.155624270439148, + "step": 4260, + "unclipped_grad_norm": 0.775193989276886 + }, + { + "bits_per_byte": 1.643573356836487, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1392382383346558, + "step": 4270, + "unclipped_grad_norm": 0.7224892973899841 + }, + { + "bits_per_byte": 1.6848497089915135, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.167848825454712, + "step": 4280, + "unclipped_grad_norm": 0.792568564414978 + }, + { + "bits_per_byte": 1.6287462165576516, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1289608478546143, + "step": 4290, + "unclipped_grad_norm": 0.7352984547615051 + }, + { + "bits_per_byte": 1.5679441581022175, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0868160724639893, + "step": 4300, + "unclipped_grad_norm": 0.7459802031517029 + }, + { + "bits_per_byte": 1.7134996108681453, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1877074241638184, + "step": 4310, + "unclipped_grad_norm": 0.8436940312385559 + }, + { + "bits_per_byte": 1.585466610485375, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.0989617109298706, + "step": 4320, + "unclipped_grad_norm": 0.7326236367225647 + }, + { + "bits_per_byte": 1.5870087789156617, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1000306606292725, + "step": 4330, + "unclipped_grad_norm": 0.7154185175895691 + }, + { + "bits_per_byte": 1.5845594020020737, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.0983328819274902, + "step": 4340, + "unclipped_grad_norm": 0.7299889326095581 + }, + { + "bits_per_byte": 1.5087386145947432, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0457779169082642, + "step": 4350, + "unclipped_grad_norm": 0.7116599082946777 + }, + { + "bits_per_byte": 1.6642632137005844, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1535793542861938, + "step": 4360, + "unclipped_grad_norm": 0.7841640114784241 + }, + { + "bits_per_byte": 1.6118272512963718, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1172335147857666, + "step": 4370, + "unclipped_grad_norm": 0.7682546973228455 + }, + { + "bits_per_byte": 1.5975416843851036, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1073315143585205, + "step": 4380, + "unclipped_grad_norm": 0.7542603611946106 + }, + { + "bits_per_byte": 1.6190350441940322, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1222295761108398, + "step": 4390, + "unclipped_grad_norm": 0.7354364991188049 + }, + { + "bits_per_byte": 1.5791293937663822, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.0945690870285034, + "step": 4400, + "unclipped_grad_norm": 0.7447035908699036 + }, + { + "bits_per_byte": 1.5777399459300616, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.0936059951782227, + "step": 4410, + "unclipped_grad_norm": 0.7433028221130371 + }, + { + "bits_per_byte": 1.5774630538621723, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.093414068222046, + "step": 4420, + "unclipped_grad_norm": 0.7514091730117798 + }, + { + "bits_per_byte": 1.5840658117940973, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.0979907512664795, + "step": 4430, + "unclipped_grad_norm": 0.748140811920166 + }, + { + "bits_per_byte": 1.5992336497042918, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.108504295349121, + "step": 4440, + "unclipped_grad_norm": 0.7693572044372559 + }, + { + "bits_per_byte": 1.5825853851354703, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.0969645977020264, + "step": 4450, + "unclipped_grad_norm": 0.7571823596954346 + }, + { + "bits_per_byte": 1.5385392523882981, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0664341449737549, + "step": 4460, + "unclipped_grad_norm": 0.7259396314620972 + }, + { + "bits_per_byte": 1.6428487939284018, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1387360095977783, + "step": 4470, + "unclipped_grad_norm": 0.7504101991653442 + }, + { + "bits_per_byte": 1.5687149843433847, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.0873503684997559, + "step": 4480, + "unclipped_grad_norm": 0.7544133067131042 + }, + { + "bits_per_byte": 1.5307804270772734, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.061056137084961, + "step": 4490, + "unclipped_grad_norm": 0.7395938634872437 + }, + { + "bits_per_byte": 1.6123227333135077, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1175769567489624, + "step": 4500, + "unclipped_grad_norm": 0.7624281644821167 + }, + { + "bits_per_byte": 1.5895855949735391, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1018167734146118, + "step": 4510, + "unclipped_grad_norm": 0.7442159652709961 + }, + { + "bits_per_byte": 1.5720557473363947, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.0896660089492798, + "step": 4520, + "unclipped_grad_norm": 0.7299280166625977 + }, + { + "bits_per_byte": 1.5930625682260326, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.10422682762146, + "step": 4530, + "unclipped_grad_norm": 0.7508688569068909 + }, + { + "bits_per_byte": 1.551802554422838, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.0756275653839111, + "step": 4540, + "unclipped_grad_norm": 0.7326505780220032 + }, + { + "bits_per_byte": 1.6245931795046165, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.126082181930542, + "step": 4550, + "unclipped_grad_norm": 0.7591264247894287 + }, + { + "bits_per_byte": 1.6222865481912445, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.124483346939087, + "step": 4560, + "unclipped_grad_norm": 0.7490679621696472 + }, + { + "bits_per_byte": 1.5636793323261218, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.083859920501709, + "step": 4570, + "unclipped_grad_norm": 0.7205959558486938 + }, + { + "bits_per_byte": 1.5242402708302625, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0565228462219238, + "step": 4580, + "unclipped_grad_norm": 0.7278034687042236 + }, + { + "bits_per_byte": 1.649223158899981, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1431543827056885, + "step": 4590, + "unclipped_grad_norm": 0.80039381980896 + }, + { + "bits_per_byte": 1.621592598195013, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1240023374557495, + "step": 4600, + "unclipped_grad_norm": 0.7594562768936157 + }, + { + "bits_per_byte": 1.591440943811047, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1031028032302856, + "step": 4610, + "unclipped_grad_norm": 0.7573962211608887 + }, + { + "bits_per_byte": 1.5204752266375738, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.0539131164550781, + "step": 4620, + "unclipped_grad_norm": 0.7093440890312195 + }, + { + "bits_per_byte": 1.6007937043302685, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1095856428146362, + "step": 4630, + "unclipped_grad_norm": 0.7716385126113892 + }, + { + "bits_per_byte": 1.566827990698118, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.0860424041748047, + "step": 4640, + "unclipped_grad_norm": 0.7477832436561584 + }, + { + "bits_per_byte": 1.5540792607549598, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.0772056579589844, + "step": 4650, + "unclipped_grad_norm": 0.7563350200653076 + }, + { + "bits_per_byte": 1.5711925664117143, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0890676975250244, + "step": 4660, + "unclipped_grad_norm": 0.7477205991744995 + }, + { + "bits_per_byte": 1.6065126434194095, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1135497093200684, + "step": 4670, + "unclipped_grad_norm": 0.7434800267219543 + }, + { + "bits_per_byte": 1.5613855997115644, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.0822700262069702, + "step": 4680, + "unclipped_grad_norm": 0.7484313249588013 + }, + { + "bits_per_byte": 1.533432227570927, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.0628942251205444, + "step": 4690, + "unclipped_grad_norm": 0.7767173647880554 + }, + { + "bits_per_byte": 1.6430371149310967, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1388665437698364, + "step": 4700, + "unclipped_grad_norm": 0.7674224972724915 + }, + { + "bits_per_byte": 1.552380416129737, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.0760281085968018, + "step": 4710, + "unclipped_grad_norm": 0.7836162447929382 + }, + { + "bits_per_byte": 1.5203768525612804, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.053844928741455, + "step": 4720, + "unclipped_grad_norm": 0.7613595128059387 + }, + { + "bits_per_byte": 1.6524877335786587, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1454172134399414, + "step": 4730, + "unclipped_grad_norm": 0.7924392223358154 + }, + { + "bits_per_byte": 1.587415861850254, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1003128290176392, + "step": 4740, + "unclipped_grad_norm": 0.7647171020507812 + }, + { + "bits_per_byte": 1.6396726183322676, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1365344524383545, + "step": 4750, + "unclipped_grad_norm": 0.8181672096252441 + }, + { + "bits_per_byte": 1.5091890371573529, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0460901260375977, + "step": 4760, + "unclipped_grad_norm": 0.7407304048538208 + }, + { + "bits_per_byte": 1.6098291568586478, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1158485412597656, + "step": 4770, + "unclipped_grad_norm": 0.7601819038391113 + }, + { + "bits_per_byte": 1.5582154435082103, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.0800726413726807, + "step": 4780, + "unclipped_grad_norm": 0.7307054996490479 + }, + { + "bits_per_byte": 1.5944504682184955, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1051888465881348, + "step": 4790, + "unclipped_grad_norm": 0.7617009282112122 + }, + { + "bits_per_byte": 1.5823225956449518, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0967824459075928, + "step": 4800, + "unclipped_grad_norm": 0.7428550124168396 + }, + { + "bits_per_byte": 1.542078311377753, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.0688872337341309, + "step": 4810, + "unclipped_grad_norm": 0.773262083530426 + }, + { + "bits_per_byte": 1.6108488419956133, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1165553331375122, + "step": 4820, + "unclipped_grad_norm": 0.8227523565292358 + }, + { + "bits_per_byte": 1.5982464692883394, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1078200340270996, + "step": 4830, + "unclipped_grad_norm": 0.783271312713623 + }, + { + "bits_per_byte": 1.6134836162068316, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1183816194534302, + "step": 4840, + "unclipped_grad_norm": 0.7595719695091248 + }, + { + "bits_per_byte": 1.5430993723759254, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.0695949792861938, + "step": 4850, + "unclipped_grad_norm": 0.8344528079032898 + }, + { + "bits_per_byte": 1.551674427347945, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0755387544631958, + "step": 4860, + "unclipped_grad_norm": 0.7924710512161255 + }, + { + "bits_per_byte": 1.5551434893984988, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.0779433250427246, + "step": 4870, + "unclipped_grad_norm": 0.7731344103813171 + }, + { + "bits_per_byte": 1.7162552888829201, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1896175146102905, + "step": 4880, + "unclipped_grad_norm": 0.8454892039299011 + }, + { + "bits_per_byte": 1.5015098398136777, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0407673120498657, + "step": 4890, + "unclipped_grad_norm": 0.7279435396194458 + }, + { + "bits_per_byte": 1.6017640304464364, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1102582216262817, + "step": 4900, + "unclipped_grad_norm": 0.7495299577713013 + }, + { + "bits_per_byte": 1.5795956387328711, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.0948922634124756, + "step": 4910, + "unclipped_grad_norm": 0.8354763388633728 + }, + { + "bits_per_byte": 1.5130457481029513, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.048763394355774, + "step": 4920, + "unclipped_grad_norm": 0.7128972411155701 + }, + { + "bits_per_byte": 1.5263470583033318, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.057983160018921, + "step": 4930, + "unclipped_grad_norm": 0.7301722168922424 + }, + { + "bits_per_byte": 1.56106777577277, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0820497274398804, + "step": 4940, + "unclipped_grad_norm": 0.7844029664993286 + }, + { + "bits_per_byte": 1.5809642766684375, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0958409309387207, + "step": 4950, + "unclipped_grad_norm": 0.7815173864364624 + }, + { + "bits_per_byte": 1.6004612618661507, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1093552112579346, + "step": 4960, + "unclipped_grad_norm": 0.8055645227432251 + }, + { + "bits_per_byte": 1.5618184800437862, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.0825700759887695, + "step": 4970, + "unclipped_grad_norm": 0.7773133516311646 + }, + { + "bits_per_byte": 1.606830467358204, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1137700080871582, + "step": 4980, + "unclipped_grad_norm": 0.7920810580253601 + }, + { + "bits_per_byte": 1.567882244347907, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.086773157119751, + "step": 4990, + "unclipped_grad_norm": 0.7929123640060425 + }, + { + "bits_per_byte": 1.5708692390280923, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.088843584060669, + "step": 5000, + "unclipped_grad_norm": 0.762309730052948 + }, + { + "bits_per_byte": 1.6157285057485438, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1199376583099365, + "step": 5010, + "unclipped_grad_norm": 0.7734562158584595 + }, + { + "bits_per_byte": 1.5685849654593325, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.0872602462768555, + "step": 5020, + "unclipped_grad_norm": 0.8221466541290283 + }, + { + "bits_per_byte": 1.5125002191344146, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0483852624893188, + "step": 5030, + "unclipped_grad_norm": 0.7515868544578552 + }, + { + "bits_per_byte": 1.5645020973278494, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.08443021774292, + "step": 5040, + "unclipped_grad_norm": 0.7403401732444763 + }, + { + "bits_per_byte": 1.5465288784167808, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.071972131729126, + "step": 5050, + "unclipped_grad_norm": 0.8082988262176514 + }, + { + "bits_per_byte": 1.6139199361920706, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1186840534210205, + "step": 5060, + "unclipped_grad_norm": 0.8066977262496948 + }, + { + "bits_per_byte": 1.5533101543403012, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.0766725540161133, + "step": 5070, + "unclipped_grad_norm": 0.7855141758918762 + }, + { + "bits_per_byte": 1.6163216739113697, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1203488111495972, + "step": 5080, + "unclipped_grad_norm": 0.776370644569397 + }, + { + "bits_per_byte": 1.5614495772576855, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.082314372062683, + "step": 5090, + "unclipped_grad_norm": 0.776827871799469 + }, + { + "bits_per_byte": 1.5419546558517827, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0688015222549438, + "step": 5100, + "unclipped_grad_norm": 0.7785685062408447 + }, + { + "bits_per_byte": 1.5800214677764075, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0951874256134033, + "step": 5110, + "unclipped_grad_norm": 0.8104537725448608 + }, + { + "bits_per_byte": 1.487382496941203, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0309749841690063, + "step": 5120, + "unclipped_grad_norm": 0.7389267086982727 + }, + { + "bits_per_byte": 1.6243671943013827, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1259255409240723, + "step": 5130, + "unclipped_grad_norm": 0.8053697943687439 + }, + { + "bits_per_byte": 1.5876562935961602, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1004794836044312, + "step": 5140, + "unclipped_grad_norm": 0.8099423050880432 + }, + { + "bits_per_byte": 1.5188835272038386, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0528098344802856, + "step": 5150, + "unclipped_grad_norm": 0.7463088035583496 + }, + { + "bits_per_byte": 1.5574305146896725, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.079528570175171, + "step": 5160, + "unclipped_grad_norm": 0.7795574069023132 + }, + { + "bits_per_byte": 1.604379370618107, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1120710372924805, + "step": 5170, + "unclipped_grad_norm": 0.8112488985061646 + }, + { + "bits_per_byte": 1.570943535533265, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0888950824737549, + "step": 5180, + "unclipped_grad_norm": 0.7960402965545654 + }, + { + "bits_per_byte": 1.4686855750353036, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0180152654647827, + "step": 5190, + "unclipped_grad_norm": 0.7735135555267334 + }, + { + "bits_per_byte": 1.5401951013508053, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0675818920135498, + "step": 5200, + "unclipped_grad_norm": 0.7935121655464172 + }, + { + "bits_per_byte": 1.522642552003747, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.055415391921997, + "step": 5210, + "unclipped_grad_norm": 0.7616920471191406 + }, + { + "bits_per_byte": 1.5296819738862126, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0602947473526, + "step": 5220, + "unclipped_grad_norm": 0.799717128276825 + }, + { + "bits_per_byte": 1.60920967535024, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1154191493988037, + "step": 5230, + "unclipped_grad_norm": 0.8263183832168579 + }, + { + "bits_per_byte": 1.5759482306733728, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0923640727996826, + "step": 5240, + "unclipped_grad_norm": 0.9089767336845398 + }, + { + "bits_per_byte": 1.5751560785834986, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0918149948120117, + "step": 5250, + "unclipped_grad_norm": 0.8223497271537781 + }, + { + "bits_per_byte": 1.518346597367845, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.052437663078308, + "step": 5260, + "unclipped_grad_norm": 0.7989961504936218 + }, + { + "bits_per_byte": 1.5652323356634128, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0849363803863525, + "step": 5270, + "unclipped_grad_norm": 0.7858845591545105 + }, + { + "bits_per_byte": 1.4824085866955998, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0275273323059082, + "step": 5280, + "unclipped_grad_norm": 0.7963522672653198 + }, + { + "bits_per_byte": 1.5109090356486319, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.047282338142395, + "step": 5290, + "unclipped_grad_norm": 0.7786324620246887 + }, + { + "bits_per_byte": 1.4973765807654917, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0379023551940918, + "step": 5300, + "unclipped_grad_norm": 0.8200122714042664 + }, + { + "bits_per_byte": 1.4979238295605373, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0382816791534424, + "step": 5310, + "unclipped_grad_norm": 0.798004686832428 + }, + { + "bits_per_byte": 1.4963071926424267, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.037161111831665, + "step": 5320, + "unclipped_grad_norm": 0.7881578207015991 + }, + { + "bits_per_byte": 1.5549873291515153, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0778350830078125, + "step": 5330, + "unclipped_grad_norm": 0.8092938661575317 + }, + { + "bits_per_byte": 1.6144941862633015, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1190820932388306, + "step": 5340, + "unclipped_grad_norm": 0.8349001407623291 + }, + { + "bits_per_byte": 1.4860422361430292, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.030045986175537, + "step": 5350, + "unclipped_grad_norm": 0.7843849062919617 + }, + { + "bits_per_byte": 1.5082066722556244, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0454092025756836, + "step": 5360, + "unclipped_grad_norm": 0.7918040752410889 + }, + { + "bits_per_byte": 1.577472340925319, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0934205055236816, + "step": 5370, + "unclipped_grad_norm": 0.8359649181365967 + }, + { + "bits_per_byte": 1.5013394050066726, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.040649175643921, + "step": 5380, + "unclipped_grad_norm": 0.7734270095825195 + }, + { + "bits_per_byte": 1.5016145772480531, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0408399105072021, + "step": 5390, + "unclipped_grad_norm": 0.7885358333587646 + }, + { + "bits_per_byte": 1.5491196250693782, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073767900466919, + "step": 5400, + "unclipped_grad_norm": 0.8202832341194153 + }, + { + "bits_per_byte": 1.4666546319112648, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0166075229644775, + "step": 5410, + "unclipped_grad_norm": 0.8083077073097229 + }, + { + "bits_per_byte": 1.6009971598062391, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1097266674041748, + "step": 5420, + "unclipped_grad_norm": 0.847724974155426 + }, + { + "bits_per_byte": 1.5289176829857782, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0597649812698364, + "step": 5430, + "unclipped_grad_norm": 0.8225540518760681 + }, + { + "bits_per_byte": 1.530357349756151, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.060762882232666, + "step": 5440, + "unclipped_grad_norm": 0.8045809864997864 + }, + { + "bits_per_byte": 1.5424515137301253, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.069145917892456, + "step": 5450, + "unclipped_grad_norm": 0.8051854968070984 + }, + { + "bits_per_byte": 1.5648894022575925, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0846986770629883, + "step": 5460, + "unclipped_grad_norm": 0.7835660576820374 + }, + { + "bits_per_byte": 1.5450825043230245, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.070969581604004, + "step": 5470, + "unclipped_grad_norm": 0.8599129915237427 + }, + { + "bits_per_byte": 1.4780804713039863, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0245273113250732, + "step": 5480, + "unclipped_grad_norm": 0.8000594973564148 + }, + { + "bits_per_byte": 1.5424891779306642, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0691720247268677, + "step": 5490, + "unclipped_grad_norm": 0.8055532574653625 + }, + { + "bits_per_byte": 1.5569413960306187, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0791895389556885, + "step": 5500, + "unclipped_grad_norm": 0.8300883769989014 + }, + { + "bits_per_byte": 1.5474785666148452, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0726304054260254, + "step": 5510, + "unclipped_grad_norm": 0.826352059841156 + }, + { + "bits_per_byte": 1.5350409532870977, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0640093088150024, + "step": 5520, + "unclipped_grad_norm": 0.853265643119812 + }, + { + "bits_per_byte": 1.5635150888945477, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.083746075630188, + "step": 5530, + "unclipped_grad_norm": 0.8060055375099182 + }, + { + "bits_per_byte": 1.5843347926600468, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.098177194595337, + "step": 5540, + "unclipped_grad_norm": 0.8706573843955994 + }, + { + "bits_per_byte": 1.5165710484803372, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0512069463729858, + "step": 5550, + "unclipped_grad_norm": 0.7865509986877441 + }, + { + "bits_per_byte": 1.5106121935932426, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0470765829086304, + "step": 5560, + "unclipped_grad_norm": 0.8064424991607666 + }, + { + "bits_per_byte": 1.5654490338035, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0850865840911865, + "step": 5570, + "unclipped_grad_norm": 0.8376733064651489 + }, + { + "bits_per_byte": 1.4567845475782482, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0097661018371582, + "step": 5580, + "unclipped_grad_norm": 0.7972759008407593 + }, + { + "bits_per_byte": 1.5257076268074239, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.057539939880371, + "step": 5590, + "unclipped_grad_norm": 0.8159511089324951 + }, + { + "bits_per_byte": 1.5137768463517691, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0492701530456543, + "step": 5600, + "unclipped_grad_norm": 0.7912799715995789 + }, + { + "bits_per_byte": 1.5312812405565859, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0614032745361328, + "step": 5610, + "unclipped_grad_norm": 0.7786760330200195 + }, + { + "bits_per_byte": 1.4626550033827992, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0138351917266846, + "step": 5620, + "unclipped_grad_norm": 0.8055174350738525 + }, + { + "bits_per_byte": 1.4690646247978054, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0182780027389526, + "step": 5630, + "unclipped_grad_norm": 0.7851108312606812 + }, + { + "bits_per_byte": 1.4663743001903584, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0164132118225098, + "step": 5640, + "unclipped_grad_norm": 0.8267287015914917 + }, + { + "bits_per_byte": 1.5738650048234715, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.090920090675354, + "step": 5650, + "unclipped_grad_norm": 0.8359448313713074 + }, + { + "bits_per_byte": 1.5300369460775933, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.060540795326233, + "step": 5660, + "unclipped_grad_norm": 0.8635727167129517 + }, + { + "bits_per_byte": 1.5230305448640935, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0556843280792236, + "step": 5670, + "unclipped_grad_norm": 0.8095638751983643 + }, + { + "bits_per_byte": 1.524155483383387, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0564640760421753, + "step": 5680, + "unclipped_grad_norm": 0.8353701233863831 + }, + { + "bits_per_byte": 1.566590998605229, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.0858781337738037, + "step": 5690, + "unclipped_grad_norm": 0.8297926187515259 + }, + { + "bits_per_byte": 1.499949785187701, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0396859645843506, + "step": 5700, + "unclipped_grad_norm": 0.8335801959037781 + }, + { + "bits_per_byte": 1.5306388853556132, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0609580278396606, + "step": 5710, + "unclipped_grad_norm": 0.8218200206756592 + }, + { + "bits_per_byte": 1.5663932185567369, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0857410430908203, + "step": 5720, + "unclipped_grad_norm": 0.8909743428230286 + }, + { + "bits_per_byte": 1.4614645394735268, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.013010025024414, + "step": 5730, + "unclipped_grad_norm": 0.7809724807739258 + }, + { + "bits_per_byte": 1.5314872757723195, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0615460872650146, + "step": 5740, + "unclipped_grad_norm": 0.8163876533508301 + }, + { + "bits_per_byte": 1.499649159513993, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0394775867462158, + "step": 5750, + "unclipped_grad_norm": 0.8667106628417969 + }, + { + "bits_per_byte": 1.4999243317553734, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.039668321609497, + "step": 5760, + "unclipped_grad_norm": 0.8516601324081421 + }, + { + "bits_per_byte": 1.450708744488567, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0055546760559082, + "step": 5770, + "unclipped_grad_norm": 0.7943668365478516 + }, + { + "bits_per_byte": 1.5466515020468459, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0720571279525757, + "step": 5780, + "unclipped_grad_norm": 0.8325088024139404 + }, + { + "bits_per_byte": 1.494320105094358, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0357837677001953, + "step": 5790, + "unclipped_grad_norm": 0.8366720080375671 + }, + { + "bits_per_byte": 1.4906410522271007, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033233642578125, + "step": 5800, + "unclipped_grad_norm": 0.8333538770675659 + }, + { + "bits_per_byte": 1.5321304908865463, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0619919300079346, + "step": 5810, + "unclipped_grad_norm": 0.8446563482284546 + }, + { + "bits_per_byte": 1.51830377368778, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.05240797996521, + "step": 5820, + "unclipped_grad_norm": 0.8496891856193542 + }, + { + "bits_per_byte": 1.4932847695361637, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0350661277770996, + "step": 5830, + "unclipped_grad_norm": 0.8205961585044861 + }, + { + "bits_per_byte": 1.456290785387621, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.009423851966858, + "step": 5840, + "unclipped_grad_norm": 0.8371089100837708 + }, + { + "bits_per_byte": 1.454681199758196, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0083081722259521, + "step": 5850, + "unclipped_grad_norm": 0.8174164295196533 + }, + { + "bits_per_byte": 1.4966561454410274, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0374029874801636, + "step": 5860, + "unclipped_grad_norm": 0.9058852791786194 + }, + { + "bits_per_byte": 1.5418609253070623, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0687365531921387, + "step": 5870, + "unclipped_grad_norm": 0.8718268275260925 + }, + { + "bits_per_byte": 1.5420466665699941, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0688652992248535, + "step": 5880, + "unclipped_grad_norm": 0.8315351009368896 + }, + { + "bits_per_byte": 1.525186175410008, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0571784973144531, + "step": 5890, + "unclipped_grad_norm": 0.8407886028289795 + }, + { + "bits_per_byte": 1.4989159974733648, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0389693975448608, + "step": 5900, + "unclipped_grad_norm": 0.8770707249641418 + }, + { + "bits_per_byte": 1.4937374278732347, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0353798866271973, + "step": 5910, + "unclipped_grad_norm": 0.8901946544647217 + }, + { + "bits_per_byte": 1.4996551789067731, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.03948175907135, + "step": 5920, + "unclipped_grad_norm": 0.85027015209198 + }, + { + "bits_per_byte": 1.5573538104273876, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.0794754028320312, + "step": 5930, + "unclipped_grad_norm": 0.8851437568664551 + }, + { + "bits_per_byte": 1.5404470559343193, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0677565336227417, + "step": 5940, + "unclipped_grad_norm": 0.8759228587150574 + }, + { + "bits_per_byte": 1.5066732749405318, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0443463325500488, + "step": 5950, + "unclipped_grad_norm": 0.8818607926368713 + }, + { + "bits_per_byte": 1.5044743047665998, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0428221225738525, + "step": 5960, + "unclipped_grad_norm": 0.816109299659729 + }, + { + "bits_per_byte": 1.5173669841885302, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0517586469650269, + "step": 5970, + "unclipped_grad_norm": 0.8590478301048279 + }, + { + "bits_per_byte": 1.4670959393933787, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0169134140014648, + "step": 5980, + "unclipped_grad_norm": 0.8544116616249084 + }, + { + "bits_per_byte": 1.4408131207318973, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 0.9986955523490906, + "step": 5990, + "unclipped_grad_norm": 0.8525132536888123 + }, + { + "bits_per_byte": 1.5172739415744134, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0516941547393799, + "step": 6000, + "unclipped_grad_norm": 0.860197126865387 + }, + { + "bits_per_byte": 1.499894750739425, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0396478176116943, + "step": 6010, + "unclipped_grad_norm": 0.8407278060913086 + }, + { + "bits_per_byte": 1.4776142263374972, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.024204134941101, + "step": 6020, + "unclipped_grad_norm": 0.8550556302070618 + }, + { + "bits_per_byte": 1.5367169242197558, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0651710033416748, + "step": 6030, + "unclipped_grad_norm": 0.8657232522964478 + }, + { + "bits_per_byte": 1.4931389282482321, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0349650382995605, + "step": 6040, + "unclipped_grad_norm": 0.8972119092941284 + }, + { + "bits_per_byte": 1.498401597364634, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0386128425598145, + "step": 6050, + "unclipped_grad_norm": 0.8974336981773376 + }, + { + "bits_per_byte": 1.5272425719663745, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0586038827896118, + "step": 6060, + "unclipped_grad_norm": 0.8948551416397095 + }, + { + "bits_per_byte": 1.5059438965182226, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.043840765953064, + "step": 6070, + "unclipped_grad_norm": 0.843971312046051 + }, + { + "bits_per_byte": 1.4791374766761891, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0252599716186523, + "step": 6080, + "unclipped_grad_norm": 0.8345775008201599 + }, + { + "bits_per_byte": 1.4884991802932552, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0317490100860596, + "step": 6090, + "unclipped_grad_norm": 0.8372970223426819 + }, + { + "bits_per_byte": 1.486896301987214, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0306379795074463, + "step": 6100, + "unclipped_grad_norm": 0.827723503112793 + }, + { + "bits_per_byte": 1.4668594632484424, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0167495012283325, + "step": 6110, + "unclipped_grad_norm": 0.871588945388794 + }, + { + "bits_per_byte": 1.4677823221529722, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.017389178276062, + "step": 6120, + "unclipped_grad_norm": 0.8457695245742798 + }, + { + "bits_per_byte": 1.5061057321926845, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0439529418945312, + "step": 6130, + "unclipped_grad_norm": 0.9012824296951294 + }, + { + "bits_per_byte": 1.5178127632195666, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0520676374435425, + "step": 6140, + "unclipped_grad_norm": 0.8450613617897034 + }, + { + "bits_per_byte": 1.4675592606548031, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0172345638275146, + "step": 6150, + "unclipped_grad_norm": 0.885280191898346 + }, + { + "bits_per_byte": 1.5554258849112155, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.078139066696167, + "step": 6160, + "unclipped_grad_norm": 0.9099593758583069 + }, + { + "bits_per_byte": 1.421598961003528, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 0.985377311706543, + "step": 6170, + "unclipped_grad_norm": 0.7732160091400146 + }, + { + "bits_per_byte": 1.5028951600663776, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0417275428771973, + "step": 6180, + "unclipped_grad_norm": 0.869156002998352 + }, + { + "bits_per_byte": 1.4956368042693635, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.036696434020996, + "step": 6190, + "unclipped_grad_norm": 0.8908746838569641 + }, + { + "bits_per_byte": 1.4533810109176732, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0074069499969482, + "step": 6200, + "unclipped_grad_norm": 0.8600793480873108 + }, + { + "bits_per_byte": 1.475978499345141, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0230703353881836, + "step": 6210, + "unclipped_grad_norm": 0.8739057779312134 + }, + { + "bits_per_byte": 1.4582183669384914, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.010759949684143, + "step": 6220, + "unclipped_grad_norm": 0.8542900681495667 + }, + { + "bits_per_byte": 1.4401650041121208, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 0.9982463121414185, + "step": 6230, + "unclipped_grad_norm": 0.8145909309387207 + }, + { + "bits_per_byte": 1.458188613939892, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0107393264770508, + "step": 6240, + "unclipped_grad_norm": 0.8962635397911072 + }, + { + "bits_per_byte": 1.4595658510080016, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0116939544677734, + "step": 6250, + "unclipped_grad_norm": 0.855821430683136 + }, + { + "bits_per_byte": 1.5147881043388425, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.049971103668213, + "step": 6260, + "unclipped_grad_norm": 0.8663405179977417 + }, + { + "bits_per_byte": 1.4764731214490225, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0234131813049316, + "step": 6270, + "unclipped_grad_norm": 0.8501848578453064 + }, + { + "bits_per_byte": 1.59573105103682, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1060764789581299, + "step": 6280, + "unclipped_grad_norm": 0.9987632036209106 + }, + { + "bits_per_byte": 1.4579987450933396, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0106077194213867, + "step": 6290, + "unclipped_grad_norm": 0.8792169690132141 + }, + { + "bits_per_byte": 1.4898052165439075, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0326542854309082, + "step": 6300, + "unclipped_grad_norm": 0.8601147532463074 + }, + { + "bits_per_byte": 1.537688110249178, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.065844178199768, + "step": 6310, + "unclipped_grad_norm": 0.907733678817749 + }, + { + "bits_per_byte": 1.487509248154889, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0310628414154053, + "step": 6320, + "unclipped_grad_norm": 0.8578711152076721 + }, + { + "bits_per_byte": 1.4237945775157679, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 0.986899197101593, + "step": 6330, + "unclipped_grad_norm": 0.845673143863678 + }, + { + "bits_per_byte": 1.4966876182661353, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0374248027801514, + "step": 6340, + "unclipped_grad_norm": 0.8802160620689392 + }, + { + "bits_per_byte": 1.485631197607467, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0297610759735107, + "step": 6350, + "unclipped_grad_norm": 0.884225606918335 + }, + { + "bits_per_byte": 1.4432266392627804, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0003684759140015, + "step": 6360, + "unclipped_grad_norm": 0.8594306707382202 + }, + { + "bits_per_byte": 1.497669639202562, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0381054878234863, + "step": 6370, + "unclipped_grad_norm": 0.9075121283531189 + }, + { + "bits_per_byte": 1.4643473126672892, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0150082111358643, + "step": 6380, + "unclipped_grad_norm": 0.8724440932273865 + }, + { + "bits_per_byte": 1.4983035672536422, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0385448932647705, + "step": 6390, + "unclipped_grad_norm": 0.9329665303230286 + }, + { + "bits_per_byte": 1.4855978329731998, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.029737949371338, + "step": 6400, + "unclipped_grad_norm": 0.8683961033821106 + }, + { + "bits_per_byte": 1.4585226042478678, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0109708309173584, + "step": 6410, + "unclipped_grad_norm": 0.8944246768951416 + }, + { + "bits_per_byte": 1.4417726979323862, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 0.9993606805801392, + "step": 6420, + "unclipped_grad_norm": 0.8450288772583008 + }, + { + "bits_per_byte": 1.474598510554618, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0221138000488281, + "step": 6430, + "unclipped_grad_norm": 0.8708812594413757 + }, + { + "bits_per_byte": 1.4773634756325393, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.024030327796936, + "step": 6440, + "unclipped_grad_norm": 0.8653560876846313 + }, + { + "bits_per_byte": 1.4567353605401014, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0097320079803467, + "step": 6450, + "unclipped_grad_norm": 0.8851431012153625 + }, + { + "bits_per_byte": 1.4433678370191387, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0004663467407227, + "step": 6460, + "unclipped_grad_norm": 0.902314305305481 + }, + { + "bits_per_byte": 1.478021997202693, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.024486780166626, + "step": 6470, + "unclipped_grad_norm": 0.8634862899780273 + }, + { + "bits_per_byte": 1.49569390250945, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.036736011505127, + "step": 6480, + "unclipped_grad_norm": 0.8877258896827698 + }, + { + "bits_per_byte": 1.4739315618345719, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0216515064239502, + "step": 6490, + "unclipped_grad_norm": 0.9130761027336121 + }, + { + "bits_per_byte": 1.4798123365981748, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0257277488708496, + "step": 6500, + "unclipped_grad_norm": 0.9244931936264038 + }, + { + "bits_per_byte": 1.429752142532981, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 0.9910286664962769, + "step": 6510, + "unclipped_grad_norm": 0.8583832383155823 + }, + { + "bits_per_byte": 1.3992985726054255, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9699198603630066, + "step": 6520, + "unclipped_grad_norm": 0.8968297243118286 + }, + { + "bits_per_byte": 1.5067831718544331, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0444225072860718, + "step": 6530, + "unclipped_grad_norm": 0.9009160399436951 + }, + { + "bits_per_byte": 1.468959715380779, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0182052850723267, + "step": 6540, + "unclipped_grad_norm": 0.8687822222709656 + }, + { + "bits_per_byte": 1.5110841139872102, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0474036931991577, + "step": 6550, + "unclipped_grad_norm": 0.9212287068367004 + }, + { + "bits_per_byte": 1.4657298811975754, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.015966534614563, + "step": 6560, + "unclipped_grad_norm": 0.8736171126365662 + }, + { + "bits_per_byte": 1.3923212364599213, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 0.9650835394859314, + "step": 6570, + "unclipped_grad_norm": 0.9459646344184875 + }, + { + "bits_per_byte": 1.5011586512406159, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.040523886680603, + "step": 6580, + "unclipped_grad_norm": 0.8902279138565063 + }, + { + "bits_per_byte": 1.4631396504929306, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0141711235046387, + "step": 6590, + "unclipped_grad_norm": 0.8934556245803833 + }, + { + "bits_per_byte": 1.3892133379761795, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9629293084144592, + "step": 6600, + "unclipped_grad_norm": 0.8376922607421875 + }, + { + "bits_per_byte": 1.5024242715683152, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0414011478424072, + "step": 6610, + "unclipped_grad_norm": 0.8684409856796265 + }, + { + "bits_per_byte": 1.5173028346597586, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0517141819000244, + "step": 6620, + "unclipped_grad_norm": 0.9338745474815369 + }, + { + "bits_per_byte": 1.4964176055042808, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0372376441955566, + "step": 6630, + "unclipped_grad_norm": 0.9484766721725464 + }, + { + "bits_per_byte": 1.4696447222791655, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0186800956726074, + "step": 6640, + "unclipped_grad_norm": 0.9511991143226624 + }, + { + "bits_per_byte": 1.4476113369465278, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0034077167510986, + "step": 6650, + "unclipped_grad_norm": 0.8744685649871826 + }, + { + "bits_per_byte": 1.5102728718230902, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0468413829803467, + "step": 6660, + "unclipped_grad_norm": 0.9144671559333801 + }, + { + "bits_per_byte": 1.4729612357184039, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0209789276123047, + "step": 6670, + "unclipped_grad_norm": 0.918013870716095 + }, + { + "bits_per_byte": 1.4976474534406006, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0380901098251343, + "step": 6680, + "unclipped_grad_norm": 0.9484038352966309 + }, + { + "bits_per_byte": 1.4621077545877539, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.013455867767334, + "step": 6690, + "unclipped_grad_norm": 0.9366878867149353 + }, + { + "bits_per_byte": 1.513278096664267, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.048924446105957, + "step": 6700, + "unclipped_grad_norm": 0.9588149785995483 + }, + { + "bits_per_byte": 1.448871969777352, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0042815208435059, + "step": 6710, + "unclipped_grad_norm": 0.8966986536979675 + }, + { + "bits_per_byte": 1.4475960304906008, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0033971071243286, + "step": 6720, + "unclipped_grad_norm": 0.9147130250930786 + }, + { + "bits_per_byte": 1.4208115384365525, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 0.9848315119743347, + "step": 6730, + "unclipped_grad_norm": 0.9434075951576233 + }, + { + "bits_per_byte": 1.4530764296429952, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0071958303451538, + "step": 6740, + "unclipped_grad_norm": 0.9062299728393555 + }, + { + "bits_per_byte": 1.4277258429405155, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 0.9896241426467896, + "step": 6750, + "unclipped_grad_norm": 0.9092971086502075 + }, + { + "bits_per_byte": 1.4128754850038137, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 0.9793306589126587, + "step": 6760, + "unclipped_grad_norm": 0.8910818099975586 + }, + { + "bits_per_byte": 1.434429210723195, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9942705631256104, + "step": 6770, + "unclipped_grad_norm": 0.8744387030601501 + }, + { + "bits_per_byte": 1.4315308730995298, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 0.9922615885734558, + "step": 6780, + "unclipped_grad_norm": 0.9047504663467407 + }, + { + "bits_per_byte": 1.53544803622169, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0642914772033691, + "step": 6790, + "unclipped_grad_norm": 0.9123643636703491 + }, + { + "bits_per_byte": 1.504298710480069, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0427004098892212, + "step": 6800, + "unclipped_grad_norm": 0.978726327419281 + }, + { + "bits_per_byte": 1.4819994399691971, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.027243733406067, + "step": 6810, + "unclipped_grad_norm": 0.9510568380355835 + }, + { + "bits_per_byte": 1.4201767504722178, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 0.9843915104866028, + "step": 6820, + "unclipped_grad_norm": 0.9054594039916992 + }, + { + "bits_per_byte": 1.441129998766112, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 0.9989151954650879, + "step": 6830, + "unclipped_grad_norm": 0.9485244154930115 + }, + { + "bits_per_byte": 1.4736460706341397, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.021453619003296, + "step": 6840, + "unclipped_grad_norm": 0.9141860604286194 + }, + { + "bits_per_byte": 1.439767810179953, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 0.9979709982872009, + "step": 6850, + "unclipped_grad_norm": 0.9174992442131042 + }, + { + "bits_per_byte": 1.4496288654237992, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0048061609268188, + "step": 6860, + "unclipped_grad_norm": 0.9317811727523804 + }, + { + "bits_per_byte": 1.4588201342338605, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0111770629882812, + "step": 6870, + "unclipped_grad_norm": 0.9191097617149353 + }, + { + "bits_per_byte": 1.4995917173086046, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0394377708435059, + "step": 6880, + "unclipped_grad_norm": 0.9317643642425537 + }, + { + "bits_per_byte": 1.3990970949299397, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9697802066802979, + "step": 6890, + "unclipped_grad_norm": 0.9130836725234985 + }, + { + "bits_per_byte": 1.4230204836042344, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 0.986362636089325, + "step": 6900, + "unclipped_grad_norm": 0.8910167217254639 + }, + { + "bits_per_byte": 1.5914774041330302, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1031280755996704, + "step": 6910, + "unclipped_grad_norm": 1.0252712965011597 + }, + { + "bits_per_byte": 1.394897880535148, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 0.9668695330619812, + "step": 6920, + "unclipped_grad_norm": 0.8750832676887512 + }, + { + "bits_per_byte": 1.42465509270936, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9874956607818604, + "step": 6930, + "unclipped_grad_norm": 0.8865536451339722 + }, + { + "bits_per_byte": 1.4536008047454758, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0075592994689941, + "step": 6940, + "unclipped_grad_norm": 0.913940966129303 + }, + { + "bits_per_byte": 1.4401447961506444, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9982323050498962, + "step": 6950, + "unclipped_grad_norm": 0.941871702671051 + }, + { + "bits_per_byte": 1.4963969675861772, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0372233390808105, + "step": 6960, + "unclipped_grad_norm": 0.9974261522293091 + }, + { + "bits_per_byte": 1.4704031657694705, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0192058086395264, + "step": 6970, + "unclipped_grad_norm": 0.9634589552879333 + }, + { + "bits_per_byte": 1.5157009882496222, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0506038665771484, + "step": 6980, + "unclipped_grad_norm": 0.9805173873901367 + }, + { + "bits_per_byte": 1.4572141602401034, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.010063886642456, + "step": 6990, + "unclipped_grad_norm": 0.9425977468490601 + }, + { + "bits_per_byte": 1.4146713278441232, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 0.980575442314148, + "step": 7000, + "unclipped_grad_norm": 0.9211949110031128 + }, + { + "bits_per_byte": 1.4423556331274856, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9997647404670715, + "step": 7010, + "unclipped_grad_norm": 0.9508169293403625 + }, + { + "bits_per_byte": 1.4268384124620634, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 0.9890090227127075, + "step": 7020, + "unclipped_grad_norm": 0.9370734095573425 + }, + { + "bits_per_byte": 1.4214180352548202, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 0.9852519035339355, + "step": 7030, + "unclipped_grad_norm": 0.8771427273750305 + }, + { + "bits_per_byte": 1.5093840654834314, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0462253093719482, + "step": 7040, + "unclipped_grad_norm": 0.9992957711219788 + }, + { + "bits_per_byte": 1.466185635222362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0162824392318726, + "step": 7050, + "unclipped_grad_norm": 0.9372990131378174 + }, + { + "bits_per_byte": 1.4783773133593756, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.024733066558838, + "step": 7060, + "unclipped_grad_norm": 1.0182687044143677 + }, + { + "bits_per_byte": 1.4142496263842075, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 0.9802831411361694, + "step": 7070, + "unclipped_grad_norm": 0.8880259394645691 + }, + { + "bits_per_byte": 1.3447492013960853, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9321091175079346, + "step": 7080, + "unclipped_grad_norm": 0.910698413848877 + }, + { + "bits_per_byte": 1.413691198716856, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.979896068572998, + "step": 7090, + "unclipped_grad_norm": 0.9244340062141418 + }, + { + "bits_per_byte": 1.4025504205679395, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9721738696098328, + "step": 7100, + "unclipped_grad_norm": 0.9016135334968567 + }, + { + "bits_per_byte": 1.4533139376838367, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0073604583740234, + "step": 7110, + "unclipped_grad_norm": 0.970280110836029 + }, + { + "bits_per_byte": 1.4606615524766484, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0124534368515015, + "step": 7120, + "unclipped_grad_norm": 0.9771916270256042 + }, + { + "bits_per_byte": 1.396041221198084, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 0.9676620364189148, + "step": 7130, + "unclipped_grad_norm": 0.8866344094276428 + }, + { + "bits_per_byte": 1.5095894127685614, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0463676452636719, + "step": 7140, + "unclipped_grad_norm": 0.9947805404663086 + }, + { + "bits_per_byte": 1.5291790966150898, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0599461793899536, + "step": 7150, + "unclipped_grad_norm": 0.9418985247612 + }, + { + "bits_per_byte": 1.4640222654571586, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0147829055786133, + "step": 7160, + "unclipped_grad_norm": 1.0358314514160156 + }, + { + "bits_per_byte": 1.4559156912260893, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0091638565063477, + "step": 7170, + "unclipped_grad_norm": 0.9539772272109985 + }, + { + "bits_per_byte": 1.3591113005869877, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 0.9420641660690308, + "step": 7180, + "unclipped_grad_norm": 0.8915972709655762 + }, + { + "bits_per_byte": 1.3840849873101015, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9593746066093445, + "step": 7190, + "unclipped_grad_norm": 0.9118176102638245 + }, + { + "bits_per_byte": 1.4715360154907038, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0199910402297974, + "step": 7200, + "unclipped_grad_norm": 0.973518431186676 + }, + { + "bits_per_byte": 1.4678313372084681, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.017423152923584, + "step": 7210, + "unclipped_grad_norm": 0.9181845188140869 + }, + { + "bits_per_byte": 1.4265923052886786, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9888384342193604, + "step": 7220, + "unclipped_grad_norm": 0.926552951335907 + }, + { + "bits_per_byte": 1.4337483314084294, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9937986135482788, + "step": 7230, + "unclipped_grad_norm": 0.9526586532592773 + }, + { + "bits_per_byte": 1.383381664259398, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 0.9588871002197266, + "step": 7240, + "unclipped_grad_norm": 0.8890237212181091 + }, + { + "bits_per_byte": 1.431437056563484, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 0.9921965599060059, + "step": 7250, + "unclipped_grad_norm": 0.9829798340797424 + }, + { + "bits_per_byte": 1.4272578781475178, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9892997741699219, + "step": 7260, + "unclipped_grad_norm": 0.9537968039512634 + }, + { + "bits_per_byte": 1.5069463833901018, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0445356369018555, + "step": 7270, + "unclipped_grad_norm": 0.9953339099884033 + }, + { + "bits_per_byte": 1.4318632295723221, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 0.9924919605255127, + "step": 7280, + "unclipped_grad_norm": 0.9815621972084045 + }, + { + "bits_per_byte": 1.435238647069481, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.9948316216468811, + "step": 7290, + "unclipped_grad_norm": 0.9373346567153931 + }, + { + "bits_per_byte": 1.44098209368637, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 0.9988126754760742, + "step": 7300, + "unclipped_grad_norm": 0.9782799482345581 + }, + { + "bits_per_byte": 1.467732619166873, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0173547267913818, + "step": 7310, + "unclipped_grad_norm": 0.9545702338218689 + }, + { + "bits_per_byte": 1.4680886232541588, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.017601490020752, + "step": 7320, + "unclipped_grad_norm": 0.9820490479469299 + }, + { + "bits_per_byte": 1.4597027491980883, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0117888450622559, + "step": 7330, + "unclipped_grad_norm": 1.0537965297698975 + }, + { + "bits_per_byte": 1.4358094574876945, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9952272772789001, + "step": 7340, + "unclipped_grad_norm": 0.9470505714416504 + }, + { + "bits_per_byte": 1.4763711357370608, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.023342490196228, + "step": 7350, + "unclipped_grad_norm": 0.93792724609375 + }, + { + "bits_per_byte": 1.4524729425211176, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0067775249481201, + "step": 7360, + "unclipped_grad_norm": 0.959013044834137 + }, + { + "bits_per_byte": 1.4678055398108387, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0174052715301514, + "step": 7370, + "unclipped_grad_norm": 0.9769075512886047 + }, + { + "bits_per_byte": 1.5064952728968888, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0442229509353638, + "step": 7380, + "unclipped_grad_norm": 1.0261726379394531 + }, + { + "bits_per_byte": 1.4743861119808024, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.021966576576233, + "step": 7390, + "unclipped_grad_norm": 0.977978527545929 + }, + { + "bits_per_byte": 1.4106443540741704, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9777841567993164, + "step": 7400, + "unclipped_grad_norm": 0.9216173887252808 + }, + { + "bits_per_byte": 1.4343313525948542, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.994202733039856, + "step": 7410, + "unclipped_grad_norm": 0.9778451919555664 + }, + { + "bits_per_byte": 1.5075760118749106, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0449720621109009, + "step": 7420, + "unclipped_grad_norm": 1.0143206119537354 + }, + { + "bits_per_byte": 1.4176087055295346, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9826114773750305, + "step": 7430, + "unclipped_grad_norm": 0.9865745902061462 + }, + { + "bits_per_byte": 1.4384498211350658, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9970574378967285, + "step": 7440, + "unclipped_grad_norm": 0.9753794074058533 + }, + { + "bits_per_byte": 1.385612537215065, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 0.9604334235191345, + "step": 7450, + "unclipped_grad_norm": 0.9882722496986389 + }, + { + "bits_per_byte": 1.40335280562554, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9727300405502319, + "step": 7460, + "unclipped_grad_norm": 0.8938714861869812 + }, + { + "bits_per_byte": 1.4650283639647061, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0154802799224854, + "step": 7470, + "unclipped_grad_norm": 0.95097815990448 + }, + { + "bits_per_byte": 1.378530205661209, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 0.9555243253707886, + "step": 7480, + "unclipped_grad_norm": 0.9593113660812378 + }, + { + "bits_per_byte": 1.4079058743244819, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 0.9758859872817993, + "step": 7490, + "unclipped_grad_norm": 0.9498793482780457 + }, + { + "bits_per_byte": 1.4795925427703722, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0255753993988037, + "step": 7500, + "unclipped_grad_norm": 0.9999135732650757 + }, + { + "bits_per_byte": 1.462181535144974, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0135070085525513, + "step": 7510, + "unclipped_grad_norm": 0.96773761510849 + }, + { + "bits_per_byte": 1.424363066168195, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 0.9872932434082031, + "step": 7520, + "unclipped_grad_norm": 0.9349227547645569 + }, + { + "bits_per_byte": 1.4575968216382733, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0103291273117065, + "step": 7530, + "unclipped_grad_norm": 0.9571959972381592 + }, + { + "bits_per_byte": 1.484410464751643, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0289149284362793, + "step": 7540, + "unclipped_grad_norm": 0.9889418482780457 + }, + { + "bits_per_byte": 1.4190770074112757, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 0.9836292266845703, + "step": 7550, + "unclipped_grad_norm": 0.9660657048225403 + }, + { + "bits_per_byte": 1.458957376389249, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0112721920013428, + "step": 7560, + "unclipped_grad_norm": 0.9621381163597107 + }, + { + "bits_per_byte": 1.4744461339259534, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0220081806182861, + "step": 7570, + "unclipped_grad_norm": 0.9809478521347046 + }, + { + "bits_per_byte": 1.424117474942763, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9871230125427246, + "step": 7580, + "unclipped_grad_norm": 1.0034832954406738 + }, + { + "bits_per_byte": 1.4167208450944553, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9819960594177246, + "step": 7590, + "unclipped_grad_norm": 0.9541960954666138 + }, + { + "bits_per_byte": 1.3862604818521906, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9608825445175171, + "step": 7600, + "unclipped_grad_norm": 0.950609564781189 + }, + { + "bits_per_byte": 1.3616721222583348, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9438391923904419, + "step": 7610, + "unclipped_grad_norm": 0.9224535226821899 + }, + { + "bits_per_byte": 1.4445553772233464, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0012894868850708, + "step": 7620, + "unclipped_grad_norm": 0.9717100858688354 + }, + { + "bits_per_byte": 1.4175787805482842, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9825907349586487, + "step": 7630, + "unclipped_grad_norm": 0.9714697599411011 + }, + { + "bits_per_byte": 1.4152020663046858, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9809433221817017, + "step": 7640, + "unclipped_grad_norm": 0.9795277118682861 + }, + { + "bits_per_byte": 1.5113582543326856, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0475937128067017, + "step": 7650, + "unclipped_grad_norm": 1.0563151836395264 + }, + { + "bits_per_byte": 1.4375178471500403, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9964114427566528, + "step": 7660, + "unclipped_grad_norm": 0.9650751948356628 + }, + { + "bits_per_byte": 1.4172367070557181, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 0.9823536276817322, + "step": 7670, + "unclipped_grad_norm": 0.9894714951515198 + }, + { + "bits_per_byte": 1.3937812831744212, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9660955667495728, + "step": 7680, + "unclipped_grad_norm": 0.9648637175559998 + }, + { + "bits_per_byte": 1.4266868097553278, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9889039397239685, + "step": 7690, + "unclipped_grad_norm": 0.9686183333396912 + }, + { + "bits_per_byte": 1.4503658110827464, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.005316972732544, + "step": 7700, + "unclipped_grad_norm": 0.9993582367897034 + }, + { + "bits_per_byte": 1.3691781330725912, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9490419626235962, + "step": 7710, + "unclipped_grad_norm": 0.9620972275733948 + }, + { + "bits_per_byte": 1.3650053180147068, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9461495876312256, + "step": 7720, + "unclipped_grad_norm": 0.9128993153572083 + }, + { + "bits_per_byte": 1.4252105106803215, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9878806471824646, + "step": 7730, + "unclipped_grad_norm": 0.9594771265983582 + }, + { + "bits_per_byte": 1.4081796707046554, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9760757684707642, + "step": 7740, + "unclipped_grad_norm": 0.9255442023277283 + }, + { + "bits_per_byte": 1.433292233418341, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9934824705123901, + "step": 7750, + "unclipped_grad_norm": 0.9785766005516052 + }, + { + "bits_per_byte": 1.3999133245909345, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9703459739685059, + "step": 7760, + "unclipped_grad_norm": 1.0032256841659546 + }, + { + "bits_per_byte": 1.4048038232508695, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9737358093261719, + "step": 7770, + "unclipped_grad_norm": 1.0204466581344604 + }, + { + "bits_per_byte": 1.536527399338505, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0650396347045898, + "step": 7780, + "unclipped_grad_norm": 1.0556561946868896 + }, + { + "bits_per_byte": 1.4128328333063997, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9793010950088501, + "step": 7790, + "unclipped_grad_norm": 1.0078489780426025 + }, + { + "bits_per_byte": 1.451735308931567, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0062662363052368, + "step": 7800, + "unclipped_grad_norm": 1.0289596319198608 + }, + { + "bits_per_byte": 1.3852072600983067, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9601525068283081, + "step": 7810, + "unclipped_grad_norm": 0.9891387224197388 + }, + { + "bits_per_byte": 1.3953124447150527, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.9671568870544434, + "step": 7820, + "unclipped_grad_norm": 0.9778857231140137 + }, + { + "bits_per_byte": 1.45392653988621, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0077850818634033, + "step": 7830, + "unclipped_grad_norm": 1.0185887813568115 + }, + { + "bits_per_byte": 1.4113820736550464, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9782955050468445, + "step": 7840, + "unclipped_grad_norm": 0.9330953359603882 + }, + { + "bits_per_byte": 1.380841996454107, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9571267366409302, + "step": 7850, + "unclipped_grad_norm": 0.9742368459701538 + }, + { + "bits_per_byte": 1.3846048048723343, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 0.9597349166870117, + "step": 7860, + "unclipped_grad_norm": 0.9645414352416992 + }, + { + "bits_per_byte": 1.4429803601067448, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0001977682113647, + "step": 7870, + "unclipped_grad_norm": 0.9958038926124573 + }, + { + "bits_per_byte": 1.4256185395194934, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9881634712219238, + "step": 7880, + "unclipped_grad_norm": 0.9739922881126404 + }, + { + "bits_per_byte": 1.4834710955126302, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.028263807296753, + "step": 7890, + "unclipped_grad_norm": 0.9912284016609192 + }, + { + "bits_per_byte": 1.4598313922209336, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0118780136108398, + "step": 7900, + "unclipped_grad_norm": 0.9979069232940674 + }, + { + "bits_per_byte": 1.465345499972897, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.015700101852417, + "step": 7910, + "unclipped_grad_norm": 1.0503500699996948 + }, + { + "bits_per_byte": 1.3660396216769959, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 0.946866512298584, + "step": 7920, + "unclipped_grad_norm": 0.9824531078338623 + }, + { + "bits_per_byte": 1.3848080023743288, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.959875762462616, + "step": 7930, + "unclipped_grad_norm": 1.0091050863265991 + }, + { + "bits_per_byte": 1.4375059803471308, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9964032173156738, + "step": 7940, + "unclipped_grad_norm": 1.0658900737762451 + }, + { + "bits_per_byte": 1.4049340141175726, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9738260507583618, + "step": 7950, + "unclipped_grad_norm": 1.0148085355758667 + }, + { + "bits_per_byte": 1.415199744538899, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9809417128562927, + "step": 7960, + "unclipped_grad_norm": 0.9737400412559509 + }, + { + "bits_per_byte": 1.4008050546356583, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9709640741348267, + "step": 7970, + "unclipped_grad_norm": 0.9626508951187134 + }, + { + "bits_per_byte": 1.4372590132604919, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 0.9962320327758789, + "step": 7980, + "unclipped_grad_norm": 0.9673776626586914 + }, + { + "bits_per_byte": 1.4403348369798477, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9983640313148499, + "step": 7990, + "unclipped_grad_norm": 0.9839751720428467 + }, + { + "bits_per_byte": 1.3635582559903472, + "learning_rate": 3e-05, + "loss_nats": 0.9451465606689453, + "step": 8000, + "unclipped_grad_norm": 0.9773924946784973 + } + ] + } + }, + "schema_version": 1, + "study": { + "architectures": [ + "baseline", + "block" + ], + "depths": [ + 16, + 32 + ], + "diagnostic_steps": [ + 0, + 100, + 500, + 2000, + 4000, + 8000 + ], + "formal_runs": 12, + "formal_target_bytes": 786432000, + "gradient_object": "RMS of d(mean token CE)/d(post-MLP Transformer-block output) over batch×time×channel", + "replay_target_bytes": 65536000, + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ] + } +}