From d1d9d22bf32404f92a7bb85f451877c95b094a73 Mon Sep 17 00:00:00 2001 From: wuyang <5700876+banisherwy@user.noreply.gitee.com> Date: Thu, 30 Jul 2026 07:15:05 +0800 Subject: [PATCH] research: audit reduced AttnRes study --- experiments/k3/attnres/README.md | 21 + experiments/k3/attnres/analyze.py | 536 + experiments/k3/attnres/reproduction.json | 109 + research/K3_ATTNRES_REDUCED_AUDIT.md | 568 + src/data/k3-attnres-reduced-compact.json | 3106 +++ src/data/k3-attnres-reduced.json | 21498 +++++++++++++++++++++ 6 files changed, 25838 insertions(+) create mode 100644 experiments/k3/attnres/analyze.py create mode 100644 experiments/k3/attnres/reproduction.json create mode 100644 research/K3_ATTNRES_REDUCED_AUDIT.md create mode 100644 src/data/k3-attnres-reduced-compact.json create mode 100644 src/data/k3-attnres-reduced.json diff --git a/experiments/k3/attnres/README.md b/experiments/k3/attnres/README.md index 66b1614..017fc2e 100644 --- a/experiments/k3/attnres/README.md +++ b/experiments/k3/attnres/README.md @@ -41,3 +41,24 @@ python experiments/k3/attnres/train.py \ Raw parquet and checkpoints stay in the local cache. Frozen manifests, metric JSON, analyses, code, checksums, and a compact website payload enter the public repository. + +## Validate and aggregate the complete study + +After the nine formal cells, the preregistered replay, and the paired smoke runs +exist in the cache: + +```bash +python experiments/k3/attnres/analyze.py \ + --formal-dir /home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/formal \ + --smoke-dir /home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/smoke \ + --replay /home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/replay/block-2026073001.json \ + --manifest experiments/k3/attnres/manifest.json \ + --output src/data/k3-attnres-reduced.json \ + --compact-output src/data/k3-attnres-reduced-compact.json \ + --reproduction-output experiments/k3/attnres/reproduction.json +``` + +The aggregator fails closed on protocol identity, grid completeness, byte +budget, schedule hashes, shared initialization, non-finite metrics, diagnostic +shape, smoke mismatch, or formal replay mismatch. Timing is recorded but is not +required to replay bit-for-bit. diff --git a/experiments/k3/attnres/analyze.py b/experiments/k3/attnres/analyze.py new file mode 100644 index 0000000..be118b8 --- /dev/null +++ b/experiments/k3/attnres/analyze.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +"""Validate, aggregate, and compact the reduced Attention Residuals study.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import statistics +from pathlib import Path +from typing import Any, Iterable + + +PROTOCOL_ID = "llm-atlas-k3-attnres-reduced-v1" +ARCHITECTURES = ("baseline", "full", "block") +SEEDS = (2026073001, 2026073002, 2026073003) +REPLAY_FIELDS = ( + "manifest", + "model", + "optimizer", + "hashes", + "evaluations", + "training_history", + "diagnostic", + "environment", +) +SMOKE_FIELDS = REPLAY_FIELDS + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--formal-dir", type=Path, required=True) + parser.add_argument("--smoke-dir", type=Path, required=True) + parser.add_argument("--replay", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--compact-output", type=Path, required=True) + parser.add_argument("--reproduction-output", type=Path, required=True) + return parser.parse_args() + + +def read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + os.replace(temporary, path) + + +def mean(values: Iterable[float]) -> float: + return statistics.fmean(values) + + +def elementwise_summary(rows: list[list[float]]) -> dict[str, list[float]]: + length = len(rows[0]) + if any(len(row) != length for row in rows): + raise ValueError("array lengths do not match") + return { + "mean": [mean(row[index] for row in rows) for index in range(length)], + "min": [min(row[index] for row in rows) for index in range(length)], + "max": [max(row[index] for row in rows) for index in range(length)], + } + + +def coefficient_of_variation(values: list[float]) -> float: + average = mean(values) + variance = mean((value - average) ** 2 for value in values) + return math.sqrt(variance) / average + + +def verdict(deltas: list[float]) -> dict[str, Any]: + average = mean(deltas) + if all(delta < 0 for delta in deltas) and average <= -0.010: + label = "directional support in this reduced protocol" + elif all(delta > 0 for delta in deltas) and average >= 0.010: + label = "directional concern in this reduced protocol" + else: + label = "inconclusive at this budget" + return { + "paired_deltas_bpc": deltas, + "mean_delta_bpc": average, + "min_delta_bpc": min(deltas), + "max_delta_bpc": max(deltas), + "same_direction": len({delta < 0 for delta in deltas}) == 1, + "threshold_bpc": 0.010, + "verdict": label, + } + + +def average_depth_weights(runs: list[dict[str, Any]]) -> dict[str, Any]: + rows_by_run = [run["diagnostic"]["depth_weights"] for run in runs] + layer_count = len(rows_by_run[0]) + if any(len(rows) != layer_count for rows in rows_by_run): + raise ValueError("depth-weight layer counts differ") + rows = [] + max_sources = 0 + for layer in range(layer_count): + source_count = rows_by_run[0][layer]["sources"] + if any(rows[layer]["sources"] != source_count for rows in rows_by_run): + raise ValueError("source count differs across seeds") + weights = [ + mean(rows_by_run[seed_index][layer]["mean_weights"][source] + for seed_index in range(len(runs))) + for source in range(source_count) + ] + entropies = [ + rows_by_run[seed_index][layer]["entropy_mean"] + for seed_index in range(len(runs)) + ] + rows.append( + { + "sublayer": layer + 1, + "sources": source_count, + "mean_weights": weights, + "entropy_mean": mean(entropies), + "entropy_min": min(entropies), + "entropy_max": max(entropies), + } + ) + max_sources = max(max_sources, source_count) + + output_rows = [run["diagnostic"]["output_weights"] for run in runs] + output_source_count = output_rows[0]["sources"] + output_weights = [ + mean(row["mean_weights"][source] for row in output_rows) + for source in range(output_source_count) + ] + return { + "rows": rows, + "max_sources": max_sources, + "output": { + "sources": output_source_count, + "mean_weights": output_weights, + "entropy_mean": mean(row["entropy_mean"] for row in output_rows), + "entropy_min": min(row["entropy_mean"] for row in output_rows), + "entropy_max": max(row["entropy_mean"] for row in output_rows), + }, + } + + +def main() -> None: + args = parse_args() + manifest = read_json(args.manifest) + if manifest["protocol_id"] != PROTOCOL_ID: + raise ValueError("manifest protocol mismatch") + + runs: dict[tuple[str, int], dict[str, Any]] = {} + formal_file_hashes: dict[str, str] = {} + for seed in SEEDS: + for architecture in ARCHITECTURES: + path = args.formal_dir / f"{architecture}-{seed}.json" + run = read_json(path) + if run["protocol_id"] != PROTOCOL_ID: + raise ValueError(f"protocol mismatch: {path}") + if run["run_kind"] != "formal": + raise ValueError(f"not a formal run: {path}") + if run["architecture"] != architecture or run["seed"] != seed: + raise ValueError(f"cell identity mismatch: {path}") + if run["steps"] != 2000 or run["batch_size"] != 32: + raise ValueError(f"formal budget mismatch: {path}") + if run["target_bytes_seen"] != 16_384_000: + raise ValueError(f"target byte count mismatch: {path}") + if run["manifest"]["file_sha256"] != file_sha256(args.manifest): + raise ValueError(f"manifest file hash mismatch: {path}") + if run["manifest"]["formal_schedule_sha256"] != ( + manifest["windows"]["formal_schedule_sha256"] + ): + raise ValueError(f"schedule mismatch: {path}") + if run["evaluations"][-1]["step"] != 2000: + raise ValueError(f"missing final evaluation: {path}") + if any( + not math.isfinite(value) + for evaluation in run["evaluations"] + for value in ( + evaluation["cross_entropy_nats"], + evaluation["bits_per_byte"], + ) + ): + raise ValueError(f"non-finite evaluation: {path}") + if len(run["diagnostic"]["layer_input_rms"]) != 32: + raise ValueError(f"diagnostic depth mismatch: {path}") + if len(run["diagnostic"]["core_parameter_grad_rms_by_block"]) != 16: + raise ValueError(f"gradient depth mismatch: {path}") + runs[(architecture, seed)] = run + formal_file_hashes[path.name] = file_sha256(path) + + common_initial_exact = {} + for seed in SEEDS: + hashes = { + architecture: runs[(architecture, seed)]["hashes"][ + "initial_common_parameters" + ] + for architecture in ARCHITECTURES + } + common_initial_exact[str(seed)] = { + "hashes": hashes, + "exact": len(set(hashes.values())) == 1, + } + if not common_initial_exact[str(seed)]["exact"]: + raise ValueError(f"common initialization mismatch for seed {seed}") + + by_seed = [] + for seed in SEEDS: + values = { + architecture: runs[(architecture, seed)]["evaluations"][-1][ + "bits_per_byte" + ] + for architecture in ARCHITECTURES + } + by_seed.append( + { + "seed": seed, + "final_bpc": values, + "full_minus_baseline": values["full"] - values["baseline"], + "block_minus_baseline": values["block"] - values["baseline"], + "block_minus_full": values["block"] - values["full"], + } + ) + + final = { + "by_seed": by_seed, + "means": { + architecture: mean( + runs[(architecture, seed)]["evaluations"][-1]["bits_per_byte"] + for seed in SEEDS + ) + for architecture in ARCHITECTURES + }, + "full_contrast": verdict( + [row["full_minus_baseline"] for row in by_seed] + ), + "block_contrast": verdict( + [row["block_minus_baseline"] for row in by_seed] + ), + "block_minus_full": { + "paired_deltas_bpc": [row["block_minus_full"] for row in by_seed], + "mean_delta_bpc": mean(row["block_minus_full"] for row in by_seed), + }, + } + + evaluation_steps = [ + evaluation["step"] for evaluation in runs[("baseline", SEEDS[0])]["evaluations"] + ] + curves = {} + for architecture in ARCHITECTURES: + curve = [] + for index, step in enumerate(evaluation_steps): + values = [ + runs[(architecture, seed)]["evaluations"][index]["bits_per_byte"] + for seed in SEEDS + ] + if any( + runs[(architecture, seed)]["evaluations"][index]["step"] != step + for seed in SEEDS + ): + raise ValueError("evaluation step mismatch") + curve.append( + { + "step": step, + "mean_bpc": mean(values), + "min_bpc": min(values), + "max_bpc": max(values), + "by_seed": values, + } + ) + curves[architecture] = curve + + timing = {} + for architecture in ARCHITECTURES: + cells = [runs[(architecture, seed)]["timing"] for seed in SEEDS] + timing[architecture] = { + "mean_step_ms": mean(cell["mean_ms"] for cell in cells), + "median_step_ms": mean(cell["median_ms"] for cell in cells), + "p95_step_ms": mean(cell["p95_ms"] for cell in cells), + "mean_peak_allocated_bytes": mean( + cell["peak_allocated_bytes"] for cell in cells + ), + "mean_peak_reserved_bytes": mean( + cell["peak_reserved_bytes"] for cell in cells + ), + "by_seed": cells, + } + timing["relative_to_baseline"] = { + architecture: { + "step_time_ratio": timing[architecture]["mean_step_ms"] + / timing["baseline"]["mean_step_ms"], + "allocated_memory_ratio": timing[architecture][ + "mean_peak_allocated_bytes" + ] + / timing["baseline"]["mean_peak_allocated_bytes"], + } + for architecture in ("full", "block") + } + + parameters = { + architecture: runs[(architecture, SEEDS[0])]["model"]["parameters"] + for architecture in ARCHITECTURES + } + parameters["mixer_overhead_fraction_of_baseline"] = ( + parameters["full"]["mixer"] / parameters["baseline"]["total"] + ) + + traces = {} + gradients = {} + for architecture in ARCHITECTURES: + architecture_runs = [runs[(architecture, seed)] for seed in SEEDS] + traces[architecture] = { + key: elementwise_summary( + [run["diagnostic"][key] for run in architecture_runs] + ) + for key in ( + "layer_input_rms", + "branch_output_rms", + "stream_state_rms", + ) + } + gradient_rows = [ + run["diagnostic"]["core_parameter_grad_rms_by_block"] + for run in architecture_runs + ] + gradients[architecture] = { + "by_block": elementwise_summary(gradient_rows), + "cv_by_seed": [ + coefficient_of_variation(row) for row in gradient_rows + ], + "mean_cv": mean(coefficient_of_variation(row) for row in gradient_rows), + "first_last_ratio_by_seed": [ + row[0] / row[-1] for row in gradient_rows + ], + "mean_first_last_ratio": mean(row[0] / row[-1] for row in gradient_rows), + } + + mixers = { + architecture: average_depth_weights( + [runs[(architecture, seed)] for seed in SEEDS] + ) + for architecture in ("full", "block") + } + + full_branch = traces["full"]["branch_output_rms"]["mean"] + largest_index = max(range(len(full_branch)), key=full_branch.__getitem__) + # Full output source 0 is the embedding; branch l is source l+1. + largest_source_weight = mixers["full"]["output"]["mean_weights"][ + largest_index + 1 + ] + uniform_output_weight = 1 / mixers["full"]["output"]["sources"] + posthoc = { + "label": "post-hoc descriptive callout; not a preregistered endpoint", + "largest_full_branch_sublayer": largest_index + 1, + "largest_full_branch_rms": full_branch[largest_index], + "corresponding_final_output_weight": largest_source_weight, + "uniform_final_output_weight": uniform_output_weight, + "weight_over_uniform": largest_source_weight / uniform_output_weight, + } + + replay = read_json(args.replay) + formal_replay_source = runs[("block", 2026073001)] + replay_exact = { + field: formal_replay_source[field] == replay[field] + for field in REPLAY_FIELDS + } + if not all(replay_exact.values()): + raise ValueError(f"formal replay mismatch: {replay_exact}") + + smoke = {} + for architecture in ARCHITECTURES: + first_path = args.smoke_dir / f"{architecture}-2026073001-a.json" + second_path = args.smoke_dir / f"{architecture}-2026073001-b.json" + first = read_json(first_path) + second = read_json(second_path) + exact = {field: first[field] == second[field] for field in SMOKE_FIELDS} + if not all(exact.values()): + raise ValueError(f"smoke mismatch for {architecture}: {exact}") + smoke[architecture] = { + "fields": exact, + "all_exact": True, + "first_sha256": file_sha256(first_path), + "second_sha256": file_sha256(second_path), + } + + reproduction = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "manifest_sha256": file_sha256(args.manifest), + "formal_files": formal_file_hashes, + "common_initial_parameters": common_initial_exact, + "smoke": smoke, + "formal_replay": { + "architecture": "block", + "seed": 2026073001, + "fields": replay_exact, + "all_numeric_and_hash_fields_exact": all(replay_exact.values()), + "timing_exact_required": False, + "timing_exact_observed": formal_replay_source["timing"] == replay["timing"], + "formal_file_sha256": formal_file_hashes[ + "block-2026073001.json" + ], + "replay_file_sha256": file_sha256(args.replay), + }, + } + reproduction["canonical_sha256_without_self"] = canonical_sha256(reproduction) + + analysis = { + "final_validation": final, + "evaluation_curves": curves, + "timing": timing, + "parameters": parameters, + "traces": traces, + "gradients": gradients, + "mixers": mixers, + "posthoc": posthoc, + "interpretation": { + "primary": ( + "Both Full and Block AttnRes satisfy the preregistered " + "directional-support rule in this reduced protocol." + ), + "bounded_depth_pattern": ( + "Block partial-state RMS resets every four residual sublayers; " + "the complete 32-point vectors are reported." + ), + "gradient_boundary": ( + "The preregistered core-parameter gradient RMS is not more " + "uniform for AttnRes here; this metric and scale do not reproduce " + "the paper's large-model gradient-magnitude result." + ), + "scope": ( + "Reduced byte-level WikiText-2 mechanism probe; not a K3 " + "checkpoint run, paper-scale reproduction, benchmark, or " + "same-FLOP comparison." + ), + }, + } + + raw = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "manifest": manifest, + "provenance": { + "manifest_file_sha256": file_sha256(args.manifest), + "formal_file_sha256": formal_file_hashes, + "reproduction_sha256": reproduction[ + "canonical_sha256_without_self" + ], + }, + "formal_runs": [ + runs[(architecture, seed)] + for seed in SEEDS + for architecture in ARCHITECTURES + ], + "analysis": analysis, + "reproduction": reproduction, + } + raw["canonical_sha256_without_self"] = canonical_sha256(raw) + + compact = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "dataset": { + "repository": manifest["dataset"]["repository"], + "revision": manifest["dataset"]["revision"], + "train_bytes": manifest["dataset"]["splits"]["train"][ + "concatenated_bytes" + ], + "schedule_sha256": manifest["windows"]["formal_schedule_sha256"], + "validation_sha256": manifest["windows"][ + "validation_tensor_sha256" + ], + }, + "grid": { + "architectures": list(ARCHITECTURES), + "seeds": list(SEEDS), + "runs": 9, + "steps_per_run": 2000, + "target_bytes_per_run": 16_384_000, + "target_bytes_total": 9 * 16_384_000, + }, + "final_validation": final, + "evaluation_curves": curves, + "timing": timing, + "parameters": parameters, + "traces": traces, + "gradients": gradients, + "mixers": mixers, + "posthoc": posthoc, + "interpretation": analysis["interpretation"], + "reproduction": reproduction, + "source_sha256": raw["canonical_sha256_without_self"], + } + compact["canonical_sha256_without_self"] = canonical_sha256(compact) + + write_json(args.reproduction_output, reproduction) + write_json(args.output, raw) + write_json(args.compact_output, compact) + print( + json.dumps( + { + "output": str(args.output), + "compact_output": str(args.compact_output), + "reproduction_output": str(args.reproduction_output), + "raw_sha256": file_sha256(args.output), + "compact_sha256": file_sha256(args.compact_output), + "reproduction_sha256": file_sha256(args.reproduction_output), + "full": final["full_contrast"], + "block": final["block_contrast"], + "formal_replay": reproduction["formal_replay"], + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/k3/attnres/reproduction.json b/experiments/k3/attnres/reproduction.json new file mode 100644 index 0000000..3ec1f8c --- /dev/null +++ b/experiments/k3/attnres/reproduction.json @@ -0,0 +1,109 @@ +{ + "canonical_sha256_without_self": "89a127e625ccefbd9b749e6ffca4ba72868e217e0c7a24773024bd2fafde427a", + "common_initial_parameters": { + "2026073001": { + "exact": true, + "hashes": { + "baseline": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "block": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "full": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + } + }, + "2026073002": { + "exact": true, + "hashes": { + "baseline": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "block": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "full": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + } + }, + "2026073003": { + "exact": true, + "hashes": { + "baseline": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "block": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "full": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + } + } + }, + "formal_files": { + "baseline-2026073001.json": "f5cddecace6a70ee3824f272811d2a6336ab6dc24ad1100b18292f4413692e9a", + "baseline-2026073002.json": "4c3b9e1ea2ed6fd3078f1212984327364832a1146bfcbdb3a6f2e61b589bfaa8", + "baseline-2026073003.json": "e5ce5a8944dad0c4811f8bf19b4cfec8ce4a9283858fd59cd739dbc37626707b", + "block-2026073001.json": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "block-2026073002.json": "be1f109a630e27c8469438e33b53806cdaecac2f712af4f885da83f00ef6843b", + "block-2026073003.json": "949ec51ca3a219a11f260171da01f737296241536f5e139cf07d444c27efba92", + "full-2026073001.json": "648cb25ea98da1868779733da155275e9a16aad5efbbeeced7812c19d75817d5", + "full-2026073002.json": "1fa715bbd81a3a04a5aa0ba0fc27feb8883c44a01e9f28e281a309a394447979", + "full-2026073003.json": "4e34fa35bbfec460cabc94bce7a08f44fc188d908705ae2e256500f00813b4e3" + }, + "formal_replay": { + "all_numeric_and_hash_fields_exact": true, + "architecture": "block", + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "formal_file_sha256": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "replay_file_sha256": "e74d3323e5fe31378bb8aad7a8efa2fb91995cd7224c158cf03006466cdea2a7", + "seed": 2026073001, + "timing_exact_observed": false, + "timing_exact_required": false + }, + "manifest_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "schema_version": 1, + "smoke": { + "baseline": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c", + "second_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c" + }, + "block": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc", + "second_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc" + }, + "full": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb", + "second_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb" + } + } +} diff --git a/research/K3_ATTNRES_REDUCED_AUDIT.md b/research/K3_ATTNRES_REDUCED_AUDIT.md new file mode 100644 index 0000000..cc69f23 --- /dev/null +++ b/research/K3_ATTNRES_REDUCED_AUDIT.md @@ -0,0 +1,568 @@ +# Kimi K3 第四轮：Attention Residuals 缩小版独立机制实验审计 + +> 协议：`llm-atlas-k3-attnres-reduced-v1` +> +> 预注册：`research/K3_ATTNRES_REDUCED_PROTOCOL.md` +> +> 数据清单：`experiments/k3/attnres/manifest.json` +> +> 执行日期：2026-07-30 +> +> 执行设备：NVIDIA GeForce RTX 5090；PyTorch `2.11.0+cu128` + +## 0. 先说结论 + +这一轮没有伪装成“跑通了 K3”。它做的是一个刻意缩小、从零训练、可公开复查的 +Attention Residuals（AttnRes）机制实验： + +```text +3 个结构 +× 3 个预先冻结的初始化 seed +× 每格 2,000 steps +× 每步 32 × 256 target bytes += 9 个正式训练格，147,456,000 target bytes +``` + +三个结构共享完全相同的 16 个 Transformer blocks、32 个残差子层、初始公共参数、 +训练窗口、优化器与验证集。唯一设计变量是“前面产生的 residual states 怎样供下一层读取”： + +```text +Baseline：只读上一残差状态 +Full AttnRes：对全部历史残差状态做按维 softmax 混合 +Block AttnRes：每 4 个残差子层形成一个局部块，块间再混合 +``` + +在冻结的 64 个验证窗口上，最终 bits per byte（BPC，越低越好）为： + +| seed | Baseline | Full | Block | Full − Base | Block − Base | +|---:|---:|---:|---:|---:|---:| +| 2026073001 | 2.00054 | 1.98454 | 1.94788 | −0.01600 | −0.05266 | +| 2026073002 | 1.99844 | 1.98523 | 1.95709 | −0.01321 | −0.04135 | +| 2026073003 | 1.99842 | 1.98392 | 1.96503 | −0.01450 | −0.03339 | +| 三 seed 均值 | 1.99913 | 1.98457 | 1.95667 | **−0.01457** | **−0.04247** | + +按照看结果前冻结的判据——三个 seed 同为负，并且均值不高于 `−0.010 BPC`——Full 与 +Block 都得到： + +> **directional support in this reduced protocol** + +中文应该读成： + +> 在这套缩小训练合同中，允许子层重新读取更早的残差状态，方向一致地改善了验证 BPC。 + +它**不应该**读成： + +- 已复现 AttnRes 论文的大模型收益； +- 已运行 Kimi K3 checkpoint； +- 已证明 Block 一般优于 Full； +- 已得到同参数、同 FLOPs 或同 wall time 的优势； +- 三个 seed 可以支持总体显著性、置信区间或 scaling-law 外推。 + +本轮还有一个同样重要的反结果：预注册的“16 个 Transformer blocks 的核心参数梯度 RMS +变异系数”在本实验中，Baseline 为 `0.3447`，Full 为 `0.5087`，Block 为 `0.6306`。 +在这个定义和尺度下，AttnRes **没有**表现出更均匀的跨深度梯度。这个结果与论文的大模型 +梯度叙述不能直接对齐，网站必须把它作为边界而不是藏起来。 + +--- + +## 1. 为什么不是直接跑 K3 + +### 1.1 公开 checkpoint 的未决形状冲突 + +第三轮工件审计在 K3 第一层 KDA 中发现： + +| 官方工件 | `A_log` 所要求或实际给出的形状 | +|---|---| +| `config.json` | `num_heads = 96` | +| Hugging Face remote code | 按 `num_heads` 构造，即 `[96]` | +| FlashKDA kernel API | `[H]`，K3 中应为 `[96]` | +| checkpoint safetensors header | `[128]` | + +本轮再次检查了当前官方 Hugging Face / GitHub 工件，并额外检查当前 vLLM 与 SGLang +Kimi K3 loader。两者都按 local head 维构造和切分 `A_log`，没有公开 `128 → 96` 的转换规则。 + +因此，以下做法都会越过证据： + +```text +裁掉最后 32 个值 +把 128 强行 reshape 成别的语义 +把 128 解释成 head_dim +绕过 loader 后把输出叫作“K3 forward” +``` + +在 Moonshot 给出转换合同、修订权重，或一个官方 loader 明确处理这 32 个额外值之前， +本站不制造“真实 K3 前向结果”。 + +### 1.2 为什么缩小版仍有价值 + +不能诚实执行 1.56 TB checkpoint，不等于只能停在架构示意图。AttnRes 的核心问题可以被 +缩成一个更小、但仍可被证伪的问题： + +> 在相同 Transformer 主干、相同输入窗口与相同初始化下，把“固定单位 residual +> connection”替换成“学习的历史 residual 混合”，短预算训练是否出现一致方向？ + +这个问题不依赖 KDA、MLA、MoE、MXFP4、视觉塔或完整 K3 参数。它只检验 AttnRes +的局部机制方向，并且可以把全部代码、清单、聚合指标和复现哈希开源。 + +--- + +## 2. 先冻结什么 + +首版协议在正式结果产生前冻结以下项目： + +- 数据仓库、revision、config、split 拼接规则与 byte tokenizer； +- 模型层数、宽度、head 数、FFN 宽度、位置编码、RMSNorm 与激活函数； +- Baseline / Full / Block 的精确定义； +- 三个初始化 seed； +- 每格训练步数、batch、context 与总 target bytes； +- AdamW、学习率曲线、weight decay、gradient clipping； +- 验证步、固定验证窗口与主指标； +- 主对比、方向判据和“不能宣称什么”； +- 深度 RMS、混合权重与梯度诊断； +- smoke 与正式独立进程 replay 合同。 + +正式协议不是从本轮曲线倒推的。关键提交顺序为： + +```text +9039de1 research: preregister reduced AttnRes study +e361753 research: lock reduced AttnRes model contract +f998720 research: add reduced AttnRes runner +1f20f81 research: freeze reduced AttnRes corpus +``` + +### 2.1 一次被公开保留的预训练故障 + +第一次 smoke 在 step 0 的验证 forward 后停止： + +```text +RuntimeError: +view size is not compatible with input tensor's size and stride +``` + +原因是切片后的 target tensor 不连续，而 loss 路径用了 `.view()`。当时： + +- 尚未执行一个 optimizer step； +- 没有正式结果文件； +- 没有任何条件的训练或最终 BPC 可供选择。 + +修复仅把 `.view()` 改为语义等价且支持非连续输入的 `.reshape()`： + +```text +5f49906 fix: flatten noncontiguous AttnRes targets +``` + +随后三个结构分别完成两次独立 smoke，冻结字段逐字段 exact。这个故障不改变实验设计， +但应留在审计链中，避免“第一次就完美运行”的虚假叙事。 + +--- + +## 3. 数据合同：文本怎样变成 byte 任务 + +### 3.1 固定数据源 + +| 项目 | 固定值 | +|---|---| +| repository | `Salesforce/wikitext` | +| revision | `b08601e04326c79dfdd32d625aee71d232d685c3` | +| config | `wikitext-2-raw-v1` | +| 行处理 | `(text or "") + "\n"` | +| 编码 | UTF-8 | +| tokenizer | byte ID `0..255` | +| vocabulary | 256 | + +拼接后的 split： + +| split | bytes | SHA-256 | +|---|---:|---| +| train | 10,951,563 | `0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4` | +| validation | 1,148,008 | `a42356f6a8ff1d25daf25ec9db49e10a537c265581b61c74604bb63231dee719` | +| test | 1,292,014 | `bfe9eb16ab9987fb88bde4ea9a30a00f2a45db01dfc14bad78d05325789c4f12` | + +这里的 byte tokenizer 不是为了追求最佳语言模型性能，而是为了移除另一个潜在变量： +不同 BPE 模型、词表和 normalization。BPC 也因此可以直接比较，而不受 tokenization +长度变化影响。 + +### 3.2 无状态窗口计划 + +训练窗口不靠进程内 RNG 顺序产生。对每个： + +```text +architecture + seed + step + batch row +``` + +协议用 SHA-256 派生 train start offset。三种结构在同一 seed 下使用相同窗口；结构名不进入 +窗口选择的有效随机盐。冻结计划包含 192,000 个起点，SHA-256 为： + +```text +81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48 +``` + +固定 64 个验证窗口 tensor hash： + +```text +5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20 +``` + +固定 16 个诊断窗口 hash： + +```text +d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818 +``` + +--- + +## 4. 模型合同：只改变 residual 读取拓扑 + +### 4.1 公共主干 + +| 项目 | 值 | +|---|---:| +| Transformer blocks | 16 | +| residual sublayers | 32（每 block attention + FFN） | +| model width | 192 | +| attention heads | 6 | +| head dimension | 32 | +| SwiGLU hidden | 768 | +| context | 256 bytes | +| dropout | 0 | +| position | learned absolute embedding | +| norm | pre-RMSNorm + final RMSNorm | +| attention softmax | causal, float32 | +| embedding / LM head | tied | + +Attention projection 与 SwiGLU 都不使用 bias。三个结构的公共 core 参数均为 +`9,541,824`，每个 seed 的公共参数初始化哈希在三种结构间 exact。 + +### 4.2 Baseline + +普通 residual 子层： + +```text +x_(l+1) = x_l + F_l(RMSNorm(x_l)) +``` + +它只保留一个随深度持续累积的 residual stream。 + +### 4.3 Full AttnRes + +第 `l` 个子层先对从 embedding 到当前深度的全部 residual states 做学习混合： + +```text +α_l = softmax(q_l · RMSNorm(states)) +x̃_l = Σ_i α_(l,i) state_i +state_(l+1) = F_l(RMSNorm(x̃_l)) +``` + +`q_l` 是按 hidden dimension 学习的 pseudoquery，初始化为 0，所以初始 softmax 为均匀 +读取。最后还有一个 output mixer，把 33 个可见 sources 混成 LM head 的输入。 + +### 4.4 Block AttnRes + +32 个 residual sublayers 被分成 8 块，每块 4 层： + +```text +块内：新 branch state 做普通局部累加 +块边界：对历史块状态做学习混合，产生下一块输入 +``` + +它保留“可以回读历史”的机制，同时把 Full 随深度增长的状态集合限制在块级。 + +### 4.5 参数公平与计算不公平 + +| 结构 | core | mixer | total | 相对 Baseline mixer overhead | +|---|---:|---:|---:|---:| +| Baseline | 9,541,824 | 0 | 9,541,824 | 0 | +| Full | 9,541,824 | 12,672 | 9,554,496 | 0.1328% | +| Block | 9,541,824 | 12,672 | 9,554,496 | 0.1328% | + +这是近似同参数，不是同 FLOPs。当前教学实现用 PyTorch eager 保存和混合历史 states， +没有使用论文的大模型优化 kernel，所以它适合机制观察，不适合推断生产吞吐。 + +--- + +## 5. 训练与主指标 + +### 5.1 每格预算 + +```text +2,000 steps +× batch 32 +× context 256 target bytes += 16,384,000 target bytes / run +``` + +优化器： + +```text +AdamW β=(0.9, 0.95), ε=1e-8 +peak LR=3e-4, min LR=3e-5 +100-step warmup + cosine decay +weight decay=0.1 for ndim>=2 +global grad clip=1.0 +BF16 autocast +``` + +验证发生在 `0, 100, 250, 500, 1000, 1500, 2000` steps。主指标只使用 step 2000 +的固定验证 BPC；曲线用于帮助理解，不用于重新选择终点。 + +### 5.2 判据为什么这么保守 + +只有三个 seed，不能可靠估计总体方差或给出有意义的 population confidence interval。 +所以预注册不用 p-value，而只问两个简单问题： + +```text +三个 paired deltas 是否同方向？ +mean delta 是否至少达到 0.010 BPC？ +``` + +如果答案都是“是”，只写作本协议内的 directional support / concern。它是一道防止 +夸大结论的阈值，不是一个通用显著性标准。 + +### 5.3 正式结果 + +Full 的三组配对差： + +```text +−0.015996 +−0.013207 +−0.014498 +mean = −0.014567 BPC +``` + +Block 的三组配对差： + +```text +−0.052660 +−0.041349 +−0.033388 +mean = −0.042466 BPC +``` + +两组都满足预注册的方向支持规则。Block 相对 Full 的均值差为 `−0.027898 BPC`，三个 +seed 也同为负；但“Block − Full”不是预注册主判据，而且本实现的计算图与优化效率不同， +所以它只适合描述，不升级成一般性排名。 + +--- + +## 6. 它花了多少计算与显存 + +排除每次前 20 个计时 warmup steps 后，三个 seed 的均值： + +| 结构 | mean step | 相对 Baseline | peak allocated | 相对 Baseline | +|---|---:|---:|---:|---:| +| Baseline | 21.62 ms | 1.00× | 3.04 GB | 1.00× | +| Full | 146.81 ms | 6.79× | 14.26 GB | 4.70× | +| Block | 54.63 ms | 2.53× | 6.52 GB | 2.15× | + +这张表应该怎样读： + +- Full 在教学实现中为每层保留并读取更多历史 states，因此最贵； +- Block 把可见历史限制在块级，成本明显下降； +- 时间与显存是“这份 PyTorch eager 实现 + RTX 5090”的观测； +- 不能把 6.79× / 2.53× 外推到论文 kernel、K3 训练系统或生产推理； +- BPC 改善不能被写成同 FLOPs 改善。 + +--- + +## 7. 看 residual stream：Block 为什么呈现锯齿 + +Baseline 的 stream-state RMS 从平均 `0.0661` 增到 `0.2110`：普通 residual +connection 把分支输出一路累积。 + +Block 的前 12 个 partial-state RMS 是： + +```text +0.0334 0.0598 0.0967 0.1181 +0.0509 0.0569 0.0745 0.0815 +0.0532 0.0581 0.1079 0.1158 +``` + +每四个值形成一个局部块： + +```text +块内：逐层累积，RMS 通常上升 +块间：重新从历史块状态混合，partial stream 被重置 +``` + +所以曲线不是训练不稳定造成的随机锯齿，而是 Block 拓扑的直接几何痕迹。完整 32 点向量、 +三个 seed 的 min / mean / max 均进入公开 JSON。 + +Full 的 layer-input RMS 则从 `0.0544` 降到 `0.00866`。这不等于信息“消失”： +每层输入是多个经过 RMSNorm 的历史 states 的学习加权和，混合可以通过方向抵消改变 +合成向量的 RMS。只看单个标量不能判断信息保留量。 + +--- + +## 8. mixer 权重：一个事后但有解释力的观察 + +Full 在第 31 个 residual sublayer 的 branch-output RMS 平均达到 `1.4964`，是 32 层中 +最大值。但最终 output mixer 给这个 source 的平均权重只有： + +```text +observed = 0.002794 +uniform = 1 / 33 = 0.030303 +ratio = 0.0922× uniform +``` + +一个直观解释是：输出混合器学会了压低这个幅值突增的 source，而不是被迫把它以单位 +residual 权重传到输出。 + +必须同时保留两个限制： + +1. “最大 spike 对应低权重”是看完完整 trace 后挑出的描述； +2. 它不是预注册 endpoint，不能作为独立确认性证据。 + +网站会明确标注 **post-hoc descriptive callout**，并展示完整深度权重图，让读者看到它 +不是从被隐藏的其他 source 中挑出的孤立数字。 + +--- + +## 9. 梯度结果没有复现论文叙述 + +预注册诊断对每个 Transformer block 的公共 core 参数计算 gradient RMS，再求 16 个 +block 间的 coefficient of variation： + +| 结构 | seed 1 | seed 2 | seed 3 | mean CV | +|---|---:|---:|---:|---:| +| Baseline | 0.3537 | 0.3409 | 0.3396 | **0.3447** | +| Full | 0.4890 | 0.5100 | 0.5272 | **0.5087** | +| Block | 0.6622 | 0.6849 | 0.5448 | **0.6306** | + +CV 越低，按这个特定定义才越均匀。因此本轮观察是： + +```text +Baseline < Full < Block +``` + +这和 AttnRes 论文在大模型训练中报告的、更平坦的跨深度梯度幅值叙述不是同一个结果。 +合理边界包括： + +- 本实验只有 width 192、16 blocks、2,000 steps； +- 本指标是“按 block 汇总的核心参数 gradient RMS”； +- 论文图可能观察 activation / residual-output gradients，聚合对象并不相同； +- byte-level WikiText-2 与论文的大规模训练数据、优化器状态和训练阶段不同。 + +正确表述是： + +> 本缩小实验的主 BPC 对比支持 AttnRes 的方向，但预注册的核心参数梯度均匀性指标不支持 +> 论文式叙述；这提示该解释可能依赖尺度、指标定义或训练阶段，需要后续专门实验。 + +不正确的做法是改换一个看起来更漂亮的梯度统计后，只展示新指标。 + +--- + +## 10. 复现链 + +### 10.1 smoke + +三个结构各执行两次独立的 20-step smoke。以下字段对每个结构都 exact： + +```text +manifest +model +optimizer +hashes +evaluations +training_history +diagnostic +environment +``` + +### 10.2 正式独立进程 replay + +预先指定： + +```text +architecture = block +seed = 2026073001 +steps = 2000 +``` + +正式格与 fresh-process replay 的最终 BPC 都是： + +```text +1.947877975922777 +``` + +上面的八组冻结字段全部 exact。计时不要求 exact，实际也不相等，因为 wall time 受系统 +调度影响。 + +| 文件 | SHA-256 | +|---|---| +| formal block / seed 1 | `5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf` | +| fresh replay | `e74d3323e5fe31378bb8aad7a8efa2fb91995cd7224c158cf03006466cdea2a7` | + +### 10.3 公开产物 + +| 产物 | 内容 | SHA-256 | +|---|---|---| +| `src/data/k3-attnres-reduced.json` | 9 个完整 run + 聚合 + 复现记录 | `44f8622654d32485f8d6e698c02ba1365ddffb10cbd73db0294136d0bd93ce88` | +| `src/data/k3-attnres-reduced-compact.json` | 网站所需完整曲线与诊断 | `44864d48eddb2ae5887fba4b74f63b5a3d6a23497886ee307decf5b4f45d9faf` | +| `experiments/k3/attnres/reproduction.json` | smoke、初始化与 replay audit | `545543b7e4a970ca3bc0e6246610a32fb53ec3d9546a98e0f17f38d7121918a2` | + +聚合器在同一批只读 run 文件上再次运行后，三个文件 SHA-256 全部不变。 + +--- + +## 11. 证据等级 + +### A. 本轮可直接主张 + +- 冻结协议下 9 个训练格的最终 BPC 与完整验证曲线； +- Full / Block 相对 Baseline 的三 seed 配对方向； +- 当前实现的参数量、实测 step time 与 peak allocated memory； +- 固定诊断窗口上的 residual RMS、mixer 权重与参数梯度统计； +- smoke 与指定正式格的独立进程 exact replay； +- 数据、窗口、初始化和产物 SHA-256。 + +### B. 只能作为机制解释 + +- Block 的四层锯齿与块边界重混合一致； +- Full mixer 可能通过降低权重抑制高 RMS source； +- Full / Block 的短预算优势可能来自更灵活的深度路由。 + +这些解释与观测相容，但不是唯一因果解释。 + +### C. 本轮明确不主张 + +- K3 checkpoint 已成功 forward 或训练； +- 论文表格、Figure 4–8 或 paper-scale scaling 已复现； +- AttnRes 在任意模型、数据和预算上都降低 loss； +- Block 一般优于 Full； +- 梯度在 AttnRes 中更均匀； +- 同 FLOPs、同 wall time 或生产系统的性价比优势； +- 三个 seed 支持总体统计显著性。 + +--- + +## 12. 下一步 + +这个实验把“能运行的独立机制 probe”完成了，但真实 K3 仍有两道门： + +1. `A_log [128]` 的官方转换或权重修订； +2. 能加载完整或官方切分 K3 的受支持执行环境。 + +AttnRes 本身的下一轮也不应只增加 seed。优先级更高的是： + +- 对齐论文实际使用的 activation / output-gradient 诊断定义； +- 增加 depth 与训练预算，检验梯度结论是否随尺度翻转； +- 做 mixer 计算的优化实现，再讨论同 wall-time 或近似同-FLOP 对比； +- 冻结一个更强 tokenizer / corpus 后检查 byte-level 结论是否保持； +- 将 Block size 作为预注册变量，而不是看完结果后挑 4。 + +--- + +## 13. 一手来源 + +- [Kimi K3 Technical Report](https://arxiv.org/abs/2607.24653) +- [Kimi K3 official checkpoint](https://huggingface.co/moonshotai/Kimi-K3) +- [Kimi K3 official code repository](https://github.com/MoonshotAI/Kimi-K3) +- [Attention Residuals](https://arxiv.org/abs/2603.15031) +- [Official Attention Residuals implementation](https://github.com/MoonshotAI/Attention-Residuals) +- [WikiText dataset repository](https://huggingface.co/datasets/Salesforce/wikitext) +- [vLLM Kimi K3 implementation](https://github.com/vllm-project/vllm) +- [SGLang Kimi K3 implementation](https://github.com/sgl-project/sglang) + +Grok CLI 在协议冻结前只承担一次对抗式方法审阅：它提出锁定参数容量、残差拓扑、数据顺序、 +指标定义和复现合同的检查项。所有论文事实和实验结论仍由一手来源、冻结代码与本地运行产物 +支持；Grok 输出不作为证据来源。 diff --git a/src/data/k3-attnres-reduced-compact.json b/src/data/k3-attnres-reduced-compact.json new file mode 100644 index 0000000..fe4a6cb --- /dev/null +++ b/src/data/k3-attnres-reduced-compact.json @@ -0,0 +1,3106 @@ +{ + "canonical_sha256_without_self": "da86221c5bc56d9d09747b72c06ccd77a5d4b49bcdaab9be5712cbf6e668a77c", + "dataset": { + "repository": "Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "train_bytes": 10951563, + "validation_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "evaluation_curves": { + "baseline": [ + { + "by_seed": [ + 8.095421586117522, + 8.037616927292651, + 8.032852405924473 + ], + "max_bpc": 8.095421586117522, + "mean_bpc": 8.055296973111549, + "min_bpc": 8.032852405924473, + "step": 0 + }, + { + "by_seed": [ + 3.86267014445088, + 3.8739961479147755, + 3.864131094074297 + ], + "max_bpc": 3.8739961479147755, + "mean_bpc": 3.8669324621466505, + "min_bpc": 3.86267014445088, + "step": 100 + }, + { + "by_seed": [ + 3.431632090385233, + 3.4334316738481987, + 3.4184366785021214 + ], + "max_bpc": 3.4334316738481987, + "mean_bpc": 3.427833480911851, + "min_bpc": 3.4184366785021214, + "step": 250 + }, + { + "by_seed": [ + 2.9448990671750175, + 2.9345221069742453, + 2.907037430552872 + ], + "max_bpc": 2.9448990671750175, + "mean_bpc": 2.9288195349007116, + "min_bpc": 2.907037430552872, + "step": 500 + }, + { + "by_seed": [ + 2.2560981455190916, + 2.2564510109229996, + 2.249317836997982 + ], + "max_bpc": 2.2564510109229996, + "mean_bpc": 2.2539556644800243, + "min_bpc": 2.249317836997982, + "step": 1000 + }, + { + "by_seed": [ + 2.0700625342803067, + 2.0709166216223225, + 2.061515899441342 + ], + "max_bpc": 2.0709166216223225, + "mean_bpc": 2.0674983517813237, + "min_bpc": 2.061515899441342, + "step": 1500 + }, + { + "by_seed": [ + 2.0005383326907027, + 1.9984403593284898, + 1.9984182165621913 + ], + "max_bpc": 2.0005383326907027, + "mean_bpc": 1.9991323028604613, + "min_bpc": 1.9984182165621913, + "step": 2000 + } + ], + "block": [ + { + "by_seed": [ + 8.091490664658076, + 8.035310725935906, + 8.029935924131141 + ], + "max_bpc": 8.091490664658076, + "mean_bpc": 8.052245771575041, + "min_bpc": 8.029935924131141, + "step": 0 + }, + { + "by_seed": [ + 3.8613770068990427, + 3.8667511637688663, + 3.861173508427409 + ], + "max_bpc": 3.8667511637688663, + "mean_bpc": 3.8631005596984394, + "min_bpc": 3.861173508427409, + "step": 100 + }, + { + "by_seed": [ + 3.4246572049925037, + 3.422946407573046, + 3.427078118777374 + ], + "max_bpc": 3.427078118777374, + "mean_bpc": 3.4248939104476412, + "min_bpc": 3.422946407573046, + "step": 250 + }, + { + "by_seed": [ + 2.890138823737888, + 2.8856276543122625, + 2.9073413023991153 + ], + "max_bpc": 2.9073413023991153, + "mean_bpc": 2.8943692601497553, + "min_bpc": 2.8856276543122625, + "step": 500 + }, + { + "by_seed": [ + 2.193909756412884, + 2.199974509617247, + 2.2110541049381194 + ], + "max_bpc": 2.2110541049381194, + "mean_bpc": 2.2016461236560834, + "min_bpc": 2.193909756412884, + "step": 1000 + }, + { + "by_seed": [ + 2.01564146212851, + 2.028303792287441, + 2.0265933173354536 + ], + "max_bpc": 2.028303792287441, + "mean_bpc": 2.023512857250468, + "min_bpc": 2.01564146212851, + "step": 1500 + }, + { + "by_seed": [ + 1.947877975922777, + 1.9570916454731129, + 1.9650305581174223 + ], + "max_bpc": 1.9650305581174223, + "mean_bpc": 1.9566667265044373, + "min_bpc": 1.947877975922777, + "step": 2000 + } + ], + "full": [ + { + "by_seed": [ + 8.053220311266156, + 8.015559292406241, + 8.006702013904155 + ], + "max_bpc": 8.053220311266156, + "mean_bpc": 8.025160539192184, + "min_bpc": 8.006702013904155, + "step": 0 + }, + { + "by_seed": [ + 3.87981092433611, + 3.8904535117411396, + 3.877691539133865 + ], + "max_bpc": 3.8904535117411396, + "mean_bpc": 3.8826519917370383, + "min_bpc": 3.877691539133865, + "step": 100 + }, + { + "by_seed": [ + 3.4658855739879395, + 3.4716255379561485, + 3.4643856702941025 + ], + "max_bpc": 3.4716255379561485, + "mean_bpc": 3.4672989274127297, + "min_bpc": 3.4643856702941025, + "step": 250 + }, + { + "by_seed": [ + 2.9766696157493864, + 2.996736035504106, + 2.967935433829656 + ], + "max_bpc": 2.996736035504106, + "mean_bpc": 2.98044702836105, + "min_bpc": 2.967935433829656, + "step": 500 + }, + { + "by_seed": [ + 2.2318612109744427, + 2.2425945407256336, + 2.2438350085906444 + ], + "max_bpc": 2.2438350085906444, + "mean_bpc": 2.2394302534302404, + "min_bpc": 2.2318612109744427, + "step": 1000 + }, + { + "by_seed": [ + 2.05077289566604, + 2.051821925342809, + 2.046430935671032 + ], + "max_bpc": 2.051821925342809, + "mean_bpc": 2.0496752522266273, + "min_bpc": 2.046430935671032, + "step": 1500 + }, + { + "by_seed": [ + 1.9845419683599765, + 1.9852328656641551, + 1.9839206165402405 + ], + "max_bpc": 1.9852328656641551, + "mean_bpc": 1.984565150188124, + "min_bpc": 1.9839206165402405, + "step": 2000 + } + ] + }, + "final_validation": { + "block_contrast": { + "max_delta_bpc": -0.033387658444768986, + "mean_delta_bpc": -0.04246557635602392, + "min_delta_bpc": -0.0526603567679258, + "paired_deltas_bpc": [ + -0.0526603567679258, + -0.04134871385537697, + -0.033387658444768986 + ], + "same_direction": true, + "threshold_bpc": 0.01, + "verdict": "directional support in this reduced protocol" + }, + "block_minus_full": { + "mean_delta_bpc": -0.027898423683686707, + "paired_deltas_bpc": [ + -0.036663992437199644, + -0.02814122019104226, + -0.018890058422818212 + ] + }, + "by_seed": [ + { + "block_minus_baseline": -0.0526603567679258, + "block_minus_full": -0.036663992437199644, + "final_bpc": { + "baseline": 2.0005383326907027, + "block": 1.947877975922777, + "full": 1.9845419683599765 + }, + "full_minus_baseline": -0.015996364330726154, + "seed": 2026073001 + }, + { + "block_minus_baseline": -0.04134871385537697, + "block_minus_full": -0.02814122019104226, + "final_bpc": { + "baseline": 1.9984403593284898, + "block": 1.9570916454731129, + "full": 1.9852328656641551 + }, + "full_minus_baseline": -0.013207493664334713, + "seed": 2026073002 + }, + { + "block_minus_baseline": -0.033387658444768986, + "block_minus_full": -0.018890058422818212, + "final_bpc": { + "baseline": 1.9984182165621913, + "block": 1.9650305581174223, + "full": 1.9839206165402405 + }, + "full_minus_baseline": -0.014497600021950774, + "seed": 2026073003 + } + ], + "full_contrast": { + "max_delta_bpc": -0.013207493664334713, + "mean_delta_bpc": -0.014567152672337214, + "min_delta_bpc": -0.015996364330726154, + "paired_deltas_bpc": [ + -0.015996364330726154, + -0.013207493664334713, + -0.014497600021950774 + ], + "same_direction": true, + "threshold_bpc": 0.01, + "verdict": "directional support in this reduced protocol" + }, + "means": { + "baseline": 1.9991323028604613, + "block": 1.9566667265044373, + "full": 1.984565150188124 + } + }, + "gradients": { + "baseline": { + "by_block": { + "max": [ + 0.00075677209292545, + 0.0005532402766490476, + 0.00043713799901330333, + 0.00033044463575291374, + 0.0003495170932308389, + 0.00035395098520716656, + 0.00041254654039553357, + 0.0005141244473775357, + 0.0006036197919264307, + 0.0006948711477092932, + 0.0008904295974468759, + 0.00101380642253527, + 0.0008315976940648109, + 0.000694730303324183, + 0.0005421243248268182, + 0.00042412043092329184 + ], + "mean": [ + 0.0007019380262137629, + 0.0004973613932330452, + 0.0003913067103051371, + 0.0003113512617537747, + 0.00031526216215974653, + 0.0003354778900867988, + 0.0003856618618192619, + 0.00046334355513234175, + 0.000584073639160152, + 0.0006482042032616321, + 0.0008321813718249159, + 0.0008802390721549605, + 0.0008241068628698583, + 0.0006579748827421658, + 0.0005357449999541818, + 0.0004032229752138717 + ], + "min": [ + 0.0006069268184991451, + 0.0004268615003720108, + 0.000323043032117742, + 0.0002914754177093503, + 0.00028453589406033573, + 0.0003157797529074047, + 0.00036398451532517026, + 0.0004021196515771015, + 0.000555786501522983, + 0.0005651240708116888, + 0.0007729757575868522, + 0.0007932171508463918, + 0.0008177916178264611, + 0.0006303107513205802, + 0.0005291293137127175, + 0.00039202598310552093 + ] + }, + "cv_by_seed": [ + 0.35371966889493056, + 0.34086455023848533, + 0.33957005977765514 + ], + "first_last_ratio_by_seed": [ + 1.4310247143196844, + 1.9304130989750008, + 1.8858264656200423 + ], + "mean_cv": 0.34471809297035705, + "mean_first_last_ratio": 1.7490880929715757 + }, + "block": { + "by_block": { + "max": [ + 0.0006136196561721271, + 0.0004981167249238301, + 0.0003300655422364756, + 0.00024049221342133134, + 0.0002907770609625826, + 0.00041448514649562707, + 0.00042775107615344534, + 0.00039679767722893174, + 0.0006871599037030366, + 0.0009600722047358557, + 0.0016852072794625043, + 0.0012425342652473243, + 0.0011452921321391885, + 0.0007695163028177718, + 0.0005286419550644225, + 0.00042053944700778587 + ], + "mean": [ + 0.0004888831050398458, + 0.00044961389290692, + 0.0003041330266206076, + 0.00023643267595969324, + 0.00027324420174070326, + 0.0002986754642657978, + 0.00036172653847793204, + 0.00034532391576277656, + 0.0006077407773615036, + 0.0008327782910825645, + 0.001588981506880309, + 0.001219213916530425, + 0.0010906193570424788, + 0.0007477400770465019, + 0.0004999503819265139, + 0.0004088396118379981 + ], + "min": [ + 0.0003999679822384072, + 0.00042300100546983135, + 0.00028404817531530786, + 0.0002340430140301531, + 0.00025161489682913623, + 0.0002331203443329733, + 0.00032361633132051105, + 0.00031515007512692854, + 0.00046013207297086815, + 0.0006049664535646685, + 0.001431364890916451, + 0.0012053624438772336, + 0.0010361348995012233, + 0.0007327317391797293, + 0.00045754647605205013, + 0.0003918054076853068 + ] + }, + "cv_by_seed": [ + 0.6622010179556602, + 0.6848619013329328, + 0.5448410226471907 + ], + "first_last_ratio_by_seed": [ + 1.156343602773591, + 0.9657004079436908, + 1.459125084550669 + ], + "mean_cv": 0.6306346473119279, + "mean_first_last_ratio": 1.1937230317559837 + }, + "full": { + "by_block": { + "max": [ + 0.0006630627553059442, + 0.000431366526533741, + 0.0002935829834809252, + 0.00027276189477100897, + 0.0002558489145159978, + 0.0004401354335484962, + 0.00044057723716284654, + 0.0004762864793838318, + 0.0008786784638329018, + 0.0010455563665680043, + 0.0014801483865081893, + 0.0012357605502724435, + 0.0009144618812121531, + 0.0007776715348883125, + 0.0006049584856703907, + 0.0005101241707709288 + ], + "mean": [ + 0.0006167513874874072, + 0.00040618462626699743, + 0.0002733799653117713, + 0.0002419244417044836, + 0.00024695137499203994, + 0.0003522638252622234, + 0.00039760768314637185, + 0.00043278080669132644, + 0.0006604308069081531, + 0.0009861650947710283, + 0.0012433582725884958, + 0.0010459425806746048, + 0.0008832753785975217, + 0.0007581138332205081, + 0.0005866601291828091, + 0.0004765533868027066 + ], + "min": [ + 0.0005889504686640395, + 0.0003745185363114891, + 0.0002580782374219271, + 0.00021098033769564252, + 0.00022993427970632987, + 0.00029749385150172825, + 0.0003254079422057709, + 0.0003844568143161905, + 0.00048201456346138433, + 0.0009251527144282604, + 0.0010490551061311498, + 0.0009276232454397661, + 0.0008520028126550238, + 0.0007384596329355597, + 0.0005570695353753757, + 0.0004474027776125591 + ] + }, + "cv_by_seed": [ + 0.48897346891847043, + 0.5100252960815966, + 0.5272225078814977 + ], + "first_last_ratio_by_seed": [ + 1.24742435749958, + 1.4820264613559055, + 1.1727359195470821 + ], + "mean_cv": 0.5087404242938549, + "mean_first_last_ratio": 1.300728912800856 + } + }, + "grid": { + "architectures": [ + "baseline", + "full", + "block" + ], + "runs": 9, + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ], + "steps_per_run": 2000, + "target_bytes_per_run": 16384000, + "target_bytes_total": 147456000 + }, + "interpretation": { + "bounded_depth_pattern": "Block partial-state RMS resets every four residual sublayers; the complete 32-point vectors are reported.", + "gradient_boundary": "The preregistered core-parameter gradient RMS is not more uniform for AttnRes here; this metric and scale do not reproduce the paper's large-model gradient-magnitude result.", + "primary": "Both Full and Block AttnRes satisfy the preregistered directional-support rule in this reduced protocol.", + "scope": "Reduced byte-level WikiText-2 mechanism probe; not a K3 checkpoint run, paper-scale reproduction, benchmark, or same-FLOP comparison." + }, + "mixers": { + "block": { + "max_sources": 9, + "output": { + "entropy_max": 1.9263067245483398, + "entropy_mean": 1.911759614944458, + "entropy_min": 1.89408540725708, + "mean_weights": [ + 0.0960588405529658, + 0.06008275722463926, + 0.053474895656108856, + 0.04561992113788923, + 0.04707453027367592, + 0.07797466466824214, + 0.09173511217037837, + 0.2304698278506597, + 0.29750946164131165 + ], + "sources": 9 + }, + "rows": [ + { + "entropy_max": 0.0, + "entropy_mean": 0.0, + "entropy_min": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1, + "sublayer": 1 + }, + { + "entropy_max": 0.676069974899292, + "entropy_mean": 0.663312554359436, + "entropy_min": 0.6499756574630737, + "mean_weights": [ + 0.568194588025411, + 0.43180541197458905 + ], + "sources": 2, + "sublayer": 2 + }, + { + "entropy_max": 0.6758064031600952, + "entropy_mean": 0.6724996566772461, + "entropy_min": 0.6664855480194092, + "mean_weights": [ + 0.5410497585932413, + 0.4589502414067586 + ], + "sources": 2, + "sublayer": 3 + }, + { + "entropy_max": 0.5902198553085327, + "entropy_mean": 0.547868569691976, + "entropy_min": 0.5004937648773193, + "mean_weights": [ + 0.7541502118110657, + 0.24584979315598807 + ], + "sources": 2, + "sublayer": 4 + }, + { + "entropy_max": 0.6855782270431519, + "entropy_mean": 0.6811449726422628, + "entropy_min": 0.6787354350090027, + "mean_weights": [ + 0.4620973467826843, + 0.5379026532173157 + ], + "sources": 2, + "sublayer": 5 + }, + { + "entropy_max": 1.025155782699585, + "entropy_mean": 1.0066618919372559, + "entropy_min": 0.973181962966919, + "mean_weights": [ + 0.5044599374135336, + 0.2516526132822037, + 0.24388746420542398 + ], + "sources": 3, + "sublayer": 6 + }, + { + "entropy_max": 1.0936508178710938, + "entropy_mean": 1.0913390318552654, + "entropy_min": 1.0888967514038086, + "mean_weights": [ + 0.30184901754061383, + 0.3493907650311788, + 0.3487602174282074 + ], + "sources": 3, + "sublayer": 7 + }, + { + "entropy_max": 1.0497782230377197, + "entropy_mean": 1.027435878912608, + "entropy_min": 0.9970768094062805, + "mean_weights": [ + 0.45536842942237854, + 0.23504690329233804, + 0.30958468715349835 + ], + "sources": 3, + "sublayer": 8 + }, + { + "entropy_max": 1.0915882587432861, + "entropy_mean": 1.0884315172831218, + "entropy_min": 1.0863194465637207, + "mean_weights": [ + 0.30525945623715717, + 0.34874869386355084, + 0.3459918598333995 + ], + "sources": 3, + "sublayer": 9 + }, + { + "entropy_max": 1.3575613498687744, + "entropy_mean": 1.3401769797007244, + "entropy_min": 1.3304085731506348, + "mean_weights": [ + 0.31245625019073486, + 0.188400000333786, + 0.24196669459342957, + 0.25717706481615704 + ], + "sources": 4, + "sublayer": 10 + }, + { + "entropy_max": 1.37892484664917, + "entropy_mean": 1.3752654393513997, + "entropy_min": 1.372523546218872, + "mean_weights": [ + 0.2376734067996343, + 0.25084111591180164, + 0.2222500592470169, + 0.2892354329427083 + ], + "sources": 4, + "sublayer": 11 + }, + { + "entropy_max": 1.3570414781570435, + "entropy_mean": 1.3392419815063477, + "entropy_min": 1.3118952512741089, + "mean_weights": [ + 0.2914406458536784, + 0.17078583439191183, + 0.22245727479457855, + 0.31531623005867004 + ], + "sources": 4, + "sublayer": 12 + }, + { + "entropy_max": 1.372481346130371, + "entropy_mean": 1.371339241663615, + "entropy_min": 1.3699870109558105, + "mean_weights": [ + 0.2318959136803945, + 0.26858144998550415, + 0.20875323315461478, + 0.29076939821243286 + ], + "sources": 4, + "sublayer": 13 + }, + { + "entropy_max": 1.5968154668807983, + "entropy_mean": 1.5836326678593953, + "entropy_min": 1.5674753189086914, + "mean_weights": [ + 0.19836588700612387, + 0.15423202017943063, + 0.17724995811780295, + 0.2274299512306849, + 0.24272218346595764 + ], + "sources": 5, + "sublayer": 14 + }, + { + "entropy_max": 1.5973689556121826, + "entropy_mean": 1.5895702838897705, + "entropy_min": 1.577967882156372, + "mean_weights": [ + 0.18468524515628815, + 0.19026227295398712, + 0.15613725781440735, + 0.19949903090794882, + 0.269416203101476 + ], + "sources": 5, + "sublayer": 15 + }, + { + "entropy_max": 1.5947315692901611, + "entropy_mean": 1.5757121245066326, + "entropy_min": 1.546515941619873, + "mean_weights": [ + 0.1997243563334147, + 0.17882535854975382, + 0.15601223707199097, + 0.17899859448273978, + 0.286439448595047 + ], + "sources": 5, + "sublayer": 16 + }, + { + "entropy_max": 1.604175090789795, + "entropy_mean": 1.5968455870946248, + "entropy_min": 1.5857822895050049, + "mean_weights": [ + 0.1956237554550171, + 0.203901469707489, + 0.17066203554471335, + 0.1839087208112081, + 0.24590401351451874 + ], + "sources": 5, + "sublayer": 17 + }, + { + "entropy_max": 1.7821681499481201, + "entropy_mean": 1.768844485282898, + "entropy_min": 1.7451198101043701, + "mean_weights": [ + 0.15534367660681406, + 0.14029951890309653, + 0.14831382284561792, + 0.15113131205240884, + 0.18528948724269867, + 0.21962219973405203 + ], + "sources": 6, + "sublayer": 18 + }, + { + "entropy_max": 1.7596988677978516, + "entropy_mean": 1.7357758680979412, + "entropy_min": 1.7161343097686768, + "mean_weights": [ + 0.1554203579823176, + 0.15391315519809723, + 0.1261703222990036, + 0.12737740576267242, + 0.14683481554190317, + 0.2902839382489522 + ], + "sources": 6, + "sublayer": 19 + }, + { + "entropy_max": 1.59963059425354, + "entropy_mean": 1.4545996189117432, + "entropy_min": 1.360633373260498, + "mean_weights": [ + 0.13907443980375925, + 0.13006270925203958, + 0.08823741724093755, + 0.06815870975454648, + 0.07939869786302249, + 0.4950680136680603 + ], + "sources": 6, + "sublayer": 20 + }, + { + "entropy_max": 1.7793185710906982, + "entropy_mean": 1.7684559027353923, + "entropy_min": 1.7599127292633057, + "mean_weights": [ + 0.16601511339346567, + 0.16745010018348694, + 0.13976068794727325, + 0.14097271859645844, + 0.15037270387013754, + 0.23542868594328561 + ], + "sources": 6, + "sublayer": 21 + }, + { + "entropy_max": 1.592297911643982, + "entropy_mean": 1.5481091340382893, + "entropy_min": 1.4908201694488525, + "mean_weights": [ + 0.09055864065885544, + 0.07739925881226857, + 0.05696734661857287, + 0.046672369043032326, + 0.05360592405001322, + 0.22773849964141846, + 0.44705795248349506 + ], + "sources": 7, + "sublayer": 22 + }, + { + "entropy_max": 1.8954918384552002, + "entropy_mean": 1.8707793553670247, + "entropy_min": 1.8509920835494995, + "mean_weights": [ + 0.12786388397216797, + 0.12726488709449768, + 0.1035188560684522, + 0.10487095763285954, + 0.10812034954627354, + 0.1509028524160385, + 0.2774582306543986 + ], + "sources": 7, + "sublayer": 23 + }, + { + "entropy_max": 1.5638015270233154, + "entropy_mean": 1.529894272486369, + "entropy_min": 1.4770772457122803, + "mean_weights": [ + 0.09439751009146373, + 0.07925632844368617, + 0.057954128831624985, + 0.049073984225591026, + 0.05442543948690096, + 0.17891186972459158, + 0.4859807292620341 + ], + "sources": 7, + "sublayer": 24 + }, + { + "entropy_max": 1.9023945331573486, + "entropy_mean": 1.880323886871338, + "entropy_min": 1.8683671951293945, + "mean_weights": [ + 0.13544990122318268, + 0.14224635561307272, + 0.10577710469563802, + 0.0981563354531924, + 0.10264134655396144, + 0.15992753704388937, + 0.25580141445000965 + ], + "sources": 7, + "sublayer": 25 + }, + { + "entropy_max": 1.7563724517822266, + "entropy_mean": 1.7308308283487956, + "entropy_min": 1.7099403142929077, + "mean_weights": [ + 0.07523585855960846, + 0.06218656276663145, + 0.045632037023703255, + 0.03874268631140391, + 0.042072441428899765, + 0.1278796618183454, + 0.3132644097010295, + 0.2949863374233246 + ], + "sources": 8, + "sublayer": 26 + }, + { + "entropy_max": 1.9232137203216553, + "entropy_mean": 1.9054110050201416, + "entropy_min": 1.8879387378692627, + "mean_weights": [ + 0.09858222305774689, + 0.10369476675987244, + 0.07179193198680878, + 0.0594374363621076, + 0.061381166179974876, + 0.12209223707516988, + 0.18827446301778158, + 0.294745792945226 + ], + "sources": 8, + "sublayer": 27 + }, + { + "entropy_max": 1.7447065114974976, + "entropy_mean": 1.72612730662028, + "entropy_min": 1.7093220949172974, + "mean_weights": [ + 0.0765189677476883, + 0.06796563292543094, + 0.04787288730343183, + 0.04231621200839678, + 0.04612377534310023, + 0.11212762693564098, + 0.2269348402818044, + 0.3801400661468506 + ], + "sources": 8, + "sublayer": 28 + }, + { + "entropy_max": 1.9634629487991333, + "entropy_mean": 1.9337302446365356, + "entropy_min": 1.886972427368164, + "mean_weights": [ + 0.10630816966295242, + 0.10707242041826248, + 0.07552854220072429, + 0.06032343705495199, + 0.05965699007113775, + 0.13685152928034464, + 0.20677822828292847, + 0.24748070041338602 + ], + "sources": 8, + "sublayer": 29 + }, + { + "entropy_max": 1.9826093912124634, + "entropy_mean": 1.9681602716445923, + "entropy_min": 1.943651795387268, + "mean_weights": [ + 0.07450833419958751, + 0.06967654327551524, + 0.04876244689027468, + 0.040633084873358406, + 0.04262504850824674, + 0.10585258652766545, + 0.19568686187267303, + 0.23621322711308798, + 0.1860418419043223 + ], + "sources": 9, + "sublayer": 30 + }, + { + "entropy_max": 2.052138090133667, + "entropy_mean": 2.0466067790985107, + "entropy_min": 2.0432467460632324, + "mean_weights": [ + 0.08745819330215454, + 0.08252991984287898, + 0.058957609037558235, + 0.04905804619193077, + 0.049261036018530525, + 0.11601439615090688, + 0.1833944966395696, + 0.20734459161758423, + 0.16598171492417654 + ], + "sources": 9, + "sublayer": 31 + }, + { + "entropy_max": 2.0141000747680664, + "entropy_mean": 1.9910953442255657, + "entropy_min": 1.9660465717315674, + "mean_weights": [ + 0.07816717276970546, + 0.07901749511559804, + 0.05229665587345759, + 0.04327899465958277, + 0.046668255080779396, + 0.10070234537124634, + 0.17064277331034342, + 0.24551014105478922, + 0.1837161829074224 + ], + "sources": 9, + "sublayer": 32 + } + ] + }, + "full": { + "max_sources": 32, + "output": { + "entropy_max": 3.2301199436187744, + "entropy_mean": 3.2225964864095054, + "entropy_min": 3.2135813236236572, + "mean_weights": [ + 0.025556615243355434, + 0.02490210657318433, + 0.026224765926599503, + 0.019764999548594158, + 0.024120309079686802, + 0.01947101578116417, + 0.020250216126441956, + 0.01827173059185346, + 0.017942425484458607, + 0.017180486271778744, + 0.01900123494366805, + 0.015923069479564827, + 0.013854833009342352, + 0.016641111423571903, + 0.013891398285826048, + 0.023777509729067486, + 0.021802425384521484, + 0.03012333872417609, + 0.025545488422115643, + 0.03698895126581192, + 0.027168714130918186, + 0.03949497267603874, + 0.027805685997009277, + 0.03334506042301655, + 0.03443349897861481, + 0.02792014181613922, + 0.056730019549528755, + 0.026892853900790215, + 0.08061449726422627, + 0.013165338585774103, + 0.0910993938644727, + 0.0027941359827915826, + 0.10730165491501491 + ], + "sources": 33 + }, + "rows": [ + { + "entropy_max": 0.0, + "entropy_mean": 0.0, + "entropy_min": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1, + "sublayer": 1 + }, + { + "entropy_max": 0.6729873418807983, + "entropy_mean": 0.6630954543749491, + "entropy_min": 0.6552057266235352, + "mean_weights": [ + 0.5729489127794901, + 0.42705105741818744 + ], + "sources": 2, + "sublayer": 2 + }, + { + "entropy_max": 1.0850456953048706, + "entropy_mean": 1.0804216861724854, + "entropy_min": 1.074110507965088, + "mean_weights": [ + 0.3627606928348541, + 0.3307747046152751, + 0.30646461248397827 + ], + "sources": 3, + "sublayer": 3 + }, + { + "entropy_max": 1.3302907943725586, + "entropy_mean": 1.3018674453099568, + "entropy_min": 1.2723480463027954, + "mean_weights": [ + 0.34105542302131653, + 0.19593795637289682, + 0.308497816324234, + 0.15450880428155264 + ], + "sources": 4, + "sublayer": 4 + }, + { + "entropy_max": 1.5865752696990967, + "entropy_mean": 1.575003186861674, + "entropy_min": 1.5582118034362793, + "mean_weights": [ + 0.18386788666248322, + 0.2390782485405604, + 0.18032612899939218, + 0.23851566016674042, + 0.1582120656967163 + ], + "sources": 5, + "sublayer": 5 + }, + { + "entropy_max": 1.7290915250778198, + "entropy_mean": 1.7206300099690754, + "entropy_min": 1.7134463787078857, + "mean_weights": [ + 0.21605777740478516, + 0.14990001420180002, + 0.2151650389035543, + 0.11052880187829335, + 0.1715006877978643, + 0.1368476798137029 + ], + "sources": 6, + "sublayer": 6 + }, + { + "entropy_max": 1.9155155420303345, + "entropy_mean": 1.8901333411534627, + "entropy_min": 1.8659067153930664, + "mean_weights": [ + 0.12854668498039246, + 0.19778794050216675, + 0.13465766112009683, + 0.18992085258165994, + 0.11008669932683308, + 0.15766242146492004, + 0.0813377524415652 + ], + "sources": 7, + "sublayer": 7 + }, + { + "entropy_max": 2.019052743911743, + "entropy_mean": 2.013235012690226, + "entropy_min": 2.0070152282714844, + "mean_weights": [ + 0.14720548192660013, + 0.1377126748363177, + 0.16037661333878836, + 0.10299472014109294, + 0.11697593828042348, + 0.11880407234032948, + 0.07762171079715093, + 0.1383087933063507 + ], + "sources": 8, + "sublayer": 8 + }, + { + "entropy_max": 2.157308578491211, + "entropy_mean": 2.154908021291097, + "entropy_min": 2.152099132537842, + "mean_weights": [ + 0.10807480166355769, + 0.15884055693944296, + 0.11500980953375499, + 0.13613408307234445, + 0.09466097255547841, + 0.12546123564243317, + 0.0702842225631078, + 0.12216035773356755, + 0.06937396277983983 + ], + "sources": 9, + "sublayer": 9 + }, + { + "entropy_max": 2.2573065757751465, + "entropy_mean": 2.2304697831471763, + "entropy_min": 2.2092034816741943, + "mean_weights": [ + 0.11872178316116333, + 0.11724316825469334, + 0.13413792848587036, + 0.08227266867955525, + 0.09661872188250224, + 0.10174320389827092, + 0.06136756141980489, + 0.11396426459153493, + 0.05892340963085493, + 0.11500727633635204 + ], + "sources": 10, + "sublayer": 10 + }, + { + "entropy_max": 2.359937906265259, + "entropy_mean": 2.352225144704183, + "entropy_min": 2.341871976852417, + "mean_weights": [ + 0.09207804997762044, + 0.13507375617822012, + 0.0997256338596344, + 0.11042191833257675, + 0.08121875921885173, + 0.09732648730278015, + 0.058663940678040184, + 0.10317053894201915, + 0.05721114327510198, + 0.10648540159066518, + 0.05862438430388769 + ], + "sources": 11, + "sublayer": 11 + }, + { + "entropy_max": 2.3737237453460693, + "entropy_mean": 2.3604607582092285, + "entropy_min": 2.336538791656494, + "mean_weights": [ + 0.07879289239645004, + 0.11672904590765636, + 0.08075895408789317, + 0.09057638794183731, + 0.05744863177339236, + 0.10357426106929779, + 0.036198447148005165, + 0.11587326476971309, + 0.03574693823854128, + 0.12555705259243646, + 0.0362573837240537, + 0.12248673538366954 + ], + "sources": 12, + "sublayer": 12 + }, + { + "entropy_max": 2.5292272567749023, + "entropy_mean": 2.523596445719401, + "entropy_min": 2.5176010131835938, + "mean_weights": [ + 0.07940395176410675, + 0.10745090742905934, + 0.08859917024771373, + 0.08839243898789088, + 0.07243171830972035, + 0.08091367284456889, + 0.05198533460497856, + 0.08363978813091914, + 0.05071114872892698, + 0.09088355551163356, + 0.051586925983428955, + 0.10110345482826233, + 0.05289792890350024 + ], + "sources": 13, + "sublayer": 13 + }, + { + "entropy_max": 2.5260555744171143, + "entropy_mean": 2.511597235997518, + "entropy_min": 2.4945454597473145, + "mean_weights": [ + 0.06728010376294453, + 0.09017306566238403, + 0.06705486277739207, + 0.07410627727707227, + 0.049341840048631035, + 0.08006220559279124, + 0.03232185977200667, + 0.09271023919185002, + 0.03153213858604431, + 0.1004645402232806, + 0.03198575973510742, + 0.11092506349086761, + 0.035584566493829094, + 0.13645747800668082 + ], + "sources": 14, + "sublayer": 14 + }, + { + "entropy_max": 2.6629085540771484, + "entropy_mean": 2.6597483158111572, + "entropy_min": 2.6540651321411133, + "mean_weights": [ + 0.0696261500318845, + 0.08718715111414592, + 0.07843850553035736, + 0.07442579418420792, + 0.0630200741191705, + 0.07242759317159653, + 0.04383365189035734, + 0.0742378830909729, + 0.042224218448003135, + 0.07677323619524638, + 0.04271993786096573, + 0.0888681411743164, + 0.04172474518418312, + 0.09675361216068268, + 0.04773929715156555 + ], + "sources": 15, + "sublayer": 15 + }, + { + "entropy_max": 2.653104782104492, + "entropy_mean": 2.6441221237182617, + "entropy_min": 2.6280198097229004, + "mean_weights": [ + 0.05986166993776957, + 0.08501092096169789, + 0.06847748905420303, + 0.06890026852488518, + 0.04862064744035403, + 0.07089548061291377, + 0.02869326372941335, + 0.07980451981226604, + 0.02707553468644619, + 0.07720348238945007, + 0.026560273642341297, + 0.08022717883189519, + 0.02696048965056737, + 0.09724790354569753, + 0.03290759275356928, + 0.12155329436063766 + ], + "sources": 16, + "sublayer": 16 + }, + { + "entropy_max": 2.7772436141967773, + "entropy_mean": 2.7734902699788413, + "entropy_min": 2.767390251159668, + "mean_weights": [ + 0.06181313097476959, + 0.0787958453098933, + 0.0740993320941925, + 0.06472191214561462, + 0.05765968312819799, + 0.06431250025828679, + 0.038311279068390526, + 0.0646959940592448, + 0.035535636047522225, + 0.06446120763818423, + 0.03540633494655291, + 0.06941856443881989, + 0.03171839565038681, + 0.07583510130643845, + 0.035414453595876694, + 0.09135590742031734, + 0.05644472067554792 + ], + "sources": 17, + "sublayer": 17 + }, + { + "entropy_max": 2.76259446144104, + "entropy_mean": 2.7232768535614014, + "entropy_min": 2.6990277767181396, + "mean_weights": [ + 0.05518728867173195, + 0.06774829079707463, + 0.06946775317192078, + 0.05141793688138326, + 0.04801270241538683, + 0.05370426674683889, + 0.027809271588921547, + 0.0562954917550087, + 0.02551322678724925, + 0.05110526581605276, + 0.024331960827112198, + 0.046864173064629235, + 0.020843032126625378, + 0.05734457323948542, + 0.024186863874395687, + 0.09807546188433965, + 0.07210367918014526, + 0.14998876055081686 + ], + "sources": 18, + "sublayer": 18 + }, + { + "entropy_max": 2.8850622177124023, + "entropy_mean": 2.8652973969777427, + "entropy_min": 2.843248128890991, + "mean_weights": [ + 0.05260627344250679, + 0.06991977492968242, + 0.06476513544718425, + 0.0581423689921697, + 0.049954539785782494, + 0.056481542686621346, + 0.03229485948880514, + 0.05682817722360293, + 0.02923661842942238, + 0.0552576519548893, + 0.028797871122757595, + 0.05738401288787524, + 0.024513501053055126, + 0.06065270428856214, + 0.02665022263924281, + 0.07460795839627583, + 0.04375817130009333, + 0.09523340314626694, + 0.06291522085666656 + ], + "sources": 19, + "sublayer": 19 + }, + { + "entropy_max": 2.7220685482025146, + "entropy_mean": 2.7043351332346597, + "entropy_min": 2.69278621673584, + "mean_weights": [ + 0.045603882521390915, + 0.05447864532470703, + 0.06410424038767815, + 0.03792111948132515, + 0.04226911440491676, + 0.03555091346303622, + 0.02289348654448986, + 0.03695064038038254, + 0.020027659212549526, + 0.031009394054611523, + 0.01926272859176, + 0.025271238759160042, + 0.014632520886758963, + 0.028143515810370445, + 0.014787437083820501, + 0.062148607025543846, + 0.05986413732171059, + 0.11426966885725658, + 0.08119676013787587, + 0.18961429099241892 + ], + "sources": 20, + "sublayer": 20 + }, + { + "entropy_max": 2.9706764221191406, + "entropy_mean": 2.957854668299357, + "entropy_min": 2.9454524517059326, + "mean_weights": [ + 0.04558223858475685, + 0.059097565710544586, + 0.05700209612647692, + 0.048397842794656754, + 0.04409012322624525, + 0.0473170280456543, + 0.028855254873633385, + 0.047425925731658936, + 0.02600932928423087, + 0.04637702306111654, + 0.02551550293962161, + 0.047576854626337685, + 0.021547649676601093, + 0.05002515638868014, + 0.02275315361718337, + 0.06288522730271022, + 0.03654991711179415, + 0.07972793529431026, + 0.047366989155610405, + 0.0914284239212672, + 0.06446876997749011 + ], + "sources": 21, + "sublayer": 21 + }, + { + "entropy_max": 2.807748794555664, + "entropy_mean": 2.770190954208374, + "entropy_min": 2.746567964553833, + "mean_weights": [ + 0.036101615677277245, + 0.04429023340344429, + 0.052328101048866905, + 0.029807519167661667, + 0.03372577950358391, + 0.02957167973121007, + 0.017642357076207798, + 0.029840663075447083, + 0.015398593929906687, + 0.025294749687115353, + 0.01460645099480947, + 0.019713498651981354, + 0.01107433003683885, + 0.02177129437526067, + 0.010899525446196398, + 0.04968325545390447, + 0.047834025075038276, + 0.08632492522398631, + 0.058039311319589615, + 0.13162659605344137, + 0.08316896359125774, + 0.1512565314769745 + ], + "sources": 22, + "sublayer": 22 + }, + { + "entropy_max": 3.0476112365722656, + "entropy_mean": 3.0367375214894614, + "entropy_min": 3.0187454223632812, + "mean_weights": [ + 0.03872521221637726, + 0.04870210215449333, + 0.04870830848813057, + 0.04040446256597837, + 0.037807745238145195, + 0.04030604908863703, + 0.025170981884002686, + 0.04038627694050471, + 0.02254965218404929, + 0.03987714648246765, + 0.022182572012146313, + 0.03952491035064062, + 0.01878617952267329, + 0.04185949141780535, + 0.019280270983775456, + 0.05313673863808314, + 0.031132920334736507, + 0.0669172207514445, + 0.039429452270269394, + 0.07793149848779042, + 0.054866241912047066, + 0.08117576936880748, + 0.07113879422346751 + ], + "sources": 23, + "sublayer": 23 + }, + { + "entropy_max": 2.9273719787597656, + "entropy_mean": 2.884519020716349, + "entropy_min": 2.858419179916382, + "mean_weights": [ + 0.030646016200383503, + 0.04199869930744171, + 0.043607903023560844, + 0.02879402848581473, + 0.027949823066592216, + 0.02709902140001456, + 0.014362686003247896, + 0.026844498390952747, + 0.012698768948515257, + 0.023212683697541554, + 0.012096195171276728, + 0.018513306354482967, + 0.009606915215651194, + 0.02043161727488041, + 0.009408737532794476, + 0.04485694939891497, + 0.03561059944331646, + 0.07173362125953038, + 0.04302813857793808, + 0.10230723023414612, + 0.06663860380649567, + 0.11446122825145721, + 0.09680233647425969, + 0.07729038720329602 + ], + "sources": 24, + "sublayer": 24 + }, + { + "entropy_max": 3.1288928985595703, + "entropy_mean": 3.1179122924804688, + "entropy_min": 3.096397638320923, + "mean_weights": [ + 0.03394729644060135, + 0.04239057252804438, + 0.043130082388718925, + 0.03574401264389356, + 0.03375358507037163, + 0.0352306067943573, + 0.02269617219765981, + 0.03447849303483963, + 0.020519376421968143, + 0.03359278105199337, + 0.019900760302941006, + 0.03270889073610306, + 0.016650297368566196, + 0.03397464441756407, + 0.016444630610446136, + 0.047457544753948845, + 0.028430978457132976, + 0.057780602325995765, + 0.0361362403879563, + 0.06799524029095967, + 0.04930434127648672, + 0.06849389274915059, + 0.06071932862202326, + 0.05846334621310234, + 0.07005627950032552 + ], + "sources": 25, + "sublayer": 25 + }, + { + "entropy_max": 2.997812271118164, + "entropy_mean": 2.9913203716278076, + "entropy_min": 2.981506109237671, + "mean_weights": [ + 0.026762607196966808, + 0.03790475676457087, + 0.03804942717154821, + 0.026963916296760242, + 0.025157970065871876, + 0.024532814199725788, + 0.014002490788698196, + 0.02430441789329052, + 0.012250386799375216, + 0.021353704233964283, + 0.011727250491579374, + 0.01740749552845955, + 0.009949984649817148, + 0.018954483792185783, + 0.009896416527529558, + 0.038485350708166756, + 0.029461168373624485, + 0.05913051341970762, + 0.034761415173610054, + 0.07939759890238444, + 0.05290326724449793, + 0.0877164974808693, + 0.07776692012945811, + 0.06165457765261332, + 0.11417745302120845, + 0.04532711332043012 + ], + "sources": 26, + "sublayer": 26 + }, + { + "entropy_max": 3.2317047119140625, + "entropy_mean": 3.190260728200277, + "entropy_min": 3.135222911834717, + "mean_weights": [ + 0.030709152420361836, + 0.038856384654839836, + 0.03852503622571627, + 0.03404424712061882, + 0.030600749577085178, + 0.029806426415840786, + 0.0211167111992836, + 0.030848986158768337, + 0.019134069482485454, + 0.029532151917616527, + 0.018607016652822495, + 0.028101677695910137, + 0.015492160183688005, + 0.02930781990289688, + 0.015279345214366913, + 0.03950384880105654, + 0.024300287167231243, + 0.05189322431882223, + 0.03257856145501137, + 0.0612580676873525, + 0.04323995237549146, + 0.06529123956958453, + 0.055030049135287605, + 0.048991107692321144, + 0.06333444888393085, + 0.041943887869517006, + 0.06267338494459788 + ], + "sources": 27, + "sublayer": 27 + }, + { + "entropy_max": 3.0881078243255615, + "entropy_mean": 3.081976811091105, + "entropy_min": 3.0709128379821777, + "mean_weights": [ + 0.024140593285361927, + 0.03355464587608973, + 0.034119228521982826, + 0.024404085551699, + 0.02367766263584296, + 0.022779367864131927, + 0.014641802137096723, + 0.02278521905342738, + 0.013004937209188938, + 0.01984588863948981, + 0.012441112038989862, + 0.016958024973670643, + 0.011285776272416115, + 0.01809990033507347, + 0.011298118780056635, + 0.033870985731482506, + 0.028080757707357407, + 0.04873387018839518, + 0.03195954114198685, + 0.06577975302934647, + 0.044237478325764336, + 0.0697377473115921, + 0.06441850339372952, + 0.05027745167414347, + 0.09281101822853088, + 0.037099356452624, + 0.10307030379772186, + 0.026886864254872005 + ], + "sources": 28, + "sublayer": 28 + }, + { + "entropy_max": 3.315805435180664, + "entropy_mean": 3.2921435038248696, + "entropy_min": 3.2788450717926025, + "mean_weights": [ + 0.02865760214626789, + 0.03586554403106371, + 0.0367035319407781, + 0.032036833465099335, + 0.03201191189388434, + 0.028546927496790886, + 0.024774956827362377, + 0.026971304789185524, + 0.022935497884949047, + 0.025910282507538795, + 0.022265223165353138, + 0.02342439629137516, + 0.018185899282495182, + 0.02431691127518813, + 0.016727660472194355, + 0.03539527704318365, + 0.02383350332578023, + 0.040299609303474426, + 0.027922411759694416, + 0.05242166916529337, + 0.0389099158346653, + 0.05958003054062525, + 0.04594611252347628, + 0.048638347536325455, + 0.05130115399758021, + 0.03639778991540273, + 0.054013351599375405, + 0.034571390599012375, + 0.051434955249230065 + ], + "sources": 29, + "sublayer": 29 + }, + { + "entropy_max": 3.1905009746551514, + "entropy_mean": 3.18230938911438, + "entropy_min": 3.1718106269836426, + "mean_weights": [ + 0.023452899729212124, + 0.0325044517715772, + 0.03281022980809212, + 0.023958238462607067, + 0.024015444020430248, + 0.021982961023847263, + 0.0159907682488362, + 0.021546220406889915, + 0.014109862968325615, + 0.019567859669526417, + 0.013529953546822071, + 0.016938431809345882, + 0.011665725770095984, + 0.01821698558827241, + 0.01173508632928133, + 0.0315526028474172, + 0.02504568360745907, + 0.04361861323316892, + 0.029006240889430046, + 0.057882855335871376, + 0.038831111043691635, + 0.06207767128944397, + 0.051631235828002296, + 0.04616564139723778, + 0.07385702927907307, + 0.03487453361352285, + 0.08937223007281621, + 0.02605181684096654, + 0.07663016517957051, + 0.011377457529306412 + ], + "sources": 30, + "sublayer": 30 + }, + { + "entropy_max": 3.4240646362304688, + "entropy_mean": 3.3962062994639077, + "entropy_min": 3.3770389556884766, + "mean_weights": [ + 0.036534909158945084, + 0.027843889469901722, + 0.029182132333517075, + 0.031089238822460175, + 0.03259412323435148, + 0.032848658661047615, + 0.03959940994779269, + 0.032999612390995026, + 0.041947100311517715, + 0.03467814748485883, + 0.0426634947458903, + 0.03830170755585035, + 0.04692706217368444, + 0.037095338106155396, + 0.04795412967602412, + 0.029356527452667553, + 0.03787682702143987, + 0.02283823365966479, + 0.02676370181143284, + 0.02388523022333781, + 0.02641577584048112, + 0.02381802164018154, + 0.02564098685979843, + 0.02967298651734988, + 0.024603380511204403, + 0.027582359810670216, + 0.0274051936964194, + 0.03037252277135849, + 0.030458840851982433, + 0.0283659677952528, + 0.032684496293465294 + ], + "sources": 31, + "sublayer": 31 + }, + { + "entropy_max": 3.1776766777038574, + "entropy_mean": 3.1591249306996665, + "entropy_min": 3.145052194595337, + "mean_weights": [ + 0.0212711604932944, + 0.028220354268948238, + 0.029740046088894207, + 0.020458971460660298, + 0.024074342101812363, + 0.020205453038215637, + 0.017948679625988007, + 0.019636886815230053, + 0.014780626632273197, + 0.017502093066771824, + 0.014497815320889154, + 0.015224847942590714, + 0.011039208620786667, + 0.015552480394641558, + 0.011189129513998827, + 0.026409130543470383, + 0.022006437182426453, + 0.03754762870570024, + 0.02449234699209531, + 0.04594653596480688, + 0.030809269597133, + 0.053079865872859955, + 0.03982332721352577, + 0.04045433799425761, + 0.05811700349052747, + 0.029337162151932716, + 0.08656761050224304, + 0.023294669886430103, + 0.09694787611564, + 0.007489877597739299, + 0.09413341929515202, + 0.0022014072940995297 + ], + "sources": 32, + "sublayer": 32 + } + ] + } + }, + "parameters": { + "baseline": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "block": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "full": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "mixer_overhead_fraction_of_baseline": 0.001328047970702457 + }, + "posthoc": { + "corresponding_final_output_weight": 0.0027941359827915826, + "label": "post-hoc descriptive callout; not a preregistered endpoint", + "largest_full_branch_rms": 1.496443748474121, + "largest_full_branch_sublayer": 31, + "uniform_final_output_weight": 0.030303030303030304, + "weight_over_uniform": 0.09220648743212222 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "reproduction": { + "canonical_sha256_without_self": "89a127e625ccefbd9b749e6ffca4ba72868e217e0c7a24773024bd2fafde427a", + "common_initial_parameters": { + "2026073001": { + "exact": true, + "hashes": { + "baseline": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "block": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "full": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + } + }, + "2026073002": { + "exact": true, + "hashes": { + "baseline": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "block": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "full": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + } + }, + "2026073003": { + "exact": true, + "hashes": { + "baseline": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "block": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "full": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + } + } + }, + "formal_files": { + "baseline-2026073001.json": "f5cddecace6a70ee3824f272811d2a6336ab6dc24ad1100b18292f4413692e9a", + "baseline-2026073002.json": "4c3b9e1ea2ed6fd3078f1212984327364832a1146bfcbdb3a6f2e61b589bfaa8", + "baseline-2026073003.json": "e5ce5a8944dad0c4811f8bf19b4cfec8ce4a9283858fd59cd739dbc37626707b", + "block-2026073001.json": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "block-2026073002.json": "be1f109a630e27c8469438e33b53806cdaecac2f712af4f885da83f00ef6843b", + "block-2026073003.json": "949ec51ca3a219a11f260171da01f737296241536f5e139cf07d444c27efba92", + "full-2026073001.json": "648cb25ea98da1868779733da155275e9a16aad5efbbeeced7812c19d75817d5", + "full-2026073002.json": "1fa715bbd81a3a04a5aa0ba0fc27feb8883c44a01e9f28e281a309a394447979", + "full-2026073003.json": "4e34fa35bbfec460cabc94bce7a08f44fc188d908705ae2e256500f00813b4e3" + }, + "formal_replay": { + "all_numeric_and_hash_fields_exact": true, + "architecture": "block", + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "formal_file_sha256": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "replay_file_sha256": "e74d3323e5fe31378bb8aad7a8efa2fb91995cd7224c158cf03006466cdea2a7", + "seed": 2026073001, + "timing_exact_observed": false, + "timing_exact_required": false + }, + "manifest_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "schema_version": 1, + "smoke": { + "baseline": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c", + "second_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c" + }, + "block": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc", + "second_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc" + }, + "full": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb", + "second_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb" + } + } + }, + "schema_version": 1, + "source_sha256": "8625a34f377a14e0d69339a78e94788cd0ce9c925e521f51e3cba31acf96b20c", + "timing": { + "baseline": { + "by_seed": [ + { + "mean_ms": 21.30391749891751, + "measured_steps": 1980, + "median_ms": 21.206228993833065, + "p95_ms": 21.648590854601935, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 21.793900100873532, + "measured_steps": 1980, + "median_ms": 21.672270508133806, + "p95_ms": 22.421945561654866, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 21.765943476063025, + "measured_steps": 1980, + "median_ms": 21.700910496292636, + "p95_ms": 22.111314281937666, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 3036005376.0, + "mean_peak_reserved_bytes": 3296722944.0, + "mean_step_ms": 21.621253691951356, + "median_step_ms": 21.526469999419835, + "p95_step_ms": 22.060616899398156 + }, + "block": { + "by_seed": [ + { + "mean_ms": 54.54293606894542, + "measured_steps": 1980, + "median_ms": 54.46750250121113, + "p95_ms": 55.017453705659136, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 54.45831580526452, + "measured_steps": 1980, + "median_ms": 54.40393550088629, + "p95_ms": 54.91135738266166, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 54.8763856947342, + "measured_steps": 1980, + "median_ms": 54.84093050472438, + "p95_ms": 55.21141063363757, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 6520143872.0, + "mean_peak_reserved_bytes": 6788481024.0, + "mean_step_ms": 54.62587918964805, + "median_step_ms": 54.57078950227393, + "p95_step_ms": 55.04674057398612 + }, + "full": { + "by_seed": [ + { + "mean_ms": 146.80656970705252, + "measured_steps": 1980, + "median_ms": 146.7325690027792, + "p95_ms": 147.17569539061515, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 146.7571435391555, + "measured_steps": 1980, + "median_ms": 146.72027151391376, + "p95_ms": 147.0799343456747, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 146.85863199750506, + "measured_steps": 1980, + "median_ms": 146.81190850387793, + "p95_ms": 147.1791482414119, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 14260571648.0, + "mean_peak_reserved_bytes": 14950596608.0, + "mean_step_ms": 146.807448414571, + "median_step_ms": 146.7549163401903, + "p95_step_ms": 147.14492599256724 + }, + "relative_to_baseline": { + "block": { + "allocated_memory_ratio": 2.1476061681387484, + "step_time_ratio": 2.526489905161368 + }, + "full": { + "allocated_memory_ratio": 4.697149669342351, + "step_time_ratio": 6.789960032207613 + } + } + }, + "traces": { + "baseline": { + "branch_output_rms": { + "max": [ + 0.03301293030381203, + 0.03252410888671875, + 0.04544902965426445, + 0.012599779292941093, + 0.03163857012987137, + 0.007620926480740309, + 0.016936156898736954, + 0.007713559083640575, + 0.014406777918338776, + 0.009163067676126957, + 0.014294588007032871, + 0.010015063919126987, + 0.016019638627767563, + 0.01165300328284502, + 0.019276386126875877, + 0.01118372566998005, + 0.021849332377314568, + 0.013802342116832733, + 0.02479873225092888, + 0.014017625711858273, + 0.039869681000709534, + 0.01826939918100834, + 0.045646462589502335, + 0.026948343962430954, + 0.03186195716261864, + 0.03553422540426254, + 0.021814068779349327, + 0.041563451290130615, + 0.011395728215575218, + 0.06082121655344963, + 0.010711577720940113, + 0.12170413136482239 + ], + "mean": [ + 0.029140000542004902, + 0.03135617325703303, + 0.04321867351730665, + 0.012356962698201338, + 0.030009073515733082, + 0.0073511286949117976, + 0.015980400145053864, + 0.0073293728443483514, + 0.013850266424318155, + 0.00872932467609644, + 0.013727998671432337, + 0.009513386835654577, + 0.015388951947291693, + 0.01081705124427875, + 0.017621537980933983, + 0.010974762650827566, + 0.020954379191001255, + 0.012728280077377955, + 0.022901471083362896, + 0.012544240181644758, + 0.034868016839027405, + 0.01587048452347517, + 0.034901623303691544, + 0.023213444277644157, + 0.028262764836351078, + 0.03317628552516302, + 0.015859666280448437, + 0.03943942735592524, + 0.009605003676066795, + 0.0564091665049394, + 0.010157584212720394, + 0.12020406872034073 + ], + "min": [ + 0.024467680603265762, + 0.030463015660643578, + 0.04058511182665825, + 0.012055210769176483, + 0.027046527713537216, + 0.006995280738919973, + 0.01488693431019783, + 0.00682956725358963, + 0.013462266884744167, + 0.00827884953469038, + 0.012803927063941956, + 0.00879740621894598, + 0.014694017358124256, + 0.010121054947376251, + 0.014741585589945316, + 0.010847908444702625, + 0.020217377692461014, + 0.011856775730848312, + 0.02128380537033081, + 0.011742421425879002, + 0.029588066041469574, + 0.014059662818908691, + 0.022691959515213966, + 0.02104850672185421, + 0.02629370056092739, + 0.030305426567792892, + 0.012291512452065945, + 0.037283238023519516, + 0.007733044680207968, + 0.05344158038496971, + 0.009596864692866802, + 0.11796895414590836 + ] + }, + "layer_input_rms": { + "max": [ + 0.053540125489234924, + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.12003147602081299, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594 + ], + "mean": [ + 0.053358995666106544, + 0.06613848606745402, + 0.07352576404809952, + 0.10093248138825099, + 0.10369285692771275, + 0.12192569176355998, + 0.12289420515298843, + 0.1264623502890269, + 0.12660532196362814, + 0.1285571108261744, + 0.12829979757467905, + 0.1300874004761378, + 0.12902945280075073, + 0.13068528473377228, + 0.1294754147529602, + 0.1288459748029709, + 0.12698591748873392, + 0.1257581685980161, + 0.12347647547721863, + 0.11898198475440343, + 0.11615285277366638, + 0.10215904066960017, + 0.10102975865205129, + 0.08804203818241756, + 0.09022414187590282, + 0.08323186387618382, + 0.0895061269402504, + 0.08785812556743622, + 0.09989260633786519, + 0.09980995953083038, + 0.1267695054411888, + 0.12696876376867294 + ], + "min": [ + 0.05312936007976532, + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.1217338889837265, + 0.12261121720075607, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12287117540836334, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.08582619577646255, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995 + ] + }, + "stream_state_rms": { + "max": [ + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.12003147602081299, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594, + 0.2156660109758377 + ], + "mean": [ + 0.06613848606745402, + 0.07352576404809952, + 0.10093248138825099, + 0.10369285692771275, + 0.12192569176355998, + 0.12289420515298843, + 0.1264623502890269, + 0.12660532196362814, + 0.1285571108261744, + 0.12829979757467905, + 0.1300874004761378, + 0.12902945280075073, + 0.13068528473377228, + 0.1294754147529602, + 0.1288459748029709, + 0.12698591748873392, + 0.1257581685980161, + 0.12347647547721863, + 0.11898198475440343, + 0.11615285277366638, + 0.10215904066960017, + 0.10102975865205129, + 0.08804203818241756, + 0.09022414187590282, + 0.08323186387618382, + 0.0895061269402504, + 0.08785812556743622, + 0.09989260633786519, + 0.09980995953083038, + 0.1267695054411888, + 0.12696876376867294, + 0.2109663486480713 + ], + "min": [ + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.1217338889837265, + 0.12261121720075607, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12287117540836334, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.08582619577646255, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995, + 0.20744846761226654 + ] + } + }, + "block": { + "branch_output_rms": { + "max": [ + 0.0370347797870636, + 0.048244379460811615, + 0.07630905508995056, + 0.040389351546764374, + 0.05223172530531883, + 0.022662153467535973, + 0.034338437020778656, + 0.018601570278406143, + 0.06659915298223495, + 0.019084658473730087, + 0.11068977415561676, + 0.017863860353827477, + 0.06974328309297562, + 0.023023422807455063, + 0.03994842991232872, + 0.024390853941440582, + 0.022348226979374886, + 0.017856158316135406, + 0.027489805594086647, + 0.026076123118400574, + 0.033282581716775894, + 0.023831162601709366, + 0.017817942425608635, + 0.033413130789995193, + 0.019294502213597298, + 0.02716788649559021, + 0.011242104694247246, + 0.03591271862387657, + 0.015554063953459263, + 0.046544428914785385, + 0.01049043145030737, + 0.05786125734448433 + ], + "mean": [ + 0.03337173908948898, + 0.045107426742712654, + 0.052010158697764076, + 0.03859390194217364, + 0.050928935408592224, + 0.021324736376603443, + 0.02902730492254098, + 0.018222644925117493, + 0.05315905436873436, + 0.016110357828438282, + 0.06097829528152943, + 0.016606629826128483, + 0.06566628689567248, + 0.018870445899665356, + 0.03398136980831623, + 0.021034574136137962, + 0.020854807148377102, + 0.017290979002912838, + 0.025822721421718597, + 0.02190040610730648, + 0.027670275419950485, + 0.02298567195733388, + 0.015916637765864532, + 0.031282839054862656, + 0.01756543169418971, + 0.026957616209983826, + 0.010056077502667904, + 0.03371469924847285, + 0.014171119158466658, + 0.04198148101568222, + 0.009337710837523142, + 0.05569397285580635 + ], + "min": [ + 0.028859306126832962, + 0.042584121227264404, + 0.030812542885541916, + 0.036034028977155685, + 0.04880734533071518, + 0.019183801487088203, + 0.02264152280986309, + 0.01778092049062252, + 0.039430491626262665, + 0.013977118767797947, + 0.027242736890912056, + 0.015299440361559391, + 0.058694127947092056, + 0.01103654783219099, + 0.02528632991015911, + 0.015162505209445953, + 0.019655495882034302, + 0.016425520181655884, + 0.02445649914443493, + 0.01807972602546215, + 0.0240048635751009, + 0.02158590778708458, + 0.014158087782561779, + 0.027974048629403114, + 0.016243331134319305, + 0.026792073622345924, + 0.009424413554370403, + 0.03243190050125122, + 0.011988751590251923, + 0.038416508585214615, + 0.008579479530453682, + 0.052862487733364105 + ] + }, + "layer_input_rms": { + "max": [ + 0.05405270308256149, + 0.03717699274420738, + 0.043796516954898834, + 0.05271390080451965, + 0.08092177659273148, + 0.05150312930345535, + 0.06945399194955826, + 0.05591938644647598, + 0.07346180826425552, + 0.0544980950653553, + 0.060793664306402206, + 0.07243207842111588, + 0.07691676914691925, + 0.06619398295879364, + 0.06854937970638275, + 0.07206644117832184, + 0.07398614287376404, + 0.05593166500329971, + 0.047586891800165176, + 0.036662496626377106, + 0.051853884011507034, + 0.01807689666748047, + 0.03394656628370285, + 0.02252720296382904, + 0.03615279868245125, + 0.019617732614278793, + 0.02297247014939785, + 0.01989392749965191, + 0.0260133296251297, + 0.02015369012951851, + 0.023471562191843987, + 0.023934273049235344 + ], + "mean": [ + 0.05387426788608233, + 0.03533576180537542, + 0.04317111149430275, + 0.04991913214325905, + 0.07097297410170238, + 0.04975094646215439, + 0.06104101364811262, + 0.05404127140839895, + 0.06731398651997249, + 0.04994910334547361, + 0.05693682904044787, + 0.06259432186683019, + 0.07018299649159114, + 0.060772753010193505, + 0.06431671231985092, + 0.06616326545675595, + 0.06897229204575221, + 0.05195336416363716, + 0.04609082266688347, + 0.031601848701635994, + 0.049942340701818466, + 0.017869824543595314, + 0.03332234298189481, + 0.0219339511046807, + 0.03447778026262919, + 0.01884455730517705, + 0.022380413487553596, + 0.019762429719169933, + 0.024852463354667027, + 0.019185719390710194, + 0.023034153506159782, + 0.02343445022900899 + ], + "min": [ + 0.053777214139699936, + 0.03439030423760414, + 0.042556535452604294, + 0.04696016386151314, + 0.06085871905088425, + 0.04705430194735527, + 0.053277771919965744, + 0.05260215699672699, + 0.0606326088309288, + 0.04669487476348877, + 0.054401230067014694, + 0.0512978732585907, + 0.06302255392074585, + 0.05218172445893288, + 0.05766461044549942, + 0.06084785237908363, + 0.06420515477657318, + 0.04857253283262253, + 0.04421957954764366, + 0.02869899943470955, + 0.04853617027401924, + 0.017642762511968613, + 0.032592616975307465, + 0.02158820629119873, + 0.03257649391889572, + 0.01816716231405735, + 0.021423330530524254, + 0.01960809901356697, + 0.024138133972883224, + 0.01791227236390114, + 0.02242189459502697, + 0.023178130388259888 + ] + }, + "stream_state_rms": { + "max": [ + 0.0370347797870636, + 0.06143631786108017, + 0.11840524524450302, + 0.136356920003891, + 0.05223172530531883, + 0.05775697901844978, + 0.07745972275733948, + 0.08341154456138611, + 0.06659915298223495, + 0.0708504468202591, + 0.15584944188594818, + 0.1662624329328537, + 0.06974328309297562, + 0.08339263498783112, + 0.11490626633167267, + 0.12939290702342987, + 0.022348226979374886, + 0.030708789825439453, + 0.04433037340641022, + 0.053864628076553345, + 0.033282581716775894, + 0.04125083237886429, + 0.045212581753730774, + 0.057350847870111465, + 0.019294502213597298, + 0.03496377170085907, + 0.03744323179125786, + 0.056224361062049866, + 0.015554063953459263, + 0.050526175647974014, + 0.051927778869867325, + 0.09535783529281616 + ], + "mean": [ + 0.03337173908948898, + 0.059812614073355995, + 0.09668255100647609, + 0.11812450736761093, + 0.050928935408592224, + 0.0568557046353817, + 0.07451550910870235, + 0.0815352201461792, + 0.05315905436873436, + 0.05806975563367208, + 0.10785960281888644, + 0.11578060686588287, + 0.06566628689567248, + 0.07592015713453293, + 0.09934087345997493, + 0.11063759525616963, + 0.020854807148377102, + 0.029423799365758896, + 0.04364252711335818, + 0.05246620997786522, + 0.027670275419950485, + 0.0368656280140082, + 0.04281706859668096, + 0.054927793641885124, + 0.01756543169418971, + 0.033875805636247, + 0.03674849991997083, + 0.05424185593922933, + 0.014171119158466658, + 0.045518445471922554, + 0.04748128727078438, + 0.09287819017966588 + ], + "min": [ + 0.028859306126832962, + 0.05888580158352852, + 0.07637989521026611, + 0.10048534721136093, + 0.04880734533071518, + 0.05540755018591881, + 0.07019630819559097, + 0.07779177278280258, + 0.039430491626262665, + 0.04417555779218674, + 0.06122447922825813, + 0.068731389939785, + 0.058694127947092056, + 0.07109052687883377, + 0.08456306159496307, + 0.09095416963100433, + 0.019655495882034302, + 0.02821449190378189, + 0.04324287921190262, + 0.05173908546566963, + 0.0240048635751009, + 0.034459229558706284, + 0.04023054614663124, + 0.05085337162017822, + 0.016243331134319305, + 0.033271633088588715, + 0.03624724969267845, + 0.05302892625331879, + 0.011988751590251923, + 0.040694523602724075, + 0.04344379901885986, + 0.0900634303689003 + ] + } + }, + "full": { + "branch_output_rms": { + "max": [ + 0.027083024382591248, + 0.05378643795847893, + 0.053479041904211044, + 0.04230710119009018, + 0.03489644080400467, + 0.0346454493701458, + 0.02454756386578083, + 0.03129609674215317, + 0.0382988266646862, + 0.030918236821889877, + 0.08029001951217651, + 0.032227013260126114, + 0.0788588896393776, + 0.046491838991642, + 0.03414471819996834, + 0.028103213757276535, + 0.028869561851024628, + 0.024058125913143158, + 0.02986966073513031, + 0.024125492200255394, + 0.03027774952352047, + 0.03037451021373272, + 0.028807632625102997, + 0.031061751767992973, + 0.030561715364456177, + 0.031379926949739456, + 0.03756574168801308, + 0.03404445946216583, + 0.2097787708044052, + 0.041366904973983765, + 1.8557170629501343, + 0.05258624255657196 + ], + "mean": [ + 0.024008604387442272, + 0.05206462492545446, + 0.03722590704758962, + 0.041239711145559944, + 0.031058302149176598, + 0.0338385080297788, + 0.02349867671728134, + 0.029538670554757118, + 0.028947196279962856, + 0.029389141748348873, + 0.0621493769188722, + 0.02872789278626442, + 0.06633553778131802, + 0.0376141636321942, + 0.026593025773763657, + 0.02356371966501077, + 0.024800419807434082, + 0.021621525287628174, + 0.027926772211988766, + 0.022361520677804947, + 0.028618198509017628, + 0.02809702232480049, + 0.027077977856000263, + 0.030229248106479645, + 0.027249973888198536, + 0.030415359263618786, + 0.030738824357589085, + 0.03279792827864488, + 0.11324572935700417, + 0.03710922226309776, + 1.496443748474121, + 0.05082246412833532 + ], + "min": [ + 0.022041022777557373, + 0.05085906386375427, + 0.019325949251651764, + 0.03948163986206055, + 0.0279708169400692, + 0.033292125910520554, + 0.021648211404681206, + 0.028076333925127983, + 0.02268683910369873, + 0.027197115123271942, + 0.05061359703540802, + 0.021779853850603104, + 0.05264769122004509, + 0.0304368045181036, + 0.02188333496451378, + 0.01987709477543831, + 0.021968834102153778, + 0.02021927572786808, + 0.025901971384882927, + 0.021093308925628662, + 0.025842806324362755, + 0.02625282295048237, + 0.025757966563105583, + 0.029710212722420692, + 0.021688953042030334, + 0.02968704327940941, + 0.02333390712738037, + 0.030768385156989098, + 0.04889204725623131, + 0.034267887473106384, + 1.202970266342163, + 0.04837163910269737 + ] + }, + "layer_input_rms": { + "max": [ + 0.05461336672306061, + 0.034621261060237885, + 0.029128335416316986, + 0.03089532069861889, + 0.026308147236704826, + 0.02630813792347908, + 0.023830953985452652, + 0.02176283486187458, + 0.020936816930770874, + 0.020500168204307556, + 0.019486865028738976, + 0.022477135062217712, + 0.0208187997341156, + 0.02693997137248516, + 0.023864183574914932, + 0.024786626920104027, + 0.02244819700717926, + 0.01847153715789318, + 0.01782904379069805, + 0.010763376951217651, + 0.014036872424185276, + 0.008387424051761627, + 0.010892084799706936, + 0.007874744012951851, + 0.009214057587087154, + 0.007483004592359066, + 0.007594018243253231, + 0.007620570715516806, + 0.0075973751954734325, + 0.007396102417260408, + 0.014656757935881615, + 0.008849309757351875 + ], + "mean": [ + 0.054420417795578636, + 0.03333831081787745, + 0.028589200228452682, + 0.02826053649187088, + 0.023984291901191074, + 0.02512436422208945, + 0.021577948704361916, + 0.021324680497248966, + 0.019697973504662514, + 0.019812118262052536, + 0.018859490131338436, + 0.021303291122118633, + 0.02054722234606743, + 0.025280661260088284, + 0.023031941304604214, + 0.022272694235046703, + 0.02007630281150341, + 0.016114353202283382, + 0.01666525937616825, + 0.010316944991548857, + 0.013107998917500177, + 0.008068556897342205, + 0.010080384400983652, + 0.007476135001828273, + 0.008358963920424381, + 0.0073659901196757955, + 0.00731347252925237, + 0.007378495919207732, + 0.007157225937892993, + 0.007153482021143039, + 0.012953085514406363, + 0.008658299222588539 + ], + "min": [ + 0.054283902049064636, + 0.03162544220685959, + 0.02765912003815174, + 0.02553493343293667, + 0.02228466607630253, + 0.024269601330161095, + 0.020151648670434952, + 0.02089775912463665, + 0.018591508269309998, + 0.018769782036542892, + 0.017839200794696808, + 0.02058389224112034, + 0.02009216882288456, + 0.022332707419991493, + 0.021584752947092056, + 0.01947355456650257, + 0.018600277602672577, + 0.014435344375669956, + 0.015265379101037979, + 0.009849075227975845, + 0.012294777669012547, + 0.0078109800815582275, + 0.009295190684497356, + 0.007262552157044411, + 0.007303531747311354, + 0.007160624954849482, + 0.00694533996284008, + 0.007063175085932016, + 0.006762783043086529, + 0.006884528324007988, + 0.010628288611769676, + 0.008319443091750145 + ] + }, + "stream_state_rms": { + "max": [ + 0.04294736310839653, + 0.04605885222554207, + 0.04797445610165596, + 0.046828098595142365, + 0.044502634555101395, + 0.04308011010289192, + 0.0412217415869236, + 0.039975181221961975, + 0.03879518434405327, + 0.03814637288451195, + 0.04060528427362442, + 0.039477188140153885, + 0.04267995432019234, + 0.0419749952852726, + 0.041285403072834015, + 0.04062855243682861, + 0.040065914392471313, + 0.039282262325286865, + 0.03886580839753151, + 0.038292769342660904, + 0.037945639342069626, + 0.03764813393354416, + 0.03725530579686165, + 0.03698969632387161, + 0.036729857325553894, + 0.03649326041340828, + 0.036105964332818985, + 0.036036837846040726, + 0.05135522037744522, + 0.050893642008304596, + 0.33184999227523804, + 0.32691144943237305 + ], + "mean": [ + 0.0420840618511041, + 0.04565946012735367, + 0.044145053873459496, + 0.04360272611180941, + 0.04180811593929926, + 0.04077240079641342, + 0.039034511893987656, + 0.0381052220861117, + 0.03734947616855303, + 0.03669755533337593, + 0.03964470078547796, + 0.038941322515408196, + 0.04158667723337809, + 0.041379332542419434, + 0.040633320808410645, + 0.03984006121754646, + 0.03916142632563909, + 0.03844020888209343, + 0.03798529009024302, + 0.03739011660218239, + 0.03703943143288294, + 0.036697146793206535, + 0.03634887933731079, + 0.03612434739867846, + 0.035830430686473846, + 0.03564548244078954, + 0.035502238819996514, + 0.035413493712743126, + 0.04190699135263761, + 0.04178603241840998, + 0.26773401101430255, + 0.26379891236623126 + ], + "min": [ + 0.04164384678006172, + 0.044926099479198456, + 0.040089137852191925, + 0.04054243862628937, + 0.03965730220079422, + 0.038980793207883835, + 0.03725782036781311, + 0.03664333373308182, + 0.035495493561029434, + 0.03482290729880333, + 0.03899230435490608, + 0.038514114916324615, + 0.039978086948394775, + 0.0404449887573719, + 0.039540987461805344, + 0.03875384107232094, + 0.03806925565004349, + 0.0373431071639061, + 0.036932460963726044, + 0.03633511811494827, + 0.03608185052871704, + 0.03575707972049713, + 0.03549469634890556, + 0.035281531512737274, + 0.0348568893969059, + 0.034734319895505905, + 0.03483940288424492, + 0.03479676693677902, + 0.03653828799724579, + 0.036703966557979584, + 0.2158803939819336, + 0.21277332305908203 + ] + } + } + } +} diff --git a/src/data/k3-attnres-reduced.json b/src/data/k3-attnres-reduced.json new file mode 100644 index 0000000..7ba67bc --- /dev/null +++ b/src/data/k3-attnres-reduced.json @@ -0,0 +1,21498 @@ +{ + "analysis": { + "evaluation_curves": { + "baseline": [ + { + "by_seed": [ + 8.095421586117522, + 8.037616927292651, + 8.032852405924473 + ], + "max_bpc": 8.095421586117522, + "mean_bpc": 8.055296973111549, + "min_bpc": 8.032852405924473, + "step": 0 + }, + { + "by_seed": [ + 3.86267014445088, + 3.8739961479147755, + 3.864131094074297 + ], + "max_bpc": 3.8739961479147755, + "mean_bpc": 3.8669324621466505, + "min_bpc": 3.86267014445088, + "step": 100 + }, + { + "by_seed": [ + 3.431632090385233, + 3.4334316738481987, + 3.4184366785021214 + ], + "max_bpc": 3.4334316738481987, + "mean_bpc": 3.427833480911851, + "min_bpc": 3.4184366785021214, + "step": 250 + }, + { + "by_seed": [ + 2.9448990671750175, + 2.9345221069742453, + 2.907037430552872 + ], + "max_bpc": 2.9448990671750175, + "mean_bpc": 2.9288195349007116, + "min_bpc": 2.907037430552872, + "step": 500 + }, + { + "by_seed": [ + 2.2560981455190916, + 2.2564510109229996, + 2.249317836997982 + ], + "max_bpc": 2.2564510109229996, + "mean_bpc": 2.2539556644800243, + "min_bpc": 2.249317836997982, + "step": 1000 + }, + { + "by_seed": [ + 2.0700625342803067, + 2.0709166216223225, + 2.061515899441342 + ], + "max_bpc": 2.0709166216223225, + "mean_bpc": 2.0674983517813237, + "min_bpc": 2.061515899441342, + "step": 1500 + }, + { + "by_seed": [ + 2.0005383326907027, + 1.9984403593284898, + 1.9984182165621913 + ], + "max_bpc": 2.0005383326907027, + "mean_bpc": 1.9991323028604613, + "min_bpc": 1.9984182165621913, + "step": 2000 + } + ], + "block": [ + { + "by_seed": [ + 8.091490664658076, + 8.035310725935906, + 8.029935924131141 + ], + "max_bpc": 8.091490664658076, + "mean_bpc": 8.052245771575041, + "min_bpc": 8.029935924131141, + "step": 0 + }, + { + "by_seed": [ + 3.8613770068990427, + 3.8667511637688663, + 3.861173508427409 + ], + "max_bpc": 3.8667511637688663, + "mean_bpc": 3.8631005596984394, + "min_bpc": 3.861173508427409, + "step": 100 + }, + { + "by_seed": [ + 3.4246572049925037, + 3.422946407573046, + 3.427078118777374 + ], + "max_bpc": 3.427078118777374, + "mean_bpc": 3.4248939104476412, + "min_bpc": 3.422946407573046, + "step": 250 + }, + { + "by_seed": [ + 2.890138823737888, + 2.8856276543122625, + 2.9073413023991153 + ], + "max_bpc": 2.9073413023991153, + "mean_bpc": 2.8943692601497553, + "min_bpc": 2.8856276543122625, + "step": 500 + }, + { + "by_seed": [ + 2.193909756412884, + 2.199974509617247, + 2.2110541049381194 + ], + "max_bpc": 2.2110541049381194, + "mean_bpc": 2.2016461236560834, + "min_bpc": 2.193909756412884, + "step": 1000 + }, + { + "by_seed": [ + 2.01564146212851, + 2.028303792287441, + 2.0265933173354536 + ], + "max_bpc": 2.028303792287441, + "mean_bpc": 2.023512857250468, + "min_bpc": 2.01564146212851, + "step": 1500 + }, + { + "by_seed": [ + 1.947877975922777, + 1.9570916454731129, + 1.9650305581174223 + ], + "max_bpc": 1.9650305581174223, + "mean_bpc": 1.9566667265044373, + "min_bpc": 1.947877975922777, + "step": 2000 + } + ], + "full": [ + { + "by_seed": [ + 8.053220311266156, + 8.015559292406241, + 8.006702013904155 + ], + "max_bpc": 8.053220311266156, + "mean_bpc": 8.025160539192184, + "min_bpc": 8.006702013904155, + "step": 0 + }, + { + "by_seed": [ + 3.87981092433611, + 3.8904535117411396, + 3.877691539133865 + ], + "max_bpc": 3.8904535117411396, + "mean_bpc": 3.8826519917370383, + "min_bpc": 3.877691539133865, + "step": 100 + }, + { + "by_seed": [ + 3.4658855739879395, + 3.4716255379561485, + 3.4643856702941025 + ], + "max_bpc": 3.4716255379561485, + "mean_bpc": 3.4672989274127297, + "min_bpc": 3.4643856702941025, + "step": 250 + }, + { + "by_seed": [ + 2.9766696157493864, + 2.996736035504106, + 2.967935433829656 + ], + "max_bpc": 2.996736035504106, + "mean_bpc": 2.98044702836105, + "min_bpc": 2.967935433829656, + "step": 500 + }, + { + "by_seed": [ + 2.2318612109744427, + 2.2425945407256336, + 2.2438350085906444 + ], + "max_bpc": 2.2438350085906444, + "mean_bpc": 2.2394302534302404, + "min_bpc": 2.2318612109744427, + "step": 1000 + }, + { + "by_seed": [ + 2.05077289566604, + 2.051821925342809, + 2.046430935671032 + ], + "max_bpc": 2.051821925342809, + "mean_bpc": 2.0496752522266273, + "min_bpc": 2.046430935671032, + "step": 1500 + }, + { + "by_seed": [ + 1.9845419683599765, + 1.9852328656641551, + 1.9839206165402405 + ], + "max_bpc": 1.9852328656641551, + "mean_bpc": 1.984565150188124, + "min_bpc": 1.9839206165402405, + "step": 2000 + } + ] + }, + "final_validation": { + "block_contrast": { + "max_delta_bpc": -0.033387658444768986, + "mean_delta_bpc": -0.04246557635602392, + "min_delta_bpc": -0.0526603567679258, + "paired_deltas_bpc": [ + -0.0526603567679258, + -0.04134871385537697, + -0.033387658444768986 + ], + "same_direction": true, + "threshold_bpc": 0.01, + "verdict": "directional support in this reduced protocol" + }, + "block_minus_full": { + "mean_delta_bpc": -0.027898423683686707, + "paired_deltas_bpc": [ + -0.036663992437199644, + -0.02814122019104226, + -0.018890058422818212 + ] + }, + "by_seed": [ + { + "block_minus_baseline": -0.0526603567679258, + "block_minus_full": -0.036663992437199644, + "final_bpc": { + "baseline": 2.0005383326907027, + "block": 1.947877975922777, + "full": 1.9845419683599765 + }, + "full_minus_baseline": -0.015996364330726154, + "seed": 2026073001 + }, + { + "block_minus_baseline": -0.04134871385537697, + "block_minus_full": -0.02814122019104226, + "final_bpc": { + "baseline": 1.9984403593284898, + "block": 1.9570916454731129, + "full": 1.9852328656641551 + }, + "full_minus_baseline": -0.013207493664334713, + "seed": 2026073002 + }, + { + "block_minus_baseline": -0.033387658444768986, + "block_minus_full": -0.018890058422818212, + "final_bpc": { + "baseline": 1.9984182165621913, + "block": 1.9650305581174223, + "full": 1.9839206165402405 + }, + "full_minus_baseline": -0.014497600021950774, + "seed": 2026073003 + } + ], + "full_contrast": { + "max_delta_bpc": -0.013207493664334713, + "mean_delta_bpc": -0.014567152672337214, + "min_delta_bpc": -0.015996364330726154, + "paired_deltas_bpc": [ + -0.015996364330726154, + -0.013207493664334713, + -0.014497600021950774 + ], + "same_direction": true, + "threshold_bpc": 0.01, + "verdict": "directional support in this reduced protocol" + }, + "means": { + "baseline": 1.9991323028604613, + "block": 1.9566667265044373, + "full": 1.984565150188124 + } + }, + "gradients": { + "baseline": { + "by_block": { + "max": [ + 0.00075677209292545, + 0.0005532402766490476, + 0.00043713799901330333, + 0.00033044463575291374, + 0.0003495170932308389, + 0.00035395098520716656, + 0.00041254654039553357, + 0.0005141244473775357, + 0.0006036197919264307, + 0.0006948711477092932, + 0.0008904295974468759, + 0.00101380642253527, + 0.0008315976940648109, + 0.000694730303324183, + 0.0005421243248268182, + 0.00042412043092329184 + ], + "mean": [ + 0.0007019380262137629, + 0.0004973613932330452, + 0.0003913067103051371, + 0.0003113512617537747, + 0.00031526216215974653, + 0.0003354778900867988, + 0.0003856618618192619, + 0.00046334355513234175, + 0.000584073639160152, + 0.0006482042032616321, + 0.0008321813718249159, + 0.0008802390721549605, + 0.0008241068628698583, + 0.0006579748827421658, + 0.0005357449999541818, + 0.0004032229752138717 + ], + "min": [ + 0.0006069268184991451, + 0.0004268615003720108, + 0.000323043032117742, + 0.0002914754177093503, + 0.00028453589406033573, + 0.0003157797529074047, + 0.00036398451532517026, + 0.0004021196515771015, + 0.000555786501522983, + 0.0005651240708116888, + 0.0007729757575868522, + 0.0007932171508463918, + 0.0008177916178264611, + 0.0006303107513205802, + 0.0005291293137127175, + 0.00039202598310552093 + ] + }, + "cv_by_seed": [ + 0.35371966889493056, + 0.34086455023848533, + 0.33957005977765514 + ], + "first_last_ratio_by_seed": [ + 1.4310247143196844, + 1.9304130989750008, + 1.8858264656200423 + ], + "mean_cv": 0.34471809297035705, + "mean_first_last_ratio": 1.7490880929715757 + }, + "block": { + "by_block": { + "max": [ + 0.0006136196561721271, + 0.0004981167249238301, + 0.0003300655422364756, + 0.00024049221342133134, + 0.0002907770609625826, + 0.00041448514649562707, + 0.00042775107615344534, + 0.00039679767722893174, + 0.0006871599037030366, + 0.0009600722047358557, + 0.0016852072794625043, + 0.0012425342652473243, + 0.0011452921321391885, + 0.0007695163028177718, + 0.0005286419550644225, + 0.00042053944700778587 + ], + "mean": [ + 0.0004888831050398458, + 0.00044961389290692, + 0.0003041330266206076, + 0.00023643267595969324, + 0.00027324420174070326, + 0.0002986754642657978, + 0.00036172653847793204, + 0.00034532391576277656, + 0.0006077407773615036, + 0.0008327782910825645, + 0.001588981506880309, + 0.001219213916530425, + 0.0010906193570424788, + 0.0007477400770465019, + 0.0004999503819265139, + 0.0004088396118379981 + ], + "min": [ + 0.0003999679822384072, + 0.00042300100546983135, + 0.00028404817531530786, + 0.0002340430140301531, + 0.00025161489682913623, + 0.0002331203443329733, + 0.00032361633132051105, + 0.00031515007512692854, + 0.00046013207297086815, + 0.0006049664535646685, + 0.001431364890916451, + 0.0012053624438772336, + 0.0010361348995012233, + 0.0007327317391797293, + 0.00045754647605205013, + 0.0003918054076853068 + ] + }, + "cv_by_seed": [ + 0.6622010179556602, + 0.6848619013329328, + 0.5448410226471907 + ], + "first_last_ratio_by_seed": [ + 1.156343602773591, + 0.9657004079436908, + 1.459125084550669 + ], + "mean_cv": 0.6306346473119279, + "mean_first_last_ratio": 1.1937230317559837 + }, + "full": { + "by_block": { + "max": [ + 0.0006630627553059442, + 0.000431366526533741, + 0.0002935829834809252, + 0.00027276189477100897, + 0.0002558489145159978, + 0.0004401354335484962, + 0.00044057723716284654, + 0.0004762864793838318, + 0.0008786784638329018, + 0.0010455563665680043, + 0.0014801483865081893, + 0.0012357605502724435, + 0.0009144618812121531, + 0.0007776715348883125, + 0.0006049584856703907, + 0.0005101241707709288 + ], + "mean": [ + 0.0006167513874874072, + 0.00040618462626699743, + 0.0002733799653117713, + 0.0002419244417044836, + 0.00024695137499203994, + 0.0003522638252622234, + 0.00039760768314637185, + 0.00043278080669132644, + 0.0006604308069081531, + 0.0009861650947710283, + 0.0012433582725884958, + 0.0010459425806746048, + 0.0008832753785975217, + 0.0007581138332205081, + 0.0005866601291828091, + 0.0004765533868027066 + ], + "min": [ + 0.0005889504686640395, + 0.0003745185363114891, + 0.0002580782374219271, + 0.00021098033769564252, + 0.00022993427970632987, + 0.00029749385150172825, + 0.0003254079422057709, + 0.0003844568143161905, + 0.00048201456346138433, + 0.0009251527144282604, + 0.0010490551061311498, + 0.0009276232454397661, + 0.0008520028126550238, + 0.0007384596329355597, + 0.0005570695353753757, + 0.0004474027776125591 + ] + }, + "cv_by_seed": [ + 0.48897346891847043, + 0.5100252960815966, + 0.5272225078814977 + ], + "first_last_ratio_by_seed": [ + 1.24742435749958, + 1.4820264613559055, + 1.1727359195470821 + ], + "mean_cv": 0.5087404242938549, + "mean_first_last_ratio": 1.300728912800856 + } + }, + "interpretation": { + "bounded_depth_pattern": "Block partial-state RMS resets every four residual sublayers; the complete 32-point vectors are reported.", + "gradient_boundary": "The preregistered core-parameter gradient RMS is not more uniform for AttnRes here; this metric and scale do not reproduce the paper's large-model gradient-magnitude result.", + "primary": "Both Full and Block AttnRes satisfy the preregistered directional-support rule in this reduced protocol.", + "scope": "Reduced byte-level WikiText-2 mechanism probe; not a K3 checkpoint run, paper-scale reproduction, benchmark, or same-FLOP comparison." + }, + "mixers": { + "block": { + "max_sources": 9, + "output": { + "entropy_max": 1.9263067245483398, + "entropy_mean": 1.911759614944458, + "entropy_min": 1.89408540725708, + "mean_weights": [ + 0.0960588405529658, + 0.06008275722463926, + 0.053474895656108856, + 0.04561992113788923, + 0.04707453027367592, + 0.07797466466824214, + 0.09173511217037837, + 0.2304698278506597, + 0.29750946164131165 + ], + "sources": 9 + }, + "rows": [ + { + "entropy_max": 0.0, + "entropy_mean": 0.0, + "entropy_min": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1, + "sublayer": 1 + }, + { + "entropy_max": 0.676069974899292, + "entropy_mean": 0.663312554359436, + "entropy_min": 0.6499756574630737, + "mean_weights": [ + 0.568194588025411, + 0.43180541197458905 + ], + "sources": 2, + "sublayer": 2 + }, + { + "entropy_max": 0.6758064031600952, + "entropy_mean": 0.6724996566772461, + "entropy_min": 0.6664855480194092, + "mean_weights": [ + 0.5410497585932413, + 0.4589502414067586 + ], + "sources": 2, + "sublayer": 3 + }, + { + "entropy_max": 0.5902198553085327, + "entropy_mean": 0.547868569691976, + "entropy_min": 0.5004937648773193, + "mean_weights": [ + 0.7541502118110657, + 0.24584979315598807 + ], + "sources": 2, + "sublayer": 4 + }, + { + "entropy_max": 0.6855782270431519, + "entropy_mean": 0.6811449726422628, + "entropy_min": 0.6787354350090027, + "mean_weights": [ + 0.4620973467826843, + 0.5379026532173157 + ], + "sources": 2, + "sublayer": 5 + }, + { + "entropy_max": 1.025155782699585, + "entropy_mean": 1.0066618919372559, + "entropy_min": 0.973181962966919, + "mean_weights": [ + 0.5044599374135336, + 0.2516526132822037, + 0.24388746420542398 + ], + "sources": 3, + "sublayer": 6 + }, + { + "entropy_max": 1.0936508178710938, + "entropy_mean": 1.0913390318552654, + "entropy_min": 1.0888967514038086, + "mean_weights": [ + 0.30184901754061383, + 0.3493907650311788, + 0.3487602174282074 + ], + "sources": 3, + "sublayer": 7 + }, + { + "entropy_max": 1.0497782230377197, + "entropy_mean": 1.027435878912608, + "entropy_min": 0.9970768094062805, + "mean_weights": [ + 0.45536842942237854, + 0.23504690329233804, + 0.30958468715349835 + ], + "sources": 3, + "sublayer": 8 + }, + { + "entropy_max": 1.0915882587432861, + "entropy_mean": 1.0884315172831218, + "entropy_min": 1.0863194465637207, + "mean_weights": [ + 0.30525945623715717, + 0.34874869386355084, + 0.3459918598333995 + ], + "sources": 3, + "sublayer": 9 + }, + { + "entropy_max": 1.3575613498687744, + "entropy_mean": 1.3401769797007244, + "entropy_min": 1.3304085731506348, + "mean_weights": [ + 0.31245625019073486, + 0.188400000333786, + 0.24196669459342957, + 0.25717706481615704 + ], + "sources": 4, + "sublayer": 10 + }, + { + "entropy_max": 1.37892484664917, + "entropy_mean": 1.3752654393513997, + "entropy_min": 1.372523546218872, + "mean_weights": [ + 0.2376734067996343, + 0.25084111591180164, + 0.2222500592470169, + 0.2892354329427083 + ], + "sources": 4, + "sublayer": 11 + }, + { + "entropy_max": 1.3570414781570435, + "entropy_mean": 1.3392419815063477, + "entropy_min": 1.3118952512741089, + "mean_weights": [ + 0.2914406458536784, + 0.17078583439191183, + 0.22245727479457855, + 0.31531623005867004 + ], + "sources": 4, + "sublayer": 12 + }, + { + "entropy_max": 1.372481346130371, + "entropy_mean": 1.371339241663615, + "entropy_min": 1.3699870109558105, + "mean_weights": [ + 0.2318959136803945, + 0.26858144998550415, + 0.20875323315461478, + 0.29076939821243286 + ], + "sources": 4, + "sublayer": 13 + }, + { + "entropy_max": 1.5968154668807983, + "entropy_mean": 1.5836326678593953, + "entropy_min": 1.5674753189086914, + "mean_weights": [ + 0.19836588700612387, + 0.15423202017943063, + 0.17724995811780295, + 0.2274299512306849, + 0.24272218346595764 + ], + "sources": 5, + "sublayer": 14 + }, + { + "entropy_max": 1.5973689556121826, + "entropy_mean": 1.5895702838897705, + "entropy_min": 1.577967882156372, + "mean_weights": [ + 0.18468524515628815, + 0.19026227295398712, + 0.15613725781440735, + 0.19949903090794882, + 0.269416203101476 + ], + "sources": 5, + "sublayer": 15 + }, + { + "entropy_max": 1.5947315692901611, + "entropy_mean": 1.5757121245066326, + "entropy_min": 1.546515941619873, + "mean_weights": [ + 0.1997243563334147, + 0.17882535854975382, + 0.15601223707199097, + 0.17899859448273978, + 0.286439448595047 + ], + "sources": 5, + "sublayer": 16 + }, + { + "entropy_max": 1.604175090789795, + "entropy_mean": 1.5968455870946248, + "entropy_min": 1.5857822895050049, + "mean_weights": [ + 0.1956237554550171, + 0.203901469707489, + 0.17066203554471335, + 0.1839087208112081, + 0.24590401351451874 + ], + "sources": 5, + "sublayer": 17 + }, + { + "entropy_max": 1.7821681499481201, + "entropy_mean": 1.768844485282898, + "entropy_min": 1.7451198101043701, + "mean_weights": [ + 0.15534367660681406, + 0.14029951890309653, + 0.14831382284561792, + 0.15113131205240884, + 0.18528948724269867, + 0.21962219973405203 + ], + "sources": 6, + "sublayer": 18 + }, + { + "entropy_max": 1.7596988677978516, + "entropy_mean": 1.7357758680979412, + "entropy_min": 1.7161343097686768, + "mean_weights": [ + 0.1554203579823176, + 0.15391315519809723, + 0.1261703222990036, + 0.12737740576267242, + 0.14683481554190317, + 0.2902839382489522 + ], + "sources": 6, + "sublayer": 19 + }, + { + "entropy_max": 1.59963059425354, + "entropy_mean": 1.4545996189117432, + "entropy_min": 1.360633373260498, + "mean_weights": [ + 0.13907443980375925, + 0.13006270925203958, + 0.08823741724093755, + 0.06815870975454648, + 0.07939869786302249, + 0.4950680136680603 + ], + "sources": 6, + "sublayer": 20 + }, + { + "entropy_max": 1.7793185710906982, + "entropy_mean": 1.7684559027353923, + "entropy_min": 1.7599127292633057, + "mean_weights": [ + 0.16601511339346567, + 0.16745010018348694, + 0.13976068794727325, + 0.14097271859645844, + 0.15037270387013754, + 0.23542868594328561 + ], + "sources": 6, + "sublayer": 21 + }, + { + "entropy_max": 1.592297911643982, + "entropy_mean": 1.5481091340382893, + "entropy_min": 1.4908201694488525, + "mean_weights": [ + 0.09055864065885544, + 0.07739925881226857, + 0.05696734661857287, + 0.046672369043032326, + 0.05360592405001322, + 0.22773849964141846, + 0.44705795248349506 + ], + "sources": 7, + "sublayer": 22 + }, + { + "entropy_max": 1.8954918384552002, + "entropy_mean": 1.8707793553670247, + "entropy_min": 1.8509920835494995, + "mean_weights": [ + 0.12786388397216797, + 0.12726488709449768, + 0.1035188560684522, + 0.10487095763285954, + 0.10812034954627354, + 0.1509028524160385, + 0.2774582306543986 + ], + "sources": 7, + "sublayer": 23 + }, + { + "entropy_max": 1.5638015270233154, + "entropy_mean": 1.529894272486369, + "entropy_min": 1.4770772457122803, + "mean_weights": [ + 0.09439751009146373, + 0.07925632844368617, + 0.057954128831624985, + 0.049073984225591026, + 0.05442543948690096, + 0.17891186972459158, + 0.4859807292620341 + ], + "sources": 7, + "sublayer": 24 + }, + { + "entropy_max": 1.9023945331573486, + "entropy_mean": 1.880323886871338, + "entropy_min": 1.8683671951293945, + "mean_weights": [ + 0.13544990122318268, + 0.14224635561307272, + 0.10577710469563802, + 0.0981563354531924, + 0.10264134655396144, + 0.15992753704388937, + 0.25580141445000965 + ], + "sources": 7, + "sublayer": 25 + }, + { + "entropy_max": 1.7563724517822266, + "entropy_mean": 1.7308308283487956, + "entropy_min": 1.7099403142929077, + "mean_weights": [ + 0.07523585855960846, + 0.06218656276663145, + 0.045632037023703255, + 0.03874268631140391, + 0.042072441428899765, + 0.1278796618183454, + 0.3132644097010295, + 0.2949863374233246 + ], + "sources": 8, + "sublayer": 26 + }, + { + "entropy_max": 1.9232137203216553, + "entropy_mean": 1.9054110050201416, + "entropy_min": 1.8879387378692627, + "mean_weights": [ + 0.09858222305774689, + 0.10369476675987244, + 0.07179193198680878, + 0.0594374363621076, + 0.061381166179974876, + 0.12209223707516988, + 0.18827446301778158, + 0.294745792945226 + ], + "sources": 8, + "sublayer": 27 + }, + { + "entropy_max": 1.7447065114974976, + "entropy_mean": 1.72612730662028, + "entropy_min": 1.7093220949172974, + "mean_weights": [ + 0.0765189677476883, + 0.06796563292543094, + 0.04787288730343183, + 0.04231621200839678, + 0.04612377534310023, + 0.11212762693564098, + 0.2269348402818044, + 0.3801400661468506 + ], + "sources": 8, + "sublayer": 28 + }, + { + "entropy_max": 1.9634629487991333, + "entropy_mean": 1.9337302446365356, + "entropy_min": 1.886972427368164, + "mean_weights": [ + 0.10630816966295242, + 0.10707242041826248, + 0.07552854220072429, + 0.06032343705495199, + 0.05965699007113775, + 0.13685152928034464, + 0.20677822828292847, + 0.24748070041338602 + ], + "sources": 8, + "sublayer": 29 + }, + { + "entropy_max": 1.9826093912124634, + "entropy_mean": 1.9681602716445923, + "entropy_min": 1.943651795387268, + "mean_weights": [ + 0.07450833419958751, + 0.06967654327551524, + 0.04876244689027468, + 0.040633084873358406, + 0.04262504850824674, + 0.10585258652766545, + 0.19568686187267303, + 0.23621322711308798, + 0.1860418419043223 + ], + "sources": 9, + "sublayer": 30 + }, + { + "entropy_max": 2.052138090133667, + "entropy_mean": 2.0466067790985107, + "entropy_min": 2.0432467460632324, + "mean_weights": [ + 0.08745819330215454, + 0.08252991984287898, + 0.058957609037558235, + 0.04905804619193077, + 0.049261036018530525, + 0.11601439615090688, + 0.1833944966395696, + 0.20734459161758423, + 0.16598171492417654 + ], + "sources": 9, + "sublayer": 31 + }, + { + "entropy_max": 2.0141000747680664, + "entropy_mean": 1.9910953442255657, + "entropy_min": 1.9660465717315674, + "mean_weights": [ + 0.07816717276970546, + 0.07901749511559804, + 0.05229665587345759, + 0.04327899465958277, + 0.046668255080779396, + 0.10070234537124634, + 0.17064277331034342, + 0.24551014105478922, + 0.1837161829074224 + ], + "sources": 9, + "sublayer": 32 + } + ] + }, + "full": { + "max_sources": 32, + "output": { + "entropy_max": 3.2301199436187744, + "entropy_mean": 3.2225964864095054, + "entropy_min": 3.2135813236236572, + "mean_weights": [ + 0.025556615243355434, + 0.02490210657318433, + 0.026224765926599503, + 0.019764999548594158, + 0.024120309079686802, + 0.01947101578116417, + 0.020250216126441956, + 0.01827173059185346, + 0.017942425484458607, + 0.017180486271778744, + 0.01900123494366805, + 0.015923069479564827, + 0.013854833009342352, + 0.016641111423571903, + 0.013891398285826048, + 0.023777509729067486, + 0.021802425384521484, + 0.03012333872417609, + 0.025545488422115643, + 0.03698895126581192, + 0.027168714130918186, + 0.03949497267603874, + 0.027805685997009277, + 0.03334506042301655, + 0.03443349897861481, + 0.02792014181613922, + 0.056730019549528755, + 0.026892853900790215, + 0.08061449726422627, + 0.013165338585774103, + 0.0910993938644727, + 0.0027941359827915826, + 0.10730165491501491 + ], + "sources": 33 + }, + "rows": [ + { + "entropy_max": 0.0, + "entropy_mean": 0.0, + "entropy_min": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1, + "sublayer": 1 + }, + { + "entropy_max": 0.6729873418807983, + "entropy_mean": 0.6630954543749491, + "entropy_min": 0.6552057266235352, + "mean_weights": [ + 0.5729489127794901, + 0.42705105741818744 + ], + "sources": 2, + "sublayer": 2 + }, + { + "entropy_max": 1.0850456953048706, + "entropy_mean": 1.0804216861724854, + "entropy_min": 1.074110507965088, + "mean_weights": [ + 0.3627606928348541, + 0.3307747046152751, + 0.30646461248397827 + ], + "sources": 3, + "sublayer": 3 + }, + { + "entropy_max": 1.3302907943725586, + "entropy_mean": 1.3018674453099568, + "entropy_min": 1.2723480463027954, + "mean_weights": [ + 0.34105542302131653, + 0.19593795637289682, + 0.308497816324234, + 0.15450880428155264 + ], + "sources": 4, + "sublayer": 4 + }, + { + "entropy_max": 1.5865752696990967, + "entropy_mean": 1.575003186861674, + "entropy_min": 1.5582118034362793, + "mean_weights": [ + 0.18386788666248322, + 0.2390782485405604, + 0.18032612899939218, + 0.23851566016674042, + 0.1582120656967163 + ], + "sources": 5, + "sublayer": 5 + }, + { + "entropy_max": 1.7290915250778198, + "entropy_mean": 1.7206300099690754, + "entropy_min": 1.7134463787078857, + "mean_weights": [ + 0.21605777740478516, + 0.14990001420180002, + 0.2151650389035543, + 0.11052880187829335, + 0.1715006877978643, + 0.1368476798137029 + ], + "sources": 6, + "sublayer": 6 + }, + { + "entropy_max": 1.9155155420303345, + "entropy_mean": 1.8901333411534627, + "entropy_min": 1.8659067153930664, + "mean_weights": [ + 0.12854668498039246, + 0.19778794050216675, + 0.13465766112009683, + 0.18992085258165994, + 0.11008669932683308, + 0.15766242146492004, + 0.0813377524415652 + ], + "sources": 7, + "sublayer": 7 + }, + { + "entropy_max": 2.019052743911743, + "entropy_mean": 2.013235012690226, + "entropy_min": 2.0070152282714844, + "mean_weights": [ + 0.14720548192660013, + 0.1377126748363177, + 0.16037661333878836, + 0.10299472014109294, + 0.11697593828042348, + 0.11880407234032948, + 0.07762171079715093, + 0.1383087933063507 + ], + "sources": 8, + "sublayer": 8 + }, + { + "entropy_max": 2.157308578491211, + "entropy_mean": 2.154908021291097, + "entropy_min": 2.152099132537842, + "mean_weights": [ + 0.10807480166355769, + 0.15884055693944296, + 0.11500980953375499, + 0.13613408307234445, + 0.09466097255547841, + 0.12546123564243317, + 0.0702842225631078, + 0.12216035773356755, + 0.06937396277983983 + ], + "sources": 9, + "sublayer": 9 + }, + { + "entropy_max": 2.2573065757751465, + "entropy_mean": 2.2304697831471763, + "entropy_min": 2.2092034816741943, + "mean_weights": [ + 0.11872178316116333, + 0.11724316825469334, + 0.13413792848587036, + 0.08227266867955525, + 0.09661872188250224, + 0.10174320389827092, + 0.06136756141980489, + 0.11396426459153493, + 0.05892340963085493, + 0.11500727633635204 + ], + "sources": 10, + "sublayer": 10 + }, + { + "entropy_max": 2.359937906265259, + "entropy_mean": 2.352225144704183, + "entropy_min": 2.341871976852417, + "mean_weights": [ + 0.09207804997762044, + 0.13507375617822012, + 0.0997256338596344, + 0.11042191833257675, + 0.08121875921885173, + 0.09732648730278015, + 0.058663940678040184, + 0.10317053894201915, + 0.05721114327510198, + 0.10648540159066518, + 0.05862438430388769 + ], + "sources": 11, + "sublayer": 11 + }, + { + "entropy_max": 2.3737237453460693, + "entropy_mean": 2.3604607582092285, + "entropy_min": 2.336538791656494, + "mean_weights": [ + 0.07879289239645004, + 0.11672904590765636, + 0.08075895408789317, + 0.09057638794183731, + 0.05744863177339236, + 0.10357426106929779, + 0.036198447148005165, + 0.11587326476971309, + 0.03574693823854128, + 0.12555705259243646, + 0.0362573837240537, + 0.12248673538366954 + ], + "sources": 12, + "sublayer": 12 + }, + { + "entropy_max": 2.5292272567749023, + "entropy_mean": 2.523596445719401, + "entropy_min": 2.5176010131835938, + "mean_weights": [ + 0.07940395176410675, + 0.10745090742905934, + 0.08859917024771373, + 0.08839243898789088, + 0.07243171830972035, + 0.08091367284456889, + 0.05198533460497856, + 0.08363978813091914, + 0.05071114872892698, + 0.09088355551163356, + 0.051586925983428955, + 0.10110345482826233, + 0.05289792890350024 + ], + "sources": 13, + "sublayer": 13 + }, + { + "entropy_max": 2.5260555744171143, + "entropy_mean": 2.511597235997518, + "entropy_min": 2.4945454597473145, + "mean_weights": [ + 0.06728010376294453, + 0.09017306566238403, + 0.06705486277739207, + 0.07410627727707227, + 0.049341840048631035, + 0.08006220559279124, + 0.03232185977200667, + 0.09271023919185002, + 0.03153213858604431, + 0.1004645402232806, + 0.03198575973510742, + 0.11092506349086761, + 0.035584566493829094, + 0.13645747800668082 + ], + "sources": 14, + "sublayer": 14 + }, + { + "entropy_max": 2.6629085540771484, + "entropy_mean": 2.6597483158111572, + "entropy_min": 2.6540651321411133, + "mean_weights": [ + 0.0696261500318845, + 0.08718715111414592, + 0.07843850553035736, + 0.07442579418420792, + 0.0630200741191705, + 0.07242759317159653, + 0.04383365189035734, + 0.0742378830909729, + 0.042224218448003135, + 0.07677323619524638, + 0.04271993786096573, + 0.0888681411743164, + 0.04172474518418312, + 0.09675361216068268, + 0.04773929715156555 + ], + "sources": 15, + "sublayer": 15 + }, + { + "entropy_max": 2.653104782104492, + "entropy_mean": 2.6441221237182617, + "entropy_min": 2.6280198097229004, + "mean_weights": [ + 0.05986166993776957, + 0.08501092096169789, + 0.06847748905420303, + 0.06890026852488518, + 0.04862064744035403, + 0.07089548061291377, + 0.02869326372941335, + 0.07980451981226604, + 0.02707553468644619, + 0.07720348238945007, + 0.026560273642341297, + 0.08022717883189519, + 0.02696048965056737, + 0.09724790354569753, + 0.03290759275356928, + 0.12155329436063766 + ], + "sources": 16, + "sublayer": 16 + }, + { + "entropy_max": 2.7772436141967773, + "entropy_mean": 2.7734902699788413, + "entropy_min": 2.767390251159668, + "mean_weights": [ + 0.06181313097476959, + 0.0787958453098933, + 0.0740993320941925, + 0.06472191214561462, + 0.05765968312819799, + 0.06431250025828679, + 0.038311279068390526, + 0.0646959940592448, + 0.035535636047522225, + 0.06446120763818423, + 0.03540633494655291, + 0.06941856443881989, + 0.03171839565038681, + 0.07583510130643845, + 0.035414453595876694, + 0.09135590742031734, + 0.05644472067554792 + ], + "sources": 17, + "sublayer": 17 + }, + { + "entropy_max": 2.76259446144104, + "entropy_mean": 2.7232768535614014, + "entropy_min": 2.6990277767181396, + "mean_weights": [ + 0.05518728867173195, + 0.06774829079707463, + 0.06946775317192078, + 0.05141793688138326, + 0.04801270241538683, + 0.05370426674683889, + 0.027809271588921547, + 0.0562954917550087, + 0.02551322678724925, + 0.05110526581605276, + 0.024331960827112198, + 0.046864173064629235, + 0.020843032126625378, + 0.05734457323948542, + 0.024186863874395687, + 0.09807546188433965, + 0.07210367918014526, + 0.14998876055081686 + ], + "sources": 18, + "sublayer": 18 + }, + { + "entropy_max": 2.8850622177124023, + "entropy_mean": 2.8652973969777427, + "entropy_min": 2.843248128890991, + "mean_weights": [ + 0.05260627344250679, + 0.06991977492968242, + 0.06476513544718425, + 0.0581423689921697, + 0.049954539785782494, + 0.056481542686621346, + 0.03229485948880514, + 0.05682817722360293, + 0.02923661842942238, + 0.0552576519548893, + 0.028797871122757595, + 0.05738401288787524, + 0.024513501053055126, + 0.06065270428856214, + 0.02665022263924281, + 0.07460795839627583, + 0.04375817130009333, + 0.09523340314626694, + 0.06291522085666656 + ], + "sources": 19, + "sublayer": 19 + }, + { + "entropy_max": 2.7220685482025146, + "entropy_mean": 2.7043351332346597, + "entropy_min": 2.69278621673584, + "mean_weights": [ + 0.045603882521390915, + 0.05447864532470703, + 0.06410424038767815, + 0.03792111948132515, + 0.04226911440491676, + 0.03555091346303622, + 0.02289348654448986, + 0.03695064038038254, + 0.020027659212549526, + 0.031009394054611523, + 0.01926272859176, + 0.025271238759160042, + 0.014632520886758963, + 0.028143515810370445, + 0.014787437083820501, + 0.062148607025543846, + 0.05986413732171059, + 0.11426966885725658, + 0.08119676013787587, + 0.18961429099241892 + ], + "sources": 20, + "sublayer": 20 + }, + { + "entropy_max": 2.9706764221191406, + "entropy_mean": 2.957854668299357, + "entropy_min": 2.9454524517059326, + "mean_weights": [ + 0.04558223858475685, + 0.059097565710544586, + 0.05700209612647692, + 0.048397842794656754, + 0.04409012322624525, + 0.0473170280456543, + 0.028855254873633385, + 0.047425925731658936, + 0.02600932928423087, + 0.04637702306111654, + 0.02551550293962161, + 0.047576854626337685, + 0.021547649676601093, + 0.05002515638868014, + 0.02275315361718337, + 0.06288522730271022, + 0.03654991711179415, + 0.07972793529431026, + 0.047366989155610405, + 0.0914284239212672, + 0.06446876997749011 + ], + "sources": 21, + "sublayer": 21 + }, + { + "entropy_max": 2.807748794555664, + "entropy_mean": 2.770190954208374, + "entropy_min": 2.746567964553833, + "mean_weights": [ + 0.036101615677277245, + 0.04429023340344429, + 0.052328101048866905, + 0.029807519167661667, + 0.03372577950358391, + 0.02957167973121007, + 0.017642357076207798, + 0.029840663075447083, + 0.015398593929906687, + 0.025294749687115353, + 0.01460645099480947, + 0.019713498651981354, + 0.01107433003683885, + 0.02177129437526067, + 0.010899525446196398, + 0.04968325545390447, + 0.047834025075038276, + 0.08632492522398631, + 0.058039311319589615, + 0.13162659605344137, + 0.08316896359125774, + 0.1512565314769745 + ], + "sources": 22, + "sublayer": 22 + }, + { + "entropy_max": 3.0476112365722656, + "entropy_mean": 3.0367375214894614, + "entropy_min": 3.0187454223632812, + "mean_weights": [ + 0.03872521221637726, + 0.04870210215449333, + 0.04870830848813057, + 0.04040446256597837, + 0.037807745238145195, + 0.04030604908863703, + 0.025170981884002686, + 0.04038627694050471, + 0.02254965218404929, + 0.03987714648246765, + 0.022182572012146313, + 0.03952491035064062, + 0.01878617952267329, + 0.04185949141780535, + 0.019280270983775456, + 0.05313673863808314, + 0.031132920334736507, + 0.0669172207514445, + 0.039429452270269394, + 0.07793149848779042, + 0.054866241912047066, + 0.08117576936880748, + 0.07113879422346751 + ], + "sources": 23, + "sublayer": 23 + }, + { + "entropy_max": 2.9273719787597656, + "entropy_mean": 2.884519020716349, + "entropy_min": 2.858419179916382, + "mean_weights": [ + 0.030646016200383503, + 0.04199869930744171, + 0.043607903023560844, + 0.02879402848581473, + 0.027949823066592216, + 0.02709902140001456, + 0.014362686003247896, + 0.026844498390952747, + 0.012698768948515257, + 0.023212683697541554, + 0.012096195171276728, + 0.018513306354482967, + 0.009606915215651194, + 0.02043161727488041, + 0.009408737532794476, + 0.04485694939891497, + 0.03561059944331646, + 0.07173362125953038, + 0.04302813857793808, + 0.10230723023414612, + 0.06663860380649567, + 0.11446122825145721, + 0.09680233647425969, + 0.07729038720329602 + ], + "sources": 24, + "sublayer": 24 + }, + { + "entropy_max": 3.1288928985595703, + "entropy_mean": 3.1179122924804688, + "entropy_min": 3.096397638320923, + "mean_weights": [ + 0.03394729644060135, + 0.04239057252804438, + 0.043130082388718925, + 0.03574401264389356, + 0.03375358507037163, + 0.0352306067943573, + 0.02269617219765981, + 0.03447849303483963, + 0.020519376421968143, + 0.03359278105199337, + 0.019900760302941006, + 0.03270889073610306, + 0.016650297368566196, + 0.03397464441756407, + 0.016444630610446136, + 0.047457544753948845, + 0.028430978457132976, + 0.057780602325995765, + 0.0361362403879563, + 0.06799524029095967, + 0.04930434127648672, + 0.06849389274915059, + 0.06071932862202326, + 0.05846334621310234, + 0.07005627950032552 + ], + "sources": 25, + "sublayer": 25 + }, + { + "entropy_max": 2.997812271118164, + "entropy_mean": 2.9913203716278076, + "entropy_min": 2.981506109237671, + "mean_weights": [ + 0.026762607196966808, + 0.03790475676457087, + 0.03804942717154821, + 0.026963916296760242, + 0.025157970065871876, + 0.024532814199725788, + 0.014002490788698196, + 0.02430441789329052, + 0.012250386799375216, + 0.021353704233964283, + 0.011727250491579374, + 0.01740749552845955, + 0.009949984649817148, + 0.018954483792185783, + 0.009896416527529558, + 0.038485350708166756, + 0.029461168373624485, + 0.05913051341970762, + 0.034761415173610054, + 0.07939759890238444, + 0.05290326724449793, + 0.0877164974808693, + 0.07776692012945811, + 0.06165457765261332, + 0.11417745302120845, + 0.04532711332043012 + ], + "sources": 26, + "sublayer": 26 + }, + { + "entropy_max": 3.2317047119140625, + "entropy_mean": 3.190260728200277, + "entropy_min": 3.135222911834717, + "mean_weights": [ + 0.030709152420361836, + 0.038856384654839836, + 0.03852503622571627, + 0.03404424712061882, + 0.030600749577085178, + 0.029806426415840786, + 0.0211167111992836, + 0.030848986158768337, + 0.019134069482485454, + 0.029532151917616527, + 0.018607016652822495, + 0.028101677695910137, + 0.015492160183688005, + 0.02930781990289688, + 0.015279345214366913, + 0.03950384880105654, + 0.024300287167231243, + 0.05189322431882223, + 0.03257856145501137, + 0.0612580676873525, + 0.04323995237549146, + 0.06529123956958453, + 0.055030049135287605, + 0.048991107692321144, + 0.06333444888393085, + 0.041943887869517006, + 0.06267338494459788 + ], + "sources": 27, + "sublayer": 27 + }, + { + "entropy_max": 3.0881078243255615, + "entropy_mean": 3.081976811091105, + "entropy_min": 3.0709128379821777, + "mean_weights": [ + 0.024140593285361927, + 0.03355464587608973, + 0.034119228521982826, + 0.024404085551699, + 0.02367766263584296, + 0.022779367864131927, + 0.014641802137096723, + 0.02278521905342738, + 0.013004937209188938, + 0.01984588863948981, + 0.012441112038989862, + 0.016958024973670643, + 0.011285776272416115, + 0.01809990033507347, + 0.011298118780056635, + 0.033870985731482506, + 0.028080757707357407, + 0.04873387018839518, + 0.03195954114198685, + 0.06577975302934647, + 0.044237478325764336, + 0.0697377473115921, + 0.06441850339372952, + 0.05027745167414347, + 0.09281101822853088, + 0.037099356452624, + 0.10307030379772186, + 0.026886864254872005 + ], + "sources": 28, + "sublayer": 28 + }, + { + "entropy_max": 3.315805435180664, + "entropy_mean": 3.2921435038248696, + "entropy_min": 3.2788450717926025, + "mean_weights": [ + 0.02865760214626789, + 0.03586554403106371, + 0.0367035319407781, + 0.032036833465099335, + 0.03201191189388434, + 0.028546927496790886, + 0.024774956827362377, + 0.026971304789185524, + 0.022935497884949047, + 0.025910282507538795, + 0.022265223165353138, + 0.02342439629137516, + 0.018185899282495182, + 0.02431691127518813, + 0.016727660472194355, + 0.03539527704318365, + 0.02383350332578023, + 0.040299609303474426, + 0.027922411759694416, + 0.05242166916529337, + 0.0389099158346653, + 0.05958003054062525, + 0.04594611252347628, + 0.048638347536325455, + 0.05130115399758021, + 0.03639778991540273, + 0.054013351599375405, + 0.034571390599012375, + 0.051434955249230065 + ], + "sources": 29, + "sublayer": 29 + }, + { + "entropy_max": 3.1905009746551514, + "entropy_mean": 3.18230938911438, + "entropy_min": 3.1718106269836426, + "mean_weights": [ + 0.023452899729212124, + 0.0325044517715772, + 0.03281022980809212, + 0.023958238462607067, + 0.024015444020430248, + 0.021982961023847263, + 0.0159907682488362, + 0.021546220406889915, + 0.014109862968325615, + 0.019567859669526417, + 0.013529953546822071, + 0.016938431809345882, + 0.011665725770095984, + 0.01821698558827241, + 0.01173508632928133, + 0.0315526028474172, + 0.02504568360745907, + 0.04361861323316892, + 0.029006240889430046, + 0.057882855335871376, + 0.038831111043691635, + 0.06207767128944397, + 0.051631235828002296, + 0.04616564139723778, + 0.07385702927907307, + 0.03487453361352285, + 0.08937223007281621, + 0.02605181684096654, + 0.07663016517957051, + 0.011377457529306412 + ], + "sources": 30, + "sublayer": 30 + }, + { + "entropy_max": 3.4240646362304688, + "entropy_mean": 3.3962062994639077, + "entropy_min": 3.3770389556884766, + "mean_weights": [ + 0.036534909158945084, + 0.027843889469901722, + 0.029182132333517075, + 0.031089238822460175, + 0.03259412323435148, + 0.032848658661047615, + 0.03959940994779269, + 0.032999612390995026, + 0.041947100311517715, + 0.03467814748485883, + 0.0426634947458903, + 0.03830170755585035, + 0.04692706217368444, + 0.037095338106155396, + 0.04795412967602412, + 0.029356527452667553, + 0.03787682702143987, + 0.02283823365966479, + 0.02676370181143284, + 0.02388523022333781, + 0.02641577584048112, + 0.02381802164018154, + 0.02564098685979843, + 0.02967298651734988, + 0.024603380511204403, + 0.027582359810670216, + 0.0274051936964194, + 0.03037252277135849, + 0.030458840851982433, + 0.0283659677952528, + 0.032684496293465294 + ], + "sources": 31, + "sublayer": 31 + }, + { + "entropy_max": 3.1776766777038574, + "entropy_mean": 3.1591249306996665, + "entropy_min": 3.145052194595337, + "mean_weights": [ + 0.0212711604932944, + 0.028220354268948238, + 0.029740046088894207, + 0.020458971460660298, + 0.024074342101812363, + 0.020205453038215637, + 0.017948679625988007, + 0.019636886815230053, + 0.014780626632273197, + 0.017502093066771824, + 0.014497815320889154, + 0.015224847942590714, + 0.011039208620786667, + 0.015552480394641558, + 0.011189129513998827, + 0.026409130543470383, + 0.022006437182426453, + 0.03754762870570024, + 0.02449234699209531, + 0.04594653596480688, + 0.030809269597133, + 0.053079865872859955, + 0.03982332721352577, + 0.04045433799425761, + 0.05811700349052747, + 0.029337162151932716, + 0.08656761050224304, + 0.023294669886430103, + 0.09694787611564, + 0.007489877597739299, + 0.09413341929515202, + 0.0022014072940995297 + ], + "sources": 32, + "sublayer": 32 + } + ] + } + }, + "parameters": { + "baseline": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "block": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "full": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "mixer_overhead_fraction_of_baseline": 0.001328047970702457 + }, + "posthoc": { + "corresponding_final_output_weight": 0.0027941359827915826, + "label": "post-hoc descriptive callout; not a preregistered endpoint", + "largest_full_branch_rms": 1.496443748474121, + "largest_full_branch_sublayer": 31, + "uniform_final_output_weight": 0.030303030303030304, + "weight_over_uniform": 0.09220648743212222 + }, + "timing": { + "baseline": { + "by_seed": [ + { + "mean_ms": 21.30391749891751, + "measured_steps": 1980, + "median_ms": 21.206228993833065, + "p95_ms": 21.648590854601935, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 21.793900100873532, + "measured_steps": 1980, + "median_ms": 21.672270508133806, + "p95_ms": 22.421945561654866, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 21.765943476063025, + "measured_steps": 1980, + "median_ms": 21.700910496292636, + "p95_ms": 22.111314281937666, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 3036005376.0, + "mean_peak_reserved_bytes": 3296722944.0, + "mean_step_ms": 21.621253691951356, + "median_step_ms": 21.526469999419835, + "p95_step_ms": 22.060616899398156 + }, + "block": { + "by_seed": [ + { + "mean_ms": 54.54293606894542, + "measured_steps": 1980, + "median_ms": 54.46750250121113, + "p95_ms": 55.017453705659136, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 54.45831580526452, + "measured_steps": 1980, + "median_ms": 54.40393550088629, + "p95_ms": 54.91135738266166, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 54.8763856947342, + "measured_steps": 1980, + "median_ms": 54.84093050472438, + "p95_ms": 55.21141063363757, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 6520143872.0, + "mean_peak_reserved_bytes": 6788481024.0, + "mean_step_ms": 54.62587918964805, + "median_step_ms": 54.57078950227393, + "p95_step_ms": 55.04674057398612 + }, + "full": { + "by_seed": [ + { + "mean_ms": 146.80656970705252, + "measured_steps": 1980, + "median_ms": 146.7325690027792, + "p95_ms": 147.17569539061515, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 146.7571435391555, + "measured_steps": 1980, + "median_ms": 146.72027151391376, + "p95_ms": 147.0799343456747, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + { + "mean_ms": 146.85863199750506, + "measured_steps": 1980, + "median_ms": 146.81190850387793, + "p95_ms": 147.1791482414119, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + } + ], + "mean_peak_allocated_bytes": 14260571648.0, + "mean_peak_reserved_bytes": 14950596608.0, + "mean_step_ms": 146.807448414571, + "median_step_ms": 146.7549163401903, + "p95_step_ms": 147.14492599256724 + }, + "relative_to_baseline": { + "block": { + "allocated_memory_ratio": 2.1476061681387484, + "step_time_ratio": 2.526489905161368 + }, + "full": { + "allocated_memory_ratio": 4.697149669342351, + "step_time_ratio": 6.789960032207613 + } + } + }, + "traces": { + "baseline": { + "branch_output_rms": { + "max": [ + 0.03301293030381203, + 0.03252410888671875, + 0.04544902965426445, + 0.012599779292941093, + 0.03163857012987137, + 0.007620926480740309, + 0.016936156898736954, + 0.007713559083640575, + 0.014406777918338776, + 0.009163067676126957, + 0.014294588007032871, + 0.010015063919126987, + 0.016019638627767563, + 0.01165300328284502, + 0.019276386126875877, + 0.01118372566998005, + 0.021849332377314568, + 0.013802342116832733, + 0.02479873225092888, + 0.014017625711858273, + 0.039869681000709534, + 0.01826939918100834, + 0.045646462589502335, + 0.026948343962430954, + 0.03186195716261864, + 0.03553422540426254, + 0.021814068779349327, + 0.041563451290130615, + 0.011395728215575218, + 0.06082121655344963, + 0.010711577720940113, + 0.12170413136482239 + ], + "mean": [ + 0.029140000542004902, + 0.03135617325703303, + 0.04321867351730665, + 0.012356962698201338, + 0.030009073515733082, + 0.0073511286949117976, + 0.015980400145053864, + 0.0073293728443483514, + 0.013850266424318155, + 0.00872932467609644, + 0.013727998671432337, + 0.009513386835654577, + 0.015388951947291693, + 0.01081705124427875, + 0.017621537980933983, + 0.010974762650827566, + 0.020954379191001255, + 0.012728280077377955, + 0.022901471083362896, + 0.012544240181644758, + 0.034868016839027405, + 0.01587048452347517, + 0.034901623303691544, + 0.023213444277644157, + 0.028262764836351078, + 0.03317628552516302, + 0.015859666280448437, + 0.03943942735592524, + 0.009605003676066795, + 0.0564091665049394, + 0.010157584212720394, + 0.12020406872034073 + ], + "min": [ + 0.024467680603265762, + 0.030463015660643578, + 0.04058511182665825, + 0.012055210769176483, + 0.027046527713537216, + 0.006995280738919973, + 0.01488693431019783, + 0.00682956725358963, + 0.013462266884744167, + 0.00827884953469038, + 0.012803927063941956, + 0.00879740621894598, + 0.014694017358124256, + 0.010121054947376251, + 0.014741585589945316, + 0.010847908444702625, + 0.020217377692461014, + 0.011856775730848312, + 0.02128380537033081, + 0.011742421425879002, + 0.029588066041469574, + 0.014059662818908691, + 0.022691959515213966, + 0.02104850672185421, + 0.02629370056092739, + 0.030305426567792892, + 0.012291512452065945, + 0.037283238023519516, + 0.007733044680207968, + 0.05344158038496971, + 0.009596864692866802, + 0.11796895414590836 + ] + }, + "layer_input_rms": { + "max": [ + 0.053540125489234924, + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.12003147602081299, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594 + ], + "mean": [ + 0.053358995666106544, + 0.06613848606745402, + 0.07352576404809952, + 0.10093248138825099, + 0.10369285692771275, + 0.12192569176355998, + 0.12289420515298843, + 0.1264623502890269, + 0.12660532196362814, + 0.1285571108261744, + 0.12829979757467905, + 0.1300874004761378, + 0.12902945280075073, + 0.13068528473377228, + 0.1294754147529602, + 0.1288459748029709, + 0.12698591748873392, + 0.1257581685980161, + 0.12347647547721863, + 0.11898198475440343, + 0.11615285277366638, + 0.10215904066960017, + 0.10102975865205129, + 0.08804203818241756, + 0.09022414187590282, + 0.08323186387618382, + 0.0895061269402504, + 0.08785812556743622, + 0.09989260633786519, + 0.09980995953083038, + 0.1267695054411888, + 0.12696876376867294 + ], + "min": [ + 0.05312936007976532, + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.1217338889837265, + 0.12261121720075607, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12287117540836334, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.08582619577646255, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995 + ] + }, + "stream_state_rms": { + "max": [ + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.12003147602081299, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594, + 0.2156660109758377 + ], + "mean": [ + 0.06613848606745402, + 0.07352576404809952, + 0.10093248138825099, + 0.10369285692771275, + 0.12192569176355998, + 0.12289420515298843, + 0.1264623502890269, + 0.12660532196362814, + 0.1285571108261744, + 0.12829979757467905, + 0.1300874004761378, + 0.12902945280075073, + 0.13068528473377228, + 0.1294754147529602, + 0.1288459748029709, + 0.12698591748873392, + 0.1257581685980161, + 0.12347647547721863, + 0.11898198475440343, + 0.11615285277366638, + 0.10215904066960017, + 0.10102975865205129, + 0.08804203818241756, + 0.09022414187590282, + 0.08323186387618382, + 0.0895061269402504, + 0.08785812556743622, + 0.09989260633786519, + 0.09980995953083038, + 0.1267695054411888, + 0.12696876376867294, + 0.2109663486480713 + ], + "min": [ + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.1217338889837265, + 0.12261121720075607, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12287117540836334, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.08582619577646255, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995, + 0.20744846761226654 + ] + } + }, + "block": { + "branch_output_rms": { + "max": [ + 0.0370347797870636, + 0.048244379460811615, + 0.07630905508995056, + 0.040389351546764374, + 0.05223172530531883, + 0.022662153467535973, + 0.034338437020778656, + 0.018601570278406143, + 0.06659915298223495, + 0.019084658473730087, + 0.11068977415561676, + 0.017863860353827477, + 0.06974328309297562, + 0.023023422807455063, + 0.03994842991232872, + 0.024390853941440582, + 0.022348226979374886, + 0.017856158316135406, + 0.027489805594086647, + 0.026076123118400574, + 0.033282581716775894, + 0.023831162601709366, + 0.017817942425608635, + 0.033413130789995193, + 0.019294502213597298, + 0.02716788649559021, + 0.011242104694247246, + 0.03591271862387657, + 0.015554063953459263, + 0.046544428914785385, + 0.01049043145030737, + 0.05786125734448433 + ], + "mean": [ + 0.03337173908948898, + 0.045107426742712654, + 0.052010158697764076, + 0.03859390194217364, + 0.050928935408592224, + 0.021324736376603443, + 0.02902730492254098, + 0.018222644925117493, + 0.05315905436873436, + 0.016110357828438282, + 0.06097829528152943, + 0.016606629826128483, + 0.06566628689567248, + 0.018870445899665356, + 0.03398136980831623, + 0.021034574136137962, + 0.020854807148377102, + 0.017290979002912838, + 0.025822721421718597, + 0.02190040610730648, + 0.027670275419950485, + 0.02298567195733388, + 0.015916637765864532, + 0.031282839054862656, + 0.01756543169418971, + 0.026957616209983826, + 0.010056077502667904, + 0.03371469924847285, + 0.014171119158466658, + 0.04198148101568222, + 0.009337710837523142, + 0.05569397285580635 + ], + "min": [ + 0.028859306126832962, + 0.042584121227264404, + 0.030812542885541916, + 0.036034028977155685, + 0.04880734533071518, + 0.019183801487088203, + 0.02264152280986309, + 0.01778092049062252, + 0.039430491626262665, + 0.013977118767797947, + 0.027242736890912056, + 0.015299440361559391, + 0.058694127947092056, + 0.01103654783219099, + 0.02528632991015911, + 0.015162505209445953, + 0.019655495882034302, + 0.016425520181655884, + 0.02445649914443493, + 0.01807972602546215, + 0.0240048635751009, + 0.02158590778708458, + 0.014158087782561779, + 0.027974048629403114, + 0.016243331134319305, + 0.026792073622345924, + 0.009424413554370403, + 0.03243190050125122, + 0.011988751590251923, + 0.038416508585214615, + 0.008579479530453682, + 0.052862487733364105 + ] + }, + "layer_input_rms": { + "max": [ + 0.05405270308256149, + 0.03717699274420738, + 0.043796516954898834, + 0.05271390080451965, + 0.08092177659273148, + 0.05150312930345535, + 0.06945399194955826, + 0.05591938644647598, + 0.07346180826425552, + 0.0544980950653553, + 0.060793664306402206, + 0.07243207842111588, + 0.07691676914691925, + 0.06619398295879364, + 0.06854937970638275, + 0.07206644117832184, + 0.07398614287376404, + 0.05593166500329971, + 0.047586891800165176, + 0.036662496626377106, + 0.051853884011507034, + 0.01807689666748047, + 0.03394656628370285, + 0.02252720296382904, + 0.03615279868245125, + 0.019617732614278793, + 0.02297247014939785, + 0.01989392749965191, + 0.0260133296251297, + 0.02015369012951851, + 0.023471562191843987, + 0.023934273049235344 + ], + "mean": [ + 0.05387426788608233, + 0.03533576180537542, + 0.04317111149430275, + 0.04991913214325905, + 0.07097297410170238, + 0.04975094646215439, + 0.06104101364811262, + 0.05404127140839895, + 0.06731398651997249, + 0.04994910334547361, + 0.05693682904044787, + 0.06259432186683019, + 0.07018299649159114, + 0.060772753010193505, + 0.06431671231985092, + 0.06616326545675595, + 0.06897229204575221, + 0.05195336416363716, + 0.04609082266688347, + 0.031601848701635994, + 0.049942340701818466, + 0.017869824543595314, + 0.03332234298189481, + 0.0219339511046807, + 0.03447778026262919, + 0.01884455730517705, + 0.022380413487553596, + 0.019762429719169933, + 0.024852463354667027, + 0.019185719390710194, + 0.023034153506159782, + 0.02343445022900899 + ], + "min": [ + 0.053777214139699936, + 0.03439030423760414, + 0.042556535452604294, + 0.04696016386151314, + 0.06085871905088425, + 0.04705430194735527, + 0.053277771919965744, + 0.05260215699672699, + 0.0606326088309288, + 0.04669487476348877, + 0.054401230067014694, + 0.0512978732585907, + 0.06302255392074585, + 0.05218172445893288, + 0.05766461044549942, + 0.06084785237908363, + 0.06420515477657318, + 0.04857253283262253, + 0.04421957954764366, + 0.02869899943470955, + 0.04853617027401924, + 0.017642762511968613, + 0.032592616975307465, + 0.02158820629119873, + 0.03257649391889572, + 0.01816716231405735, + 0.021423330530524254, + 0.01960809901356697, + 0.024138133972883224, + 0.01791227236390114, + 0.02242189459502697, + 0.023178130388259888 + ] + }, + "stream_state_rms": { + "max": [ + 0.0370347797870636, + 0.06143631786108017, + 0.11840524524450302, + 0.136356920003891, + 0.05223172530531883, + 0.05775697901844978, + 0.07745972275733948, + 0.08341154456138611, + 0.06659915298223495, + 0.0708504468202591, + 0.15584944188594818, + 0.1662624329328537, + 0.06974328309297562, + 0.08339263498783112, + 0.11490626633167267, + 0.12939290702342987, + 0.022348226979374886, + 0.030708789825439453, + 0.04433037340641022, + 0.053864628076553345, + 0.033282581716775894, + 0.04125083237886429, + 0.045212581753730774, + 0.057350847870111465, + 0.019294502213597298, + 0.03496377170085907, + 0.03744323179125786, + 0.056224361062049866, + 0.015554063953459263, + 0.050526175647974014, + 0.051927778869867325, + 0.09535783529281616 + ], + "mean": [ + 0.03337173908948898, + 0.059812614073355995, + 0.09668255100647609, + 0.11812450736761093, + 0.050928935408592224, + 0.0568557046353817, + 0.07451550910870235, + 0.0815352201461792, + 0.05315905436873436, + 0.05806975563367208, + 0.10785960281888644, + 0.11578060686588287, + 0.06566628689567248, + 0.07592015713453293, + 0.09934087345997493, + 0.11063759525616963, + 0.020854807148377102, + 0.029423799365758896, + 0.04364252711335818, + 0.05246620997786522, + 0.027670275419950485, + 0.0368656280140082, + 0.04281706859668096, + 0.054927793641885124, + 0.01756543169418971, + 0.033875805636247, + 0.03674849991997083, + 0.05424185593922933, + 0.014171119158466658, + 0.045518445471922554, + 0.04748128727078438, + 0.09287819017966588 + ], + "min": [ + 0.028859306126832962, + 0.05888580158352852, + 0.07637989521026611, + 0.10048534721136093, + 0.04880734533071518, + 0.05540755018591881, + 0.07019630819559097, + 0.07779177278280258, + 0.039430491626262665, + 0.04417555779218674, + 0.06122447922825813, + 0.068731389939785, + 0.058694127947092056, + 0.07109052687883377, + 0.08456306159496307, + 0.09095416963100433, + 0.019655495882034302, + 0.02821449190378189, + 0.04324287921190262, + 0.05173908546566963, + 0.0240048635751009, + 0.034459229558706284, + 0.04023054614663124, + 0.05085337162017822, + 0.016243331134319305, + 0.033271633088588715, + 0.03624724969267845, + 0.05302892625331879, + 0.011988751590251923, + 0.040694523602724075, + 0.04344379901885986, + 0.0900634303689003 + ] + } + }, + "full": { + "branch_output_rms": { + "max": [ + 0.027083024382591248, + 0.05378643795847893, + 0.053479041904211044, + 0.04230710119009018, + 0.03489644080400467, + 0.0346454493701458, + 0.02454756386578083, + 0.03129609674215317, + 0.0382988266646862, + 0.030918236821889877, + 0.08029001951217651, + 0.032227013260126114, + 0.0788588896393776, + 0.046491838991642, + 0.03414471819996834, + 0.028103213757276535, + 0.028869561851024628, + 0.024058125913143158, + 0.02986966073513031, + 0.024125492200255394, + 0.03027774952352047, + 0.03037451021373272, + 0.028807632625102997, + 0.031061751767992973, + 0.030561715364456177, + 0.031379926949739456, + 0.03756574168801308, + 0.03404445946216583, + 0.2097787708044052, + 0.041366904973983765, + 1.8557170629501343, + 0.05258624255657196 + ], + "mean": [ + 0.024008604387442272, + 0.05206462492545446, + 0.03722590704758962, + 0.041239711145559944, + 0.031058302149176598, + 0.0338385080297788, + 0.02349867671728134, + 0.029538670554757118, + 0.028947196279962856, + 0.029389141748348873, + 0.0621493769188722, + 0.02872789278626442, + 0.06633553778131802, + 0.0376141636321942, + 0.026593025773763657, + 0.02356371966501077, + 0.024800419807434082, + 0.021621525287628174, + 0.027926772211988766, + 0.022361520677804947, + 0.028618198509017628, + 0.02809702232480049, + 0.027077977856000263, + 0.030229248106479645, + 0.027249973888198536, + 0.030415359263618786, + 0.030738824357589085, + 0.03279792827864488, + 0.11324572935700417, + 0.03710922226309776, + 1.496443748474121, + 0.05082246412833532 + ], + "min": [ + 0.022041022777557373, + 0.05085906386375427, + 0.019325949251651764, + 0.03948163986206055, + 0.0279708169400692, + 0.033292125910520554, + 0.021648211404681206, + 0.028076333925127983, + 0.02268683910369873, + 0.027197115123271942, + 0.05061359703540802, + 0.021779853850603104, + 0.05264769122004509, + 0.0304368045181036, + 0.02188333496451378, + 0.01987709477543831, + 0.021968834102153778, + 0.02021927572786808, + 0.025901971384882927, + 0.021093308925628662, + 0.025842806324362755, + 0.02625282295048237, + 0.025757966563105583, + 0.029710212722420692, + 0.021688953042030334, + 0.02968704327940941, + 0.02333390712738037, + 0.030768385156989098, + 0.04889204725623131, + 0.034267887473106384, + 1.202970266342163, + 0.04837163910269737 + ] + }, + "layer_input_rms": { + "max": [ + 0.05461336672306061, + 0.034621261060237885, + 0.029128335416316986, + 0.03089532069861889, + 0.026308147236704826, + 0.02630813792347908, + 0.023830953985452652, + 0.02176283486187458, + 0.020936816930770874, + 0.020500168204307556, + 0.019486865028738976, + 0.022477135062217712, + 0.0208187997341156, + 0.02693997137248516, + 0.023864183574914932, + 0.024786626920104027, + 0.02244819700717926, + 0.01847153715789318, + 0.01782904379069805, + 0.010763376951217651, + 0.014036872424185276, + 0.008387424051761627, + 0.010892084799706936, + 0.007874744012951851, + 0.009214057587087154, + 0.007483004592359066, + 0.007594018243253231, + 0.007620570715516806, + 0.0075973751954734325, + 0.007396102417260408, + 0.014656757935881615, + 0.008849309757351875 + ], + "mean": [ + 0.054420417795578636, + 0.03333831081787745, + 0.028589200228452682, + 0.02826053649187088, + 0.023984291901191074, + 0.02512436422208945, + 0.021577948704361916, + 0.021324680497248966, + 0.019697973504662514, + 0.019812118262052536, + 0.018859490131338436, + 0.021303291122118633, + 0.02054722234606743, + 0.025280661260088284, + 0.023031941304604214, + 0.022272694235046703, + 0.02007630281150341, + 0.016114353202283382, + 0.01666525937616825, + 0.010316944991548857, + 0.013107998917500177, + 0.008068556897342205, + 0.010080384400983652, + 0.007476135001828273, + 0.008358963920424381, + 0.0073659901196757955, + 0.00731347252925237, + 0.007378495919207732, + 0.007157225937892993, + 0.007153482021143039, + 0.012953085514406363, + 0.008658299222588539 + ], + "min": [ + 0.054283902049064636, + 0.03162544220685959, + 0.02765912003815174, + 0.02553493343293667, + 0.02228466607630253, + 0.024269601330161095, + 0.020151648670434952, + 0.02089775912463665, + 0.018591508269309998, + 0.018769782036542892, + 0.017839200794696808, + 0.02058389224112034, + 0.02009216882288456, + 0.022332707419991493, + 0.021584752947092056, + 0.01947355456650257, + 0.018600277602672577, + 0.014435344375669956, + 0.015265379101037979, + 0.009849075227975845, + 0.012294777669012547, + 0.0078109800815582275, + 0.009295190684497356, + 0.007262552157044411, + 0.007303531747311354, + 0.007160624954849482, + 0.00694533996284008, + 0.007063175085932016, + 0.006762783043086529, + 0.006884528324007988, + 0.010628288611769676, + 0.008319443091750145 + ] + }, + "stream_state_rms": { + "max": [ + 0.04294736310839653, + 0.04605885222554207, + 0.04797445610165596, + 0.046828098595142365, + 0.044502634555101395, + 0.04308011010289192, + 0.0412217415869236, + 0.039975181221961975, + 0.03879518434405327, + 0.03814637288451195, + 0.04060528427362442, + 0.039477188140153885, + 0.04267995432019234, + 0.0419749952852726, + 0.041285403072834015, + 0.04062855243682861, + 0.040065914392471313, + 0.039282262325286865, + 0.03886580839753151, + 0.038292769342660904, + 0.037945639342069626, + 0.03764813393354416, + 0.03725530579686165, + 0.03698969632387161, + 0.036729857325553894, + 0.03649326041340828, + 0.036105964332818985, + 0.036036837846040726, + 0.05135522037744522, + 0.050893642008304596, + 0.33184999227523804, + 0.32691144943237305 + ], + "mean": [ + 0.0420840618511041, + 0.04565946012735367, + 0.044145053873459496, + 0.04360272611180941, + 0.04180811593929926, + 0.04077240079641342, + 0.039034511893987656, + 0.0381052220861117, + 0.03734947616855303, + 0.03669755533337593, + 0.03964470078547796, + 0.038941322515408196, + 0.04158667723337809, + 0.041379332542419434, + 0.040633320808410645, + 0.03984006121754646, + 0.03916142632563909, + 0.03844020888209343, + 0.03798529009024302, + 0.03739011660218239, + 0.03703943143288294, + 0.036697146793206535, + 0.03634887933731079, + 0.03612434739867846, + 0.035830430686473846, + 0.03564548244078954, + 0.035502238819996514, + 0.035413493712743126, + 0.04190699135263761, + 0.04178603241840998, + 0.26773401101430255, + 0.26379891236623126 + ], + "min": [ + 0.04164384678006172, + 0.044926099479198456, + 0.040089137852191925, + 0.04054243862628937, + 0.03965730220079422, + 0.038980793207883835, + 0.03725782036781311, + 0.03664333373308182, + 0.035495493561029434, + 0.03482290729880333, + 0.03899230435490608, + 0.038514114916324615, + 0.039978086948394775, + 0.0404449887573719, + 0.039540987461805344, + 0.03875384107232094, + 0.03806925565004349, + 0.0373431071639061, + 0.036932460963726044, + 0.03633511811494827, + 0.03608185052871704, + 0.03575707972049713, + 0.03549469634890556, + 0.035281531512737274, + 0.0348568893969059, + 0.034734319895505905, + 0.03483940288424492, + 0.03479676693677902, + 0.03653828799724579, + 0.036703966557979584, + 0.2158803939819336, + 0.21277332305908203 + ] + } + } + } + }, + "canonical_sha256_without_self": "8625a34f377a14e0d69339a78e94788cd0ce9c925e521f51e3cba31acf96b20c", + "formal_runs": [ + { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "08dde18cc4cd5dbbbe3edda9b02f393fa8cd26ee52d75ba07af58c843848c49d", + "diagnostic": { + "bits_per_byte": 2.1535406127411587, + "branch_output_rms": [ + 0.03301293030381203, + 0.030463015660643578, + 0.04362187907099724, + 0.012599779292941093, + 0.027046527713537216, + 0.006995280738919973, + 0.01488693431019783, + 0.007444992195814848, + 0.014406777918338776, + 0.008746056817471981, + 0.012803927063941956, + 0.00879740621894598, + 0.014694017358124256, + 0.01165300328284502, + 0.014741585589945316, + 0.010892653837800026, + 0.021849332377314568, + 0.01252572238445282, + 0.02128380537033081, + 0.011742421425879002, + 0.03514630347490311, + 0.01528239157050848, + 0.03636644780635834, + 0.02104850672185421, + 0.03186195716261864, + 0.03368920460343361, + 0.013473417609930038, + 0.039471592754125595, + 0.007733044680207968, + 0.05344158038496971, + 0.010711577720940113, + 0.12170413136482239 + ], + "core_parameter_grad_rms_by_block": [ + 0.0006069268184991451, + 0.0004268615003720108, + 0.000323043032117742, + 0.0002914754177093503, + 0.00028453589406033573, + 0.0003157797529074047, + 0.0003804545297370818, + 0.0004021196515771015, + 0.000555786501522983, + 0.0005651240708116888, + 0.0007729757575868522, + 0.00083369364308322, + 0.0008315976940648109, + 0.0006488835935817343, + 0.0005359813613230099, + 0.00042412043092329184 + ], + "depth_weights": [], + "layer_input_rms": [ + 0.05340750142931938, + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12325409054756165, + 0.11969955265522003, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0875694528222084, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08705572783946991, + 0.09912198781967163, + 0.09870348125696182, + 0.12346243113279343, + 0.12389487028121948 + ], + "loss_nats": 1.492720603942871, + "output_weights": null, + "stream_state_rms": [ + 0.06873997300863266, + 0.07564551383256912, + 0.10404959321022034, + 0.1065979078412056, + 0.1221257820725441, + 0.12304633110761642, + 0.12577508389949799, + 0.12567545473575592, + 0.12801343202590942, + 0.12763085961341858, + 0.12953580915927887, + 0.12862826883792877, + 0.12898454070091248, + 0.12794172763824463, + 0.12716630101203918, + 0.12565097212791443, + 0.1247769221663475, + 0.12325409054756165, + 0.11969955265522003, + 0.11714839935302734, + 0.10485497117042542, + 0.1029127761721611, + 0.0875694528222084, + 0.08903128653764725, + 0.0807143896818161, + 0.08791504055261612, + 0.08705572783946991, + 0.09912198781967163, + 0.09870348125696182, + 0.12346243113279343, + 0.12389487028121948, + 0.20978456735610962 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.095421586117522, + "cross_entropy_nats": 5.611318647861481, + "step": 0 + }, + { + "bits_per_byte": 3.86267014445088, + "cross_entropy_nats": 2.677398920059204, + "step": 100 + }, + { + "bits_per_byte": 3.431632090385233, + "cross_entropy_nats": 2.3786261081695557, + "step": 250 + }, + { + "bits_per_byte": 2.9448990671750175, + "cross_entropy_nats": 2.0412484854459763, + "step": 500 + }, + { + "bits_per_byte": 2.2560981455190916, + "cross_entropy_nats": 1.5638080686330795, + "step": 1000 + }, + { + "bits_per_byte": 2.0700625342803067, + "cross_entropy_nats": 1.4348580092191696, + "step": 1500 + }, + { + "bits_per_byte": 2.0005383326907027, + "cross_entropy_nats": 1.3866675049066544, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "3d4ad01179d432cd0a6a93d23e63072126f63673a966015bd4f9f19e923f5108", + "final_mixer_parameters": null, + "initial_common_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "initial_mixer_parameters": null + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 21.30391749891751, + "measured_steps": 1980, + "median_ms": 21.206228993833065, + "p95_ms": 21.648590854601935, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.082520049589073, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.6023759841918945, + "step": 1, + "unclipped_grad_norm": 19.278701782226562 + }, + { + "bits_per_byte": 6.858694945702343, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.754085063934326, + "step": 10, + "unclipped_grad_norm": 5.061380863189697 + }, + { + "bits_per_byte": 6.454015640638533, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.473582744598389, + "step": 20, + "unclipped_grad_norm": 2.890021800994873 + }, + { + "bits_per_byte": 6.203172065049093, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.299711227416992, + "step": 30, + "unclipped_grad_norm": 2.66562819480896 + }, + { + "bits_per_byte": 5.679915537729601, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.9370174407958984, + "step": 40, + "unclipped_grad_norm": 2.3282792568206787 + }, + { + "bits_per_byte": 5.254645781773893, + "learning_rate": 0.00015, + "loss_nats": 3.642242908477783, + "step": 50, + "unclipped_grad_norm": 2.040722131729126 + }, + { + "bits_per_byte": 4.779658249210859, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.313006639480591, + "step": 60, + "unclipped_grad_norm": 1.7241066694259644 + }, + { + "bits_per_byte": 4.424262636543597, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.0666651725769043, + "step": 70, + "unclipped_grad_norm": 1.284820318222046 + }, + { + "bits_per_byte": 4.1880592560915035, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.9029414653778076, + "step": 80, + "unclipped_grad_norm": 0.9151132106781006 + }, + { + "bits_per_byte": 3.9606658589125083, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.7453243732452393, + "step": 90, + "unclipped_grad_norm": 0.7345118522644043 + }, + { + "bits_per_byte": 3.824000533334981, + "learning_rate": 0.0003, + "loss_nats": 2.650595188140869, + "step": 100, + "unclipped_grad_norm": 0.5508835315704346 + }, + { + "bits_per_byte": 3.779783449832851, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.619946241378784, + "step": 110, + "unclipped_grad_norm": 0.5057564377784729 + }, + { + "bits_per_byte": 3.713208621718556, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.5738000869750977, + "step": 120, + "unclipped_grad_norm": 0.5957204103469849 + }, + { + "bits_per_byte": 3.6704819398688033, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5441842079162598, + "step": 130, + "unclipped_grad_norm": 0.5626782774925232 + }, + { + "bits_per_byte": 3.612546488237951, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.504026412963867, + "step": 140, + "unclipped_grad_norm": 0.3983434736728668 + }, + { + "bits_per_byte": 3.6181579381703024, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.50791597366333, + "step": 150, + "unclipped_grad_norm": 0.36885207891464233 + }, + { + "bits_per_byte": 3.58535190355292, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4851765632629395, + "step": 160, + "unclipped_grad_norm": 0.5443975329399109 + }, + { + "bits_per_byte": 3.5546392417618398, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.463888168334961, + "step": 170, + "unclipped_grad_norm": 0.5308160185813904 + }, + { + "bits_per_byte": 3.518414879975908, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.438779354095459, + "step": 180, + "unclipped_grad_norm": 0.45907488465309143 + }, + { + "bits_per_byte": 3.5099375111495785, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.432903289794922, + "step": 190, + "unclipped_grad_norm": 0.520439624786377 + }, + { + "bits_per_byte": 3.4387769376326776, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.3835785388946533, + "step": 200, + "unclipped_grad_norm": 0.7100375294685364 + }, + { + "bits_per_byte": 3.498519238993495, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.4249887466430664, + "step": 210, + "unclipped_grad_norm": 0.5540094971656799 + }, + { + "bits_per_byte": 3.463703071152828, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.4008560180664062, + "step": 220, + "unclipped_grad_norm": 0.7454037666320801 + }, + { + "bits_per_byte": 3.4467916731281862, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.389133930206299, + "step": 230, + "unclipped_grad_norm": 0.545799732208252 + }, + { + "bits_per_byte": 3.4789998960511697, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.411458969116211, + "step": 240, + "unclipped_grad_norm": 0.7584995627403259 + }, + { + "bits_per_byte": 3.435720117996242, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.3814597129821777, + "step": 250, + "unclipped_grad_norm": 0.6863992214202881 + }, + { + "bits_per_byte": 3.394907259085291, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.353170394897461, + "step": 260, + "unclipped_grad_norm": 0.5891655683517456 + }, + { + "bits_per_byte": 3.390574672144755, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.3501672744750977, + "step": 270, + "unclipped_grad_norm": 0.526258647441864 + }, + { + "bits_per_byte": 3.375952019237795, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.340031623840332, + "step": 280, + "unclipped_grad_norm": 1.1747543811798096 + }, + { + "bits_per_byte": 3.3352409740561546, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3118128776550293, + "step": 290, + "unclipped_grad_norm": 0.5939303636550903 + }, + { + "bits_per_byte": 3.3165203185444363, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.2988367080688477, + "step": 300, + "unclipped_grad_norm": 0.5224592089653015 + }, + { + "bits_per_byte": 3.3100609941433308, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.2943594455718994, + "step": 310, + "unclipped_grad_norm": 0.720944344997406 + }, + { + "bits_per_byte": 3.3574566609987073, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.327211618423462, + "step": 320, + "unclipped_grad_norm": 0.596331775188446 + }, + { + "bits_per_byte": 3.3054742168448197, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.29118013381958, + "step": 330, + "unclipped_grad_norm": 0.8132797479629517 + }, + { + "bits_per_byte": 3.321859691923123, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.302537679672241, + "step": 340, + "unclipped_grad_norm": 0.7523438930511475 + }, + { + "bits_per_byte": 3.2988319029031965, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.28657603263855, + "step": 350, + "unclipped_grad_norm": 0.7955809831619263 + }, + { + "bits_per_byte": 3.3143371707743836, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.29732346534729, + "step": 360, + "unclipped_grad_norm": 0.6717050671577454 + }, + { + "bits_per_byte": 3.260947908536439, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.260316848754883, + "step": 370, + "unclipped_grad_norm": 0.9467564225196838 + }, + { + "bits_per_byte": 3.2622783663235135, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.2612390518188477, + "step": 380, + "unclipped_grad_norm": 1.130399465560913 + }, + { + "bits_per_byte": 3.217816035561253, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.2304201126098633, + "step": 390, + "unclipped_grad_norm": 0.9525595903396606 + }, + { + "bits_per_byte": 3.197722270565046, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.216492176055908, + "step": 400, + "unclipped_grad_norm": 1.304746389389038 + }, + { + "bits_per_byte": 3.1037082984361937, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.1513266563415527, + "step": 410, + "unclipped_grad_norm": 0.9077643752098083 + }, + { + "bits_per_byte": 3.1124470808918345, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.157383918762207, + "step": 420, + "unclipped_grad_norm": 0.9870430827140808 + }, + { + "bits_per_byte": 3.0903411188805325, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.142061233520508, + "step": 430, + "unclipped_grad_norm": 1.2044774293899536 + }, + { + "bits_per_byte": 3.0673250246808648, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.126107692718506, + "step": 440, + "unclipped_grad_norm": 0.7101535201072693 + }, + { + "bits_per_byte": 3.0553330184015026, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.117795467376709, + "step": 450, + "unclipped_grad_norm": 1.375364899635315 + }, + { + "bits_per_byte": 2.9995236163006163, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.079111337661743, + "step": 460, + "unclipped_grad_norm": 1.2433860301971436 + }, + { + "bits_per_byte": 2.9800073690460063, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.0655837059020996, + "step": 470, + "unclipped_grad_norm": 1.1379404067993164 + }, + { + "bits_per_byte": 3.0331118280141225, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.1023929119110107, + "step": 480, + "unclipped_grad_norm": 0.977416455745697 + }, + { + "bits_per_byte": 2.932088874932007, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.0323691368103027, + "step": 490, + "unclipped_grad_norm": 1.0012375116348267 + }, + { + "bits_per_byte": 2.88244866250022, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.997961163520813, + "step": 500, + "unclipped_grad_norm": 1.1916924715042114 + }, + { + "bits_per_byte": 2.8386042654025103, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.9675705432891846, + "step": 510, + "unclipped_grad_norm": 1.0254079103469849 + }, + { + "bits_per_byte": 2.857028594806792, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.9803413152694702, + "step": 520, + "unclipped_grad_norm": 0.8719282150268555 + }, + { + "bits_per_byte": 2.836969570306059, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.9664374589920044, + "step": 530, + "unclipped_grad_norm": 1.1046723127365112 + }, + { + "bits_per_byte": 2.817192769335392, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.9527292251586914, + "step": 540, + "unclipped_grad_norm": 1.4310210943222046 + }, + { + "bits_per_byte": 2.8188448346795805, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.9538743495941162, + "step": 550, + "unclipped_grad_norm": 1.0653572082519531 + }, + { + "bits_per_byte": 2.7444260498420836, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.902291178703308, + "step": 560, + "unclipped_grad_norm": 0.988775372505188 + }, + { + "bits_per_byte": 2.752570288273692, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.9079363346099854, + "step": 570, + "unclipped_grad_norm": 1.1247562170028687 + }, + { + "bits_per_byte": 2.7907552522794594, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.9344041347503662, + "step": 580, + "unclipped_grad_norm": 1.1135188341140747 + }, + { + "bits_per_byte": 2.6978061967247493, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8699767589569092, + "step": 590, + "unclipped_grad_norm": 1.1573277711868286 + }, + { + "bits_per_byte": 2.731025505652155, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8930026292800903, + "step": 600, + "unclipped_grad_norm": 1.0118002891540527 + }, + { + "bits_per_byte": 2.714647425827839, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8816502094268799, + "step": 610, + "unclipped_grad_norm": 1.4108420610427856 + }, + { + "bits_per_byte": 2.667507325191645, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.8489751815795898, + "step": 620, + "unclipped_grad_norm": 1.356989860534668 + }, + { + "bits_per_byte": 2.6646273037202963, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8469789028167725, + "step": 630, + "unclipped_grad_norm": 1.0785958766937256 + }, + { + "bits_per_byte": 2.587041458367161, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.7932004928588867, + "step": 640, + "unclipped_grad_norm": 1.0370774269104004 + }, + { + "bits_per_byte": 2.621500246259335, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.8170855045318604, + "step": 650, + "unclipped_grad_norm": 1.3772567510604858 + }, + { + "bits_per_byte": 2.58724061427686, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.7933385372161865, + "step": 660, + "unclipped_grad_norm": 1.1089061498641968 + }, + { + "bits_per_byte": 2.6021377514945967, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.8036644458770752, + "step": 670, + "unclipped_grad_norm": 1.2382783889770508 + }, + { + "bits_per_byte": 2.535791660305949, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7576768398284912, + "step": 680, + "unclipped_grad_norm": 1.0874292850494385 + }, + { + "bits_per_byte": 2.5901048133443294, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.7953238487243652, + "step": 690, + "unclipped_grad_norm": 1.129819393157959 + }, + { + "bits_per_byte": 2.4864622205242717, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7234842777252197, + "step": 700, + "unclipped_grad_norm": 1.1056071519851685 + }, + { + "bits_per_byte": 2.5464105571008226, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.7650372982025146, + "step": 710, + "unclipped_grad_norm": 1.1447758674621582 + }, + { + "bits_per_byte": 2.5127927643713677, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7417352199554443, + "step": 720, + "unclipped_grad_norm": 1.1710957288742065 + }, + { + "bits_per_byte": 2.413335541342707, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.6727967262268066, + "step": 730, + "unclipped_grad_norm": 1.1179577112197876 + }, + { + "bits_per_byte": 2.387620351420398, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6549723148345947, + "step": 740, + "unclipped_grad_norm": 1.1390100717544556 + }, + { + "bits_per_byte": 2.530797284124893, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7542150020599365, + "step": 750, + "unclipped_grad_norm": 1.2838371992111206 + }, + { + "bits_per_byte": 2.369598633401786, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.6424806118011475, + "step": 760, + "unclipped_grad_norm": 1.336150884628296 + }, + { + "bits_per_byte": 2.450027351973684, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.6982295513153076, + "step": 770, + "unclipped_grad_norm": 1.2392343282699585 + }, + { + "bits_per_byte": 2.5490327765785277, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7668548822402954, + "step": 780, + "unclipped_grad_norm": 1.1718425750732422 + }, + { + "bits_per_byte": 2.3740495444061156, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.6455657482147217, + "step": 790, + "unclipped_grad_norm": 1.2897855043411255 + }, + { + "bits_per_byte": 2.4331341841100333, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6865200996398926, + "step": 800, + "unclipped_grad_norm": 1.1979949474334717 + }, + { + "bits_per_byte": 2.374416211417755, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.645819902420044, + "step": 810, + "unclipped_grad_norm": 1.2329429388046265 + }, + { + "bits_per_byte": 2.4079268589557223, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.6690477132797241, + "step": 820, + "unclipped_grad_norm": 1.7955585718154907 + }, + { + "bits_per_byte": 2.3311477842177846, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.615828514099121, + "step": 830, + "unclipped_grad_norm": 1.149026870727539 + }, + { + "bits_per_byte": 2.3024265095410446, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.595920443534851, + "step": 840, + "unclipped_grad_norm": 1.3038955926895142 + }, + { + "bits_per_byte": 2.397476677141346, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.66180419921875, + "step": 850, + "unclipped_grad_norm": 1.3544203042984009 + }, + { + "bits_per_byte": 2.3581428690278146, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.63454008102417, + "step": 860, + "unclipped_grad_norm": 1.1749086380004883 + }, + { + "bits_per_byte": 2.389466069229458, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6562516689300537, + "step": 870, + "unclipped_grad_norm": 1.256735920906067 + }, + { + "bits_per_byte": 2.3681801204974695, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.6414973735809326, + "step": 880, + "unclipped_grad_norm": 1.3939846754074097 + }, + { + "bits_per_byte": 2.365631509594334, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.6397308111190796, + "step": 890, + "unclipped_grad_norm": 1.2991011142730713 + }, + { + "bits_per_byte": 2.25590227877759, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5636723041534424, + "step": 900, + "unclipped_grad_norm": 1.040560245513916 + }, + { + "bits_per_byte": 2.3482356364422117, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.6276729106903076, + "step": 910, + "unclipped_grad_norm": 1.2334232330322266 + }, + { + "bits_per_byte": 2.2714704922989934, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5744633674621582, + "step": 920, + "unclipped_grad_norm": 1.338327407836914 + }, + { + "bits_per_byte": 2.2269675736311294, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5436162948608398, + "step": 930, + "unclipped_grad_norm": 1.2011080980300903 + }, + { + "bits_per_byte": 2.2917099266004812, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5884922742843628, + "step": 940, + "unclipped_grad_norm": 1.1391189098358154 + }, + { + "bits_per_byte": 2.2633119793073635, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5688083171844482, + "step": 950, + "unclipped_grad_norm": 1.1283504962921143 + }, + { + "bits_per_byte": 2.311904989378047, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.6024904251098633, + "step": 960, + "unclipped_grad_norm": 1.1545019149780273 + }, + { + "bits_per_byte": 2.1837827299994785, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.5136828422546387, + "step": 970, + "unclipped_grad_norm": 1.1795276403427124 + }, + { + "bits_per_byte": 2.271298853613432, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5743443965911865, + "step": 980, + "unclipped_grad_norm": 1.2672607898712158 + }, + { + "bits_per_byte": 2.317997990732814, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.6067137718200684, + "step": 990, + "unclipped_grad_norm": 1.2962802648544312 + }, + { + "bits_per_byte": 2.162906099976544, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.4992122650146484, + "step": 1000, + "unclipped_grad_norm": 1.2881999015808105 + }, + { + "bits_per_byte": 2.201945129826496, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.5262720584869385, + "step": 1010, + "unclipped_grad_norm": 1.1237962245941162 + }, + { + "bits_per_byte": 2.24223430156557, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5541983842849731, + "step": 1020, + "unclipped_grad_norm": 1.2514863014221191 + }, + { + "bits_per_byte": 2.1296558341719827, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.4761649370193481, + "step": 1030, + "unclipped_grad_norm": 1.168424129486084 + }, + { + "bits_per_byte": 2.2345234594141363, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5488536357879639, + "step": 1040, + "unclipped_grad_norm": 1.1286299228668213 + }, + { + "bits_per_byte": 2.1963825229669895, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.522416353225708, + "step": 1050, + "unclipped_grad_norm": 1.0536384582519531 + }, + { + "bits_per_byte": 2.166973489669449, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.5020315647125244, + "step": 1060, + "unclipped_grad_norm": 1.278731346130371 + }, + { + "bits_per_byte": 2.1453139946197872, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.487018346786499, + "step": 1070, + "unclipped_grad_norm": 1.156399130821228 + }, + { + "bits_per_byte": 2.208945511647216, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.5311243534088135, + "step": 1080, + "unclipped_grad_norm": 1.137290596961975 + }, + { + "bits_per_byte": 2.1481881686810063, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.4890105724334717, + "step": 1090, + "unclipped_grad_norm": 1.1462913751602173 + }, + { + "bits_per_byte": 2.2885430380674934, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.5862971544265747, + "step": 1100, + "unclipped_grad_norm": 1.2327862977981567 + }, + { + "bits_per_byte": 2.260198577378794, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.5666502714157104, + "step": 1110, + "unclipped_grad_norm": 1.2376710176467896 + }, + { + "bits_per_byte": 2.1645426868821542, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.5003466606140137, + "step": 1120, + "unclipped_grad_norm": 1.2120041847229004 + }, + { + "bits_per_byte": 2.128523500398702, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.4753800630569458, + "step": 1130, + "unclipped_grad_norm": 1.1611560583114624 + }, + { + "bits_per_byte": 2.1978904668497545, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.5234615802764893, + "step": 1140, + "unclipped_grad_norm": 1.2637544870376587 + }, + { + "bits_per_byte": 2.1217606266188236, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4706923961639404, + "step": 1150, + "unclipped_grad_norm": 1.389503002166748 + }, + { + "bits_per_byte": 2.113784415237108, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4651637077331543, + "step": 1160, + "unclipped_grad_norm": 1.2648072242736816 + }, + { + "bits_per_byte": 2.1227447113470608, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.47137451171875, + "step": 1170, + "unclipped_grad_norm": 1.1031522750854492 + }, + { + "bits_per_byte": 2.1223940387219513, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.471131443977356, + "step": 1180, + "unclipped_grad_norm": 1.3680875301361084 + }, + { + "bits_per_byte": 2.1246487312747626, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.4726942777633667, + "step": 1190, + "unclipped_grad_norm": 1.1871044635772705 + }, + { + "bits_per_byte": 2.145305395487244, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.4870123863220215, + "step": 1200, + "unclipped_grad_norm": 1.2274872064590454 + }, + { + "bits_per_byte": 2.20696100383891, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.529748797416687, + "step": 1210, + "unclipped_grad_norm": 1.1122627258300781 + }, + { + "bits_per_byte": 2.062349563843568, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4295117855072021, + "step": 1220, + "unclipped_grad_norm": 1.1313010454177856 + }, + { + "bits_per_byte": 2.0806880739050695, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.442223072052002, + "step": 1230, + "unclipped_grad_norm": 1.224672555923462 + }, + { + "bits_per_byte": 2.1280366175141094, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4750425815582275, + "step": 1240, + "unclipped_grad_norm": 1.1895887851715088 + }, + { + "bits_per_byte": 2.1452462334553473, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.486971378326416, + "step": 1250, + "unclipped_grad_norm": 1.2189933061599731 + }, + { + "bits_per_byte": 2.0779416829534414, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.4403194189071655, + "step": 1260, + "unclipped_grad_norm": 1.189552903175354 + }, + { + "bits_per_byte": 2.147902677480574, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4888126850128174, + "step": 1270, + "unclipped_grad_norm": 1.331443190574646 + }, + { + "bits_per_byte": 2.1365745242335428, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4809606075286865, + "step": 1280, + "unclipped_grad_norm": 1.1365375518798828 + }, + { + "bits_per_byte": 2.1713016050610627, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.5050315856933594, + "step": 1290, + "unclipped_grad_norm": 1.1142851114273071 + }, + { + "bits_per_byte": 2.0535875637300607, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4234384298324585, + "step": 1300, + "unclipped_grad_norm": 1.2023956775665283 + }, + { + "bits_per_byte": 2.113203113877192, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4647607803344727, + "step": 1310, + "unclipped_grad_norm": 1.1256351470947266 + }, + { + "bits_per_byte": 2.0973841496508308, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4537959098815918, + "step": 1320, + "unclipped_grad_norm": 1.2931033372879028 + }, + { + "bits_per_byte": 2.110522248315543, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.462902545928955, + "step": 1330, + "unclipped_grad_norm": 1.1746976375579834 + }, + { + "bits_per_byte": 2.0757989511063415, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4388341903686523, + "step": 1340, + "unclipped_grad_norm": 1.1528301239013672 + }, + { + "bits_per_byte": 2.1029072005006393, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4576241970062256, + "step": 1350, + "unclipped_grad_norm": 1.1862523555755615 + }, + { + "bits_per_byte": 2.015066373641856, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.3967375755310059, + "step": 1360, + "unclipped_grad_norm": 1.2617039680480957 + }, + { + "bits_per_byte": 2.0999384359814455, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.45556640625, + "step": 1370, + "unclipped_grad_norm": 1.2175390720367432 + }, + { + "bits_per_byte": 2.196251472187032, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.5223255157470703, + "step": 1380, + "unclipped_grad_norm": 1.30815589427948 + }, + { + "bits_per_byte": 1.9927942763898208, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.3812997341156006, + "step": 1390, + "unclipped_grad_norm": 1.1892951726913452 + }, + { + "bits_per_byte": 2.0442975768784044, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.416999101638794, + "step": 1400, + "unclipped_grad_norm": 1.228626012802124 + }, + { + "bits_per_byte": 1.985456464608108, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.376213550567627, + "step": 1410, + "unclipped_grad_norm": 1.071134090423584 + }, + { + "bits_per_byte": 2.0232785452205553, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.4024298191070557, + "step": 1420, + "unclipped_grad_norm": 1.1106035709381104 + }, + { + "bits_per_byte": 2.00920692472696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.3926761150360107, + "step": 1430, + "unclipped_grad_norm": 1.154021143913269 + }, + { + "bits_per_byte": 2.0410753099318386, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.4147655963897705, + "step": 1440, + "unclipped_grad_norm": 1.1190521717071533 + }, + { + "bits_per_byte": 2.1245646317584908, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.4726359844207764, + "step": 1450, + "unclipped_grad_norm": 1.1753525733947754 + }, + { + "bits_per_byte": 1.999700067752566, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.3860864639282227, + "step": 1460, + "unclipped_grad_norm": 1.1662393808364868 + }, + { + "bits_per_byte": 2.0258692918731525, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.4042255878448486, + "step": 1470, + "unclipped_grad_norm": 1.2186145782470703 + }, + { + "bits_per_byte": 2.089011002310924, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4479920864105225, + "step": 1480, + "unclipped_grad_norm": 1.1694611310958862 + }, + { + "bits_per_byte": 2.066873051526562, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.4326472282409668, + "step": 1490, + "unclipped_grad_norm": 1.1517577171325684 + }, + { + "bits_per_byte": 1.9803764410669225, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.372692346572876, + "step": 1500, + "unclipped_grad_norm": 1.1810340881347656 + }, + { + "bits_per_byte": 2.0236120195805785, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4026609659194946, + "step": 1510, + "unclipped_grad_norm": 1.0960806608200073 + }, + { + "bits_per_byte": 2.009075529981701, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.392585039138794, + "step": 1520, + "unclipped_grad_norm": 1.1778756380081177 + }, + { + "bits_per_byte": 2.1078860982431173, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4610753059387207, + "step": 1530, + "unclipped_grad_norm": 1.3102644681930542 + }, + { + "bits_per_byte": 2.066523066832056, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.432404637336731, + "step": 1540, + "unclipped_grad_norm": 1.6690092086791992 + }, + { + "bits_per_byte": 1.9648137309903468, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3619050979614258, + "step": 1550, + "unclipped_grad_norm": 1.1498525142669678 + }, + { + "bits_per_byte": 2.0466048961223797, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.4185984134674072, + "step": 1560, + "unclipped_grad_norm": 1.2708338499069214 + }, + { + "bits_per_byte": 2.001680619959902, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.3874592781066895, + "step": 1570, + "unclipped_grad_norm": 1.132536768913269 + }, + { + "bits_per_byte": 2.029526846909052, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.406760811805725, + "step": 1580, + "unclipped_grad_norm": 1.2013366222381592 + }, + { + "bits_per_byte": 2.0275629770188495, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.4053995609283447, + "step": 1590, + "unclipped_grad_norm": 1.2054722309112549 + }, + { + "bits_per_byte": 2.0915341597817325, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.449741005897522, + "step": 1600, + "unclipped_grad_norm": 1.2556712627410889 + }, + { + "bits_per_byte": 2.022758469684346, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.402069330215454, + "step": 1610, + "unclipped_grad_norm": 1.2762330770492554 + }, + { + "bits_per_byte": 2.0397445081794623, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.4138431549072266, + "step": 1620, + "unclipped_grad_norm": 1.2387187480926514 + }, + { + "bits_per_byte": 1.9943696374717241, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.3823916912078857, + "step": 1630, + "unclipped_grad_norm": 1.304571509361267 + }, + { + "bits_per_byte": 2.0494484572717453, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.4205694198608398, + "step": 1640, + "unclipped_grad_norm": 1.1713929176330566 + }, + { + "bits_per_byte": 2.0229333760402737, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.4021905660629272, + "step": 1650, + "unclipped_grad_norm": 1.1892685890197754 + }, + { + "bits_per_byte": 1.9633522224233149, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.3608920574188232, + "step": 1660, + "unclipped_grad_norm": 1.1836787462234497 + }, + { + "bits_per_byte": 1.9956928719874625, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3833088874816895, + "step": 1670, + "unclipped_grad_norm": 1.200740933418274 + }, + { + "bits_per_byte": 1.9488145289458816, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3508152961730957, + "step": 1680, + "unclipped_grad_norm": 1.1854628324508667 + }, + { + "bits_per_byte": 2.0864009936014303, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.4461829662322998, + "step": 1690, + "unclipped_grad_norm": 1.3681666851043701 + }, + { + "bits_per_byte": 1.9817603854584156, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.3736516237258911, + "step": 1700, + "unclipped_grad_norm": 1.296437382698059 + }, + { + "bits_per_byte": 1.9808034739890148, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3729883432388306, + "step": 1710, + "unclipped_grad_norm": 1.2013524770736694 + }, + { + "bits_per_byte": 1.9722798418296033, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.3670802116394043, + "step": 1720, + "unclipped_grad_norm": 1.1360468864440918 + }, + { + "bits_per_byte": 1.9954861488411255, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3831655979156494, + "step": 1730, + "unclipped_grad_norm": 1.2313921451568604 + }, + { + "bits_per_byte": 2.023764224226592, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.402766466140747, + "step": 1740, + "unclipped_grad_norm": 1.1889103651046753 + }, + { + "bits_per_byte": 1.9979706102154895, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3848876953125, + "step": 1750, + "unclipped_grad_norm": 1.1490246057510376 + }, + { + "bits_per_byte": 2.05144070429934, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.421950340270996, + "step": 1760, + "unclipped_grad_norm": 1.210589051246643 + }, + { + "bits_per_byte": 1.9460597108443611, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.3489058017730713, + "step": 1770, + "unclipped_grad_norm": 1.2303162813186646 + }, + { + "bits_per_byte": 1.9591059707735121, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3579487800598145, + "step": 1780, + "unclipped_grad_norm": 1.2075440883636475 + }, + { + "bits_per_byte": 1.9767319567124888, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3701661825180054, + "step": 1790, + "unclipped_grad_norm": 1.177036166191101 + }, + { + "bits_per_byte": 2.022314926427771, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.4017618894577026, + "step": 1800, + "unclipped_grad_norm": 1.3064422607421875 + }, + { + "bits_per_byte": 1.9852208483764262, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.3760502338409424, + "step": 1810, + "unclipped_grad_norm": 1.2134971618652344 + }, + { + "bits_per_byte": 1.9359139383220114, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.341873288154602, + "step": 1820, + "unclipped_grad_norm": 1.1297476291656494 + }, + { + "bits_per_byte": 1.9091556736222197, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3233258724212646, + "step": 1830, + "unclipped_grad_norm": 1.1944961547851562 + }, + { + "bits_per_byte": 1.9665731135086735, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3631246089935303, + "step": 1840, + "unclipped_grad_norm": 1.176443099975586 + }, + { + "bits_per_byte": 1.9440027983400419, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.347480058670044, + "step": 1850, + "unclipped_grad_norm": 1.203460693359375 + }, + { + "bits_per_byte": 2.0439780331131012, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.4167776107788086, + "step": 1860, + "unclipped_grad_norm": 1.2028536796569824 + }, + { + "bits_per_byte": 1.9767374601573164, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.370169997215271, + "step": 1870, + "unclipped_grad_norm": 1.2155165672302246 + }, + { + "bits_per_byte": 2.026455752712595, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.4046320915222168, + "step": 1880, + "unclipped_grad_norm": 1.2128735780715942 + }, + { + "bits_per_byte": 2.0724893169731375, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.436540126800537, + "step": 1890, + "unclipped_grad_norm": 1.281884789466858 + }, + { + "bits_per_byte": 1.9597055022944199, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3583643436431885, + "step": 1900, + "unclipped_grad_norm": 1.2043333053588867 + }, + { + "bits_per_byte": 2.000661450770889, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.3867528438568115, + "step": 1910, + "unclipped_grad_norm": 1.1812796592712402 + }, + { + "bits_per_byte": 1.9502626228661464, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3518190383911133, + "step": 1920, + "unclipped_grad_norm": 1.180728554725647 + }, + { + "bits_per_byte": 1.8931537198553405, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3122341632843018, + "step": 1930, + "unclipped_grad_norm": 1.2632079124450684 + }, + { + "bits_per_byte": 1.9857192540986266, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3763957023620605, + "step": 1940, + "unclipped_grad_norm": 1.2192130088806152 + }, + { + "bits_per_byte": 1.9815562420518413, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3735101222991943, + "step": 1950, + "unclipped_grad_norm": 1.1729612350463867 + }, + { + "bits_per_byte": 1.9707252906484543, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3660026788711548, + "step": 1960, + "unclipped_grad_norm": 1.2030855417251587 + }, + { + "bits_per_byte": 2.0371467102381797, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.412042498588562, + "step": 1970, + "unclipped_grad_norm": 1.252558946609497 + }, + { + "bits_per_byte": 1.9560663494021628, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.355841875076294, + "step": 1980, + "unclipped_grad_norm": 1.2411595582962036 + }, + { + "bits_per_byte": 1.9922193383879863, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3809012174606323, + "step": 1990, + "unclipped_grad_norm": 1.252178430557251 + }, + { + "bits_per_byte": 2.038916583698209, + "learning_rate": 3e-05, + "loss_nats": 1.413269281387329, + "step": 2000, + "unclipped_grad_norm": 1.311530590057373 + } + ] + }, + { + "architecture": "full", + "batch_size": 32, + "canonical_sha256_without_self": "e14a6f92b16d7d5f17d0e6a7cdee3471ea717988528e081360d851296a0a7f5a", + "diagnostic": { + "bits_per_byte": 2.14482418803013, + "branch_output_rms": [ + 0.027083024382591248, + 0.05154837295413017, + 0.053479041904211044, + 0.041930392384529114, + 0.030307648703455925, + 0.033292125910520554, + 0.02454756386578083, + 0.028076333925127983, + 0.02585592307150364, + 0.030918236821889877, + 0.05061359703540802, + 0.03217681124806404, + 0.05264769122004509, + 0.046491838991642, + 0.02188333496451378, + 0.022710850462317467, + 0.02356286346912384, + 0.02021927572786808, + 0.028008684515953064, + 0.021093308925628662, + 0.03027774952352047, + 0.027663733810186386, + 0.028807632625102997, + 0.029710212722420692, + 0.021688953042030334, + 0.031379926949739456, + 0.03756574168801308, + 0.03358094021677971, + 0.2097787708044052, + 0.034267887473106384, + 1.8557170629501343, + 0.05258624255657196 + ], + "core_parameter_grad_rms_by_block": [ + 0.0005889504686640395, + 0.000431366526533741, + 0.0002935829834809252, + 0.00027276189477100897, + 0.00022993427970632987, + 0.00029749385150172825, + 0.0003254079422057709, + 0.000437599126373957, + 0.0006205993934301731, + 0.0009251527144282604, + 0.0012008713251261485, + 0.0009744439463116046, + 0.0008833614419253884, + 0.0007582103318376523, + 0.0005570695353753757, + 0.00047213321202463197 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6552057266235352, + "mean_weights": [ + 0.5899416208267212, + 0.41005831956863403 + ], + "sources": 2 + }, + { + "entropy_mean": 1.074110507965088, + "mean_weights": [ + 0.3676150441169739, + 0.332220196723938, + 0.30016472935676575 + ], + "sources": 3 + }, + { + "entropy_mean": 1.2723480463027954, + "mean_weights": [ + 0.3555198907852173, + 0.16011321544647217, + 0.3348948359489441, + 0.14947205781936646 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5582118034362793, + "mean_weights": [ + 0.16559171676635742, + 0.2511868476867676, + 0.16271471977233887, + 0.285312294960022, + 0.13519440591335297 + ], + "sources": 5 + }, + { + "entropy_mean": 1.719352126121521, + "mean_weights": [ + 0.21674910187721252, + 0.1298384815454483, + 0.21351651847362518, + 0.11778293550014496, + 0.16921769082546234, + 0.1528952568769455 + ], + "sources": 6 + }, + { + "entropy_mean": 1.8659067153930664, + "mean_weights": [ + 0.12279738485813141, + 0.209499329328537, + 0.12951356172561646, + 0.22417230904102325, + 0.09914661198854446, + 0.14670151472091675, + 0.06816928833723068 + ], + "sources": 7 + }, + { + "entropy_mean": 2.019052743911743, + "mean_weights": [ + 0.138309046626091, + 0.12189304083585739, + 0.15173543989658356, + 0.10711444914340973, + 0.10736483335494995, + 0.13547836244106293, + 0.0768684446811676, + 0.16123637557029724 + ], + "sources": 8 + }, + { + "entropy_mean": 2.1553163528442383, + "mean_weights": [ + 0.10782437771558762, + 0.15511271357536316, + 0.11365120112895966, + 0.14651325345039368, + 0.09214359521865845, + 0.12269628047943115, + 0.06948265433311462, + 0.12447832524776459, + 0.06809761375188828 + ], + "sources": 9 + }, + { + "entropy_mean": 2.2248992919921875, + "mean_weights": [ + 0.1250765323638916, + 0.09274249523878098, + 0.12701454758644104, + 0.07507169246673584, + 0.08977675437927246, + 0.11032186448574066, + 0.06382834911346436, + 0.1372935175895691, + 0.05465774983167648, + 0.1242164820432663 + ], + "sources": 10 + }, + { + "entropy_mean": 2.359937906265259, + "mean_weights": [ + 0.09064529091119766, + 0.12523706257343292, + 0.09787359833717346, + 0.11289006471633911, + 0.07962331175804138, + 0.09815013408660889, + 0.060369715094566345, + 0.11304862797260284, + 0.057952508330345154, + 0.10411867499351501, + 0.06009101867675781 + ], + "sources": 11 + }, + { + "entropy_mean": 2.371119737625122, + "mean_weights": [ + 0.08177289366722107, + 0.0973823294043541, + 0.07850118726491928, + 0.09761933237314224, + 0.05462679639458656, + 0.10870781540870667, + 0.03703199699521065, + 0.12642592191696167, + 0.0350656695663929, + 0.12671604752540588, + 0.036303386092185974, + 0.11984661221504211 + ], + "sources": 12 + }, + { + "entropy_mean": 2.523961067199707, + "mean_weights": [ + 0.07941006124019623, + 0.1025254875421524, + 0.08785355091094971, + 0.09459104388952255, + 0.07015340030193329, + 0.07883293926715851, + 0.0514906570315361, + 0.08862217515707016, + 0.049287330359220505, + 0.09146402776241302, + 0.050756484270095825, + 0.10278001427650452, + 0.05223281681537628 + ], + "sources": 13 + }, + { + "entropy_mean": 2.5260555744171143, + "mean_weights": [ + 0.07178902626037598, + 0.08412610739469528, + 0.0694701224565506, + 0.07757194340229034, + 0.05084136128425598, + 0.08818310499191284, + 0.03413278982043266, + 0.10019787400960922, + 0.03067246451973915, + 0.0932452604174614, + 0.031987275928258896, + 0.10189531743526459, + 0.034488312900066376, + 0.13139905035495758 + ], + "sources": 14 + }, + { + "entropy_mean": 2.66227126121521, + "mean_weights": [ + 0.07101236283779144, + 0.07927534729242325, + 0.07929423451423645, + 0.07709577679634094, + 0.06179986894130707, + 0.07244092226028442, + 0.044221073389053345, + 0.08079221844673157, + 0.041108809411525726, + 0.077225461602211, + 0.042375072836875916, + 0.08864865452051163, + 0.04135163128376007, + 0.09543520212173462, + 0.04792335256934166 + ], + "sources": 15 + }, + { + "entropy_mean": 2.6512417793273926, + "mean_weights": [ + 0.0608191192150116, + 0.08778031170368195, + 0.07251773774623871, + 0.07472798228263855, + 0.05009090155363083, + 0.07822239398956299, + 0.030434954911470413, + 0.08796542882919312, + 0.025094006210565567, + 0.07832452654838562, + 0.025485597550868988, + 0.07227373123168945, + 0.025366825982928276, + 0.08716164529323578, + 0.02966739982366562, + 0.1140674501657486 + ], + "sources": 16 + }, + { + "entropy_mean": 2.775836944580078, + "mean_weights": [ + 0.06218317896127701, + 0.07270783931016922, + 0.0734800398349762, + 0.06857287883758545, + 0.05565473064780235, + 0.06780414283275604, + 0.03857433795928955, + 0.07381865382194519, + 0.03338298201560974, + 0.06580726057291031, + 0.03422714024782181, + 0.0671156719326973, + 0.031492769718170166, + 0.07390124350786209, + 0.03565242886543274, + 0.09292156994342804, + 0.052703142166137695 + ], + "sources": 17 + }, + { + "entropy_mean": 2.76259446144104, + "mean_weights": [ + 0.05494416505098343, + 0.06508217751979828, + 0.06912961602210999, + 0.05489455908536911, + 0.04735521599650383, + 0.06342022120952606, + 0.02933962643146515, + 0.06950756907463074, + 0.02359858527779579, + 0.05526488646864891, + 0.0241834819316864, + 0.04860550910234451, + 0.02301885187625885, + 0.05820363759994507, + 0.024978550150990486, + 0.09920422732830048, + 0.056571729481220245, + 0.13269740343093872 + ], + "sources": 18 + }, + { + "entropy_mean": 2.867581844329834, + "mean_weights": [ + 0.05244980752468109, + 0.06523977220058441, + 0.06313808262348175, + 0.060504764318466187, + 0.04746279865503311, + 0.059726499021053314, + 0.03205186873674393, + 0.06600943207740784, + 0.026977211236953735, + 0.056454066187143326, + 0.027098361402750015, + 0.05530134588479996, + 0.02379560098052025, + 0.0578027069568634, + 0.026392817497253418, + 0.0750894695520401, + 0.03944184631109238, + 0.08907561004161835, + 0.07598794251680374 + ], + "sources": 19 + }, + { + "entropy_mean": 2.7220685482025146, + "mean_weights": [ + 0.04507756978273392, + 0.05042458325624466, + 0.06334839761257172, + 0.03487860783934593, + 0.03997787460684776, + 0.04022468626499176, + 0.023146267980337143, + 0.04642810672521591, + 0.017794962972402573, + 0.032394684851169586, + 0.017961964011192322, + 0.025713298469781876, + 0.015309647656977177, + 0.027951309457421303, + 0.01385954674333334, + 0.06821286678314209, + 0.045731861144304276, + 0.10842365026473999, + 0.09600982069969177, + 0.18713030219078064 + ], + "sources": 20 + }, + { + "entropy_mean": 2.9706764221191406, + "mean_weights": [ + 0.04438478499650955, + 0.05772855877876282, + 0.05557219684123993, + 0.05148957669734955, + 0.04265815019607544, + 0.052152879536151886, + 0.029726460576057434, + 0.05611037835478783, + 0.025108125060796738, + 0.047879625111818314, + 0.02500380575656891, + 0.047868698835372925, + 0.02236107736825943, + 0.048469364643096924, + 0.023287266492843628, + 0.06311047822237015, + 0.03218208625912666, + 0.07191765308380127, + 0.054885007441043854, + 0.08266504108905792, + 0.06543879210948944 + ], + "sources": 21 + }, + { + "entropy_mean": 2.807748794555664, + "mean_weights": [ + 0.035533905029296875, + 0.04258962348103523, + 0.052646808326244354, + 0.02946171537041664, + 0.03273501247167587, + 0.03544780984520912, + 0.018608057871460915, + 0.03881338983774185, + 0.01461428590118885, + 0.028256740421056747, + 0.014569966122508049, + 0.021484043449163437, + 0.012886431068181992, + 0.021817432716488838, + 0.010389651171863079, + 0.0528411865234375, + 0.03315703943371773, + 0.08006380498409271, + 0.06689971685409546, + 0.12302057445049286, + 0.08468171209096909, + 0.14948110282421112 + ], + "sources": 22 + }, + { + "entropy_mean": 3.043855905532837, + "mean_weights": [ + 0.037522993981838226, + 0.046761415898799896, + 0.047712475061416626, + 0.039633505046367645, + 0.03645867109298706, + 0.042903006076812744, + 0.026118159294128418, + 0.046902693808078766, + 0.021814875304698944, + 0.04030716419219971, + 0.021950285881757736, + 0.03951451927423477, + 0.019624117761850357, + 0.04043006896972656, + 0.019534897059202194, + 0.056119292974472046, + 0.027303311973810196, + 0.060620568692684174, + 0.0447116456925869, + 0.07258439809083939, + 0.05768676847219467, + 0.08186041563749313, + 0.07192474603652954 + ], + "sources": 23 + }, + { + "entropy_mean": 2.9273719787597656, + "mean_weights": [ + 0.02985021099448204, + 0.04255836457014084, + 0.04390624910593033, + 0.0302383154630661, + 0.02726692706346512, + 0.033654406666755676, + 0.015117933042347431, + 0.03556524217128754, + 0.012127472087740898, + 0.026151910424232483, + 0.012214772403240204, + 0.020746367052197456, + 0.011011885479092598, + 0.02168337255716324, + 0.00922815315425396, + 0.04956081137061119, + 0.024936452507972717, + 0.06564131379127502, + 0.04552200436592102, + 0.09413468837738037, + 0.06643196940422058, + 0.11070455610752106, + 0.09393187612295151, + 0.07781475782394409 + ], + "sources": 24 + }, + { + "entropy_mean": 3.096397638320923, + "mean_weights": [ + 0.032453835010528564, + 0.04033014923334122, + 0.041946642100811005, + 0.030874401330947876, + 0.03128889203071594, + 0.033737268298864365, + 0.021996498107910156, + 0.03627324849367142, + 0.018692100420594215, + 0.03084566630423069, + 0.018477097153663635, + 0.028095830231904984, + 0.016697287559509277, + 0.027684343978762627, + 0.014903861097991467, + 0.047992873936891556, + 0.025130677968263626, + 0.05354497954249382, + 0.04114660248160362, + 0.07086223363876343, + 0.0541878342628479, + 0.07850205898284912, + 0.06536564230918884, + 0.0636153370141983, + 0.07535463571548462 + ], + "sources": 25 + }, + { + "entropy_mean": 2.997812271118164, + "mean_weights": [ + 0.02616383507847786, + 0.037428222596645355, + 0.03764750063419342, + 0.026574721559882164, + 0.023854296654462814, + 0.028589164838194847, + 0.014235948212444782, + 0.030330747365951538, + 0.011487850919365883, + 0.02271238900721073, + 0.011588271707296371, + 0.018345903605222702, + 0.010859362781047821, + 0.019163543358445168, + 0.009489243850111961, + 0.04241595417261124, + 0.023671936243772507, + 0.05662977695465088, + 0.03964083641767502, + 0.07622891664505005, + 0.055274978280067444, + 0.08794642984867096, + 0.08163244277238846, + 0.05967992544174194, + 0.11678636074066162, + 0.031621433794498444 + ], + "sources": 26 + }, + { + "entropy_mean": 3.2317047119140625, + "mean_weights": [ + 0.031362950801849365, + 0.03973812609910965, + 0.03811698406934738, + 0.035181086510419846, + 0.030816296115517616, + 0.034574005752801895, + 0.02347460202872753, + 0.03967968747019768, + 0.02002294361591339, + 0.03612414747476578, + 0.019827425479888916, + 0.03531107306480408, + 0.017541782930493355, + 0.034270524978637695, + 0.017486033961176872, + 0.04436960071325302, + 0.02294747903943062, + 0.048037633299827576, + 0.03643457964062691, + 0.051959600299596786, + 0.04280269145965576, + 0.05617425590753555, + 0.05096263810992241, + 0.046745024621486664, + 0.05715267360210419, + 0.03279910981655121, + 0.05608703941106796 + ], + "sources": 27 + }, + { + "entropy_mean": 3.086909770965576, + "mean_weights": [ + 0.02353682741522789, + 0.0323694571852684, + 0.03388986364006996, + 0.024155544117093086, + 0.02298620715737343, + 0.025800812989473343, + 0.015456659719347954, + 0.027840476483106613, + 0.012814443558454514, + 0.021053779870271683, + 0.012840271927416325, + 0.01791493408381939, + 0.012872975319623947, + 0.018450215458869934, + 0.01152428612112999, + 0.0360846184194088, + 0.02403624914586544, + 0.045380353927612305, + 0.03381213918328285, + 0.06214917451143265, + 0.047034166753292084, + 0.06954774260520935, + 0.06689899414777756, + 0.05115798860788345, + 0.09373119473457336, + 0.02853286638855934, + 0.10822182148694992, + 0.019905947148799896 + ], + "sources": 28 + }, + { + "entropy_mean": 3.315805435180664, + "mean_weights": [ + 0.02935282327234745, + 0.03673504665493965, + 0.03679622709751129, + 0.03257634490728378, + 0.033802762627601624, + 0.028226884081959724, + 0.02924294024705887, + 0.02899230271577835, + 0.02730604074895382, + 0.02819140814244747, + 0.026536326855421066, + 0.02373933419585228, + 0.02345927432179451, + 0.021810201928019524, + 0.01910841092467308, + 0.033235181123018265, + 0.02363448590040207, + 0.034036993980407715, + 0.02865864336490631, + 0.04625985398888588, + 0.039478849619627, + 0.05723576992750168, + 0.04284664988517761, + 0.05078060179948807, + 0.04738301783800125, + 0.03202638775110245, + 0.051890138536691666, + 0.03457200527191162, + 0.05208509415388107 + ], + "sources": 29 + }, + { + "entropy_mean": 3.1718106269836426, + "mean_weights": [ + 0.0228898823261261, + 0.032648585736751556, + 0.032548993825912476, + 0.02478966861963272, + 0.02345195785164833, + 0.025297708809375763, + 0.01653383858501911, + 0.025769304484128952, + 0.013373354449868202, + 0.020823869854211807, + 0.013316335156559944, + 0.01746056228876114, + 0.012606586329638958, + 0.017618611454963684, + 0.01099008321762085, + 0.03218454867601395, + 0.020701495930552483, + 0.0405520424246788, + 0.02972545474767685, + 0.05470733344554901, + 0.04089745134115219, + 0.062037162482738495, + 0.05491635948419571, + 0.048106055706739426, + 0.07804838567972183, + 0.027585100382566452, + 0.0937582403421402, + 0.022382214665412903, + 0.07786072790622711, + 0.006418092176318169 + ], + "sources": 30 + }, + { + "entropy_mean": 3.3875153064727783, + "mean_weights": [ + 0.038437843322753906, + 0.028551608324050903, + 0.02790694311261177, + 0.03092133067548275, + 0.032153114676475525, + 0.032235465943813324, + 0.04118969663977623, + 0.03342031314969063, + 0.04516782611608505, + 0.037846751511096954, + 0.046404458582401276, + 0.04342315346002579, + 0.05148046836256981, + 0.03953840211033821, + 0.05360480770468712, + 0.024233557283878326, + 0.03574083745479584, + 0.022590674459934235, + 0.026733383536338806, + 0.01917802169919014, + 0.02454635687172413, + 0.02167828008532524, + 0.02426629327237606, + 0.02386978641152382, + 0.022893395274877548, + 0.028569117188453674, + 0.026349421590566635, + 0.02942071110010147, + 0.030438879504799843, + 0.023745929822325706, + 0.033463168889284134 + ], + "sources": 31 + }, + { + "entropy_mean": 3.145052194595337, + "mean_weights": [ + 0.020978696644306183, + 0.02698666602373123, + 0.02920648083090782, + 0.019914908334612846, + 0.024827096611261368, + 0.023302476853132248, + 0.020407848060131073, + 0.02314101532101631, + 0.014423400163650513, + 0.018985718488693237, + 0.01506085880100727, + 0.014740705490112305, + 0.01204362977296114, + 0.014488669112324715, + 0.011569652706384659, + 0.027938414365053177, + 0.018632810562849045, + 0.03279075026512146, + 0.024732567369937897, + 0.042812280356884, + 0.02916465327143669, + 0.05146252363920212, + 0.04100646451115608, + 0.0426330529153347, + 0.06057015061378479, + 0.025632426142692566, + 0.08685624599456787, + 0.022757193073630333, + 0.10413752496242523, + 0.0030479258857667446, + 0.09382546693086624, + 0.0019217166118323803 + ], + "sources": 32 + } + ], + "layer_input_rms": [ + 0.05436398461461067, + 0.034621261060237885, + 0.02898014523088932, + 0.03089532069861889, + 0.026308147236704826, + 0.02630813792347908, + 0.023830953985452652, + 0.02176283486187458, + 0.020936816930770874, + 0.02016640454530716, + 0.019486865028738976, + 0.02058389224112034, + 0.02009216882288456, + 0.022332707419991493, + 0.021584752947092056, + 0.01947355456650257, + 0.018600277602672577, + 0.01543617807328701, + 0.015265379101037979, + 0.009849075227975845, + 0.012294777669012547, + 0.0078109800815582275, + 0.009295190684497356, + 0.007291108835488558, + 0.007303531747311354, + 0.007454340811818838, + 0.007594018243253231, + 0.007620570715516806, + 0.0075973751954734325, + 0.007396102417260408, + 0.014656757935881615, + 0.008849309757351875 + ], + "loss_nats": 1.4866788387298584, + "output_weights": { + "entropy_mean": 3.2135813236236572, + "mean_weights": [ + 0.02472158893942833, + 0.024527523666620255, + 0.026205478236079216, + 0.020940078422427177, + 0.024859845638275146, + 0.021043118089437485, + 0.021455809473991394, + 0.020458368584513664, + 0.01645323634147644, + 0.017987605184316635, + 0.018875984475016594, + 0.016256874427199364, + 0.01387819368392229, + 0.01676936075091362, + 0.014496944844722748, + 0.0242566280066967, + 0.019732363522052765, + 0.027034074068069458, + 0.027489308267831802, + 0.03677881509065628, + 0.023889372125267982, + 0.03709936514496803, + 0.029002245515584946, + 0.03442607820034027, + 0.036184292286634445, + 0.025571145117282867, + 0.05934413522481918, + 0.030483966693282127, + 0.085813008248806, + 0.005895273759961128, + 0.08629065752029419, + 0.0024080341681838036, + 0.10937123000621796 + ], + "sources": 33 + }, + "stream_state_rms": [ + 0.04294736310839653, + 0.04599342867732048, + 0.04797445610165596, + 0.046828098595142365, + 0.044502634555101395, + 0.04308011010289192, + 0.0412217415869236, + 0.039975181221961975, + 0.03879518434405327, + 0.03814637288451195, + 0.039336513727903366, + 0.038832664489746094, + 0.039978086948394775, + 0.0404449887573719, + 0.039540987461805344, + 0.03875384107232094, + 0.03806925565004349, + 0.0373431071639061, + 0.036932460963726044, + 0.03633511811494827, + 0.03608185052871704, + 0.03575707972049713, + 0.03549469634890556, + 0.035281531512737274, + 0.0348568893969059, + 0.034734319895505905, + 0.03483940288424492, + 0.03479676693677902, + 0.05135522037744522, + 0.050893642008304596, + 0.33184999227523804, + 0.32691144943237305 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.053220311266156, + "cross_entropy_nats": 5.5820669531822205, + "step": 0 + }, + { + "bits_per_byte": 3.87981092433611, + "cross_entropy_nats": 2.68928000330925, + "step": 100 + }, + { + "bits_per_byte": 3.4658855739879395, + "cross_entropy_nats": 2.402368813753128, + "step": 250 + }, + { + "bits_per_byte": 2.9766696157493864, + "cross_entropy_nats": 2.063270151615143, + "step": 500 + }, + { + "bits_per_byte": 2.2318612109744427, + "cross_entropy_nats": 1.5470083057880402, + "step": 1000 + }, + { + "bits_per_byte": 2.05077289566604, + "cross_entropy_nats": 1.4214874505996704, + "step": 1500 + }, + { + "bits_per_byte": 1.9845419683599765, + "cross_entropy_nats": 1.3755796700716019, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "5bf82568b88ca65bc4881674251c7eb6e2fbf73223a8e25b09ea26227966d76d", + "final_mixer_parameters": "e908b5ae6bd682e2905bd3c48334d71f989af89162ee5a2345820c45cd2e6a1b", + "initial_common_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 146.80656970705252, + "measured_steps": 1980, + "median_ms": 146.7325690027792, + "p95_ms": 147.17569539061515, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.04291519681778, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.574923992156982, + "step": 1, + "unclipped_grad_norm": 12.666659355163574 + }, + { + "bits_per_byte": 6.94569133774539, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.814386367797852, + "step": 10, + "unclipped_grad_norm": 5.22608757019043 + }, + { + "bits_per_byte": 6.4733815750562895, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.487006187438965, + "step": 20, + "unclipped_grad_norm": 2.8815624713897705 + }, + { + "bits_per_byte": 6.25022101488033, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.33232307434082, + "step": 30, + "unclipped_grad_norm": 2.630570411682129 + }, + { + "bits_per_byte": 5.7805274522761545, + "learning_rate": 0.00011999999999999999, + "loss_nats": 4.00675630569458, + "step": 40, + "unclipped_grad_norm": 2.4446051120758057 + }, + { + "bits_per_byte": 5.308193268050632, + "learning_rate": 0.00015, + "loss_nats": 3.679359197616577, + "step": 50, + "unclipped_grad_norm": 2.090557813644409 + }, + { + "bits_per_byte": 4.8050088359133385, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.330578327178955, + "step": 60, + "unclipped_grad_norm": 1.684688687324524 + }, + { + "bits_per_byte": 4.451415257461817, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.0854859352111816, + "step": 70, + "unclipped_grad_norm": 1.2390650510787964 + }, + { + "bits_per_byte": 4.205407146084035, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.914966106414795, + "step": 80, + "unclipped_grad_norm": 1.0080320835113525 + }, + { + "bits_per_byte": 3.976652678206112, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.7564055919647217, + "step": 90, + "unclipped_grad_norm": 0.6797000169754028 + }, + { + "bits_per_byte": 3.84149942409497, + "learning_rate": 0.0003, + "loss_nats": 2.662724494934082, + "step": 100, + "unclipped_grad_norm": 0.5370410084724426 + }, + { + "bits_per_byte": 3.7975977567745227, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.632294178009033, + "step": 110, + "unclipped_grad_norm": 0.5649384260177612 + }, + { + "bits_per_byte": 3.7296680493367305, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.5852088928222656, + "step": 120, + "unclipped_grad_norm": 0.7387393712997437 + }, + { + "bits_per_byte": 3.689995091435698, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5577096939086914, + "step": 130, + "unclipped_grad_norm": 0.645742654800415 + }, + { + "bits_per_byte": 3.6437637471266626, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5256645679473877, + "step": 140, + "unclipped_grad_norm": 0.7681083679199219 + }, + { + "bits_per_byte": 3.6446305396870113, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.5262653827667236, + "step": 150, + "unclipped_grad_norm": 0.5295066237449646 + }, + { + "bits_per_byte": 3.605972623391371, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.499469757080078, + "step": 160, + "unclipped_grad_norm": 0.5723434090614319 + }, + { + "bits_per_byte": 3.570140382049406, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.474632740020752, + "step": 170, + "unclipped_grad_norm": 0.7854509949684143 + }, + { + "bits_per_byte": 3.544894360798651, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.4571335315704346, + "step": 180, + "unclipped_grad_norm": 0.520199179649353 + }, + { + "bits_per_byte": 3.535139160876411, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.450371742248535, + "step": 190, + "unclipped_grad_norm": 0.7861291170120239 + }, + { + "bits_per_byte": 3.467205325854998, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.4032835960388184, + "step": 200, + "unclipped_grad_norm": 0.7966506481170654 + }, + { + "bits_per_byte": 3.54077400044928, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.454277515411377, + "step": 210, + "unclipped_grad_norm": 0.7065793871879578 + }, + { + "bits_per_byte": 3.5074465144344815, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.4311766624450684, + "step": 220, + "unclipped_grad_norm": 0.9572800993919373 + }, + { + "bits_per_byte": 3.4917176691171727, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.420274257659912, + "step": 230, + "unclipped_grad_norm": 0.7477400302886963 + }, + { + "bits_per_byte": 3.5213117557470412, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.4407873153686523, + "step": 240, + "unclipped_grad_norm": 1.8658361434936523 + }, + { + "bits_per_byte": 3.4630082612433424, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.400374412536621, + "step": 250, + "unclipped_grad_norm": 0.7070822715759277 + }, + { + "bits_per_byte": 3.425318951237361, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3742501735687256, + "step": 260, + "unclipped_grad_norm": 0.8998879194259644 + }, + { + "bits_per_byte": 3.419324667924189, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.3700952529907227, + "step": 270, + "unclipped_grad_norm": 0.7938683032989502 + }, + { + "bits_per_byte": 3.4087043952681086, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.362733840942383, + "step": 280, + "unclipped_grad_norm": 1.3725367784500122 + }, + { + "bits_per_byte": 3.3607721425420403, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.329509735107422, + "step": 290, + "unclipped_grad_norm": 0.663077175617218 + }, + { + "bits_per_byte": 3.3506292937247553, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.322479248046875, + "step": 300, + "unclipped_grad_norm": 1.5422239303588867 + }, + { + "bits_per_byte": 3.3323203646792026, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.309788465499878, + "step": 310, + "unclipped_grad_norm": 0.9811199307441711 + }, + { + "bits_per_byte": 3.3997000715995354, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.356492519378662, + "step": 320, + "unclipped_grad_norm": 0.9832785725593567 + }, + { + "bits_per_byte": 3.325735492942967, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.3052241802215576, + "step": 330, + "unclipped_grad_norm": 0.8850824236869812 + }, + { + "bits_per_byte": 3.3549013427721874, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.3254404067993164, + "step": 340, + "unclipped_grad_norm": 1.1324429512023926 + }, + { + "bits_per_byte": 3.3256223283586994, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.305145740509033, + "step": 350, + "unclipped_grad_norm": 1.1854218244552612 + }, + { + "bits_per_byte": 3.3488926129163428, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.321275472640991, + "step": 360, + "unclipped_grad_norm": 0.9185400605201721 + }, + { + "bits_per_byte": 3.287911004573408, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.279006242752075, + "step": 370, + "unclipped_grad_norm": 0.8177536129951477 + }, + { + "bits_per_byte": 3.2841648784723145, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.27640962600708, + "step": 380, + "unclipped_grad_norm": 1.2166821956634521 + }, + { + "bits_per_byte": 3.2522779191411426, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.254307270050049, + "step": 390, + "unclipped_grad_norm": 2.8050966262817383 + }, + { + "bits_per_byte": 3.2271061943955606, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.2368595600128174, + "step": 400, + "unclipped_grad_norm": 2.1616663932800293 + }, + { + "bits_per_byte": 3.147015593715259, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.181344985961914, + "step": 410, + "unclipped_grad_norm": 1.1989624500274658 + }, + { + "bits_per_byte": 3.156460536935343, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.187891721725464, + "step": 420, + "unclipped_grad_norm": 1.124785304069519 + }, + { + "bits_per_byte": 3.1228706053953275, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.164608955383301, + "step": 430, + "unclipped_grad_norm": 1.1144435405731201 + }, + { + "bits_per_byte": 3.1051673992461137, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.1523380279541016, + "step": 440, + "unclipped_grad_norm": 0.8945496678352356 + }, + { + "bits_per_byte": 3.10164450662584, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.1498961448669434, + "step": 450, + "unclipped_grad_norm": 1.8599950075149536 + }, + { + "bits_per_byte": 3.035198665499692, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.10383939743042, + "step": 460, + "unclipped_grad_norm": 1.228452205657959 + }, + { + "bits_per_byte": 3.0171559655976745, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.0913331508636475, + "step": 470, + "unclipped_grad_norm": 1.5541294813156128 + }, + { + "bits_per_byte": 3.0920134781775226, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.1432204246520996, + "step": 480, + "unclipped_grad_norm": 1.4661445617675781 + }, + { + "bits_per_byte": 2.9856814206632722, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.069516658782959, + "step": 490, + "unclipped_grad_norm": 1.5669759511947632 + }, + { + "bits_per_byte": 2.9340539487007655, + "learning_rate": 0.0002715339687685131, + "loss_nats": 2.03373122215271, + "step": 500, + "unclipped_grad_norm": 1.541242003440857 + }, + { + "bits_per_byte": 2.8803503021770434, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.996506690979004, + "step": 510, + "unclipped_grad_norm": 1.2288976907730103 + }, + { + "bits_per_byte": 2.894343498581845, + "learning_rate": 0.00026873414996567596, + "loss_nats": 2.0062060356140137, + "step": 520, + "unclipped_grad_norm": 1.6336432695388794 + }, + { + "bits_per_byte": 2.8663058549422846, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.986771821975708, + "step": 530, + "unclipped_grad_norm": 1.58860445022583 + }, + { + "bits_per_byte": 2.8583403064849224, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.981250524520874, + "step": 540, + "unclipped_grad_norm": 1.9627076387405396 + }, + { + "bits_per_byte": 2.844007616327318, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.971315860748291, + "step": 550, + "unclipped_grad_norm": 1.3802709579467773 + }, + { + "bits_per_byte": 2.7883144744984145, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.9327123165130615, + "step": 560, + "unclipped_grad_norm": 1.9945849180221558 + }, + { + "bits_per_byte": 2.778748627474774, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.9260817766189575, + "step": 570, + "unclipped_grad_norm": 1.726778507232666 + }, + { + "bits_per_byte": 2.813214294672983, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.9499715566635132, + "step": 580, + "unclipped_grad_norm": 1.687652826309204 + }, + { + "bits_per_byte": 2.7173795422194456, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8835439682006836, + "step": 590, + "unclipped_grad_norm": 1.3619040250778198 + }, + { + "bits_per_byte": 2.744553488986373, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.9023795127868652, + "step": 600, + "unclipped_grad_norm": 1.555774211883545 + }, + { + "bits_per_byte": 2.7051845964120655, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8750910758972168, + "step": 610, + "unclipped_grad_norm": 1.578161597251892 + }, + { + "bits_per_byte": 2.6740369904969534, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.8535012006759644, + "step": 620, + "unclipped_grad_norm": 1.6466553211212158 + }, + { + "bits_per_byte": 2.680800036259482, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8581889867782593, + "step": 630, + "unclipped_grad_norm": 1.7413678169250488 + }, + { + "bits_per_byte": 2.6103540506569165, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.8093595504760742, + "step": 640, + "unclipped_grad_norm": 2.0753581523895264 + }, + { + "bits_per_byte": 2.630097486993316, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.8230446577072144, + "step": 650, + "unclipped_grad_norm": 1.6899276971817017 + }, + { + "bits_per_byte": 2.610523453568016, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.8094769716262817, + "step": 660, + "unclipped_grad_norm": 2.3933722972869873 + }, + { + "bits_per_byte": 2.613311980269106, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.811409831047058, + "step": 670, + "unclipped_grad_norm": 1.9564707279205322 + }, + { + "bits_per_byte": 2.5324404063712365, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7553539276123047, + "step": 680, + "unclipped_grad_norm": 1.5214877128601074 + }, + { + "bits_per_byte": 2.5979031946650526, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.8007292747497559, + "step": 690, + "unclipped_grad_norm": 2.014816999435425 + }, + { + "bits_per_byte": 2.5026700375242337, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.734718680381775, + "step": 700, + "unclipped_grad_norm": 2.0726895332336426 + }, + { + "bits_per_byte": 2.54265273617947, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.76243257522583, + "step": 710, + "unclipped_grad_norm": 1.9414186477661133 + }, + { + "bits_per_byte": 2.4905184313448707, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.726295828819275, + "step": 720, + "unclipped_grad_norm": 1.761396050453186 + }, + { + "bits_per_byte": 2.397057727403844, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.6615138053894043, + "step": 730, + "unclipped_grad_norm": 1.7791234254837036 + }, + { + "bits_per_byte": 2.380290794805927, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6498918533325195, + "step": 740, + "unclipped_grad_norm": 1.8914049863815308 + }, + { + "bits_per_byte": 2.5147248174711603, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7430744171142578, + "step": 750, + "unclipped_grad_norm": 1.7577917575836182 + }, + { + "bits_per_byte": 2.358124294901521, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.6345272064208984, + "step": 760, + "unclipped_grad_norm": 1.653015375137329 + }, + { + "bits_per_byte": 2.4395315947568283, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.6909544467926025, + "step": 770, + "unclipped_grad_norm": 1.6864970922470093 + }, + { + "bits_per_byte": 2.5334920802812624, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7560828924179077, + "step": 780, + "unclipped_grad_norm": 1.6279327869415283 + }, + { + "bits_per_byte": 2.3544111894693933, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.631953477859497, + "step": 790, + "unclipped_grad_norm": 1.8251627683639526 + }, + { + "bits_per_byte": 2.4022681137943835, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6651253700256348, + "step": 800, + "unclipped_grad_norm": 1.8714652061462402 + }, + { + "bits_per_byte": 2.3596497810146744, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6355845928192139, + "step": 810, + "unclipped_grad_norm": 1.9191111326217651 + }, + { + "bits_per_byte": 2.3940191379284, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.659407615661621, + "step": 820, + "unclipped_grad_norm": 1.872559905052185 + }, + { + "bits_per_byte": 2.30431126741185, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.597226858139038, + "step": 830, + "unclipped_grad_norm": 1.6266796588897705 + }, + { + "bits_per_byte": 2.280569750390843, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.580770492553711, + "step": 840, + "unclipped_grad_norm": 1.5997295379638672 + }, + { + "bits_per_byte": 2.364882869115128, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.639211893081665, + "step": 850, + "unclipped_grad_norm": 2.033557653427124 + }, + { + "bits_per_byte": 2.338472869283333, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.620905876159668, + "step": 860, + "unclipped_grad_norm": 1.4284406900405884 + }, + { + "bits_per_byte": 2.3655563531759065, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.639678716659546, + "step": 870, + "unclipped_grad_norm": 1.748233437538147 + }, + { + "bits_per_byte": 2.352845975363891, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.6308685541152954, + "step": 880, + "unclipped_grad_norm": 1.559957504272461 + }, + { + "bits_per_byte": 2.3258477948661453, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.6121548414230347, + "step": 890, + "unclipped_grad_norm": 1.8586716651916504 + }, + { + "bits_per_byte": 2.2548714147683184, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.562957763671875, + "step": 900, + "unclipped_grad_norm": 1.7450528144836426 + }, + { + "bits_per_byte": 2.3205823740273295, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.608505129814148, + "step": 910, + "unclipped_grad_norm": 1.802466630935669 + }, + { + "bits_per_byte": 2.2460665909747464, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5568547248840332, + "step": 920, + "unclipped_grad_norm": 1.7096110582351685 + }, + { + "bits_per_byte": 2.209676437913383, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5316309928894043, + "step": 930, + "unclipped_grad_norm": 1.827244520187378 + }, + { + "bits_per_byte": 2.271329638507937, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5743657350540161, + "step": 940, + "unclipped_grad_norm": 1.7139626741409302 + }, + { + "bits_per_byte": 2.2674117017386313, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5716500282287598, + "step": 950, + "unclipped_grad_norm": 1.9270174503326416 + }, + { + "bits_per_byte": 2.273380187654174, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.57578706741333, + "step": 960, + "unclipped_grad_norm": 1.528204083442688 + }, + { + "bits_per_byte": 2.1610737968142515, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.4979422092437744, + "step": 970, + "unclipped_grad_norm": 1.8320369720458984 + }, + { + "bits_per_byte": 2.254816724285344, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5629198551177979, + "step": 980, + "unclipped_grad_norm": 1.9189670085906982 + }, + { + "bits_per_byte": 2.278162853192018, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.5791021585464478, + "step": 990, + "unclipped_grad_norm": 1.6875810623168945 + }, + { + "bits_per_byte": 2.144177189297584, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.4862303733825684, + "step": 1000, + "unclipped_grad_norm": 1.6283420324325562 + }, + { + "bits_per_byte": 2.1704492590433864, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.5044407844543457, + "step": 1010, + "unclipped_grad_norm": 1.5491986274719238 + }, + { + "bits_per_byte": 2.2150374811060782, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5353469848632812, + "step": 1020, + "unclipped_grad_norm": 1.511672019958496 + }, + { + "bits_per_byte": 2.0960844767582607, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.4528950452804565, + "step": 1030, + "unclipped_grad_norm": 1.3766611814498901 + }, + { + "bits_per_byte": 2.2182786661442386, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5375936031341553, + "step": 1040, + "unclipped_grad_norm": 1.728167176246643 + }, + { + "bits_per_byte": 2.1874555914913043, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.5162286758422852, + "step": 1050, + "unclipped_grad_norm": 1.799404501914978 + }, + { + "bits_per_byte": 2.146398173250826, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.4877698421478271, + "step": 1060, + "unclipped_grad_norm": 1.7711206674575806 + }, + { + "bits_per_byte": 2.1162038671694465, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4668407440185547, + "step": 1070, + "unclipped_grad_norm": 1.6511199474334717 + }, + { + "bits_per_byte": 2.1854497578342915, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.5148383378982544, + "step": 1080, + "unclipped_grad_norm": 1.5779398679733276 + }, + { + "bits_per_byte": 2.1157890450155654, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.4665532112121582, + "step": 1090, + "unclipped_grad_norm": 1.4929699897766113 + }, + { + "bits_per_byte": 2.2712235252123545, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.5742921829223633, + "step": 1100, + "unclipped_grad_norm": 1.5628890991210938 + }, + { + "bits_per_byte": 2.2346772119040077, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.5489602088928223, + "step": 1110, + "unclipped_grad_norm": 1.6343225240707397 + }, + { + "bits_per_byte": 2.1397589549969185, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.4831678867340088, + "step": 1120, + "unclipped_grad_norm": 1.5828447341918945 + }, + { + "bits_per_byte": 2.1089957303064844, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.4618444442749023, + "step": 1130, + "unclipped_grad_norm": 1.3475432395935059 + }, + { + "bits_per_byte": 2.1658198300474614, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.5012319087982178, + "step": 1140, + "unclipped_grad_norm": 1.6770285367965698 + }, + { + "bits_per_byte": 2.087386454190874, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4468660354614258, + "step": 1150, + "unclipped_grad_norm": 1.7341156005859375 + }, + { + "bits_per_byte": 2.0881105011510064, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4473679065704346, + "step": 1160, + "unclipped_grad_norm": 1.5863031148910522 + }, + { + "bits_per_byte": 2.0976665451635474, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.4539916515350342, + "step": 1170, + "unclipped_grad_norm": 1.4200273752212524 + }, + { + "bits_per_byte": 2.0885588599218057, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.4476786851882935, + "step": 1180, + "unclipped_grad_norm": 1.6728936433792114 + }, + { + "bits_per_byte": 2.104145991534804, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.4584828615188599, + "step": 1190, + "unclipped_grad_norm": 1.7168540954589844 + }, + { + "bits_per_byte": 2.118632262199629, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.4685239791870117, + "step": 1200, + "unclipped_grad_norm": 1.4414699077606201 + }, + { + "bits_per_byte": 2.1805246906615334, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.5114245414733887, + "step": 1210, + "unclipped_grad_norm": 1.5858746767044067 + }, + { + "bits_per_byte": 2.0302080701891194, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4072329998016357, + "step": 1220, + "unclipped_grad_norm": 1.6370232105255127 + }, + { + "bits_per_byte": 2.0442186368416584, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4169443845748901, + "step": 1230, + "unclipped_grad_norm": 1.6951018571853638 + }, + { + "bits_per_byte": 2.096816262937682, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4534022808074951, + "step": 1240, + "unclipped_grad_norm": 1.6222275495529175 + }, + { + "bits_per_byte": 2.1103076139672656, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.4627537727355957, + "step": 1250, + "unclipped_grad_norm": 1.562079906463623 + }, + { + "bits_per_byte": 2.0702380640733433, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.4349796772003174, + "step": 1260, + "unclipped_grad_norm": 1.6702338457107544 + }, + { + "bits_per_byte": 2.105623838453668, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4595072269439697, + "step": 1270, + "unclipped_grad_norm": 1.6034959554672241 + }, + { + "bits_per_byte": 2.095730364480134, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4526495933532715, + "step": 1280, + "unclipped_grad_norm": 1.4786161184310913 + }, + { + "bits_per_byte": 2.154164221833187, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.4931528568267822, + "step": 1290, + "unclipped_grad_norm": 1.3592665195465088 + }, + { + "bits_per_byte": 2.025295557749874, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4038279056549072, + "step": 1300, + "unclipped_grad_norm": 1.5583913326263428 + }, + { + "bits_per_byte": 2.0860291671102646, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.445925235748291, + "step": 1310, + "unclipped_grad_norm": 1.48281991481781 + }, + { + "bits_per_byte": 2.073239849261503, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4370603561401367, + "step": 1320, + "unclipped_grad_norm": 1.4058562517166138 + }, + { + "bits_per_byte": 2.0829166250949496, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.443767786026001, + "step": 1330, + "unclipped_grad_norm": 1.8254177570343018 + }, + { + "bits_per_byte": 2.0509285399650707, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4215953350067139, + "step": 1340, + "unclipped_grad_norm": 1.3947100639343262 + }, + { + "bits_per_byte": 2.0681381559063086, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4335241317749023, + "step": 1350, + "unclipped_grad_norm": 1.4867082834243774 + }, + { + "bits_per_byte": 1.9937013128904713, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.3819284439086914, + "step": 1360, + "unclipped_grad_norm": 1.5672006607055664 + }, + { + "bits_per_byte": 2.0650343850061876, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.4313727617263794, + "step": 1370, + "unclipped_grad_norm": 1.4830292463302612 + }, + { + "bits_per_byte": 2.165891030864919, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.5012812614440918, + "step": 1380, + "unclipped_grad_norm": 1.570335865020752 + }, + { + "bits_per_byte": 1.962535820779669, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.3603261709213257, + "step": 1390, + "unclipped_grad_norm": 1.3890300989151 + }, + { + "bits_per_byte": 2.0275364916906167, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.405381202697754, + "step": 1400, + "unclipped_grad_norm": 1.5505964756011963 + }, + { + "bits_per_byte": 1.9468675133554638, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3494657278060913, + "step": 1410, + "unclipped_grad_norm": 1.4491393566131592 + }, + { + "bits_per_byte": 1.9836485829822383, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.3749604225158691, + "step": 1420, + "unclipped_grad_norm": 1.4373064041137695 + }, + { + "bits_per_byte": 1.9810798501089515, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.3731799125671387, + "step": 1430, + "unclipped_grad_norm": 1.5067963600158691 + }, + { + "bits_per_byte": 2.007225340623719, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.3913025856018066, + "step": 1440, + "unclipped_grad_norm": 1.5309967994689941 + }, + { + "bits_per_byte": 2.0929743425000575, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.4507392644882202, + "step": 1450, + "unclipped_grad_norm": 1.6200082302093506 + }, + { + "bits_per_byte": 1.9832760685604696, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.3747022151947021, + "step": 1460, + "unclipped_grad_norm": 1.534468412399292 + }, + { + "bits_per_byte": 1.9976655129928589, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.384676218032837, + "step": 1470, + "unclipped_grad_norm": 1.418226957321167 + }, + { + "bits_per_byte": 2.049181884162908, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4203846454620361, + "step": 1480, + "unclipped_grad_norm": 1.4733176231384277 + }, + { + "bits_per_byte": 2.050229086524012, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.4211105108261108, + "step": 1490, + "unclipped_grad_norm": 1.6039254665374756 + }, + { + "bits_per_byte": 1.9527814807706831, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.353564977645874, + "step": 1500, + "unclipped_grad_norm": 1.521325945854187 + }, + { + "bits_per_byte": 2.000215327774551, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.3864436149597168, + "step": 1510, + "unclipped_grad_norm": 1.464277744293213 + }, + { + "bits_per_byte": 1.995248468817633, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.3830008506774902, + "step": 1520, + "unclipped_grad_norm": 1.6233909130096436 + }, + { + "bits_per_byte": 2.0854175968037967, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4455013275146484, + "step": 1530, + "unclipped_grad_norm": 1.6695823669433594 + }, + { + "bits_per_byte": 2.0229414592248642, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.4021961688995361, + "step": 1540, + "unclipped_grad_norm": 1.6660358905792236 + }, + { + "bits_per_byte": 1.9468549586219508, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.349457025527954, + "step": 1550, + "unclipped_grad_norm": 1.434714913368225 + }, + { + "bits_per_byte": 2.0166628885898157, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3978441953659058, + "step": 1560, + "unclipped_grad_norm": 1.6183284521102905 + }, + { + "bits_per_byte": 1.975387740313345, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.3692344427108765, + "step": 1570, + "unclipped_grad_norm": 1.4537051916122437 + }, + { + "bits_per_byte": 2.006008047420912, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.3904588222503662, + "step": 1580, + "unclipped_grad_norm": 1.467302680015564 + }, + { + "bits_per_byte": 1.9946215920552381, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.3825663328170776, + "step": 1590, + "unclipped_grad_norm": 1.4796717166900635 + }, + { + "bits_per_byte": 2.0482769114540678, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.41975736618042, + "step": 1600, + "unclipped_grad_norm": 1.4657248258590698 + }, + { + "bits_per_byte": 1.9915594409566257, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.380443811416626, + "step": 1610, + "unclipped_grad_norm": 1.6163296699523926 + }, + { + "bits_per_byte": 2.0137513942933594, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.3958261013031006, + "step": 1620, + "unclipped_grad_norm": 1.5134508609771729 + }, + { + "bits_per_byte": 1.9735719474855355, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.3679758310317993, + "step": 1630, + "unclipped_grad_norm": 1.591439127922058 + }, + { + "bits_per_byte": 2.0194574346836855, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3997812271118164, + "step": 1640, + "unclipped_grad_norm": 1.501707673072815 + }, + { + "bits_per_byte": 2.005634157137936, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3901996612548828, + "step": 1650, + "unclipped_grad_norm": 1.369825005531311 + }, + { + "bits_per_byte": 1.9363162057423795, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.3421521186828613, + "step": 1660, + "unclipped_grad_norm": 1.4967777729034424 + }, + { + "bits_per_byte": 1.981807336722101, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3736841678619385, + "step": 1670, + "unclipped_grad_norm": 1.4533015489578247 + }, + { + "bits_per_byte": 1.920158091728517, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3309521675109863, + "step": 1680, + "unclipped_grad_norm": 1.4766992330551147 + }, + { + "bits_per_byte": 2.0589319246056226, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.427142858505249, + "step": 1690, + "unclipped_grad_norm": 1.637188196182251 + }, + { + "bits_per_byte": 1.9574146933849272, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.356776475906372, + "step": 1700, + "unclipped_grad_norm": 1.5077614784240723 + }, + { + "bits_per_byte": 1.957954030978033, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3571503162384033, + "step": 1710, + "unclipped_grad_norm": 1.4464768171310425 + }, + { + "bits_per_byte": 1.946319404647164, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.349085807800293, + "step": 1720, + "unclipped_grad_norm": 1.4415843486785889 + }, + { + "bits_per_byte": 1.9696433477918764, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3652527332305908, + "step": 1730, + "unclipped_grad_norm": 1.4256844520568848 + }, + { + "bits_per_byte": 2.0201328105536236, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.4002493619918823, + "step": 1740, + "unclipped_grad_norm": 1.5626119375228882 + }, + { + "bits_per_byte": 1.9680060729556623, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3641178607940674, + "step": 1750, + "unclipped_grad_norm": 1.3952401876449585 + }, + { + "bits_per_byte": 2.0278474363233765, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.4055967330932617, + "step": 1760, + "unclipped_grad_norm": 1.530795693397522 + }, + { + "bits_per_byte": 1.9266499208506354, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.3354519605636597, + "step": 1770, + "unclipped_grad_norm": 1.4610179662704468 + }, + { + "bits_per_byte": 1.9219721647298178, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.332209587097168, + "step": 1780, + "unclipped_grad_norm": 1.5180143117904663 + }, + { + "bits_per_byte": 1.9554356890214488, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3554047346115112, + "step": 1790, + "unclipped_grad_norm": 1.6236528158187866 + }, + { + "bits_per_byte": 2.002663672792234, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3881406784057617, + "step": 1800, + "unclipped_grad_norm": 1.7409987449645996 + }, + { + "bits_per_byte": 1.9587403356577777, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.3576953411102295, + "step": 1810, + "unclipped_grad_norm": 1.4391273260116577 + }, + { + "bits_per_byte": 1.9084956042082082, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.3228683471679688, + "step": 1820, + "unclipped_grad_norm": 1.4066379070281982 + }, + { + "bits_per_byte": 1.8967103210751834, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.314699411392212, + "step": 1830, + "unclipped_grad_norm": 1.604218602180481 + }, + { + "bits_per_byte": 1.9522235690512841, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3531782627105713, + "step": 1840, + "unclipped_grad_norm": 1.4110183715820312 + }, + { + "bits_per_byte": 1.9108531423862356, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.3245024681091309, + "step": 1850, + "unclipped_grad_norm": 1.4574594497680664 + }, + { + "bits_per_byte": 2.001000944523692, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3869881629943848, + "step": 1860, + "unclipped_grad_norm": 1.4711050987243652 + }, + { + "bits_per_byte": 1.9527935195562436, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3535733222961426, + "step": 1870, + "unclipped_grad_norm": 1.518902063369751 + }, + { + "bits_per_byte": 1.9961823346118182, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.383648157119751, + "step": 1880, + "unclipped_grad_norm": 1.5269694328308105 + }, + { + "bits_per_byte": 2.035499976356168, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.4109010696411133, + "step": 1890, + "unclipped_grad_norm": 1.4457623958587646 + }, + { + "bits_per_byte": 1.9359392197716883, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.341890811920166, + "step": 1900, + "unclipped_grad_norm": 1.5047556161880493 + }, + { + "bits_per_byte": 1.9921639599744085, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.380862832069397, + "step": 1910, + "unclipped_grad_norm": 1.5278395414352417 + }, + { + "bits_per_byte": 1.9355305889932382, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3416075706481934, + "step": 1920, + "unclipped_grad_norm": 1.4905645847320557 + }, + { + "bits_per_byte": 1.8760744667634566, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3003957271575928, + "step": 1930, + "unclipped_grad_norm": 1.5404988527297974 + }, + { + "bits_per_byte": 1.959410724030841, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3581600189208984, + "step": 1940, + "unclipped_grad_norm": 1.6135326623916626 + }, + { + "bits_per_byte": 1.9634289266855995, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.360945224761963, + "step": 1950, + "unclipped_grad_norm": 1.5894566774368286 + }, + { + "bits_per_byte": 1.9293448889896556, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3373199701309204, + "step": 1960, + "unclipped_grad_norm": 1.5030673742294312 + }, + { + "bits_per_byte": 2.008814804282993, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.392404317855835, + "step": 1970, + "unclipped_grad_norm": 1.5342503786087036 + }, + { + "bits_per_byte": 1.9313775519402032, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.338728904724121, + "step": 1980, + "unclipped_grad_norm": 1.5579596757888794 + }, + { + "bits_per_byte": 1.967195518722146, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3635560274124146, + "step": 1990, + "unclipped_grad_norm": 1.5720075368881226 + }, + { + "bits_per_byte": 2.0095716999294404, + "learning_rate": 3e-05, + "loss_nats": 1.392928957939148, + "step": 2000, + "unclipped_grad_norm": 1.6950316429138184 + } + ] + }, + { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "e3332e094a30779902da401b9d220d117ad778323dfb62b9c3ed09a128392ae6", + "diagnostic": { + "bits_per_byte": 2.105546274278129, + "branch_output_rms": [ + 0.0370347797870636, + 0.04449377954006195, + 0.04890887811779976, + 0.040389351546764374, + 0.05223172530531883, + 0.019183801487088203, + 0.034338437020778656, + 0.01778092049062252, + 0.06659915298223495, + 0.015269296243786812, + 0.045002374798059464, + 0.015299440361559391, + 0.06974328309297562, + 0.022551367059350014, + 0.03994842991232872, + 0.024390853941440582, + 0.020560698583722115, + 0.017591258510947227, + 0.02445649914443493, + 0.026076123118400574, + 0.0240048635751009, + 0.023831162601709366, + 0.01577388308942318, + 0.027974048629403114, + 0.01715846173465252, + 0.026792073622345924, + 0.011242104694247246, + 0.03591271862387657, + 0.011988751590251923, + 0.038416508585214615, + 0.01049043145030737, + 0.05786125734448433 + ], + "core_parameter_grad_rms_by_block": [ + 0.0004530616767090032, + 0.00042772394832709857, + 0.00028404817531530786, + 0.00024049221342133134, + 0.000277340647430391, + 0.00024842090196879307, + 0.00032361633132051105, + 0.00032402399493246936, + 0.0006871599037030366, + 0.0009332962149471692, + 0.0016503723502619711, + 0.0012425342652473243, + 0.0010904310394870248, + 0.0007409721891420046, + 0.00045754647605205013, + 0.0003918054076853068 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6499756574630737, + "mean_weights": [ + 0.6032090187072754, + 0.396791011095047 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6664855480194092, + "mean_weights": [ + 0.5453359484672546, + 0.45466405153274536 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5528920888900757, + "mean_weights": [ + 0.752701461315155, + 0.24729855358600616 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6791212558746338, + "mean_weights": [ + 0.4506688714027405, + 0.5493311285972595 + ], + "sources": 2 + }, + { + "entropy_mean": 1.025155782699585, + "mean_weights": [ + 0.475564569234848, + 0.26032447814941406, + 0.2641109824180603 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0936508178710938, + "mean_weights": [ + 0.3082738220691681, + 0.3514207601547241, + 0.3403053879737854 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0354526042938232, + "mean_weights": [ + 0.4407365322113037, + 0.24781671166419983, + 0.31144678592681885 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0863194465637207, + "mean_weights": [ + 0.30351266264915466, + 0.35466521978378296, + 0.3418221175670624 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3304085731506348, + "mean_weights": [ + 0.3333660960197449, + 0.19735223054885864, + 0.23942604660987854, + 0.22985565662384033 + ], + "sources": 4 + }, + { + "entropy_mean": 1.37892484664917, + "mean_weights": [ + 0.23244304955005646, + 0.24598246812820435, + 0.23363947868347168, + 0.2879350185394287 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3487892150878906, + "mean_weights": [ + 0.2930099368095398, + 0.1652420610189438, + 0.23996411263942719, + 0.30178385972976685 + ], + "sources": 4 + }, + { + "entropy_mean": 1.371549367904663, + "mean_weights": [ + 0.23558703064918518, + 0.2554919123649597, + 0.21197238564491272, + 0.2969486713409424 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5968154668807983, + "mean_weights": [ + 0.19339415431022644, + 0.16176673769950867, + 0.18629853427410126, + 0.22672678530216217, + 0.23181380331516266 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5933740139007568, + "mean_weights": [ + 0.18254993855953217, + 0.18572580814361572, + 0.16277235746383667, + 0.2061573565006256, + 0.26279452443122864 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5858888626098633, + "mean_weights": [ + 0.20496419072151184, + 0.18132969737052917, + 0.1575242578983307, + 0.18580476939678192, + 0.2703770697116852 + ], + "sources": 5 + }, + { + "entropy_mean": 1.604175090789795, + "mean_weights": [ + 0.19806772470474243, + 0.20342552661895752, + 0.1811276376247406, + 0.19512757658958435, + 0.2222515344619751 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7792454957962036, + "mean_weights": [ + 0.1583983153104782, + 0.14851580560207367, + 0.1587386280298233, + 0.1588030308485031, + 0.15571442246437073, + 0.21982979774475098 + ], + "sources": 6 + }, + { + "entropy_mean": 1.731494426727295, + "mean_weights": [ + 0.15878748893737793, + 0.15536850690841675, + 0.12781691551208496, + 0.12908712029457092, + 0.13050273060798645, + 0.298437237739563 + ], + "sources": 6 + }, + { + "entropy_mean": 1.360633373260498, + "mean_weights": [ + 0.13053587079048157, + 0.124850794672966, + 0.08268078416585922, + 0.05873226001858711, + 0.05429460480809212, + 0.5489056706428528 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7661364078521729, + "mean_weights": [ + 0.16622644662857056, + 0.1671392172574997, + 0.14266835153102875, + 0.1436060667037964, + 0.1386961042881012, + 0.24166381359100342 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4908201694488525, + "mean_weights": [ + 0.08490929007530212, + 0.0734928697347641, + 0.05038502812385559, + 0.03916036710143089, + 0.03833489865064621, + 0.265095055103302, + 0.448622465133667 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8509920835494995, + "mean_weights": [ + 0.12194234132766724, + 0.11873871088027954, + 0.1029520034790039, + 0.10706067085266113, + 0.10167687386274338, + 0.1477353870868683, + 0.2998940348625183 + ], + "sources": 7 + }, + { + "entropy_mean": 1.4770772457122803, + "mean_weights": [ + 0.09011663496494293, + 0.07491730898618698, + 0.05552466958761215, + 0.04490774869918823, + 0.04155133292078972, + 0.1841764897108078, + 0.5088058114051819 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8702099323272705, + "mean_weights": [ + 0.13297535479068756, + 0.1282171905040741, + 0.1036367192864418, + 0.10012050718069077, + 0.100038543343544, + 0.16192477941513062, + 0.27308687567710876 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7099403142929077, + "mean_weights": [ + 0.07141797989606857, + 0.06031838804483414, + 0.045201994478702545, + 0.037236541509628296, + 0.034026384353637695, + 0.1353147178888321, + 0.32203125953674316, + 0.2944527268409729 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9050805568695068, + "mean_weights": [ + 0.09686028957366943, + 0.10802871733903885, + 0.07663879543542862, + 0.05910831317305565, + 0.05269083380699158, + 0.1281621903181076, + 0.1914730817079544, + 0.28703778982162476 + ], + "sources": 8 + }, + { + "entropy_mean": 1.724353313446045, + "mean_weights": [ + 0.07505077868700027, + 0.0685117095708847, + 0.04867492616176605, + 0.041080038994550705, + 0.038436442613601685, + 0.11583653092384338, + 0.24690571427345276, + 0.36550387740135193 + ], + "sources": 8 + }, + { + "entropy_mean": 1.886972427368164, + "mean_weights": [ + 0.10057500004768372, + 0.09312880784273148, + 0.06796184182167053, + 0.052606403827667236, + 0.04661618918180466, + 0.1467602550983429, + 0.2306298017501831, + 0.2617217004299164 + ], + "sources": 8 + }, + { + "entropy_mean": 1.943651795387268, + "mean_weights": [ + 0.06928816437721252, + 0.06278382241725922, + 0.044835302978754044, + 0.037070780992507935, + 0.03571176528930664, + 0.11271253973245621, + 0.20676016807556152, + 0.20467519760131836, + 0.22616222500801086 + ], + "sources": 9 + }, + { + "entropy_mean": 2.052138090133667, + "mean_weights": [ + 0.08626779168844223, + 0.09223690629005432, + 0.061754584312438965, + 0.048465337604284286, + 0.04434200003743172, + 0.11889863014221191, + 0.18807727098464966, + 0.19575801491737366, + 0.16419945657253265 + ], + "sources": 9 + }, + { + "entropy_mean": 1.993139386177063, + "mean_weights": [ + 0.07762785255908966, + 0.07757362723350525, + 0.05421649292111397, + 0.04235970973968506, + 0.03923260420560837, + 0.10997684299945831, + 0.17934441566467285, + 0.22717401385307312, + 0.1924944519996643 + ], + "sources": 9 + } + ], + "layer_input_rms": [ + 0.053792886435985565, + 0.03717699274420738, + 0.043796516954898834, + 0.050083331763744354, + 0.0711384266614914, + 0.05069540813565254, + 0.06039127707481384, + 0.05591938644647598, + 0.06784754246473312, + 0.0544980950653553, + 0.060793664306402206, + 0.064053013920784, + 0.0706096664071083, + 0.063942551612854, + 0.0667361468076706, + 0.07206644117832184, + 0.07398614287376404, + 0.05593166500329971, + 0.047586891800165176, + 0.02869899943470955, + 0.051853884011507034, + 0.01788981445133686, + 0.03394656628370285, + 0.02158820629119873, + 0.034704048186540604, + 0.01816716231405735, + 0.02297247014939785, + 0.01960809901356697, + 0.024138133972883224, + 0.01791227236390114, + 0.023471562191843987, + 0.023178130388259888 + ], + "loss_nats": 1.4594534635543823, + "output_weights": { + "entropy_mean": 1.914886713027954, + "mean_weights": [ + 0.0926462858915329, + 0.06027670204639435, + 0.05347089469432831, + 0.044498883187770844, + 0.04402533546090126, + 0.08183147758245468, + 0.09443851560354233, + 0.25589877367019653, + 0.2729131579399109 + ], + "sources": 9 + }, + "stream_state_rms": [ + 0.0370347797870636, + 0.06143631786108017, + 0.09526251256465912, + 0.11753125488758087, + 0.05223172530531883, + 0.057402584701776505, + 0.07745972275733948, + 0.08341154456138611, + 0.06659915298223495, + 0.0708504468202591, + 0.106504887342453, + 0.11234799772500992, + 0.06974328309297562, + 0.08339263498783112, + 0.11490626633167267, + 0.12939290702342987, + 0.020560698583722115, + 0.029348116368055344, + 0.043354328721761703, + 0.053864628076553345, + 0.0240048635751009, + 0.03488682210445404, + 0.04023054614663124, + 0.05085337162017822, + 0.01715846173465252, + 0.03339201211929321, + 0.03655501827597618, + 0.056224361062049866, + 0.011988751590251923, + 0.040694523602724075, + 0.04344379901885986, + 0.0900634303689003 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.091490664658076, + "cross_entropy_nats": 5.608593940734863, + "step": 0 + }, + { + "bits_per_byte": 3.8613770068990427, + "cross_entropy_nats": 2.6765025854110718, + "step": 100 + }, + { + "bits_per_byte": 3.4246572049925037, + "cross_entropy_nats": 2.3737914860248566, + "step": 250 + }, + { + "bits_per_byte": 2.890138823737888, + "cross_entropy_nats": 2.003291577100754, + "step": 500 + }, + { + "bits_per_byte": 2.193909756412884, + "cross_entropy_nats": 1.5207023620605469, + "step": 1000 + }, + { + "bits_per_byte": 2.01564146212851, + "cross_entropy_nats": 1.3971361964941025, + "step": 1500 + }, + { + "bits_per_byte": 1.947877975922777, + "cross_entropy_nats": 1.3501661270856857, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "23cf952be02ee339b463408da310689ddba6524f88ee1596924a99bb417b55fd", + "final_mixer_parameters": "b1e8728f82b7174089121674d1809428e651cc593465515b644b85263956a140", + "initial_common_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 54.54293606894542, + "measured_steps": 1980, + "median_ms": 54.46750250121113, + "p95_ms": 55.017453705659136, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.07876807607785, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.599775314331055, + "step": 1, + "unclipped_grad_norm": 18.59675407409668 + }, + { + "bits_per_byte": 6.864387571445902, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.758030891418457, + "step": 10, + "unclipped_grad_norm": 5.112982273101807 + }, + { + "bits_per_byte": 6.45473040053552, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.474078178405762, + "step": 20, + "unclipped_grad_norm": 2.898599624633789 + }, + { + "bits_per_byte": 6.205213843080136, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.301126480102539, + "step": 30, + "unclipped_grad_norm": 2.6787800788879395 + }, + { + "bits_per_byte": 5.684354753913672, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.940094470977783, + "step": 40, + "unclipped_grad_norm": 2.367745876312256 + }, + { + "bits_per_byte": 5.257686779006449, + "learning_rate": 0.00015, + "loss_nats": 3.64435076713562, + "step": 50, + "unclipped_grad_norm": 2.168215036392212 + }, + { + "bits_per_byte": 4.781810784069057, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.3144986629486084, + "step": 60, + "unclipped_grad_norm": 1.7586382627487183 + }, + { + "bits_per_byte": 4.423358695730662, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.0660386085510254, + "step": 70, + "unclipped_grad_norm": 1.3044919967651367 + }, + { + "bits_per_byte": 4.186769730175334, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.902047634124756, + "step": 80, + "unclipped_grad_norm": 0.964425802230835 + }, + { + "bits_per_byte": 3.9575405901810297, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.7431581020355225, + "step": 90, + "unclipped_grad_norm": 0.7076364755630493 + }, + { + "bits_per_byte": 3.8208394922121225, + "learning_rate": 0.0003, + "loss_nats": 2.648404121398926, + "step": 100, + "unclipped_grad_norm": 0.5602594017982483 + }, + { + "bits_per_byte": 3.7782235671895257, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.6188650131225586, + "step": 110, + "unclipped_grad_norm": 0.4652419686317444 + }, + { + "bits_per_byte": 3.711833448442257, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.5728468894958496, + "step": 120, + "unclipped_grad_norm": 0.6155686974525452 + }, + { + "bits_per_byte": 3.67204801388756, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.545269727706909, + "step": 130, + "unclipped_grad_norm": 0.5482766628265381 + }, + { + "bits_per_byte": 3.6151936452000313, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.505861282348633, + "step": 140, + "unclipped_grad_norm": 0.3733767569065094 + }, + { + "bits_per_byte": 3.6241505016569664, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.5120697021484375, + "step": 150, + "unclipped_grad_norm": 0.7431890964508057 + }, + { + "bits_per_byte": 3.5806388909886757, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.48190975189209, + "step": 160, + "unclipped_grad_norm": 0.49960198998451233 + }, + { + "bits_per_byte": 3.5469612482967205, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.458566188812256, + "step": 170, + "unclipped_grad_norm": 0.6090258359909058 + }, + { + "bits_per_byte": 3.52141322551105, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.4408576488494873, + "step": 180, + "unclipped_grad_norm": 0.5286015868186951 + }, + { + "bits_per_byte": 3.506946732851074, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.430830240249634, + "step": 190, + "unclipped_grad_norm": 0.5678954720497131 + }, + { + "bits_per_byte": 3.4337657071518373, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.3801050186157227, + "step": 200, + "unclipped_grad_norm": 0.6558411717414856 + }, + { + "bits_per_byte": 3.5012778407133345, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.426900863647461, + "step": 210, + "unclipped_grad_norm": 0.566231369972229 + }, + { + "bits_per_byte": 3.4616551017463535, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.3994364738464355, + "step": 220, + "unclipped_grad_norm": 0.8727293014526367 + }, + { + "bits_per_byte": 3.4555648521139997, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.3952150344848633, + "step": 230, + "unclipped_grad_norm": 0.7154321670532227 + }, + { + "bits_per_byte": 3.467175400873748, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.4032628536224365, + "step": 240, + "unclipped_grad_norm": 0.7503302693367004 + }, + { + "bits_per_byte": 3.427351442205258, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.3756589889526367, + "step": 250, + "unclipped_grad_norm": 0.5864220857620239 + }, + { + "bits_per_byte": 3.387560848171035, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3480782508850098, + "step": 260, + "unclipped_grad_norm": 0.8161523342132568 + }, + { + "bits_per_byte": 3.396980681924093, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.354607582092285, + "step": 270, + "unclipped_grad_norm": 0.7761324644088745 + }, + { + "bits_per_byte": 3.371118618817947, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.336681365966797, + "step": 280, + "unclipped_grad_norm": 1.0774776935577393 + }, + { + "bits_per_byte": 3.3329522289384723, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3102264404296875, + "step": 290, + "unclipped_grad_norm": 0.5604715347290039 + }, + { + "bits_per_byte": 3.315032324649171, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.2978053092956543, + "step": 300, + "unclipped_grad_norm": 0.7430016398429871 + }, + { + "bits_per_byte": 3.3108338841763083, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.2948951721191406, + "step": 310, + "unclipped_grad_norm": 1.3589977025985718 + }, + { + "bits_per_byte": 3.3521888324027795, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.3235602378845215, + "step": 320, + "unclipped_grad_norm": 1.1168256998062134 + }, + { + "bits_per_byte": 3.287714600386123, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.278870105743408, + "step": 330, + "unclipped_grad_norm": 1.4894192218780518 + }, + { + "bits_per_byte": 3.3015564520581653, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.2884645462036133, + "step": 340, + "unclipped_grad_norm": 0.7649078369140625 + }, + { + "bits_per_byte": 3.2613070083114404, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.260565757751465, + "step": 350, + "unclipped_grad_norm": 0.8027856945991516 + }, + { + "bits_per_byte": 3.2849408641930076, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.276947498321533, + "step": 360, + "unclipped_grad_norm": 0.9242268800735474 + }, + { + "bits_per_byte": 3.237909112626857, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.24434757232666, + "step": 370, + "unclipped_grad_norm": 1.0474706888198853 + }, + { + "bits_per_byte": 3.212451896680842, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.2267019748687744, + "step": 380, + "unclipped_grad_norm": 1.1611087322235107 + }, + { + "bits_per_byte": 3.161528521690968, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.1914045810699463, + "step": 390, + "unclipped_grad_norm": 0.7992200255393982 + }, + { + "bits_per_byte": 3.143635790660503, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.179002285003662, + "step": 400, + "unclipped_grad_norm": 1.5802223682403564 + }, + { + "bits_per_byte": 3.076085476950514, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.1321799755096436, + "step": 410, + "unclipped_grad_norm": 1.733614206314087 + }, + { + "bits_per_byte": 3.0692457269257005, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.127439022064209, + "step": 420, + "unclipped_grad_norm": 1.5657434463500977 + }, + { + "bits_per_byte": 3.0424577092273095, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.1088709831237793, + "step": 430, + "unclipped_grad_norm": 1.245201826095581 + }, + { + "bits_per_byte": 3.0265118218046116, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.09781813621521, + "step": 440, + "unclipped_grad_norm": 1.1346021890640259 + }, + { + "bits_per_byte": 3.0076301905316853, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.084730386734009, + "step": 450, + "unclipped_grad_norm": 1.6088787317276 + }, + { + "bits_per_byte": 2.9472205964855203, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.0428576469421387, + "step": 460, + "unclipped_grad_norm": 1.1036934852600098 + }, + { + "bits_per_byte": 2.9401266561027315, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.037940502166748, + "step": 470, + "unclipped_grad_norm": 1.790906310081482 + }, + { + "bits_per_byte": 2.9910827077962696, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.073260545730591, + "step": 480, + "unclipped_grad_norm": 1.110121250152588 + }, + { + "bits_per_byte": 2.8963157956219394, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.007573127746582, + "step": 490, + "unclipped_grad_norm": 1.201914668083191 + }, + { + "bits_per_byte": 2.833933216605076, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.9643328189849854, + "step": 500, + "unclipped_grad_norm": 1.5553689002990723 + }, + { + "bits_per_byte": 2.8020500408922238, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.9422330856323242, + "step": 510, + "unclipped_grad_norm": 1.6392277479171753 + }, + { + "bits_per_byte": 2.810952550831486, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.9484038352966309, + "step": 520, + "unclipped_grad_norm": 1.4169108867645264 + }, + { + "bits_per_byte": 2.7904621938423895, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.9342010021209717, + "step": 530, + "unclipped_grad_norm": 1.2503232955932617 + }, + { + "bits_per_byte": 2.7599605547639174, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.913058876991272, + "step": 540, + "unclipped_grad_norm": 1.2111908197402954 + }, + { + "bits_per_byte": 2.759557599412946, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.9127795696258545, + "step": 550, + "unclipped_grad_norm": 1.2925400733947754 + }, + { + "bits_per_byte": 2.7071771874049624, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.8764722347259521, + "step": 560, + "unclipped_grad_norm": 1.5252132415771484 + }, + { + "bits_per_byte": 2.6985670479721664, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.8705041408538818, + "step": 570, + "unclipped_grad_norm": 1.7691043615341187 + }, + { + "bits_per_byte": 2.7415795649876533, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.9003181457519531, + "step": 580, + "unclipped_grad_norm": 1.529776692390442 + }, + { + "bits_per_byte": 2.647439701558369, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8350653648376465, + "step": 590, + "unclipped_grad_norm": 1.431287169456482 + }, + { + "bits_per_byte": 2.6711495737716175, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8514997959136963, + "step": 600, + "unclipped_grad_norm": 1.4023431539535522 + }, + { + "bits_per_byte": 2.6565551260194, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.841383695602417, + "step": 610, + "unclipped_grad_norm": 1.9441742897033691 + }, + { + "bits_per_byte": 2.616552305394012, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.8136558532714844, + "step": 620, + "unclipped_grad_norm": 1.9690080881118774 + }, + { + "bits_per_byte": 2.595699065011595, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.799201488494873, + "step": 630, + "unclipped_grad_norm": 1.519966721534729 + }, + { + "bits_per_byte": 2.5282686232092573, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.7524622678756714, + "step": 640, + "unclipped_grad_norm": 1.6391072273254395 + }, + { + "bits_per_byte": 2.566986389467398, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.7792993783950806, + "step": 650, + "unclipped_grad_norm": 1.6319849491119385 + }, + { + "bits_per_byte": 2.5236648196283107, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.7492711544036865, + "step": 660, + "unclipped_grad_norm": 1.6076115369796753 + }, + { + "bits_per_byte": 2.5477496140204403, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.765965461730957, + "step": 670, + "unclipped_grad_norm": 1.6309348344802856 + }, + { + "bits_per_byte": 2.4695473828466126, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7117598056793213, + "step": 680, + "unclipped_grad_norm": 1.4799654483795166 + }, + { + "bits_per_byte": 2.5410412587408855, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.7613155841827393, + "step": 690, + "unclipped_grad_norm": 1.5887646675109863 + }, + { + "bits_per_byte": 2.4261492807278913, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.6816785335540771, + "step": 700, + "unclipped_grad_norm": 1.4776864051818848 + }, + { + "bits_per_byte": 2.4751299396936193, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.7156293392181396, + "step": 710, + "unclipped_grad_norm": 1.5311601161956787 + }, + { + "bits_per_byte": 2.453164831473374, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7004042863845825, + "step": 720, + "unclipped_grad_norm": 1.6596695184707642 + }, + { + "bits_per_byte": 2.3446387912820668, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.6251797676086426, + "step": 730, + "unclipped_grad_norm": 1.5184777975082397 + }, + { + "bits_per_byte": 2.330378677803126, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.61529541015625, + "step": 740, + "unclipped_grad_norm": 1.7135038375854492 + }, + { + "bits_per_byte": 2.458158003775874, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7038652896881104, + "step": 750, + "unclipped_grad_norm": 1.5154943466186523 + }, + { + "bits_per_byte": 2.310748578033645, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.6016888618469238, + "step": 760, + "unclipped_grad_norm": 1.4598900079727173 + }, + { + "bits_per_byte": 2.3697214290145023, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.6425657272338867, + "step": 770, + "unclipped_grad_norm": 1.5468471050262451 + }, + { + "bits_per_byte": 2.471202887843818, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.712907314300537, + "step": 780, + "unclipped_grad_norm": 1.7015125751495361 + }, + { + "bits_per_byte": 2.306392601452592, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.5986695289611816, + "step": 790, + "unclipped_grad_norm": 2.0924503803253174 + }, + { + "bits_per_byte": 2.3541026525937454, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.631739616394043, + "step": 800, + "unclipped_grad_norm": 1.6427921056747437 + }, + { + "bits_per_byte": 2.3105473583321356, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6015493869781494, + "step": 810, + "unclipped_grad_norm": 1.61247980594635 + }, + { + "bits_per_byte": 2.347201848727875, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.6269563436508179, + "step": 820, + "unclipped_grad_norm": 1.7861231565475464 + }, + { + "bits_per_byte": 2.2918626471944474, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.5885981321334839, + "step": 830, + "unclipped_grad_norm": 1.7954814434051514 + }, + { + "bits_per_byte": 2.2401615666573718, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.5527616739273071, + "step": 840, + "unclipped_grad_norm": 1.5943883657455444 + }, + { + "bits_per_byte": 2.3249048139914645, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.6115012168884277, + "step": 850, + "unclipped_grad_norm": 1.5687501430511475 + }, + { + "bits_per_byte": 2.2970565232505042, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.5921982526779175, + "step": 860, + "unclipped_grad_norm": 1.4173455238342285 + }, + { + "bits_per_byte": 2.3317949549329815, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6162770986557007, + "step": 870, + "unclipped_grad_norm": 1.7126595973968506 + }, + { + "bits_per_byte": 2.3151632006986427, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.6047488451004028, + "step": 880, + "unclipped_grad_norm": 1.8572194576263428 + }, + { + "bits_per_byte": 2.2823798677911737, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.582025170326233, + "step": 890, + "unclipped_grad_norm": 1.6456862688064575 + }, + { + "bits_per_byte": 2.1999996620799362, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.52492356300354, + "step": 900, + "unclipped_grad_norm": 1.5481798648834229 + }, + { + "bits_per_byte": 2.2945495321488774, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.5904605388641357, + "step": 910, + "unclipped_grad_norm": 1.651327133178711 + }, + { + "bits_per_byte": 2.211077752552613, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.532602310180664, + "step": 920, + "unclipped_grad_norm": 2.3538103103637695 + }, + { + "bits_per_byte": 2.16709318959445, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5021145343780518, + "step": 930, + "unclipped_grad_norm": 1.6614834070205688 + }, + { + "bits_per_byte": 2.240750607236577, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5531699657440186, + "step": 940, + "unclipped_grad_norm": 1.58348548412323 + }, + { + "bits_per_byte": 2.202367863182317, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5265650749206543, + "step": 950, + "unclipped_grad_norm": 1.4685944318771362 + }, + { + "bits_per_byte": 2.238989504891742, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5519492626190186, + "step": 960, + "unclipped_grad_norm": 1.65218186378479 + }, + { + "bits_per_byte": 2.1173002565686967, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.467600703239441, + "step": 970, + "unclipped_grad_norm": 1.461379051208496 + }, + { + "bits_per_byte": 2.2293381824906224, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5452594757080078, + "step": 980, + "unclipped_grad_norm": 1.7656797170639038 + }, + { + "bits_per_byte": 2.249700928352779, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.5593738555908203, + "step": 990, + "unclipped_grad_norm": 1.5937530994415283 + }, + { + "bits_per_byte": 2.0991646860352136, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.455030083656311, + "step": 1000, + "unclipped_grad_norm": 1.4669684171676636 + }, + { + "bits_per_byte": 2.1370534959161955, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.4812926054000854, + "step": 1010, + "unclipped_grad_norm": 1.5588597059249878 + }, + { + "bits_per_byte": 2.1782768774147563, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.50986647605896, + "step": 1020, + "unclipped_grad_norm": 1.4847451448440552 + }, + { + "bits_per_byte": 2.0752300324972874, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.4384398460388184, + "step": 1030, + "unclipped_grad_norm": 1.6708414554595947 + }, + { + "bits_per_byte": 2.1716367992475942, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5052639245986938, + "step": 1040, + "unclipped_grad_norm": 1.6902719736099243 + }, + { + "bits_per_byte": 2.1229655370707685, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.4715275764465332, + "step": 1050, + "unclipped_grad_norm": 1.479420781135559 + }, + { + "bits_per_byte": 2.1079953072264153, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.4611510038375854, + "step": 1060, + "unclipped_grad_norm": 1.7523038387298584 + }, + { + "bits_per_byte": 2.096720812566453, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4533361196517944, + "step": 1070, + "unclipped_grad_norm": 1.7101633548736572 + }, + { + "bits_per_byte": 2.1524237574064555, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.4919464588165283, + "step": 1080, + "unclipped_grad_norm": 1.7029392719268799 + }, + { + "bits_per_byte": 2.0833231920815893, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.444049596786499, + "step": 1090, + "unclipped_grad_norm": 1.40860116481781 + }, + { + "bits_per_byte": 2.2266007346368393, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.543362021446228, + "step": 1100, + "unclipped_grad_norm": 1.557461142539978 + }, + { + "bits_per_byte": 2.1918181034130906, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.5192525386810303, + "step": 1110, + "unclipped_grad_norm": 1.6624177694320679 + }, + { + "bits_per_byte": 2.0951641975934936, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.4522571563720703, + "step": 1120, + "unclipped_grad_norm": 1.5764100551605225 + }, + { + "bits_per_byte": 2.0688929877609454, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.4340473413467407, + "step": 1130, + "unclipped_grad_norm": 1.5417845249176025 + }, + { + "bits_per_byte": 2.1320438132792128, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.4778201580047607, + "step": 1140, + "unclipped_grad_norm": 1.5727717876434326 + }, + { + "bits_per_byte": 2.054807952620583, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4242843389511108, + "step": 1150, + "unclipped_grad_norm": 1.8212707042694092 + }, + { + "bits_per_byte": 2.062402190534732, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4295482635498047, + "step": 1160, + "unclipped_grad_norm": 1.5676357746124268 + }, + { + "bits_per_byte": 2.0673067917720376, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.4329478740692139, + "step": 1170, + "unclipped_grad_norm": 1.5299118757247925 + }, + { + "bits_per_byte": 2.058333768945922, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.4267282485961914, + "step": 1180, + "unclipped_grad_norm": 1.687406063079834 + }, + { + "bits_per_byte": 2.0643885901521974, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.4309251308441162, + "step": 1190, + "unclipped_grad_norm": 1.5954183340072632 + }, + { + "bits_per_byte": 2.0744939467515944, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.437929630279541, + "step": 1200, + "unclipped_grad_norm": 1.7698386907577515 + }, + { + "bits_per_byte": 2.1445201227034043, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.4864680767059326, + "step": 1210, + "unclipped_grad_norm": 1.7341222763061523 + }, + { + "bits_per_byte": 2.014112901825473, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.3960766792297363, + "step": 1220, + "unclipped_grad_norm": 1.7863450050354004 + }, + { + "bits_per_byte": 2.006989724392037, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.391139268875122, + "step": 1230, + "unclipped_grad_norm": 1.452402949333191 + }, + { + "bits_per_byte": 2.066209542459533, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4321873188018799, + "step": 1240, + "unclipped_grad_norm": 1.5674009323120117 + }, + { + "bits_per_byte": 2.088879607565665, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.4479010105133057, + "step": 1250, + "unclipped_grad_norm": 1.4704411029815674 + }, + { + "bits_per_byte": 2.0374315135080083, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.412239909172058, + "step": 1260, + "unclipped_grad_norm": 1.7344322204589844 + }, + { + "bits_per_byte": 2.067760138039712, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4332621097564697, + "step": 1270, + "unclipped_grad_norm": 1.8369526863098145 + }, + { + "bits_per_byte": 2.077964556646006, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4403352737426758, + "step": 1280, + "unclipped_grad_norm": 1.5832710266113281 + }, + { + "bits_per_byte": 2.127252376626175, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.474498987197876, + "step": 1290, + "unclipped_grad_norm": 1.5207411050796509 + }, + { + "bits_per_byte": 1.9924577063420823, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.3810664415359497, + "step": 1300, + "unclipped_grad_norm": 1.6240516901016235 + }, + { + "bits_per_byte": 2.0664596052338875, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4323606491088867, + "step": 1310, + "unclipped_grad_norm": 1.8139339685440063 + }, + { + "bits_per_byte": 2.0363533542697496, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4114925861358643, + "step": 1320, + "unclipped_grad_norm": 1.7409800291061401 + }, + { + "bits_per_byte": 2.0590003737006657, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.4271903038024902, + "step": 1330, + "unclipped_grad_norm": 1.675992488861084 + }, + { + "bits_per_byte": 2.019964439538429, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.400132656097412, + "step": 1340, + "unclipped_grad_norm": 1.5063552856445312 + }, + { + "bits_per_byte": 2.03203005439236, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4084959030151367, + "step": 1350, + "unclipped_grad_norm": 1.4544939994812012 + }, + { + "bits_per_byte": 1.9585071271832077, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.3575336933135986, + "step": 1360, + "unclipped_grad_norm": 1.5741589069366455 + }, + { + "bits_per_byte": 2.037214987350572, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.4120898246765137, + "step": 1370, + "unclipped_grad_norm": 1.6352499723434448 + }, + { + "bits_per_byte": 2.133034949296135, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.478507161140442, + "step": 1380, + "unclipped_grad_norm": 1.4854929447174072 + }, + { + "bits_per_byte": 1.945326376821082, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.3483974933624268, + "step": 1390, + "unclipped_grad_norm": 1.5653549432754517 + }, + { + "bits_per_byte": 1.991236973486258, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.3802202939987183, + "step": 1400, + "unclipped_grad_norm": 1.909532070159912 + }, + { + "bits_per_byte": 1.9282474676945, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3365592956542969, + "step": 1410, + "unclipped_grad_norm": 1.4449039697647095 + }, + { + "bits_per_byte": 1.9583843315704916, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.3574485778808594, + "step": 1420, + "unclipped_grad_norm": 1.6120092868804932 + }, + { + "bits_per_byte": 1.9395640981039237, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.3444033861160278, + "step": 1430, + "unclipped_grad_norm": 1.731689214706421 + }, + { + "bits_per_byte": 1.9762347548688444, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.369821548461914, + "step": 1440, + "unclipped_grad_norm": 1.571950912475586 + }, + { + "bits_per_byte": 2.068603884924845, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.4338469505310059, + "step": 1450, + "unclipped_grad_norm": 1.674243688583374 + }, + { + "bits_per_byte": 1.9409650687778521, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.3453744649887085, + "step": 1460, + "unclipped_grad_norm": 1.547331690788269 + }, + { + "bits_per_byte": 1.9662871063602887, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.3629263639450073, + "step": 1470, + "unclipped_grad_norm": 1.430796504020691 + }, + { + "bits_per_byte": 2.025417493449336, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4039124250411987, + "step": 1480, + "unclipped_grad_norm": 1.5492368936538696 + }, + { + "bits_per_byte": 2.0081788124401023, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.3919634819030762, + "step": 1490, + "unclipped_grad_norm": 1.5567013025283813 + }, + { + "bits_per_byte": 1.915413434356514, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.3276634216308594, + "step": 1500, + "unclipped_grad_norm": 1.4337656497955322 + }, + { + "bits_per_byte": 1.9696994141360575, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.3652915954589844, + "step": 1510, + "unclipped_grad_norm": 1.5614243745803833 + }, + { + "bits_per_byte": 1.9524416430525782, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.3533294200897217, + "step": 1520, + "unclipped_grad_norm": 1.6640793085098267 + }, + { + "bits_per_byte": 2.064014527886571, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4306658506393433, + "step": 1530, + "unclipped_grad_norm": 1.716952919960022 + }, + { + "bits_per_byte": 1.9989715492435112, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.3855814933776855, + "step": 1540, + "unclipped_grad_norm": 1.5794824361801147 + }, + { + "bits_per_byte": 1.9082897409751254, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3227256536483765, + "step": 1550, + "unclipped_grad_norm": 1.4743911027908325 + }, + { + "bits_per_byte": 1.9964554430613881, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3838374614715576, + "step": 1560, + "unclipped_grad_norm": 1.6802877187728882 + }, + { + "bits_per_byte": 1.9491115429839216, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.35102117061615, + "step": 1570, + "unclipped_grad_norm": 1.6226329803466797 + }, + { + "bits_per_byte": 1.9697898770104114, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.365354299545288, + "step": 1580, + "unclipped_grad_norm": 1.5252572298049927 + }, + { + "bits_per_byte": 1.9695903771354106, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.3652160167694092, + "step": 1590, + "unclipped_grad_norm": 1.6579627990722656 + }, + { + "bits_per_byte": 2.03182075150626, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.4083508253097534, + "step": 1600, + "unclipped_grad_norm": 1.614857792854309 + }, + { + "bits_per_byte": 1.9655869649886262, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.362441062927246, + "step": 1610, + "unclipped_grad_norm": 1.7011200189590454 + }, + { + "bits_per_byte": 1.9771732641946027, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.3704720735549927, + "step": 1620, + "unclipped_grad_norm": 1.5937589406967163 + }, + { + "bits_per_byte": 1.9360828252851587, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.341990351676941, + "step": 1630, + "unclipped_grad_norm": 1.6730507612228394 + }, + { + "bits_per_byte": 1.980311775590198, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3726475238800049, + "step": 1640, + "unclipped_grad_norm": 1.7941962480545044 + }, + { + "bits_per_byte": 1.9660537259030677, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.362764596939087, + "step": 1650, + "unclipped_grad_norm": 1.5169929265975952 + }, + { + "bits_per_byte": 1.9117374771769722, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.325115442276001, + "step": 1660, + "unclipped_grad_norm": 1.4485381841659546 + }, + { + "bits_per_byte": 1.9408042649992954, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3452630043029785, + "step": 1670, + "unclipped_grad_norm": 1.5443097352981567 + }, + { + "bits_per_byte": 1.889259344709203, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3095347881317139, + "step": 1680, + "unclipped_grad_norm": 1.3715620040893555 + }, + { + "bits_per_byte": 2.0258176970778936, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.4041898250579834, + "step": 1690, + "unclipped_grad_norm": 1.7622767686843872 + }, + { + "bits_per_byte": 1.9112861947011082, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.3248026371002197, + "step": 1700, + "unclipped_grad_norm": 1.675323724746704 + }, + { + "bits_per_byte": 1.9289749543076498, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3370635509490967, + "step": 1710, + "unclipped_grad_norm": 1.5818912982940674 + }, + { + "bits_per_byte": 1.9171609500719309, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.3288747072219849, + "step": 1720, + "unclipped_grad_norm": 1.4991596937179565 + }, + { + "bits_per_byte": 1.9336463470703853, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.340301513671875, + "step": 1730, + "unclipped_grad_norm": 1.5862295627593994 + }, + { + "bits_per_byte": 1.9860921124856972, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.3766541481018066, + "step": 1740, + "unclipped_grad_norm": 1.5270146131515503 + }, + { + "bits_per_byte": 1.9393112836071553, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3442281484603882, + "step": 1750, + "unclipped_grad_norm": 1.4840595722198486 + }, + { + "bits_per_byte": 1.999193578845775, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3857353925704956, + "step": 1760, + "unclipped_grad_norm": 1.7699408531188965 + }, + { + "bits_per_byte": 1.8956746415516874, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.313981533050537, + "step": 1770, + "unclipped_grad_norm": 1.6489516496658325 + }, + { + "bits_per_byte": 1.8979709539060077, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3155732154846191, + "step": 1780, + "unclipped_grad_norm": 1.6557731628417969 + }, + { + "bits_per_byte": 1.903561765920256, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.319448471069336, + "step": 1790, + "unclipped_grad_norm": 1.4928221702575684 + }, + { + "bits_per_byte": 1.9742321888821979, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3684334754943848, + "step": 1800, + "unclipped_grad_norm": 1.929703950881958 + }, + { + "bits_per_byte": 1.9416813765186958, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.3458709716796875, + "step": 1810, + "unclipped_grad_norm": 1.6634047031402588 + }, + { + "bits_per_byte": 1.8691863896137504, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.2956212759017944, + "step": 1820, + "unclipped_grad_norm": 1.4967890977859497 + }, + { + "bits_per_byte": 1.8511954564896427, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.2831509113311768, + "step": 1830, + "unclipped_grad_norm": 1.5580559968948364 + }, + { + "bits_per_byte": 1.9175561662036136, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3291486501693726, + "step": 1840, + "unclipped_grad_norm": 1.4777631759643555 + }, + { + "bits_per_byte": 1.882499738599691, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.30484938621521, + "step": 1850, + "unclipped_grad_norm": 1.4754829406738281 + }, + { + "bits_per_byte": 1.9890001671291364, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3786698579788208, + "step": 1860, + "unclipped_grad_norm": 1.6812334060668945 + }, + { + "bits_per_byte": 1.919764251458041, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.330679178237915, + "step": 1870, + "unclipped_grad_norm": 1.6466583013534546 + }, + { + "bits_per_byte": 1.9591183535243741, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.357957363128662, + "step": 1880, + "unclipped_grad_norm": 1.6580866575241089 + }, + { + "bits_per_byte": 2.0050218989008646, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.389775276184082, + "step": 1890, + "unclipped_grad_norm": 1.7456547021865845 + }, + { + "bits_per_byte": 1.903465111670471, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3193814754486084, + "step": 1900, + "unclipped_grad_norm": 1.575989842414856 + }, + { + "bits_per_byte": 1.9530339513021497, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.3537399768829346, + "step": 1910, + "unclipped_grad_norm": 1.573873519897461 + }, + { + "bits_per_byte": 1.8872726011264358, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3081576824188232, + "step": 1920, + "unclipped_grad_norm": 1.7693248987197876 + }, + { + "bits_per_byte": 1.8288439032529087, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.267657995223999, + "step": 1930, + "unclipped_grad_norm": 1.7306783199310303 + }, + { + "bits_per_byte": 1.9361397515425942, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3420298099517822, + "step": 1940, + "unclipped_grad_norm": 1.6159120798110962 + }, + { + "bits_per_byte": 1.9293003454830822, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3372890949249268, + "step": 1950, + "unclipped_grad_norm": 1.6223891973495483 + }, + { + "bits_per_byte": 1.906635439856476, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3215789794921875, + "step": 1960, + "unclipped_grad_norm": 1.6303201913833618 + }, + { + "bits_per_byte": 1.9771626012702492, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.3704646825790405, + "step": 1970, + "unclipped_grad_norm": 1.554176688194275 + }, + { + "bits_per_byte": 1.8914390528262384, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.3110456466674805, + "step": 1980, + "unclipped_grad_norm": 1.5244910717010498 + }, + { + "bits_per_byte": 1.9375727109895833, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3430230617523193, + "step": 1990, + "unclipped_grad_norm": 1.6129659414291382 + }, + { + "bits_per_byte": 1.9773000154082885, + "learning_rate": 3e-05, + "loss_nats": 1.3705599308013916, + "step": 2000, + "unclipped_grad_norm": 1.7478593587875366 + } + ] + }, + { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "ea303ed3653cbeae2d629f4dcc6457ce842c59dc1d0f1ee674a4eb8c25734c37", + "diagnostic": { + "bits_per_byte": 2.1380384405576867, + "branch_output_rms": [ + 0.02993939071893692, + 0.031081395223736763, + 0.04058511182665825, + 0.012415898032486439, + 0.03163857012987137, + 0.007437178865075111, + 0.016118109226226807, + 0.007713559083640575, + 0.013462266884744167, + 0.009163067676126957, + 0.014294588007032871, + 0.010015063919126987, + 0.015453199855983257, + 0.010677095502614975, + 0.01884664222598076, + 0.01118372566998005, + 0.020796427503228188, + 0.013802342116832733, + 0.02479873225092888, + 0.014017625711858273, + 0.039869681000709534, + 0.01826939918100834, + 0.022691959515213966, + 0.02164348214864731, + 0.02629370056092739, + 0.030305426567792892, + 0.021814068779349327, + 0.037283238023519516, + 0.011395728215575218, + 0.05496470257639885, + 0.009596864692866802, + 0.12093912065029144 + ], + "core_parameter_grad_rms_by_block": [ + 0.00075677209292545, + 0.0005119824026780773, + 0.000413739099784366, + 0.00031213373179906, + 0.00031173349918806503, + 0.0003367029321458252, + 0.00036398451532517026, + 0.00047378656644238797, + 0.0006036197919264307, + 0.0006846173912639145, + 0.0008904295974468759, + 0.0007932171508463918, + 0.0008177916178264611, + 0.000694730303324183, + 0.0005421243248268182, + 0.00039202598310552093 + ], + "depth_weights": [], + "layer_input_rms": [ + 0.05312936007976532, + 0.06633149087429047, + 0.07449059933423996, + 0.09969589114189148, + 0.1025252491235733, + 0.1217338889837265, + 0.12261121720075607, + 0.12622471153736115, + 0.126390740275383, + 0.12871913611888885, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.08971640467643738, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995 + ], + "loss_nats": 1.4819753170013428, + "output_weights": null, + "stream_state_rms": [ + 0.06633149087429047, + 0.07449059933423996, + 0.09969589114189148, + 0.1025252491235733, + 0.1217338889837265, + 0.12261121720075607, + 0.12622471153736115, + 0.126390740275383, + 0.12871913611888885, + 0.12898001074790955, + 0.13049590587615967, + 0.12970700860023499, + 0.13177675008773804, + 0.13056829571723938, + 0.12983354926109314, + 0.1278049200773239, + 0.12707357108592987, + 0.1243041604757309, + 0.11721492558717728, + 0.11439044773578644, + 0.09722385555505753, + 0.09743587672710419, + 0.0907304659485817, + 0.09247039258480072, + 0.08623334765434265, + 0.08971640467643738, + 0.08635294437408447, + 0.09693730622529984, + 0.0970691367983818, + 0.12251615524291992, + 0.12258730083703995, + 0.20744846761226654 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.037616927292651, + "cross_entropy_nats": 5.5712515115737915, + "step": 0 + }, + { + "bits_per_byte": 3.8739961479147755, + "cross_entropy_nats": 2.6852495074272156, + "step": 100 + }, + { + "bits_per_byte": 3.4334316738481987, + "cross_entropy_nats": 2.3798734843730927, + "step": 250 + }, + { + "bits_per_byte": 2.9345221069742453, + "cross_entropy_nats": 2.0340557247400284, + "step": 500 + }, + { + "bits_per_byte": 2.2564510109229996, + "cross_entropy_nats": 1.5640526562929153, + "step": 1000 + }, + { + "bits_per_byte": 2.0709166216223225, + "cross_entropy_nats": 1.43545001745224, + "step": 1500 + }, + { + "bits_per_byte": 1.9984403593284898, + "cross_entropy_nats": 1.3852133005857468, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "c611fc50dc4a2bef304ca726530860ba0e0fab5bc73c75bafb7df3932b17b7fe", + "final_mixer_parameters": null, + "initial_common_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "initial_mixer_parameters": null + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 21.793900100873532, + "measured_steps": 1980, + "median_ms": 21.672270508133806, + "p95_ms": 22.421945561654866, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.048772237975564, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.578983783721924, + "step": 1, + "unclipped_grad_norm": 14.304728507995605 + }, + { + "bits_per_byte": 7.020919300955196, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.866530418395996, + "step": 10, + "unclipped_grad_norm": 4.827667713165283 + }, + { + "bits_per_byte": 6.678003781330445, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.628839492797852, + "step": 20, + "unclipped_grad_norm": 3.0976028442382812 + }, + { + "bits_per_byte": 6.329492634177222, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.387269973754883, + "step": 30, + "unclipped_grad_norm": 3.038156032562256 + }, + { + "bits_per_byte": 5.774885045466648, + "learning_rate": 0.00011999999999999999, + "loss_nats": 4.002845287322998, + "step": 40, + "unclipped_grad_norm": 2.760685920715332 + }, + { + "bits_per_byte": 5.360209420804088, + "learning_rate": 0.00015, + "loss_nats": 3.715414047241211, + "step": 50, + "unclipped_grad_norm": 2.370234966278076 + }, + { + "bits_per_byte": 4.955962136159246, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.435211181640625, + "step": 60, + "unclipped_grad_norm": 2.0363569259643555 + }, + { + "bits_per_byte": 4.569005643266849, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.1669933795928955, + "step": 70, + "unclipped_grad_norm": 1.702130913734436 + }, + { + "bits_per_byte": 4.223097969482387, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.9272284507751465, + "step": 80, + "unclipped_grad_norm": 1.3255577087402344 + }, + { + "bits_per_byte": 4.041268279961778, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.8011937141418457, + "step": 90, + "unclipped_grad_norm": 0.9536243677139282 + }, + { + "bits_per_byte": 3.9076687171185305, + "learning_rate": 0.0003, + "loss_nats": 2.708589553833008, + "step": 100, + "unclipped_grad_norm": 0.6362040042877197 + }, + { + "bits_per_byte": 3.7938739884180412, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.6297130584716797, + "step": 110, + "unclipped_grad_norm": 0.4706369936466217 + }, + { + "bits_per_byte": 3.7698270302091013, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.6130449771881104, + "step": 120, + "unclipped_grad_norm": 0.3721176087856293 + }, + { + "bits_per_byte": 3.68605875252275, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.554981231689453, + "step": 130, + "unclipped_grad_norm": 0.5578080415725708 + }, + { + "bits_per_byte": 3.6636459734623084, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5394458770751953, + "step": 140, + "unclipped_grad_norm": 0.3776422142982483 + }, + { + "bits_per_byte": 3.571004422887341, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.475231647491455, + "step": 150, + "unclipped_grad_norm": 0.41638413071632385 + }, + { + "bits_per_byte": 3.5927554127072625, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4903082847595215, + "step": 160, + "unclipped_grad_norm": 0.5145140886306763 + }, + { + "bits_per_byte": 3.5366794374975385, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.451439380645752, + "step": 170, + "unclipped_grad_norm": 0.7138541340827942 + }, + { + "bits_per_byte": 3.502904108659893, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.428028106689453, + "step": 180, + "unclipped_grad_norm": 0.5306854248046875 + }, + { + "bits_per_byte": 3.5297292746108706, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.446621894836426, + "step": 190, + "unclipped_grad_norm": 0.576228141784668 + }, + { + "bits_per_byte": 3.476859743943833, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.409975528717041, + "step": 200, + "unclipped_grad_norm": 0.524835467338562 + }, + { + "bits_per_byte": 3.5175556546521975, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.4381837844848633, + "step": 210, + "unclipped_grad_norm": 0.5929914712905884 + }, + { + "bits_per_byte": 3.4430899185510153, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.386568069458008, + "step": 220, + "unclipped_grad_norm": 0.5476546883583069 + }, + { + "bits_per_byte": 3.4586240795075476, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.3973355293273926, + "step": 230, + "unclipped_grad_norm": 0.7207314372062683 + }, + { + "bits_per_byte": 3.4313889069169132, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.378457546234131, + "step": 240, + "unclipped_grad_norm": 0.47386032342910767 + }, + { + "bits_per_byte": 3.419826857264708, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.370443344116211, + "step": 250, + "unclipped_grad_norm": 0.6409332156181335 + }, + { + "bits_per_byte": 3.42538052102637, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3742928504943848, + "step": 260, + "unclipped_grad_norm": 0.6748315095901489 + }, + { + "bits_per_byte": 3.3714969806498454, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.3369436264038086, + "step": 270, + "unclipped_grad_norm": 0.9327797889709473 + }, + { + "bits_per_byte": 3.33110995078243, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.3089494705200195, + "step": 280, + "unclipped_grad_norm": 0.781674861907959 + }, + { + "bits_per_byte": 3.4015103609825172, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3577473163604736, + "step": 290, + "unclipped_grad_norm": 0.5359944105148315 + }, + { + "bits_per_byte": 3.298379932496729, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.2862627506256104, + "step": 300, + "unclipped_grad_norm": 0.6357212066650391 + }, + { + "bits_per_byte": 3.349501775465699, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.32169771194458, + "step": 310, + "unclipped_grad_norm": 0.6252917647361755 + }, + { + "bits_per_byte": 3.3178652228741834, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.2997689247131348, + "step": 320, + "unclipped_grad_norm": 0.7440266609191895 + }, + { + "bits_per_byte": 3.2787979878694897, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.2726895809173584, + "step": 330, + "unclipped_grad_norm": 0.6670364737510681 + }, + { + "bits_per_byte": 3.275393763278311, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.2703299522399902, + "step": 340, + "unclipped_grad_norm": 0.6893839836120605 + }, + { + "bits_per_byte": 3.2806560884294114, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.273977518081665, + "step": 350, + "unclipped_grad_norm": 0.7526028156280518 + }, + { + "bits_per_byte": 3.2655646108161998, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.263516902923584, + "step": 360, + "unclipped_grad_norm": 0.8451880812644958 + }, + { + "bits_per_byte": 3.2754914494240013, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.270397663116455, + "step": 370, + "unclipped_grad_norm": 0.913070559501648 + }, + { + "bits_per_byte": 3.256528642339868, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.257253646850586, + "step": 380, + "unclipped_grad_norm": 0.9695722460746765 + }, + { + "bits_per_byte": 3.216058372869435, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.2292017936706543, + "step": 390, + "unclipped_grad_norm": 0.6540295481681824 + }, + { + "bits_per_byte": 3.1761632133834876, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.2015485763549805, + "step": 400, + "unclipped_grad_norm": 0.9143710136413574 + }, + { + "bits_per_byte": 3.123554408415158, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.1650829315185547, + "step": 410, + "unclipped_grad_norm": 0.7914019823074341 + }, + { + "bits_per_byte": 3.167341879255433, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.195434093475342, + "step": 420, + "unclipped_grad_norm": 0.9806389808654785 + }, + { + "bits_per_byte": 3.061858728105841, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.122318744659424, + "step": 430, + "unclipped_grad_norm": 0.9321194291114807 + }, + { + "bits_per_byte": 3.0496493357557886, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.1138558387756348, + "step": 440, + "unclipped_grad_norm": 0.8642765879631042 + }, + { + "bits_per_byte": 3.0622394976948515, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.1225826740264893, + "step": 450, + "unclipped_grad_norm": 0.8904853463172913 + }, + { + "bits_per_byte": 2.996246658871076, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.0768399238586426, + "step": 460, + "unclipped_grad_norm": 1.1931920051574707 + }, + { + "bits_per_byte": 2.9716042967248493, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.0597591400146484, + "step": 470, + "unclipped_grad_norm": 0.9736221432685852 + }, + { + "bits_per_byte": 3.0077179016836255, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.0847911834716797, + "step": 480, + "unclipped_grad_norm": 1.1788429021835327 + }, + { + "bits_per_byte": 2.877249111016685, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.9943571090698242, + "step": 490, + "unclipped_grad_norm": 1.0083208084106445 + }, + { + "bits_per_byte": 2.9443684362036118, + "learning_rate": 0.0002715339687685131, + "loss_nats": 2.0408806800842285, + "step": 500, + "unclipped_grad_norm": 1.1510714292526245 + }, + { + "bits_per_byte": 2.914550772127621, + "learning_rate": 0.00027014843263288514, + "loss_nats": 2.0202126502990723, + "step": 510, + "unclipped_grad_norm": 0.8749134540557861 + }, + { + "bits_per_byte": 2.8766994544645277, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.99397611618042, + "step": 520, + "unclipped_grad_norm": 0.8847914934158325 + }, + { + "bits_per_byte": 2.8809904216035545, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.996950387954712, + "step": 530, + "unclipped_grad_norm": 1.0293059349060059 + }, + { + "bits_per_byte": 2.833532325045915, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.9640549421310425, + "step": 540, + "unclipped_grad_norm": 1.0963666439056396 + }, + { + "bits_per_byte": 2.7702695388219363, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.920204520225525, + "step": 550, + "unclipped_grad_norm": 1.0329253673553467 + }, + { + "bits_per_byte": 2.8496768524303597, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.975245475769043, + "step": 560, + "unclipped_grad_norm": 0.9986040592193604 + }, + { + "bits_per_byte": 2.788238458166733, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.93265962600708, + "step": 570, + "unclipped_grad_norm": 1.270523190498352 + }, + { + "bits_per_byte": 2.716837796869228, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.8831684589385986, + "step": 580, + "unclipped_grad_norm": 1.3667137622833252 + }, + { + "bits_per_byte": 2.73454065905314, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8954391479492188, + "step": 590, + "unclipped_grad_norm": 1.0585418939590454 + }, + { + "bits_per_byte": 2.724275014623139, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8883235454559326, + "step": 600, + "unclipped_grad_norm": 1.2652713060379028 + }, + { + "bits_per_byte": 2.7131061173108066, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8805818557739258, + "step": 610, + "unclipped_grad_norm": 1.3640809059143066 + }, + { + "bits_per_byte": 2.745209086851462, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.9028339385986328, + "step": 620, + "unclipped_grad_norm": 1.2050763368606567 + }, + { + "bits_per_byte": 2.658219230149148, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8425371646881104, + "step": 630, + "unclipped_grad_norm": 1.0446637868881226 + }, + { + "bits_per_byte": 2.5910808148879756, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.796000361442566, + "step": 640, + "unclipped_grad_norm": 1.1966780424118042 + }, + { + "bits_per_byte": 2.6482122476260446, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.8356008529663086, + "step": 650, + "unclipped_grad_norm": 1.2831916809082031 + }, + { + "bits_per_byte": 2.597430930305783, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.8004019260406494, + "step": 660, + "unclipped_grad_norm": 1.1526252031326294 + }, + { + "bits_per_byte": 2.541777860434531, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.7618261575698853, + "step": 670, + "unclipped_grad_norm": 1.0984342098236084 + }, + { + "bits_per_byte": 2.5201109701308817, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7468078136444092, + "step": 680, + "unclipped_grad_norm": 1.162545084953308 + }, + { + "bits_per_byte": 2.609117495397213, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.808502435684204, + "step": 690, + "unclipped_grad_norm": 1.2434271574020386 + }, + { + "bits_per_byte": 2.5297813826062465, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.75351083278656, + "step": 700, + "unclipped_grad_norm": 1.1946849822998047 + }, + { + "bits_per_byte": 2.567022849789381, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.7793246507644653, + "step": 710, + "unclipped_grad_norm": 1.0742769241333008 + }, + { + "bits_per_byte": 2.5078410398877256, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7383029460906982, + "step": 720, + "unclipped_grad_norm": 1.2059558629989624 + }, + { + "bits_per_byte": 2.567514204222896, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.779665231704712, + "step": 730, + "unclipped_grad_norm": 1.19185209274292 + }, + { + "bits_per_byte": 2.4548160369043077, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.7015488147735596, + "step": 740, + "unclipped_grad_norm": 1.318044662475586 + }, + { + "bits_per_byte": 2.449465828618617, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.6978403329849243, + "step": 750, + "unclipped_grad_norm": 1.0698200464248657 + }, + { + "bits_per_byte": 2.4087351774147776, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.6696079969406128, + "step": 760, + "unclipped_grad_norm": 1.1879687309265137 + }, + { + "bits_per_byte": 2.5190773543991964, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.746091365814209, + "step": 770, + "unclipped_grad_norm": 1.0185415744781494 + }, + { + "bits_per_byte": 2.5713784824051324, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7823437452316284, + "step": 780, + "unclipped_grad_norm": 1.2141838073730469 + }, + { + "bits_per_byte": 2.398081712107081, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.6622235774993896, + "step": 790, + "unclipped_grad_norm": 1.1480369567871094 + }, + { + "bits_per_byte": 2.4537002134655097, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.700775384902954, + "step": 800, + "unclipped_grad_norm": 1.2310506105422974 + }, + { + "bits_per_byte": 2.374427734255363, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6458278894424438, + "step": 810, + "unclipped_grad_norm": 1.4261181354522705 + }, + { + "bits_per_byte": 2.3443326621635308, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.6249675750732422, + "step": 820, + "unclipped_grad_norm": 1.5021579265594482 + }, + { + "bits_per_byte": 2.4524106875493406, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6998815536499023, + "step": 830, + "unclipped_grad_norm": 1.2496007680892944 + }, + { + "bits_per_byte": 2.375233988922608, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.646386742591858, + "step": 840, + "unclipped_grad_norm": 1.1895817518234253 + }, + { + "bits_per_byte": 2.3844410801365483, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.652768611907959, + "step": 850, + "unclipped_grad_norm": 1.187380075454712 + }, + { + "bits_per_byte": 2.365942970175046, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.639946699142456, + "step": 860, + "unclipped_grad_norm": 1.2607887983322144 + }, + { + "bits_per_byte": 2.3312026466834097, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6158665418624878, + "step": 870, + "unclipped_grad_norm": 1.4391508102416992 + }, + { + "bits_per_byte": 2.4154529917401297, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.6742644309997559, + "step": 880, + "unclipped_grad_norm": 1.1788713932037354 + }, + { + "bits_per_byte": 2.2404874737807567, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.5529875755310059, + "step": 890, + "unclipped_grad_norm": 1.2682801485061646 + }, + { + "bits_per_byte": 2.2686119686590027, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5724819898605347, + "step": 900, + "unclipped_grad_norm": 1.251353144645691 + }, + { + "bits_per_byte": 2.291240241980975, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.5881667137145996, + "step": 910, + "unclipped_grad_norm": 1.1715742349624634 + }, + { + "bits_per_byte": 2.2675518675990842, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5717471837997437, + "step": 920, + "unclipped_grad_norm": 1.1323882341384888 + }, + { + "bits_per_byte": 2.2996458940418947, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.593993067741394, + "step": 930, + "unclipped_grad_norm": 1.2065128087997437 + }, + { + "bits_per_byte": 2.3487042891658128, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.6279977560043335, + "step": 940, + "unclipped_grad_norm": 1.1545193195343018 + }, + { + "bits_per_byte": 2.313180584699496, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.6033746004104614, + "step": 950, + "unclipped_grad_norm": 1.4039849042892456 + }, + { + "bits_per_byte": 2.301272849919057, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5951207876205444, + "step": 960, + "unclipped_grad_norm": 1.2631624937057495 + }, + { + "bits_per_byte": 2.278702018802473, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.5794758796691895, + "step": 970, + "unclipped_grad_norm": 1.1297482252120972 + }, + { + "bits_per_byte": 2.2859507435710387, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5845003128051758, + "step": 980, + "unclipped_grad_norm": 1.1788796186447144 + }, + { + "bits_per_byte": 2.369809484131744, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.6426267623901367, + "step": 990, + "unclipped_grad_norm": 1.131057858467102 + }, + { + "bits_per_byte": 2.2367193339003526, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.5503756999969482, + "step": 1000, + "unclipped_grad_norm": 1.1219744682312012 + }, + { + "bits_per_byte": 2.202485327332856, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.5266464948654175, + "step": 1010, + "unclipped_grad_norm": 1.398995280265808 + }, + { + "bits_per_byte": 2.3112877436441, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.6020625829696655, + "step": 1020, + "unclipped_grad_norm": 1.2934035062789917 + }, + { + "bits_per_byte": 2.2340518829854705, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.5485267639160156, + "step": 1030, + "unclipped_grad_norm": 1.1515471935272217 + }, + { + "bits_per_byte": 2.3471115578361723, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.6268937587738037, + "step": 1040, + "unclipped_grad_norm": 1.2014071941375732 + }, + { + "bits_per_byte": 2.228570451937171, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.5447273254394531, + "step": 1050, + "unclipped_grad_norm": 1.0397356748580933 + }, + { + "bits_per_byte": 2.221294553909769, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.5396840572357178, + "step": 1060, + "unclipped_grad_norm": 1.1596951484680176 + }, + { + "bits_per_byte": 2.1589362244466774, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4964605569839478, + "step": 1070, + "unclipped_grad_norm": 1.1384665966033936 + }, + { + "bits_per_byte": 2.235138985321574, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.5492802858352661, + "step": 1080, + "unclipped_grad_norm": 1.1616183519363403 + }, + { + "bits_per_byte": 2.23459397230099, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.5489025115966797, + "step": 1090, + "unclipped_grad_norm": 1.1226704120635986 + }, + { + "bits_per_byte": 2.1831126855917167, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.5132184028625488, + "step": 1100, + "unclipped_grad_norm": 1.1548521518707275 + }, + { + "bits_per_byte": 2.1923849582303343, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.5196454524993896, + "step": 1110, + "unclipped_grad_norm": 1.2880154848098755 + }, + { + "bits_per_byte": 2.1734961036860723, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.5065526962280273, + "step": 1120, + "unclipped_grad_norm": 1.2053217887878418 + }, + { + "bits_per_byte": 2.2488805711081636, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.558805227279663, + "step": 1130, + "unclipped_grad_norm": 1.4863512516021729 + }, + { + "bits_per_byte": 2.1800503625104537, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.5110957622528076, + "step": 1140, + "unclipped_grad_norm": 1.210019588470459 + }, + { + "bits_per_byte": 2.213614840618141, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.5343608856201172, + "step": 1150, + "unclipped_grad_norm": 1.172415018081665 + }, + { + "bits_per_byte": 2.1275285807634607, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4746904373168945, + "step": 1160, + "unclipped_grad_norm": 1.2415862083435059 + }, + { + "bits_per_byte": 2.0838432676177985, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.4444100856781006, + "step": 1170, + "unclipped_grad_norm": 1.5139342546463013 + }, + { + "bits_per_byte": 2.142127156099299, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.484809398651123, + "step": 1180, + "unclipped_grad_norm": 1.3185919523239136 + }, + { + "bits_per_byte": 2.1397169912301077, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.4831387996673584, + "step": 1190, + "unclipped_grad_norm": 1.0855211019515991 + }, + { + "bits_per_byte": 2.232181055709385, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.5472300052642822, + "step": 1200, + "unclipped_grad_norm": 1.242047905921936 + }, + { + "bits_per_byte": 2.1909659293780654, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.5186618566513062, + "step": 1210, + "unclipped_grad_norm": 1.26553475856781 + }, + { + "bits_per_byte": 2.1632999402470197, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4994852542877197, + "step": 1220, + "unclipped_grad_norm": 1.0957709550857544 + }, + { + "bits_per_byte": 2.138755264246483, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4824721813201904, + "step": 1230, + "unclipped_grad_norm": 1.1655855178833008 + }, + { + "bits_per_byte": 2.1382630498997135, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.482131004333496, + "step": 1240, + "unclipped_grad_norm": 1.2601433992385864 + }, + { + "bits_per_byte": 2.1067903967744703, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.4603158235549927, + "step": 1250, + "unclipped_grad_norm": 1.234211802482605 + }, + { + "bits_per_byte": 2.1929776104452077, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.5200562477111816, + "step": 1260, + "unclipped_grad_norm": 1.2063928842544556 + }, + { + "bits_per_byte": 2.083411247198831, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.444110631942749, + "step": 1270, + "unclipped_grad_norm": 1.139397144317627 + }, + { + "bits_per_byte": 2.1212494941804594, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4703381061553955, + "step": 1280, + "unclipped_grad_norm": 1.1573246717453003 + }, + { + "bits_per_byte": 2.0000753338967487, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.3863465785980225, + "step": 1290, + "unclipped_grad_norm": 1.1543303728103638 + }, + { + "bits_per_byte": 2.0406659912227854, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4144818782806396, + "step": 1300, + "unclipped_grad_norm": 1.117952823638916 + }, + { + "bits_per_byte": 2.161868700626539, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4984931945800781, + "step": 1310, + "unclipped_grad_norm": 1.2032109498977661 + }, + { + "bits_per_byte": 2.081628647022638, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4428750276565552, + "step": 1320, + "unclipped_grad_norm": 1.4258967638015747 + }, + { + "bits_per_byte": 2.1457773158812112, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.4873394966125488, + "step": 1330, + "unclipped_grad_norm": 1.2065571546554565 + }, + { + "bits_per_byte": 2.162113775904019, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.498663067817688, + "step": 1340, + "unclipped_grad_norm": 1.2289713621139526 + }, + { + "bits_per_byte": 2.2152041322947644, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.535462498664856, + "step": 1350, + "unclipped_grad_norm": 1.3211535215377808 + }, + { + "bits_per_byte": 2.1386080470973443, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.482370138168335, + "step": 1360, + "unclipped_grad_norm": 1.186156153678894 + }, + { + "bits_per_byte": 2.124485519739094, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.472581148147583, + "step": 1370, + "unclipped_grad_norm": 1.1724953651428223 + }, + { + "bits_per_byte": 2.1025169718658314, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.4573537111282349, + "step": 1380, + "unclipped_grad_norm": 1.1783956289291382 + }, + { + "bits_per_byte": 2.066211778233994, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.432188868522644, + "step": 1390, + "unclipped_grad_norm": 1.2431398630142212 + }, + { + "bits_per_byte": 2.0440210287758167, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.4168074131011963, + "step": 1400, + "unclipped_grad_norm": 1.2186048030853271 + }, + { + "bits_per_byte": 2.0588571121524972, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.4270910024642944, + "step": 1410, + "unclipped_grad_norm": 1.1462137699127197 + }, + { + "bits_per_byte": 2.073934315205687, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.4375417232513428, + "step": 1420, + "unclipped_grad_norm": 1.0748997926712036 + }, + { + "bits_per_byte": 1.999609948843514, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.386023998260498, + "step": 1430, + "unclipped_grad_norm": 1.1649432182312012 + }, + { + "bits_per_byte": 1.9900313751037098, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.3793846368789673, + "step": 1440, + "unclipped_grad_norm": 1.1390821933746338 + }, + { + "bits_per_byte": 2.03257420749969, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.4088730812072754, + "step": 1450, + "unclipped_grad_norm": 1.1950715780258179 + }, + { + "bits_per_byte": 2.047695438111501, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.4193543195724487, + "step": 1460, + "unclipped_grad_norm": 1.14659583568573 + }, + { + "bits_per_byte": 2.038467536996806, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.412958025932312, + "step": 1470, + "unclipped_grad_norm": 1.2611298561096191 + }, + { + "bits_per_byte": 1.970922554748994, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.3661394119262695, + "step": 1480, + "unclipped_grad_norm": 1.1823135614395142 + }, + { + "bits_per_byte": 2.0994871535055815, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.4552536010742188, + "step": 1490, + "unclipped_grad_norm": 1.0860791206359863 + }, + { + "bits_per_byte": 1.9642393089364651, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.3615069389343262, + "step": 1500, + "unclipped_grad_norm": 1.1416178941726685 + }, + { + "bits_per_byte": 2.070517019933043, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4351730346679688, + "step": 1510, + "unclipped_grad_norm": 1.2412186861038208 + }, + { + "bits_per_byte": 1.9775996091860917, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.370767593383789, + "step": 1520, + "unclipped_grad_norm": 1.2847481966018677 + }, + { + "bits_per_byte": 2.062187728169106, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4293996095657349, + "step": 1530, + "unclipped_grad_norm": 1.2148329019546509 + }, + { + "bits_per_byte": 2.0604512193433444, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.4281959533691406, + "step": 1540, + "unclipped_grad_norm": 1.1148723363876343 + }, + { + "bits_per_byte": 2.0380919268873217, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.412697672843933, + "step": 1550, + "unclipped_grad_norm": 1.1971123218536377 + }, + { + "bits_per_byte": 1.998576849059781, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3853079080581665, + "step": 1560, + "unclipped_grad_norm": 1.1818065643310547 + }, + { + "bits_per_byte": 2.020246319103193, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.4003280401229858, + "step": 1570, + "unclipped_grad_norm": 1.151665210723877 + }, + { + "bits_per_byte": 1.9841407973290077, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.3753015995025635, + "step": 1580, + "unclipped_grad_norm": 1.1393790245056152 + }, + { + "bits_per_byte": 1.967821191605985, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.3639897108078003, + "step": 1590, + "unclipped_grad_norm": 1.203925609588623 + }, + { + "bits_per_byte": 2.0126751268642598, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.3950800895690918, + "step": 1600, + "unclipped_grad_norm": 1.1557369232177734 + }, + { + "bits_per_byte": 1.9572392710810471, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.3566548824310303, + "step": 1610, + "unclipped_grad_norm": 1.1665189266204834 + }, + { + "bits_per_byte": 2.046706193903738, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.4186686277389526, + "step": 1620, + "unclipped_grad_norm": 1.2666994333267212 + }, + { + "bits_per_byte": 2.0035583265420223, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.3887608051300049, + "step": 1630, + "unclipped_grad_norm": 1.1706013679504395 + }, + { + "bits_per_byte": 1.9908439931290365, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3799479007720947, + "step": 1640, + "unclipped_grad_norm": 1.247759461402893 + }, + { + "bits_per_byte": 1.9743463853623706, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3685126304626465, + "step": 1650, + "unclipped_grad_norm": 1.1583964824676514 + }, + { + "bits_per_byte": 1.9976192496597769, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.3846441507339478, + "step": 1660, + "unclipped_grad_norm": 1.2154792547225952 + }, + { + "bits_per_byte": 1.9695886573089019, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3652148246765137, + "step": 1670, + "unclipped_grad_norm": 1.2398847341537476 + }, + { + "bits_per_byte": 2.018381167254586, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3990352153778076, + "step": 1680, + "unclipped_grad_norm": 1.2556869983673096 + }, + { + "bits_per_byte": 2.0180645471943475, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.3988157510757446, + "step": 1690, + "unclipped_grad_norm": 1.2960360050201416 + }, + { + "bits_per_byte": 1.9893582350082326, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.3789180517196655, + "step": 1700, + "unclipped_grad_norm": 1.1914209127426147 + }, + { + "bits_per_byte": 1.9702848430795945, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3656973838806152, + "step": 1710, + "unclipped_grad_norm": 1.1784889698028564 + }, + { + "bits_per_byte": 2.028123640460662, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.4057881832122803, + "step": 1720, + "unclipped_grad_norm": 1.2393465042114258 + }, + { + "bits_per_byte": 2.03155366244947, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.408165693283081, + "step": 1730, + "unclipped_grad_norm": 1.4333479404449463 + }, + { + "bits_per_byte": 1.8824990506690877, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.3048489093780518, + "step": 1740, + "unclipped_grad_norm": 1.1225255727767944 + }, + { + "bits_per_byte": 2.0374005566308533, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.412218451499939, + "step": 1750, + "unclipped_grad_norm": 1.221250295639038 + }, + { + "bits_per_byte": 1.9567917722235022, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3563446998596191, + "step": 1760, + "unclipped_grad_norm": 1.1573785543441772 + }, + { + "bits_per_byte": 1.9577080957872992, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.3569798469543457, + "step": 1770, + "unclipped_grad_norm": 1.2117408514022827 + }, + { + "bits_per_byte": 1.9837094648406437, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3750026226043701, + "step": 1780, + "unclipped_grad_norm": 1.253016710281372 + }, + { + "bits_per_byte": 2.007098417427382, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3912146091461182, + "step": 1790, + "unclipped_grad_norm": 1.2688742876052856 + }, + { + "bits_per_byte": 2.0020761800568865, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3877334594726562, + "step": 1800, + "unclipped_grad_norm": 1.1731582880020142 + }, + { + "bits_per_byte": 2.042598044322578, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.4158210754394531, + "step": 1810, + "unclipped_grad_norm": 1.1794661283493042 + }, + { + "bits_per_byte": 1.9974202657327285, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.3845062255859375, + "step": 1820, + "unclipped_grad_norm": 1.300487756729126 + }, + { + "bits_per_byte": 1.9886962737850618, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3784592151641846, + "step": 1830, + "unclipped_grad_norm": 1.282581090927124 + }, + { + "bits_per_byte": 1.9444819420053456, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3478121757507324, + "step": 1840, + "unclipped_grad_norm": 1.1378467082977295 + }, + { + "bits_per_byte": 1.9545769796456909, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.3548095226287842, + "step": 1850, + "unclipped_grad_norm": 1.154492974281311 + }, + { + "bits_per_byte": 1.951268721373694, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3525164127349854, + "step": 1860, + "unclipped_grad_norm": 1.1866050958633423 + }, + { + "bits_per_byte": 2.0048340938461227, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3896450996398926, + "step": 1870, + "unclipped_grad_norm": 1.2070223093032837 + }, + { + "bits_per_byte": 2.0120465302753563, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.3946443796157837, + "step": 1880, + "unclipped_grad_norm": 1.200329303741455 + }, + { + "bits_per_byte": 2.06381898361254, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.430530309677124, + "step": 1890, + "unclipped_grad_norm": 1.261162519454956 + }, + { + "bits_per_byte": 1.994678174347372, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3826055526733398, + "step": 1900, + "unclipped_grad_norm": 1.189945936203003 + }, + { + "bits_per_byte": 2.0353407204214693, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.4107906818389893, + "step": 1910, + "unclipped_grad_norm": 1.241178035736084 + }, + { + "bits_per_byte": 2.0024674405875995, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3880046606063843, + "step": 1920, + "unclipped_grad_norm": 1.2449201345443726 + }, + { + "bits_per_byte": 1.932977162575878, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.339837670326233, + "step": 1930, + "unclipped_grad_norm": 1.252579927444458 + }, + { + "bits_per_byte": 1.977642948814109, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3707976341247559, + "step": 1940, + "unclipped_grad_norm": 1.2378138303756714 + }, + { + "bits_per_byte": 1.9667585108063035, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.363253116607666, + "step": 1950, + "unclipped_grad_norm": 1.2995620965957642 + }, + { + "bits_per_byte": 1.92886935696002, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3369903564453125, + "step": 1960, + "unclipped_grad_norm": 1.2833969593048096 + }, + { + "bits_per_byte": 1.9106790959435624, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.3243818283081055, + "step": 1970, + "unclipped_grad_norm": 1.2371577024459839 + }, + { + "bits_per_byte": 2.0177800878898204, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.3986185789108276, + "step": 1980, + "unclipped_grad_norm": 1.2813061475753784 + }, + { + "bits_per_byte": 1.9569788893476407, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3564743995666504, + "step": 1990, + "unclipped_grad_norm": 1.2118487358093262 + }, + { + "bits_per_byte": 2.0067369098952685, + "learning_rate": 3e-05, + "loss_nats": 1.3909640312194824, + "step": 2000, + "unclipped_grad_norm": 1.3561897277832031 + } + ] + }, + { + "architecture": "full", + "batch_size": 32, + "canonical_sha256_without_self": "2325e5a135e2248851288c934534eb7bd75dea189c67664716026f7268f02b06", + "diagnostic": { + "bits_per_byte": 2.1472904192435025, + "branch_output_rms": [ + 0.022901766002178192, + 0.05378643795847893, + 0.03887272998690605, + 0.03948163986206055, + 0.0279708169400692, + 0.033577948808670044, + 0.02430025488138199, + 0.029243580996990204, + 0.0382988266646862, + 0.030052073299884796, + 0.05554451420903206, + 0.032227013260126114, + 0.0788588896393776, + 0.0304368045181036, + 0.023751024156808853, + 0.01987709477543831, + 0.021968834102153778, + 0.024058125913143158, + 0.025901971384882927, + 0.021865760907530785, + 0.025842806324362755, + 0.02625282295048237, + 0.025757966563105583, + 0.031061751767992973, + 0.030561715364456177, + 0.030179107561707497, + 0.03131682425737381, + 0.030768385156989098, + 0.08106637001037598, + 0.03569287434220314, + 1.202970266342163, + 0.05150951072573662 + ], + "core_parameter_grad_rms_by_block": [ + 0.0006630627553059442, + 0.00041266881595576214, + 0.0002580782374219271, + 0.00024203109264679933, + 0.00025507093075379216, + 0.00031916219073644596, + 0.0004268378700704982, + 0.0004762864793838318, + 0.00048201456346138433, + 0.0010455563665680043, + 0.0010490551061311498, + 0.0012357605502724435, + 0.0009144618812121531, + 0.0007776715348883125, + 0.0005979523665026609, + 0.0004474027776125591 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6610932946205139, + "mean_weights": [ + 0.5873371958732605, + 0.4126627743244171 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0821088552474976, + "mean_weights": [ + 0.36868736147880554, + 0.31933116912841797, + 0.3119814991950989 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3029634952545166, + "mean_weights": [ + 0.34345370531082153, + 0.19264069199562073, + 0.30673086643218994, + 0.15717476606369019 + ], + "sources": 4 + }, + { + "entropy_mean": 1.580222487449646, + "mean_weights": [ + 0.18849551677703857, + 0.22032099962234497, + 0.18304306268692017, + 0.2473510503768921, + 0.160789355635643 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7290915250778198, + "mean_weights": [ + 0.2112603336572647, + 0.14511778950691223, + 0.21301302313804626, + 0.119179368019104, + 0.1727142632007599, + 0.1387152373790741 + ], + "sources": 6 + }, + { + "entropy_mean": 1.8889777660369873, + "mean_weights": [ + 0.12642228603363037, + 0.1895863115787506, + 0.13460946083068848, + 0.20421472191810608, + 0.10785487294197083, + 0.15911489725112915, + 0.07819747924804688 + ], + "sources": 7 + }, + { + "entropy_mean": 2.0070152282714844, + "mean_weights": [ + 0.14340901374816895, + 0.12556535005569458, + 0.15920504927635193, + 0.1186026781797409, + 0.1179700642824173, + 0.11930164694786072, + 0.07458579540252686, + 0.14136040210723877 + ], + "sources": 8 + }, + { + "entropy_mean": 2.157308578491211, + "mean_weights": [ + 0.1106209009885788, + 0.1534285843372345, + 0.12050987780094147, + 0.13998161256313324, + 0.09713210165500641, + 0.11417277157306671, + 0.07004732638597488, + 0.12356560677289963, + 0.07054120302200317 + ], + "sources": 9 + }, + { + "entropy_mean": 2.2573065757751465, + "mean_weights": [ + 0.11577743291854858, + 0.10619580745697021, + 0.13512566685676575, + 0.09955395013093948, + 0.10158410668373108, + 0.10607872903347015, + 0.06600029021501541, + 0.10816103219985962, + 0.0682200938463211, + 0.09330287575721741 + ], + "sources": 10 + }, + { + "entropy_mean": 2.341871976852417, + "mean_weights": [ + 0.08971703052520752, + 0.1367042064666748, + 0.09522739052772522, + 0.1273912638425827, + 0.07572510838508606, + 0.09440195560455322, + 0.05390384793281555, + 0.1072179526090622, + 0.054397523403167725, + 0.11095138639211655, + 0.05436237156391144 + ], + "sources": 11 + }, + { + "entropy_mean": 2.3737237453460693, + "mean_weights": [ + 0.07912933826446533, + 0.11576858907938004, + 0.08520004898309708, + 0.11106310784816742, + 0.06018906831741333, + 0.09740950167179108, + 0.03642479330301285, + 0.11728426069021225, + 0.03795357793569565, + 0.11479517072439194, + 0.036721304059028625, + 0.10806122422218323 + ], + "sources": 12 + }, + { + "entropy_mean": 2.5292272567749023, + "mean_weights": [ + 0.07913178205490112, + 0.10298915207386017, + 0.08880285918712616, + 0.09412343800067902, + 0.07212315499782562, + 0.0764903649687767, + 0.05238033086061478, + 0.08541153371334076, + 0.05263702571392059, + 0.09003354609012604, + 0.05277290940284729, + 0.09631574898958206, + 0.05678815022110939 + ], + "sources": 13 + }, + { + "entropy_mean": 2.514190673828125, + "mean_weights": [ + 0.0644359141588211, + 0.09284229576587677, + 0.06484058499336243, + 0.09166993200778961, + 0.0474497526884079, + 0.07633886486291885, + 0.030342692509293556, + 0.09496964514255524, + 0.031635671854019165, + 0.09682802855968475, + 0.03065469115972519, + 0.11059603095054626, + 0.03692394867539406, + 0.13047194480895996 + ], + "sources": 14 + }, + { + "entropy_mean": 2.6629085540771484, + "mean_weights": [ + 0.06900448352098465, + 0.08480898290872574, + 0.07787647843360901, + 0.08246792107820511, + 0.06204821541905403, + 0.06835808604955673, + 0.04372628033161163, + 0.07734093815088272, + 0.04397857189178467, + 0.07598622143268585, + 0.043274376541376114, + 0.08487202227115631, + 0.04435092955827713, + 0.09583833813667297, + 0.04606814682483673 + ], + "sources": 15 + }, + { + "entropy_mean": 2.653104782104492, + "mean_weights": [ + 0.0604591891169548, + 0.08386895060539246, + 0.06835561245679855, + 0.08302801847457886, + 0.04794828221201897, + 0.07101002335548401, + 0.027204647660255432, + 0.08424179255962372, + 0.028142455965280533, + 0.077792689204216, + 0.02640642039477825, + 0.08525443822145462, + 0.02825278602540493, + 0.09787338972091675, + 0.031746119260787964, + 0.09841520339250565 + ], + "sources": 16 + }, + { + "entropy_mean": 2.7772436141967773, + "mean_weights": [ + 0.06147155910730362, + 0.07450656592845917, + 0.0743790864944458, + 0.06956259906291962, + 0.05683615803718567, + 0.057332009077072144, + 0.03786619380116463, + 0.061565324664115906, + 0.037090327590703964, + 0.05890658125281334, + 0.03529685363173485, + 0.0626591220498085, + 0.033140867948532104, + 0.07304809987545013, + 0.03377080336213112, + 0.09310596436262131, + 0.07946188747882843 + ], + "sources": 17 + }, + { + "entropy_mean": 2.6990277767181396, + "mean_weights": [ + 0.056461941450834274, + 0.06405939906835556, + 0.07223860919475555, + 0.05435881018638611, + 0.04876486212015152, + 0.048237256705760956, + 0.026802271604537964, + 0.04862886667251587, + 0.027082327753305435, + 0.03658577799797058, + 0.023210477083921432, + 0.03817088156938553, + 0.018570829182863235, + 0.04487498849630356, + 0.02099861577153206, + 0.10265590250492096, + 0.1291292905807495, + 0.1391688883304596 + ], + "sources": 18 + }, + { + "entropy_mean": 2.8850622177124023, + "mean_weights": [ + 0.053245700895786285, + 0.06737062335014343, + 0.06535325944423676, + 0.06576033681631088, + 0.049778640270233154, + 0.053432270884513855, + 0.032843656837940216, + 0.056226540356874466, + 0.0316985622048378, + 0.053124167025089264, + 0.02983715757727623, + 0.05492723733186722, + 0.027601035311818123, + 0.05999501422047615, + 0.027355981990695, + 0.0776371955871582, + 0.06564674526453018, + 0.07830309122800827, + 0.04986279457807541 + ], + "sources": 19 + }, + { + "entropy_mean": 2.698150634765625, + "mean_weights": [ + 0.045373253524303436, + 0.0503397136926651, + 0.062431301921606064, + 0.0427018441259861, + 0.03966908156871796, + 0.03479406237602234, + 0.020493030548095703, + 0.0339934341609478, + 0.02004154771566391, + 0.026699328795075417, + 0.01738707721233368, + 0.025924768298864365, + 0.01341485045850277, + 0.02767963707447052, + 0.014693047851324081, + 0.07510045915842056, + 0.1108316034078598, + 0.09404437243938446, + 0.04819926619529724, + 0.1961883157491684 + ], + "sources": 20 + }, + { + "entropy_mean": 2.957435131072998, + "mean_weights": [ + 0.045839495956897736, + 0.056279055774211884, + 0.05738268792629242, + 0.053333066403865814, + 0.04335351288318634, + 0.0428837314248085, + 0.027807818725705147, + 0.046022575348615646, + 0.026661880314350128, + 0.043173499405384064, + 0.024672038853168488, + 0.044520266354084015, + 0.022247208282351494, + 0.04877126216888428, + 0.021882155910134315, + 0.06817448139190674, + 0.05482880398631096, + 0.06671217828989029, + 0.03836766630411148, + 0.10467275977134705, + 0.06241385266184807 + ], + "sources": 21 + }, + { + "entropy_mean": 2.756256103515625, + "mean_weights": [ + 0.03610382601618767, + 0.04033929854631424, + 0.050901830196380615, + 0.03310665488243103, + 0.03170297294855118, + 0.027360480278730392, + 0.015733443200588226, + 0.026819299906492233, + 0.015536394901573658, + 0.01994069293141365, + 0.012959280982613564, + 0.018103551119565964, + 0.009620130993425846, + 0.018999528139829636, + 0.010526305064558983, + 0.059594281017780304, + 0.09403049200773239, + 0.0681060403585434, + 0.0353500135242939, + 0.1466471403837204, + 0.09099472314119339, + 0.1375236213207245 + ], + "sources": 22 + }, + { + "entropy_mean": 3.0476112365722656, + "mean_weights": [ + 0.039203427731990814, + 0.04830028489232063, + 0.048976264894008636, + 0.0474427305161953, + 0.03720191866159439, + 0.03926095366477966, + 0.024000924080610275, + 0.04243619740009308, + 0.02319120243191719, + 0.039987944066524506, + 0.021333817392587662, + 0.04074931889772415, + 0.01917889714241028, + 0.044727664440870285, + 0.01877583935856819, + 0.05854877829551697, + 0.04624300077557564, + 0.05501851439476013, + 0.031379103660583496, + 0.08305835723876953, + 0.050938501954078674, + 0.07472094893455505, + 0.06532540917396545 + ], + "sources": 23 + }, + { + "entropy_mean": 2.8677659034729004, + "mean_weights": [ + 0.030936183407902718, + 0.03822886943817139, + 0.04300358146429062, + 0.03167930990457535, + 0.026784149929881096, + 0.024197740480303764, + 0.012831062078475952, + 0.0238640196621418, + 0.012949369847774506, + 0.018164820969104767, + 0.010670850053429604, + 0.0170684102922678, + 0.008061101660132408, + 0.0179086122661829, + 0.008555206470191479, + 0.052304357290267944, + 0.06743433326482773, + 0.05793391168117523, + 0.02602890133857727, + 0.11429835855960846, + 0.07092531025409698, + 0.10347026586532593, + 0.08230829238891602, + 0.10039296746253967 + ], + "sources": 24 + }, + { + "entropy_mean": 3.1288928985595703, + "mean_weights": [ + 0.03403628617525101, + 0.04324691742658615, + 0.04293183237314224, + 0.042291585355997086, + 0.03374169021844864, + 0.03508210554718971, + 0.022262774407863617, + 0.03554892912507057, + 0.021267227828502655, + 0.03265773504972458, + 0.019191008061170578, + 0.03249663859605789, + 0.01657738909125328, + 0.034995026886463165, + 0.01627766340970993, + 0.051272328943014145, + 0.041780948638916016, + 0.04925507307052612, + 0.02853207103908062, + 0.06859545409679413, + 0.04598146677017212, + 0.06080058217048645, + 0.055632010102272034, + 0.06713511049747467, + 0.06841014325618744 + ], + "sources": 25 + }, + { + "entropy_mean": 2.994642734527588, + "mean_weights": [ + 0.02719314768910408, + 0.03602296859025955, + 0.03890581056475639, + 0.02931005135178566, + 0.02502881921827793, + 0.023071683943271637, + 0.013303018175065517, + 0.022924184799194336, + 0.012922639958560467, + 0.01789134554564953, + 0.010892805643379688, + 0.016570929437875748, + 0.009028110653162003, + 0.0171334408223629, + 0.009445561096072197, + 0.044392384588718414, + 0.05104982852935791, + 0.047084663063287735, + 0.023645609617233276, + 0.08583007752895355, + 0.053282707929611206, + 0.08101452142000198, + 0.06313814967870712, + 0.07910145074129105, + 0.1149144172668457, + 0.046901676803827286 + ], + "sources": 26 + }, + { + "entropy_mean": 3.203854560852051, + "mean_weights": [ + 0.03144796937704086, + 0.04156367480754852, + 0.038725536316633224, + 0.04010641202330589, + 0.03018674999475479, + 0.029956158250570297, + 0.019708961248397827, + 0.031102072447538376, + 0.019150394946336746, + 0.028180096298456192, + 0.017677150666713715, + 0.02831239439547062, + 0.015457532368600368, + 0.031108714640140533, + 0.014941886067390442, + 0.04642908275127411, + 0.034512992948293686, + 0.041869938373565674, + 0.024187423288822174, + 0.062354739755392075, + 0.039829712361097336, + 0.06000286713242531, + 0.05009996145963669, + 0.05667608976364136, + 0.06087540462613106, + 0.04342607408761978, + 0.06210999935865402 + ], + "sources": 27 + }, + { + "entropy_mean": 3.0881078243255615, + "mean_weights": [ + 0.02474183402955532, + 0.03244230896234512, + 0.03458905965089798, + 0.027093201875686646, + 0.023510852828621864, + 0.021073386073112488, + 0.01371028646826744, + 0.021601323038339615, + 0.013048699125647545, + 0.017744731158018112, + 0.011399159207940102, + 0.016998980194330215, + 0.009630578570067883, + 0.017313780263066292, + 0.009958595037460327, + 0.039067283272743225, + 0.044651709496974945, + 0.04017344117164612, + 0.0207708440721035, + 0.07147198915481567, + 0.04349767044186592, + 0.06473076343536377, + 0.053407568484544754, + 0.06283475458621979, + 0.09158071875572205, + 0.03870266675949097, + 0.10204371064901352, + 0.0322100967168808 + ], + "sources": 28 + }, + { + "entropy_mean": 3.2817800045013428, + "mean_weights": [ + 0.028711076825857162, + 0.03426762670278549, + 0.03638794273138046, + 0.03224906325340271, + 0.031040817499160767, + 0.026756800711154938, + 0.02253268100321293, + 0.02567785046994686, + 0.02117670699954033, + 0.02352394163608551, + 0.019808782264590263, + 0.02086496725678444, + 0.01579754427075386, + 0.02514159306883812, + 0.015399666503071785, + 0.043684519827365875, + 0.031492527574300766, + 0.033337052911520004, + 0.02284751459956169, + 0.058106373995542526, + 0.03844648599624634, + 0.05734840780496597, + 0.04404130205512047, + 0.05852805823087692, + 0.05198270082473755, + 0.03850783407688141, + 0.0559009350836277, + 0.03525950387120247, + 0.05117972940206528 + ], + "sources": 29 + }, + { + "entropy_mean": 3.1846165657043457, + "mean_weights": [ + 0.023867003619670868, + 0.032078906893730164, + 0.03266797587275505, + 0.026273462921380997, + 0.02359132468700409, + 0.020456084981560707, + 0.014875156804919243, + 0.021093418821692467, + 0.013776535168290138, + 0.017742369323968887, + 0.012449836358428001, + 0.016635145992040634, + 0.010298751294612885, + 0.01769561506807804, + 0.010703487321734428, + 0.037344031035900116, + 0.03760749101638794, + 0.03633996099233627, + 0.019647100940346718, + 0.06284023821353912, + 0.03698626160621643, + 0.05665316432714462, + 0.043707869946956635, + 0.05543707311153412, + 0.072479248046875, + 0.03635432571172714, + 0.08983463793992996, + 0.030204840004444122, + 0.07789166271686554, + 0.012467027641832829 + ], + "sources": 30 + }, + { + "entropy_mean": 3.4240646362304688, + "mean_weights": [ + 0.032822657376527786, + 0.030061306431889534, + 0.03276324272155762, + 0.031903937458992004, + 0.03488016873598099, + 0.033655643463134766, + 0.0372784361243248, + 0.031786054372787476, + 0.036068785935640335, + 0.03242940828204155, + 0.036825031042099, + 0.03317154198884964, + 0.0345335379242897, + 0.033187512308359146, + 0.03310008719563484, + 0.0326094776391983, + 0.027491040527820587, + 0.026009205728769302, + 0.02900158427655697, + 0.0325239934027195, + 0.028335273265838623, + 0.029862137511372566, + 0.03020450845360756, + 0.03926743566989899, + 0.03019871562719345, + 0.029446609318256378, + 0.03360181301832199, + 0.028513632714748383, + 0.03574787825345993, + 0.025597158819437027, + 0.0371222048997879 + ], + "sources": 31 + }, + { + "entropy_mean": 3.1776766777038574, + "mean_weights": [ + 0.02197316288948059, + 0.028604047372937202, + 0.029586657881736755, + 0.02294374629855156, + 0.0241982601583004, + 0.01929701492190361, + 0.016942687332630157, + 0.019281990826129913, + 0.01530586276203394, + 0.015875980257987976, + 0.014155237935483456, + 0.01603291742503643, + 0.010362261906266212, + 0.015811767429113388, + 0.010319267399609089, + 0.030282333493232727, + 0.033683404326438904, + 0.03052596189081669, + 0.017536330968141556, + 0.04871707782149315, + 0.03060685098171234, + 0.0480114221572876, + 0.03484051674604416, + 0.05006794631481171, + 0.053272124379873276, + 0.03087802790105343, + 0.08416040241718292, + 0.02518470026552677, + 0.09448906779289246, + 0.008371583186089993, + 0.09626471996307373, + 0.002416683826595545 + ], + "sources": 32 + } + ], + "layer_input_rms": [ + 0.054283902049064636, + 0.03376822918653488, + 0.029128335416316986, + 0.028351355344057083, + 0.023360062390565872, + 0.024795353412628174, + 0.020751243457198143, + 0.021313447505235672, + 0.01956559531390667, + 0.020500168204307556, + 0.01925240457057953, + 0.020848846063017845, + 0.020730698481202126, + 0.0265693049877882, + 0.023864183574914932, + 0.022557901218533516, + 0.019180433824658394, + 0.014435344375669956, + 0.016901355236768723, + 0.010338382795453072, + 0.012992346659302711, + 0.00800726655870676, + 0.010892084799706936, + 0.007262552157044411, + 0.008559302426874638, + 0.007160624954849482, + 0.007401059381663799, + 0.007063175085932016, + 0.006762783043086529, + 0.006884528324007988, + 0.010628288611769676, + 0.008319443091750145 + ], + "loss_nats": 1.4883882999420166, + "output_weights": { + "entropy_mean": 3.224088191986084, + "mean_weights": [ + 0.025607774034142494, + 0.024873515591025352, + 0.025323878973722458, + 0.021044600754976273, + 0.023150064051151276, + 0.019167067483067513, + 0.019175967201590538, + 0.01836075447499752, + 0.0179922953248024, + 0.016426686197519302, + 0.018669692799448967, + 0.015922464430332184, + 0.013900435529649258, + 0.016718219965696335, + 0.013835347257554531, + 0.02698158100247383, + 0.03005104325711727, + 0.025885293260216713, + 0.021133774891495705, + 0.03821934014558792, + 0.027606457471847534, + 0.03734864294528961, + 0.026637498289346695, + 0.03630806505680084, + 0.03180483356118202, + 0.030001256614923477, + 0.05533213913440704, + 0.025778252631425858, + 0.0781014934182167, + 0.014023041352629662, + 0.10132946074008942, + 0.002933121519163251, + 0.10035593062639236 + ], + "sources": 33 + }, + "stream_state_rms": [ + 0.04166097566485405, + 0.04605885222554207, + 0.04437156766653061, + 0.043437641113996506, + 0.04126441106200218, + 0.04025629907846451, + 0.03862397372722626, + 0.03769715130329132, + 0.0377577506005764, + 0.037123385816812515, + 0.03899230435490608, + 0.038514114916324615, + 0.04267995432019234, + 0.0419749952852726, + 0.041073571890592575, + 0.04013779014348984, + 0.039349108934402466, + 0.038695257157087326, + 0.03815760090947151, + 0.03754246234893799, + 0.03709080442786217, + 0.03668622672557831, + 0.03629663586616516, + 0.0361018143594265, + 0.035904545336961746, + 0.03570886701345444, + 0.035561349242925644, + 0.03540687635540962, + 0.03782746568322182, + 0.03776048868894577, + 0.2158803939819336, + 0.21277332305908203 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.015559292406241, + "cross_entropy_nats": 5.555962324142456, + "step": 0 + }, + { + "bits_per_byte": 3.8904535117411396, + "cross_entropy_nats": 2.696656882762909, + "step": 100 + }, + { + "bits_per_byte": 3.4716255379561485, + "cross_entropy_nats": 2.4063474535942078, + "step": 250 + }, + { + "bits_per_byte": 2.996736035504106, + "cross_entropy_nats": 2.0771791338920593, + "step": 500 + }, + { + "bits_per_byte": 2.2425945407256336, + "cross_entropy_nats": 1.5544480830430984, + "step": 1000 + }, + { + "bits_per_byte": 2.051821925342809, + "cross_entropy_nats": 1.4222145825624466, + "step": 1500 + }, + { + "bits_per_byte": 1.9852328656641551, + "cross_entropy_nats": 1.3760585635900497, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "14b8ac0a095ed9066091ce628e177816f8f20d0fd17d545c12fc523c44560407", + "final_mixer_parameters": "e10b3e882743a7d5753ddfce464a96872b9685a4869657137611e1b379f7d051", + "initial_common_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 146.7571435391555, + "measured_steps": 1980, + "median_ms": 146.72027151391376, + "p95_ms": 147.0799343456747, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.025075780409084, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.562558650970459, + "step": 1, + "unclipped_grad_norm": 9.305364608764648 + }, + { + "bits_per_byte": 7.0981002991482, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.920028209686279, + "step": 10, + "unclipped_grad_norm": 4.877148628234863 + }, + { + "bits_per_byte": 6.696467150796473, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.641637325286865, + "step": 20, + "unclipped_grad_norm": 3.07029128074646 + }, + { + "bits_per_byte": 6.410801215990934, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.443628787994385, + "step": 30, + "unclipped_grad_norm": 2.9954757690429688 + }, + { + "bits_per_byte": 5.861706703136417, + "learning_rate": 0.00011999999999999999, + "loss_nats": 4.06302547454834, + "step": 40, + "unclipped_grad_norm": 2.7888526916503906 + }, + { + "bits_per_byte": 5.3938691773003535, + "learning_rate": 0.00015, + "loss_nats": 3.7387452125549316, + "step": 50, + "unclipped_grad_norm": 2.346360921859741 + }, + { + "bits_per_byte": 4.974188857497686, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.447844982147217, + "step": 60, + "unclipped_grad_norm": 2.001997709274292 + }, + { + "bits_per_byte": 4.591901005645611, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.182863235473633, + "step": 70, + "unclipped_grad_norm": 1.635903239250183 + }, + { + "bits_per_byte": 4.2339153342563565, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.9347264766693115, + "step": 80, + "unclipped_grad_norm": 1.3019540309906006 + }, + { + "bits_per_byte": 4.05991395103772, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.814117908477783, + "step": 90, + "unclipped_grad_norm": 0.9900341033935547 + }, + { + "bits_per_byte": 3.9217025014289355, + "learning_rate": 0.0003, + "loss_nats": 2.7183170318603516, + "step": 100, + "unclipped_grad_norm": 0.6851668357849121 + }, + { + "bits_per_byte": 3.8055756879827465, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.637824058532715, + "step": 110, + "unclipped_grad_norm": 0.43919727206230164 + }, + { + "bits_per_byte": 3.78331528555097, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.622394323348999, + "step": 120, + "unclipped_grad_norm": 0.38338637351989746 + }, + { + "bits_per_byte": 3.693747064946921, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5603103637695312, + "step": 130, + "unclipped_grad_norm": 0.413405179977417 + }, + { + "bits_per_byte": 3.686043274084172, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5549705028533936, + "step": 140, + "unclipped_grad_norm": 0.3583759367465973 + }, + { + "bits_per_byte": 3.584025573349466, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.484257221221924, + "step": 150, + "unclipped_grad_norm": 0.47908228635787964 + }, + { + "bits_per_byte": 3.6063778145168035, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4997506141662598, + "step": 160, + "unclipped_grad_norm": 0.37799450755119324 + }, + { + "bits_per_byte": 3.5577160113857755, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.4660208225250244, + "step": 170, + "unclipped_grad_norm": 0.6691992282867432 + }, + { + "bits_per_byte": 3.5311285254582905, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.447591781616211, + "step": 180, + "unclipped_grad_norm": 0.46057069301605225 + }, + { + "bits_per_byte": 3.5605946569959173, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.4680161476135254, + "step": 190, + "unclipped_grad_norm": 0.6580953598022461 + }, + { + "bits_per_byte": 3.5110409518375145, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.4336681365966797, + "step": 200, + "unclipped_grad_norm": 0.6617440581321716 + }, + { + "bits_per_byte": 3.554246433387269, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.463615894317627, + "step": 210, + "unclipped_grad_norm": 0.5749781727790833 + }, + { + "bits_per_byte": 3.4843994633576583, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.4152016639709473, + "step": 220, + "unclipped_grad_norm": 0.8568923473358154 + }, + { + "bits_per_byte": 3.4877840819266384, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.4175477027893066, + "step": 230, + "unclipped_grad_norm": 0.7070962190628052 + }, + { + "bits_per_byte": 3.4702239653429423, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.4053759574890137, + "step": 240, + "unclipped_grad_norm": 0.8608964681625366 + }, + { + "bits_per_byte": 3.446946457513963, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.3892412185668945, + "step": 250, + "unclipped_grad_norm": 1.1204286813735962 + }, + { + "bits_per_byte": 3.4571133839023687, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3962883949279785, + "step": 260, + "unclipped_grad_norm": 0.7315152883529663 + }, + { + "bits_per_byte": 3.3818510241623896, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.344120502471924, + "step": 270, + "unclipped_grad_norm": 0.6489255428314209 + }, + { + "bits_per_byte": 3.366465456216203, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.333456039428711, + "step": 280, + "unclipped_grad_norm": 1.0565967559814453 + }, + { + "bits_per_byte": 3.445399301586801, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3881688117980957, + "step": 290, + "unclipped_grad_norm": 0.9785327911376953 + }, + { + "bits_per_byte": 3.3274412168742247, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.3064064979553223, + "step": 300, + "unclipped_grad_norm": 0.6214700937271118 + }, + { + "bits_per_byte": 3.3833142525559303, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.345134735107422, + "step": 310, + "unclipped_grad_norm": 0.888323962688446 + }, + { + "bits_per_byte": 3.352708220008385, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.323920249938965, + "step": 320, + "unclipped_grad_norm": 1.0714529752731323 + }, + { + "bits_per_byte": 3.310818061772429, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.294884204864502, + "step": 330, + "unclipped_grad_norm": 1.0220837593078613 + }, + { + "bits_per_byte": 3.309257835163802, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.2938027381896973, + "step": 340, + "unclipped_grad_norm": 0.7234710454940796 + }, + { + "bits_per_byte": 3.3182539036651333, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.3000383377075195, + "step": 350, + "unclipped_grad_norm": 1.2935594320297241 + }, + { + "bits_per_byte": 3.2996550118702257, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.28714656829834, + "step": 360, + "unclipped_grad_norm": 1.0127232074737549 + }, + { + "bits_per_byte": 3.321108815669456, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.3020172119140625, + "step": 370, + "unclipped_grad_norm": 2.5480756759643555 + }, + { + "bits_per_byte": 3.3120446420383827, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.295734405517578, + "step": 380, + "unclipped_grad_norm": 1.759360671043396 + }, + { + "bits_per_byte": 3.266914674625473, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.2644526958465576, + "step": 390, + "unclipped_grad_norm": 1.7389194965362549 + }, + { + "bits_per_byte": 3.2684157392022035, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.265493154525757, + "step": 400, + "unclipped_grad_norm": 5.185736179351807 + }, + { + "bits_per_byte": 3.2055725906463297, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.221933603286743, + "step": 410, + "unclipped_grad_norm": 2.350154399871826 + }, + { + "bits_per_byte": 3.2192913027403542, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.23144268989563, + "step": 420, + "unclipped_grad_norm": 1.3248378038406372 + }, + { + "bits_per_byte": 3.141010991443035, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.177182912826538, + "step": 430, + "unclipped_grad_norm": 1.3705254793167114 + }, + { + "bits_per_byte": 3.1402556436404456, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.176659345626831, + "step": 440, + "unclipped_grad_norm": 1.9491982460021973 + }, + { + "bits_per_byte": 3.122428953947912, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.1643028259277344, + "step": 450, + "unclipped_grad_norm": 1.207443118095398 + }, + { + "bits_per_byte": 3.0695773094765637, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.127668857574463, + "step": 460, + "unclipped_grad_norm": 1.3383755683898926 + }, + { + "bits_per_byte": 3.0493848264387617, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.1136724948883057, + "step": 470, + "unclipped_grad_norm": 1.900756597518921 + }, + { + "bits_per_byte": 3.072635848939508, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.129788875579834, + "step": 480, + "unclipped_grad_norm": 2.40047025680542 + }, + { + "bits_per_byte": 2.9499382663344544, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.04474139213562, + "step": 490, + "unclipped_grad_norm": 1.6569600105285645 + }, + { + "bits_per_byte": 3.01048544650131, + "learning_rate": 0.0002715339687685131, + "loss_nats": 2.086709499359131, + "step": 500, + "unclipped_grad_norm": 1.3414586782455444 + }, + { + "bits_per_byte": 3.004979249951286, + "learning_rate": 0.00027014843263288514, + "loss_nats": 2.082892894744873, + "step": 510, + "unclipped_grad_norm": 1.8155630826950073 + }, + { + "bits_per_byte": 2.9403206525329044, + "learning_rate": 0.00026873414996567596, + "loss_nats": 2.0380749702453613, + "step": 520, + "unclipped_grad_norm": 1.2509859800338745 + }, + { + "bits_per_byte": 2.9436602116473587, + "learning_rate": 0.0002672915074176452, + "loss_nats": 2.0403897762298584, + "step": 530, + "unclipped_grad_norm": 1.681069016456604 + }, + { + "bits_per_byte": 2.899078180960098, + "learning_rate": 0.00026582089939286075, + "loss_nats": 2.0094878673553467, + "step": 540, + "unclipped_grad_norm": 2.4520139694213867 + }, + { + "bits_per_byte": 2.83453790760551, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.964751958847046, + "step": 550, + "unclipped_grad_norm": 1.3797680139541626 + }, + { + "bits_per_byte": 2.895264637659866, + "learning_rate": 0.000262797402646797, + "loss_nats": 2.0068445205688477, + "step": 560, + "unclipped_grad_norm": 1.11052405834198 + }, + { + "bits_per_byte": 2.8679235237563, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.9878931045532227, + "step": 570, + "unclipped_grad_norm": 1.6181483268737793 + }, + { + "bits_per_byte": 2.780145642547733, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.9270501136779785, + "step": 580, + "unclipped_grad_norm": 1.5519311428070068 + }, + { + "bits_per_byte": 2.8082854438819056, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.9465551376342773, + "step": 590, + "unclipped_grad_norm": 1.626272201538086 + }, + { + "bits_per_byte": 2.779646204929627, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.926703929901123, + "step": 600, + "unclipped_grad_norm": 1.53666090965271 + }, + { + "bits_per_byte": 2.764624724255317, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.9162918329238892, + "step": 610, + "unclipped_grad_norm": 2.209996461868286 + }, + { + "bits_per_byte": 2.7927074273494035, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.9357572793960571, + "step": 620, + "unclipped_grad_norm": 1.5756334066390991 + }, + { + "bits_per_byte": 2.7104061616749116, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.878710389137268, + "step": 630, + "unclipped_grad_norm": 1.6814439296722412 + }, + { + "bits_per_byte": 2.654353060157752, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.8398573398590088, + "step": 640, + "unclipped_grad_norm": 1.7441400289535522 + }, + { + "bits_per_byte": 2.6871977908715787, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.8626235723495483, + "step": 650, + "unclipped_grad_norm": 1.500356674194336 + }, + { + "bits_per_byte": 2.6456349156202146, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.8338143825531006, + "step": 660, + "unclipped_grad_norm": 1.86016845703125 + }, + { + "bits_per_byte": 2.5917840519473536, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.796487808227539, + "step": 670, + "unclipped_grad_norm": 1.4692447185516357 + }, + { + "bits_per_byte": 2.551894395906234, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7688384056091309, + "step": 680, + "unclipped_grad_norm": 1.7879689931869507 + }, + { + "bits_per_byte": 2.6330117330121863, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.8250646591186523, + "step": 690, + "unclipped_grad_norm": 1.38472580909729 + }, + { + "bits_per_byte": 2.5644964246482065, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7775734663009644, + "step": 700, + "unclipped_grad_norm": 1.6946994066238403 + }, + { + "bits_per_byte": 2.5943577723175157, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.7982717752456665, + "step": 710, + "unclipped_grad_norm": 1.5931897163391113 + }, + { + "bits_per_byte": 2.5268695443444886, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7514925003051758, + "step": 720, + "unclipped_grad_norm": 1.87969970703125 + }, + { + "bits_per_byte": 2.5850292613520662, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.7918057441711426, + "step": 730, + "unclipped_grad_norm": 1.6175384521484375 + }, + { + "bits_per_byte": 2.478968936426179, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.7182903289794922, + "step": 740, + "unclipped_grad_norm": 1.9508171081542969 + }, + { + "bits_per_byte": 2.476348952722935, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7164742946624756, + "step": 750, + "unclipped_grad_norm": 1.6979271173477173 + }, + { + "bits_per_byte": 2.4202882839691378, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.677616000175476, + "step": 760, + "unclipped_grad_norm": 2.0603036880493164 + }, + { + "bits_per_byte": 2.5413771408580206, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.761548399925232, + "step": 770, + "unclipped_grad_norm": 1.6721434593200684 + }, + { + "bits_per_byte": 2.587655436430741, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.793626070022583, + "step": 780, + "unclipped_grad_norm": 2.149043321609497 + }, + { + "bits_per_byte": 2.4056893646679973, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.6674968004226685, + "step": 790, + "unclipped_grad_norm": 1.5452014207839966 + }, + { + "bits_per_byte": 2.4699297002794807, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.7120248079299927, + "step": 800, + "unclipped_grad_norm": 1.7299747467041016 + }, + { + "bits_per_byte": 2.371144929415694, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6435524225234985, + "step": 810, + "unclipped_grad_norm": 1.675118327140808 + }, + { + "bits_per_byte": 2.327094325119599, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.6130188703536987, + "step": 820, + "unclipped_grad_norm": 1.489935278892517 + }, + { + "bits_per_byte": 2.4473478622732414, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6963722705841064, + "step": 830, + "unclipped_grad_norm": 1.7736183404922485 + }, + { + "bits_per_byte": 2.3644549762797813, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.6389153003692627, + "step": 840, + "unclipped_grad_norm": 1.5628443956375122 + }, + { + "bits_per_byte": 2.3979807582910246, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.6621536016464233, + "step": 850, + "unclipped_grad_norm": 1.7924773693084717 + }, + { + "bits_per_byte": 2.3594478733825617, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.6354446411132812, + "step": 860, + "unclipped_grad_norm": 1.7623909711837769 + }, + { + "bits_per_byte": 2.3338152351326666, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6176774501800537, + "step": 870, + "unclipped_grad_norm": 1.7694491147994995 + }, + { + "bits_per_byte": 2.4029701469752056, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.665611982345581, + "step": 880, + "unclipped_grad_norm": 1.509064793586731 + }, + { + "bits_per_byte": 2.242668385776348, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.5544992685317993, + "step": 890, + "unclipped_grad_norm": 2.7527503967285156 + }, + { + "bits_per_byte": 2.252844083279948, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5615525245666504, + "step": 900, + "unclipped_grad_norm": 1.7615165710449219 + }, + { + "bits_per_byte": 2.27098739303272, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.57412850856781, + "step": 910, + "unclipped_grad_norm": 1.8199325799942017 + }, + { + "bits_per_byte": 2.2750470435063366, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5769424438476562, + "step": 920, + "unclipped_grad_norm": 1.957559585571289 + }, + { + "bits_per_byte": 2.2770523612153966, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5783324241638184, + "step": 930, + "unclipped_grad_norm": 1.543131709098816 + }, + { + "bits_per_byte": 2.3390297491068273, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.6212918758392334, + "step": 940, + "unclipped_grad_norm": 1.854478359222412 + }, + { + "bits_per_byte": 2.3033708662769325, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5965750217437744, + "step": 950, + "unclipped_grad_norm": 1.668323040008545 + }, + { + "bits_per_byte": 2.2807247067592704, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5808779001235962, + "step": 960, + "unclipped_grad_norm": 1.7748920917510986 + }, + { + "bits_per_byte": 2.2775880871728345, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.578703761100769, + "step": 970, + "unclipped_grad_norm": 1.8448166847229004 + }, + { + "bits_per_byte": 2.277732380616908, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5788037776947021, + "step": 980, + "unclipped_grad_norm": 1.550948143005371 + }, + { + "bits_per_byte": 2.359567573307562, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.6355276107788086, + "step": 990, + "unclipped_grad_norm": 1.8681533336639404 + }, + { + "bits_per_byte": 2.220158952466122, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.538896918296814, + "step": 1000, + "unclipped_grad_norm": 1.737576961517334 + }, + { + "bits_per_byte": 2.1773493749786534, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.5092235803604126, + "step": 1010, + "unclipped_grad_norm": 1.532644271850586 + }, + { + "bits_per_byte": 2.282893923934603, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5823814868927002, + "step": 1020, + "unclipped_grad_norm": 1.6802674531936646 + }, + { + "bits_per_byte": 2.2078255606247974, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.5303480625152588, + "step": 1030, + "unclipped_grad_norm": 1.4638360738754272 + }, + { + "bits_per_byte": 2.337395054010376, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.6201587915420532, + "step": 1040, + "unclipped_grad_norm": 2.052419662475586 + }, + { + "bits_per_byte": 2.2261840206737986, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.5430731773376465, + "step": 1050, + "unclipped_grad_norm": 1.7121658325195312 + }, + { + "bits_per_byte": 2.196306334652657, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.522363543510437, + "step": 1060, + "unclipped_grad_norm": 1.6127790212631226 + }, + { + "bits_per_byte": 2.1455406677536244, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.487175464630127, + "step": 1070, + "unclipped_grad_norm": 1.5833548307418823 + }, + { + "bits_per_byte": 2.2279255169964354, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.5442802906036377, + "step": 1080, + "unclipped_grad_norm": 1.6587281227111816 + }, + { + "bits_per_byte": 2.2191482104270013, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.538196325302124, + "step": 1090, + "unclipped_grad_norm": 1.6756495237350464 + }, + { + "bits_per_byte": 2.1684497887444554, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.5030548572540283, + "step": 1100, + "unclipped_grad_norm": 1.653583288192749 + }, + { + "bits_per_byte": 2.154817755906466, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.4936058521270752, + "step": 1110, + "unclipped_grad_norm": 1.624094843864441 + }, + { + "bits_per_byte": 2.157416757726305, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.4954073429107666, + "step": 1120, + "unclipped_grad_norm": 1.6921063661575317 + }, + { + "bits_per_byte": 2.234742049363383, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.549005150794983, + "step": 1130, + "unclipped_grad_norm": 1.7546871900558472 + }, + { + "bits_per_byte": 2.1554886602274816, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.4940708875656128, + "step": 1140, + "unclipped_grad_norm": 1.6136784553527832 + }, + { + "bits_per_byte": 2.2050597356336215, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.5284309387207031, + "step": 1150, + "unclipped_grad_norm": 1.7648067474365234 + }, + { + "bits_per_byte": 2.1088689790927986, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4617565870285034, + "step": 1160, + "unclipped_grad_norm": 1.6633940935134888 + }, + { + "bits_per_byte": 2.0715355011914527, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.4358789920806885, + "step": 1170, + "unclipped_grad_norm": 1.7017462253570557 + }, + { + "bits_per_byte": 2.135756574746039, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.480393648147583, + "step": 1180, + "unclipped_grad_norm": 1.8441929817199707 + }, + { + "bits_per_byte": 2.1367332642202888, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.481070637702942, + "step": 1190, + "unclipped_grad_norm": 1.6285960674285889 + }, + { + "bits_per_byte": 2.2125108839822527, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.5335956811904907, + "step": 1200, + "unclipped_grad_norm": 1.7019469738006592 + }, + { + "bits_per_byte": 2.167908903307492, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.5026799440383911, + "step": 1210, + "unclipped_grad_norm": 1.6109663248062134 + }, + { + "bits_per_byte": 2.159513226240322, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4968605041503906, + "step": 1220, + "unclipped_grad_norm": 1.5659048557281494 + }, + { + "bits_per_byte": 2.1090259992530362, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4618654251098633, + "step": 1230, + "unclipped_grad_norm": 1.5079011917114258 + }, + { + "bits_per_byte": 2.1199441458604107, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.469433307647705, + "step": 1240, + "unclipped_grad_norm": 1.6655142307281494 + }, + { + "bits_per_byte": 2.093163351433356, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.4508702754974365, + "step": 1250, + "unclipped_grad_norm": 1.6711302995681763 + }, + { + "bits_per_byte": 2.172678842129172, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.505986213684082, + "step": 1260, + "unclipped_grad_norm": 1.5373055934906006 + }, + { + "bits_per_byte": 2.0630916689820413, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4300261735916138, + "step": 1270, + "unclipped_grad_norm": 1.5858079195022583 + }, + { + "bits_per_byte": 2.096708601798242, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4533276557922363, + "step": 1280, + "unclipped_grad_norm": 1.5007655620574951 + }, + { + "bits_per_byte": 1.991981658364494, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.3807364702224731, + "step": 1290, + "unclipped_grad_norm": 1.6829313039779663 + }, + { + "bits_per_byte": 2.027708130376178, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4055001735687256, + "step": 1300, + "unclipped_grad_norm": 1.503808617591858 + }, + { + "bits_per_byte": 2.1423151331366923, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.484939694404602, + "step": 1310, + "unclipped_grad_norm": 1.462040662765503 + }, + { + "bits_per_byte": 2.0684843569824953, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.433764100074768, + "step": 1320, + "unclipped_grad_norm": 1.5657871961593628 + }, + { + "bits_per_byte": 2.115981493601881, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.4666866064071655, + "step": 1330, + "unclipped_grad_norm": 1.4948500394821167 + }, + { + "bits_per_byte": 2.1462255026693597, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4876501560211182, + "step": 1340, + "unclipped_grad_norm": 1.609449028968811 + }, + { + "bits_per_byte": 2.1875231806730935, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.5162755250930786, + "step": 1350, + "unclipped_grad_norm": 1.9372320175170898 + }, + { + "bits_per_byte": 2.107255781827705, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.460638403892517, + "step": 1360, + "unclipped_grad_norm": 1.6968649625778198 + }, + { + "bits_per_byte": 2.107099449598071, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.4605300426483154, + "step": 1370, + "unclipped_grad_norm": 1.6523107290267944 + }, + { + "bits_per_byte": 2.0841438932915066, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.4446184635162354, + "step": 1380, + "unclipped_grad_norm": 1.424194574356079 + }, + { + "bits_per_byte": 2.0486624965573026, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.4200246334075928, + "step": 1390, + "unclipped_grad_norm": 1.4947720766067505 + }, + { + "bits_per_byte": 2.0184128120623446, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.399057149887085, + "step": 1400, + "unclipped_grad_norm": 1.6217533349990845 + }, + { + "bits_per_byte": 2.034293002112413, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.410064458847046, + "step": 1410, + "unclipped_grad_norm": 1.6703604459762573 + }, + { + "bits_per_byte": 2.0595723879974357, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.4275867938995361, + "step": 1420, + "unclipped_grad_norm": 1.457814335823059 + }, + { + "bits_per_byte": 1.984870003768666, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.3758070468902588, + "step": 1430, + "unclipped_grad_norm": 1.4897449016571045 + }, + { + "bits_per_byte": 1.9715315453156992, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.3665615320205688, + "step": 1440, + "unclipped_grad_norm": 1.645474910736084 + }, + { + "bits_per_byte": 2.0119866803128557, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.39460289478302, + "step": 1450, + "unclipped_grad_norm": 1.567344307899475 + }, + { + "bits_per_byte": 2.0352543851307363, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.4107308387756348, + "step": 1460, + "unclipped_grad_norm": 1.5357178449630737 + }, + { + "bits_per_byte": 2.020113376514076, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.400235891342163, + "step": 1470, + "unclipped_grad_norm": 1.6168160438537598 + }, + { + "bits_per_byte": 1.953244114101504, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.3538856506347656, + "step": 1480, + "unclipped_grad_norm": 1.554640769958496 + }, + { + "bits_per_byte": 2.0728305305524493, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.4367766380310059, + "step": 1490, + "unclipped_grad_norm": 1.452438473701477 + }, + { + "bits_per_byte": 1.9471304748286329, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.3496479988098145, + "step": 1500, + "unclipped_grad_norm": 1.4332044124603271 + }, + { + "bits_per_byte": 2.053871335103984, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4236351251602173, + "step": 1510, + "unclipped_grad_norm": 1.6625758409500122 + }, + { + "bits_per_byte": 1.9516742564644283, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.352797508239746, + "step": 1520, + "unclipped_grad_norm": 1.4334182739257812 + }, + { + "bits_per_byte": 2.0307264258988202, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4075922966003418, + "step": 1530, + "unclipped_grad_norm": 1.5977627038955688 + }, + { + "bits_per_byte": 2.039129498219977, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.413416862487793, + "step": 1540, + "unclipped_grad_norm": 1.467684268951416 + }, + { + "bits_per_byte": 2.0194213183270042, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3997561931610107, + "step": 1550, + "unclipped_grad_norm": 1.5047897100448608 + }, + { + "bits_per_byte": 1.975260989099659, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3691465854644775, + "step": 1560, + "unclipped_grad_norm": 1.8182013034820557 + }, + { + "bits_per_byte": 2.000922520434899, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.3869338035583496, + "step": 1570, + "unclipped_grad_norm": 1.5143927335739136 + }, + { + "bits_per_byte": 1.962038102988072, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.3599811792373657, + "step": 1580, + "unclipped_grad_norm": 1.6671526432037354 + }, + { + "bits_per_byte": 1.9569162876627268, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.356431007385254, + "step": 1590, + "unclipped_grad_norm": 1.4555517435073853 + }, + { + "bits_per_byte": 1.988921571057692, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.378615379333496, + "step": 1600, + "unclipped_grad_norm": 1.5734272003173828 + }, + { + "bits_per_byte": 1.936036217986775, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.3419580459594727, + "step": 1610, + "unclipped_grad_norm": 1.624767780303955 + }, + { + "bits_per_byte": 2.021858656455032, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.4014456272125244, + "step": 1620, + "unclipped_grad_norm": 1.8971368074417114 + }, + { + "bits_per_byte": 1.9838522104408598, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.3751015663146973, + "step": 1630, + "unclipped_grad_norm": 1.5017338991165161 + }, + { + "bits_per_byte": 1.9546235869440747, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3548418283462524, + "step": 1640, + "unclipped_grad_norm": 1.5532703399658203 + }, + { + "bits_per_byte": 1.9563552802556123, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3560421466827393, + "step": 1650, + "unclipped_grad_norm": 1.5462849140167236 + }, + { + "bits_per_byte": 1.982108478343762, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.373892903327942, + "step": 1660, + "unclipped_grad_norm": 1.7786948680877686 + }, + { + "bits_per_byte": 1.9491151546195897, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3510236740112305, + "step": 1670, + "unclipped_grad_norm": 1.5200086832046509 + }, + { + "bits_per_byte": 1.9922392883754865, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3809150457382202, + "step": 1680, + "unclipped_grad_norm": 1.6116204261779785 + }, + { + "bits_per_byte": 1.9901976823270942, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.379499912261963, + "step": 1690, + "unclipped_grad_norm": 1.5754514932632446 + }, + { + "bits_per_byte": 1.9800716878095936, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.372481107711792, + "step": 1700, + "unclipped_grad_norm": 1.7042871713638306 + }, + { + "bits_per_byte": 1.948211213806655, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3503971099853516, + "step": 1710, + "unclipped_grad_norm": 1.4325282573699951 + }, + { + "bits_per_byte": 1.988147993094111, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.3780791759490967, + "step": 1720, + "unclipped_grad_norm": 1.5585598945617676 + }, + { + "bits_per_byte": 2.0137507063627558, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3958256244659424, + "step": 1730, + "unclipped_grad_norm": 1.6188148260116577 + }, + { + "bits_per_byte": 1.8691497573091165, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.2955958843231201, + "step": 1740, + "unclipped_grad_norm": 1.4358497858047485 + }, + { + "bits_per_byte": 2.0081022801604687, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.391910433769226, + "step": 1750, + "unclipped_grad_norm": 1.4696511030197144 + }, + { + "bits_per_byte": 1.929275579981358, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3372719287872314, + "step": 1760, + "unclipped_grad_norm": 1.4995495080947876 + }, + { + "bits_per_byte": 1.9422454796135258, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.346261978149414, + "step": 1770, + "unclipped_grad_norm": 1.4796631336212158 + }, + { + "bits_per_byte": 1.95940453265541, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3581557273864746, + "step": 1780, + "unclipped_grad_norm": 1.705535888671875 + }, + { + "bits_per_byte": 1.9797082884683206, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3722292184829712, + "step": 1790, + "unclipped_grad_norm": 1.7329541444778442 + }, + { + "bits_per_byte": 1.9739181485617223, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.368215799331665, + "step": 1800, + "unclipped_grad_norm": 1.4475159645080566 + }, + { + "bits_per_byte": 2.02591263150117, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.4042556285858154, + "step": 1810, + "unclipped_grad_norm": 1.7348294258117676 + }, + { + "bits_per_byte": 1.964777270668364, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.361879825592041, + "step": 1820, + "unclipped_grad_norm": 1.497507929801941 + }, + { + "bits_per_byte": 1.9678882648398215, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.364036202430725, + "step": 1830, + "unclipped_grad_norm": 1.5670346021652222 + }, + { + "bits_per_byte": 1.9166611684885235, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3285282850265503, + "step": 1840, + "unclipped_grad_norm": 1.557868242263794 + }, + { + "bits_per_byte": 1.9402534045685818, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.3448811769485474, + "step": 1850, + "unclipped_grad_norm": 1.4524617195129395 + }, + { + "bits_per_byte": 1.9187920335327138, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3300052881240845, + "step": 1860, + "unclipped_grad_norm": 1.5609016418457031 + }, + { + "bits_per_byte": 1.9832234418693053, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3746657371520996, + "step": 1870, + "unclipped_grad_norm": 1.6460332870483398 + }, + { + "bits_per_byte": 2.005298790968754, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.3899672031402588, + "step": 1880, + "unclipped_grad_norm": 1.5417602062225342 + }, + { + "bits_per_byte": 2.039331061886788, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.4135565757751465, + "step": 1890, + "unclipped_grad_norm": 1.531503438949585 + }, + { + "bits_per_byte": 1.9767701368609802, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3701926469802856, + "step": 1900, + "unclipped_grad_norm": 1.6723964214324951 + }, + { + "bits_per_byte": 2.0329095736688725, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.4091055393218994, + "step": 1910, + "unclipped_grad_norm": 1.7043548822402954 + }, + { + "bits_per_byte": 1.9743474172582758, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3685133457183838, + "step": 1920, + "unclipped_grad_norm": 1.494638442993164 + }, + { + "bits_per_byte": 1.930537932638691, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3381469249725342, + "step": 1930, + "unclipped_grad_norm": 1.5504201650619507 + }, + { + "bits_per_byte": 1.9391363772512278, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.344106912612915, + "step": 1940, + "unclipped_grad_norm": 1.4306025505065918 + }, + { + "bits_per_byte": 1.9391656143018745, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3441271781921387, + "step": 1950, + "unclipped_grad_norm": 1.6829544305801392 + }, + { + "bits_per_byte": 1.9152562422136252, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.32755446434021, + "step": 1960, + "unclipped_grad_norm": 1.6364843845367432 + }, + { + "bits_per_byte": 1.893279783138423, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.3123215436935425, + "step": 1970, + "unclipped_grad_norm": 1.6044296026229858 + }, + { + "bits_per_byte": 1.9981876523208784, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.385038137435913, + "step": 1980, + "unclipped_grad_norm": 1.6767337322235107 + }, + { + "bits_per_byte": 1.9352966925880648, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3414454460144043, + "step": 1990, + "unclipped_grad_norm": 1.4996469020843506 + }, + { + "bits_per_byte": 1.978050375714003, + "learning_rate": 3e-05, + "loss_nats": 1.3710800409317017, + "step": 2000, + "unclipped_grad_norm": 1.5616031885147095 + } + ] + }, + { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "705a6cb8a18db4e97ee47c2cc53a2cb0ef68d6207108c911f96e3cefc45c9859", + "diagnostic": { + "bits_per_byte": 2.1160220815074844, + "branch_output_rms": [ + 0.03422113135457039, + 0.042584121227264404, + 0.07630905508995056, + 0.03935832530260086, + 0.05174773558974266, + 0.022662153467535973, + 0.02264152280986309, + 0.018601570278406143, + 0.039430491626262665, + 0.013977118767797947, + 0.027242736890912056, + 0.01665658876299858, + 0.058694127947092056, + 0.023023422807455063, + 0.03670934960246086, + 0.02355036325752735, + 0.019655495882034302, + 0.017856158316135406, + 0.025521859526634216, + 0.01807972602546215, + 0.033282581716775894, + 0.023539945483207703, + 0.014158087782561779, + 0.03246133774518967, + 0.019294502213597298, + 0.026912888512015343, + 0.009501714259386063, + 0.03243190050125122, + 0.014970541931688786, + 0.04098350554704666, + 0.008943221531808376, + 0.05635817348957062 + ], + "core_parameter_grad_rms_by_block": [ + 0.0003999679822384072, + 0.0004981167249238301, + 0.0003300655422364756, + 0.0002340430140301531, + 0.00025161489682913623, + 0.0002331203443329733, + 0.0003338122079598397, + 0.00031515007512692854, + 0.00046013207297086815, + 0.0006049664535646685, + 0.0016852072794625043, + 0.0012097450404667171, + 0.0011452921321391885, + 0.0007695163028177718, + 0.0005286419550644225, + 0.00041417398082090176 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.676069974899292, + "mean_weights": [ + 0.5256662368774414, + 0.4743337631225586 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6752070188522339, + "mean_weights": [ + 0.526084303855896, + 0.473915696144104 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5004937648773193, + "mean_weights": [ + 0.7952219247817993, + 0.20477813482284546 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6855782270431519, + "mean_weights": [ + 0.45605403184890747, + 0.5439459681510925 + ], + "sources": 2 + }, + { + "entropy_mean": 0.973181962966919, + "mean_weights": [ + 0.5508121252059937, + 0.20714516937732697, + 0.24204272031784058 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0914695262908936, + "mean_weights": [ + 0.29142463207244873, + 0.36466842889785767, + 0.3439069092273712 + ], + "sources": 3 + }, + { + "entropy_mean": 0.9970768094062805, + "mean_weights": [ + 0.4954737424850464, + 0.17813757061958313, + 0.3263886868953705 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0915882587432861, + "mean_weights": [ + 0.29986491799354553, + 0.3621853291988373, + 0.3379497528076172 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3325610160827637, + "mean_weights": [ + 0.28639328479766846, + 0.1406242549419403, + 0.26381438970565796, + 0.30916810035705566 + ], + "sources": 4 + }, + { + "entropy_mean": 1.372523546218872, + "mean_weights": [ + 0.235001802444458, + 0.23495016992092133, + 0.21656674146652222, + 0.31348127126693726 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3118952512741089, + "mean_weights": [ + 0.2822558581829071, + 0.14495812356472015, + 0.21341675519943237, + 0.35936927795410156 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3699870109558105, + "mean_weights": [ + 0.22112420201301575, + 0.26209473609924316, + 0.2080908715724945, + 0.3086901903152466 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5866072177886963, + "mean_weights": [ + 0.20069512724876404, + 0.139734148979187, + 0.19753095507621765, + 0.26452502608299255, + 0.19751472771167755 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5973689556121826, + "mean_weights": [ + 0.19069713354110718, + 0.19748152792453766, + 0.16151165962219238, + 0.19743436574935913, + 0.25287532806396484 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5947315692901611, + "mean_weights": [ + 0.19719600677490234, + 0.17207738757133484, + 0.1734003722667694, + 0.2008262574672699, + 0.2564999759197235 + ], + "sources": 5 + }, + { + "entropy_mean": 1.6005793809890747, + "mean_weights": [ + 0.19282269477844238, + 0.20231595635414124, + 0.18008708953857422, + 0.1822172999382019, + 0.24255694448947906 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7821681499481201, + "mean_weights": [ + 0.15993690490722656, + 0.1374683678150177, + 0.16724741458892822, + 0.1658557653427124, + 0.1655316799879074, + 0.2039598971605301 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7596988677978516, + "mean_weights": [ + 0.1570039540529251, + 0.15772980451583862, + 0.1422097533941269, + 0.13380974531173706, + 0.1495790183544159, + 0.25966769456863403 + ], + "sources": 6 + }, + { + "entropy_mean": 1.59963059425354, + "mean_weights": [ + 0.15447966754436493, + 0.12336395680904388, + 0.11268190294504166, + 0.09744387865066528, + 0.09779104590415955, + 0.4142395257949829 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7793185710906982, + "mean_weights": [ + 0.16669687628746033, + 0.17260831594467163, + 0.15142369270324707, + 0.14505012333393097, + 0.15068548917770386, + 0.21353553235530853 + ], + "sources": 6 + }, + { + "entropy_mean": 1.592297911643982, + "mean_weights": [ + 0.09717687964439392, + 0.07443678379058838, + 0.07146455347537994, + 0.06425752490758896, + 0.05914592742919922, + 0.17170733213424683, + 0.46181100606918335 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8954918384552002, + "mean_weights": [ + 0.13514602184295654, + 0.1371152102947235, + 0.1126670390367508, + 0.10706128925085068, + 0.11022193729877472, + 0.14614711701869965, + 0.2516413927078247 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5488040447235107, + "mean_weights": [ + 0.09989520907402039, + 0.07713347673416138, + 0.06720373034477234, + 0.061107393354177475, + 0.05421334505081177, + 0.1470748782157898, + 0.49337196350097656 + ], + "sources": 7 + }, + { + "entropy_mean": 1.9023945331573486, + "mean_weights": [ + 0.14041754603385925, + 0.152886301279068, + 0.11610820144414902, + 0.10672424733638763, + 0.10685749351978302, + 0.14317843317985535, + 0.23382778465747833 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7261797189712524, + "mean_weights": [ + 0.07879909873008728, + 0.05790713056921959, + 0.05081409215927124, + 0.045005545020103455, + 0.04007049649953842, + 0.10400360077619553, + 0.3018913269042969, + 0.3215087056159973 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9232137203216553, + "mean_weights": [ + 0.104021817445755, + 0.09768764674663544, + 0.0772961750626564, + 0.06826606392860413, + 0.06893711537122726, + 0.10266698896884918, + 0.18262885510921478, + 0.298495352268219 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7093220949172974, + "mean_weights": [ + 0.07989830523729324, + 0.0624738372862339, + 0.0516524538397789, + 0.048858530819416046, + 0.046183206140995026, + 0.09405303001403809, + 0.2071082890033722, + 0.4097723662853241 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9634629487991333, + "mean_weights": [ + 0.10993535816669464, + 0.11451396346092224, + 0.08913694322109222, + 0.06949189305305481, + 0.0641917884349823, + 0.11329004168510437, + 0.1931304633617401, + 0.2463095784187317 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9782196283340454, + "mean_weights": [ + 0.07671404629945755, + 0.07277035713195801, + 0.057552408427000046, + 0.048507869243621826, + 0.04175746068358421, + 0.08779345452785492, + 0.20467188954353333, + 0.2411869764328003, + 0.16904552280902863 + ], + "sources": 9 + }, + { + "entropy_mean": 2.044435501098633, + "mean_weights": [ + 0.09215540438890457, + 0.07394982129335403, + 0.06361652165651321, + 0.055349722504615784, + 0.04765096679329872, + 0.0947403758764267, + 0.17295636236667633, + 0.21584057807922363, + 0.1837402582168579 + ], + "sources": 9 + }, + { + "entropy_mean": 1.9660465717315674, + "mean_weights": [ + 0.0797988772392273, + 0.07157003879547119, + 0.05443256348371506, + 0.048355042934417725, + 0.043302472680807114, + 0.07537667453289032, + 0.1577366441488266, + 0.2724228501319885, + 0.19700486958026886 + ], + "sources": 9 + } + ], + "layer_input_rms": [ + 0.053777214139699936, + 0.03443998843431473, + 0.04316028207540512, + 0.05271390080451965, + 0.08092177659273148, + 0.05150312930345535, + 0.06945399194955826, + 0.053602270781993866, + 0.07346180826425552, + 0.04669487476348877, + 0.05561559274792671, + 0.0512978732585907, + 0.06302255392074585, + 0.05218172445893288, + 0.05766461044549942, + 0.06084785237908363, + 0.06420515477657318, + 0.04857253283262253, + 0.04646599665284157, + 0.036662496626377106, + 0.04943696781992912, + 0.01807689666748047, + 0.03342784568667412, + 0.02252720296382904, + 0.03615279868245125, + 0.018748776987195015, + 0.022745439782738686, + 0.01989392749965191, + 0.0260133296251297, + 0.02015369012951851, + 0.02320900373160839, + 0.023934273049235344 + ], + "loss_nats": 1.4667147397994995, + "output_weights": { + "entropy_mean": 1.89408540725708, + "mean_weights": [ + 0.09608907252550125, + 0.05759216845035553, + 0.056019775569438934, + 0.04851848632097244, + 0.045551691204309464, + 0.06574775278568268, + 0.09080863744020462, + 0.2241908609867096, + 0.315481573343277 + ], + "sources": 9 + }, + "stream_state_rms": [ + 0.03422113135457039, + 0.05911572277545929, + 0.11840524524450302, + 0.136356920003891, + 0.05174773558974266, + 0.05775697901844978, + 0.07019630819559097, + 0.07779177278280258, + 0.039430491626262665, + 0.04417555779218674, + 0.06122447922825813, + 0.068731389939785, + 0.058694127947092056, + 0.07109052687883377, + 0.09855329245328903, + 0.11156570911407471, + 0.019655495882034302, + 0.02821449190378189, + 0.04324287921190262, + 0.05173908546566963, + 0.033282581716775894, + 0.04125083237886429, + 0.045212581753730774, + 0.057350847870111465, + 0.019294502213597298, + 0.03496377170085907, + 0.03744323179125786, + 0.053472280502319336, + 0.014970541931688786, + 0.04533463716506958, + 0.047072283923625946, + 0.09321330487728119 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.035310725935906, + "cross_entropy_nats": 5.56965297460556, + "step": 0 + }, + { + "bits_per_byte": 3.8667511637688663, + "cross_entropy_nats": 2.680227667093277, + "step": 100 + }, + { + "bits_per_byte": 3.422946407573046, + "cross_entropy_nats": 2.37260565161705, + "step": 250 + }, + { + "bits_per_byte": 2.8856276543122625, + "cross_entropy_nats": 2.000164672732353, + "step": 500 + }, + { + "bits_per_byte": 2.199974509617247, + "cross_entropy_nats": 1.5249061286449432, + "step": 1000 + }, + { + "bits_per_byte": 2.028303792287441, + "cross_entropy_nats": 1.4059130549430847, + "step": 1500 + }, + { + "bits_per_byte": 1.9570916454731129, + "cross_entropy_nats": 1.3565525561571121, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "1754b714261c61a7085cfac60cd639c5b81e3f7015049bf4a23f015e4184428e", + "final_mixer_parameters": "b49cc8c6495b3ea8bfa327e77d39096b5ec95f6a3db53341b8a26009a2ff68f3", + "initial_common_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 54.45831580526452, + "measured_steps": 1980, + "median_ms": 54.40393550088629, + "p95_ms": 54.91135738266166, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.046501379053572, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.577409744262695, + "step": 1, + "unclipped_grad_norm": 13.799291610717773 + }, + { + "bits_per_byte": 7.026312676886254, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.870268821716309, + "step": 10, + "unclipped_grad_norm": 4.8699822425842285 + }, + { + "bits_per_byte": 6.678567884425275, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.629230499267578, + "step": 20, + "unclipped_grad_norm": 3.1052775382995605 + }, + { + "bits_per_byte": 6.336600333172081, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.3921966552734375, + "step": 30, + "unclipped_grad_norm": 3.0494046211242676 + }, + { + "bits_per_byte": 5.760058765101066, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.9925684928894043, + "step": 40, + "unclipped_grad_norm": 2.6845459938049316 + }, + { + "bits_per_byte": 5.346855311929892, + "learning_rate": 0.00015, + "loss_nats": 3.706157684326172, + "step": 50, + "unclipped_grad_norm": 2.4176599979400635 + }, + { + "bits_per_byte": 4.9448358905443275, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.4274990558624268, + "step": 60, + "unclipped_grad_norm": 2.0677621364593506 + }, + { + "bits_per_byte": 4.558067546671975, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.159411668777466, + "step": 70, + "unclipped_grad_norm": 1.6371504068374634 + }, + { + "bits_per_byte": 4.214488174014893, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.9212605953216553, + "step": 80, + "unclipped_grad_norm": 1.2613117694854736 + }, + { + "bits_per_byte": 4.0338389734098055, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.796044111251831, + "step": 90, + "unclipped_grad_norm": 0.8912747502326965 + }, + { + "bits_per_byte": 3.9015265287256162, + "learning_rate": 0.0003, + "loss_nats": 2.704332113265991, + "step": 100, + "unclipped_grad_norm": 0.6827722191810608 + }, + { + "bits_per_byte": 3.7887237959553035, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.626143217086792, + "step": 110, + "unclipped_grad_norm": 0.4674619436264038 + }, + { + "bits_per_byte": 3.7636645478633852, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.6087734699249268, + "step": 120, + "unclipped_grad_norm": 0.3633909821510315 + }, + { + "bits_per_byte": 3.677213340823574, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5488500595092773, + "step": 130, + "unclipped_grad_norm": 0.6861963272094727 + }, + { + "bits_per_byte": 3.661360324032342, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5378615856170654, + "step": 140, + "unclipped_grad_norm": 0.4464603364467621 + }, + { + "bits_per_byte": 3.5632191122480834, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.4698352813720703, + "step": 150, + "unclipped_grad_norm": 0.46651002764701843 + }, + { + "bits_per_byte": 3.5787309154600035, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4805872440338135, + "step": 160, + "unclipped_grad_norm": 0.5629671812057495 + }, + { + "bits_per_byte": 3.518842772811255, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.4390759468078613, + "step": 170, + "unclipped_grad_norm": 0.7118674516677856 + }, + { + "bits_per_byte": 3.487357564952499, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.4172520637512207, + "step": 180, + "unclipped_grad_norm": 0.4673098027706146 + }, + { + "bits_per_byte": 3.514603056502185, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.4361371994018555, + "step": 190, + "unclipped_grad_norm": 0.5881734490394592 + }, + { + "bits_per_byte": 3.4633549782674815, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.4006147384643555, + "step": 200, + "unclipped_grad_norm": 0.6767106056213379 + }, + { + "bits_per_byte": 3.517347211679352, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.4380393028259277, + "step": 210, + "unclipped_grad_norm": 0.9985042214393616 + }, + { + "bits_per_byte": 3.4266824296934013, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.375195264816284, + "step": 220, + "unclipped_grad_norm": 0.5032101273536682 + }, + { + "bits_per_byte": 3.4354979164113275, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.381305694580078, + "step": 230, + "unclipped_grad_norm": 0.6975435018539429 + }, + { + "bits_per_byte": 3.4307501633516084, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.3780148029327393, + "step": 240, + "unclipped_grad_norm": 0.9623647332191467 + }, + { + "bits_per_byte": 3.411411402192689, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.364610195159912, + "step": 250, + "unclipped_grad_norm": 0.8256871104240417 + }, + { + "bits_per_byte": 3.4157446770638287, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3676137924194336, + "step": 260, + "unclipped_grad_norm": 0.9798179864883423 + }, + { + "bits_per_byte": 3.3548776091663686, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.3254239559173584, + "step": 270, + "unclipped_grad_norm": 1.119627594947815 + }, + { + "bits_per_byte": 3.3259060997326233, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.305342435836792, + "step": 280, + "unclipped_grad_norm": 1.063017725944519 + }, + { + "bits_per_byte": 3.400357045325831, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.356947898864746, + "step": 290, + "unclipped_grad_norm": 1.0752443075180054 + }, + { + "bits_per_byte": 3.285246993311543, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.2771596908569336, + "step": 300, + "unclipped_grad_norm": 0.7791656255722046 + }, + { + "bits_per_byte": 3.348213281445435, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.3208045959472656, + "step": 310, + "unclipped_grad_norm": 1.7170469760894775 + }, + { + "bits_per_byte": 3.307036163279956, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.2922627925872803, + "step": 320, + "unclipped_grad_norm": 0.7804914116859436 + }, + { + "bits_per_byte": 3.2564216691310315, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.2571794986724854, + "step": 330, + "unclipped_grad_norm": 0.7880367040634155 + }, + { + "bits_per_byte": 3.2690964465343186, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.265964984893799, + "step": 340, + "unclipped_grad_norm": 1.0493488311767578 + }, + { + "bits_per_byte": 3.255702437685123, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.256680965423584, + "step": 350, + "unclipped_grad_norm": 0.944348156452179 + }, + { + "bits_per_byte": 3.2311013513751035, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.239628791809082, + "step": 360, + "unclipped_grad_norm": 0.9164798259735107 + }, + { + "bits_per_byte": 3.245024378858353, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.249279499053955, + "step": 370, + "unclipped_grad_norm": 0.9579949378967285 + }, + { + "bits_per_byte": 3.2342486338858927, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.2418103218078613, + "step": 380, + "unclipped_grad_norm": 1.071757435798645 + }, + { + "bits_per_byte": 3.198696380299533, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.217167377471924, + "step": 390, + "unclipped_grad_norm": 1.567230224609375 + }, + { + "bits_per_byte": 3.1229985604875696, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.1646976470947266, + "step": 400, + "unclipped_grad_norm": 1.1201071739196777 + }, + { + "bits_per_byte": 3.0955501294098653, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.145671844482422, + "step": 410, + "unclipped_grad_norm": 1.9664565324783325 + }, + { + "bits_per_byte": 3.111488793561227, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.15671968460083, + "step": 420, + "unclipped_grad_norm": 1.1844605207443237 + }, + { + "bits_per_byte": 3.0245013446160254, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.0964245796203613, + "step": 430, + "unclipped_grad_norm": 1.2996922731399536 + }, + { + "bits_per_byte": 3.028208258672722, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.098994016647339, + "step": 440, + "unclipped_grad_norm": 1.5297218561172485 + }, + { + "bits_per_byte": 3.01815965634811, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.092028856277466, + "step": 450, + "unclipped_grad_norm": 2.020317554473877 + }, + { + "bits_per_byte": 2.9509319820911397, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.0454301834106445, + "step": 460, + "unclipped_grad_norm": 0.9621044397354126 + }, + { + "bits_per_byte": 2.9337994143774884, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.033554792404175, + "step": 470, + "unclipped_grad_norm": 1.0360201597213745 + }, + { + "bits_per_byte": 2.9442225949156797, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.0407795906066895, + "step": 480, + "unclipped_grad_norm": 1.0649659633636475 + }, + { + "bits_per_byte": 2.835526119917368, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.9654369354248047, + "step": 490, + "unclipped_grad_norm": 1.3808231353759766 + }, + { + "bits_per_byte": 2.8995941289126863, + "learning_rate": 0.0002715339687685131, + "loss_nats": 2.009845495223999, + "step": 500, + "unclipped_grad_norm": 1.3657233715057373 + }, + { + "bits_per_byte": 2.874300296484991, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.9923131465911865, + "step": 510, + "unclipped_grad_norm": 1.2534769773483276 + }, + { + "bits_per_byte": 2.8221104412541633, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.9561378955841064, + "step": 520, + "unclipped_grad_norm": 1.1826246976852417 + }, + { + "bits_per_byte": 2.8163675965765527, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.9521572589874268, + "step": 530, + "unclipped_grad_norm": 1.2498561143875122 + }, + { + "bits_per_byte": 2.778361838492984, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.9258136749267578, + "step": 540, + "unclipped_grad_norm": 1.7357478141784668 + }, + { + "bits_per_byte": 2.7239372406968445, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.8880894184112549, + "step": 550, + "unclipped_grad_norm": 1.7417893409729004 + }, + { + "bits_per_byte": 2.791677423253386, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.9350433349609375, + "step": 560, + "unclipped_grad_norm": 1.544079065322876 + }, + { + "bits_per_byte": 2.744612479035619, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.9024204015731812, + "step": 570, + "unclipped_grad_norm": 1.7899011373519897 + }, + { + "bits_per_byte": 2.652013064210113, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.8382353782653809, + "step": 580, + "unclipped_grad_norm": 1.4172289371490479 + }, + { + "bits_per_byte": 2.6699637533939184, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8506778478622437, + "step": 590, + "unclipped_grad_norm": 1.2538301944732666 + }, + { + "bits_per_byte": 2.659127986476307, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8431670665740967, + "step": 600, + "unclipped_grad_norm": 1.3936573266983032 + }, + { + "bits_per_byte": 2.6414768910703046, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8309322595596313, + "step": 610, + "unclipped_grad_norm": 1.5266214609146118 + }, + { + "bits_per_byte": 2.6744640234190458, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.853797197341919, + "step": 620, + "unclipped_grad_norm": 1.7098969221115112 + }, + { + "bits_per_byte": 2.606005985277803, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8063457012176514, + "step": 630, + "unclipped_grad_norm": 1.2531689405441284 + }, + { + "bits_per_byte": 2.5300759888871744, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.7537150382995605, + "step": 640, + "unclipped_grad_norm": 1.3655412197113037 + }, + { + "bits_per_byte": 2.570826246113212, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.7819609642028809, + "step": 650, + "unclipped_grad_norm": 1.4527359008789062 + }, + { + "bits_per_byte": 2.5271502200306966, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.7516870498657227, + "step": 660, + "unclipped_grad_norm": 1.7026007175445557 + }, + { + "bits_per_byte": 2.4690955844227958, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.7114466428756714, + "step": 670, + "unclipped_grad_norm": 1.8035180568695068 + }, + { + "bits_per_byte": 2.4378941479379637, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.6898194551467896, + "step": 680, + "unclipped_grad_norm": 1.5442042350769043 + }, + { + "bits_per_byte": 2.5359672545924803, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.7577985525131226, + "step": 690, + "unclipped_grad_norm": 1.44375479221344 + }, + { + "bits_per_byte": 2.4724752154949012, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7137892246246338, + "step": 700, + "unclipped_grad_norm": 1.696199655532837 + }, + { + "bits_per_byte": 2.4807511926370704, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.719525694847107, + "step": 710, + "unclipped_grad_norm": 1.594708800315857 + }, + { + "bits_per_byte": 2.441510083172354, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.6923258304595947, + "step": 720, + "unclipped_grad_norm": 1.9483309984207153 + }, + { + "bits_per_byte": 2.4982777726038483, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.7316741943359375, + "step": 730, + "unclipped_grad_norm": 1.4037096500396729 + }, + { + "bits_per_byte": 2.3675489441688033, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6410598754882812, + "step": 740, + "unclipped_grad_norm": 1.4826147556304932 + }, + { + "bits_per_byte": 2.3704874397414453, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.643096685409546, + "step": 750, + "unclipped_grad_norm": 1.3413487672805786 + }, + { + "bits_per_byte": 2.3328975357076627, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.6170413494110107, + "step": 760, + "unclipped_grad_norm": 1.5907566547393799 + }, + { + "bits_per_byte": 2.460552346241186, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.7055249214172363, + "step": 770, + "unclipped_grad_norm": 1.313509225845337 + }, + { + "bits_per_byte": 2.5034444754010687, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.735255479812622, + "step": 780, + "unclipped_grad_norm": 1.5776115655899048 + }, + { + "bits_per_byte": 2.3060207749614263, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.5984117984771729, + "step": 790, + "unclipped_grad_norm": 1.5052303075790405 + }, + { + "bits_per_byte": 2.3783111025118453, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6485196352005005, + "step": 800, + "unclipped_grad_norm": 1.413939356803894 + }, + { + "bits_per_byte": 2.3051240574198277, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.597790241241455, + "step": 810, + "unclipped_grad_norm": 1.5896333456039429 + }, + { + "bits_per_byte": 2.2494271319726056, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.5591840744018555, + "step": 820, + "unclipped_grad_norm": 1.635888695716858 + }, + { + "bits_per_byte": 2.40091787800246, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6641894578933716, + "step": 830, + "unclipped_grad_norm": 1.9857487678527832 + }, + { + "bits_per_byte": 2.2799924046318965, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.580370306968689, + "step": 840, + "unclipped_grad_norm": 1.517561912536621 + }, + { + "bits_per_byte": 2.3261114442699182, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.612337589263916, + "step": 850, + "unclipped_grad_norm": 1.8287367820739746 + }, + { + "bits_per_byte": 2.286530841052399, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.5849024057388306, + "step": 860, + "unclipped_grad_norm": 1.7752363681793213 + }, + { + "bits_per_byte": 2.266942705049728, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.5713249444961548, + "step": 870, + "unclipped_grad_norm": 1.4794734716415405 + }, + { + "bits_per_byte": 2.3413964023653504, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.6229323148727417, + "step": 880, + "unclipped_grad_norm": 1.5235475301742554 + }, + { + "bits_per_byte": 2.1760942455926564, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.508353590965271, + "step": 890, + "unclipped_grad_norm": 2.2208006381988525 + }, + { + "bits_per_byte": 2.199139748825622, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5243275165557861, + "step": 900, + "unclipped_grad_norm": 1.8758368492126465 + }, + { + "bits_per_byte": 2.215898770221599, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.5359439849853516, + "step": 910, + "unclipped_grad_norm": 1.4233596324920654 + }, + { + "bits_per_byte": 2.1969751751818625, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5228271484375, + "step": 920, + "unclipped_grad_norm": 1.5109971761703491 + }, + { + "bits_per_byte": 2.224798184473146, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5421125888824463, + "step": 930, + "unclipped_grad_norm": 1.5682772397994995 + }, + { + "bits_per_byte": 2.2819645296893403, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5817372798919678, + "step": 940, + "unclipped_grad_norm": 1.5304511785507202 + }, + { + "bits_per_byte": 2.2496259439170028, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5593218803405762, + "step": 950, + "unclipped_grad_norm": 1.7026301622390747 + }, + { + "bits_per_byte": 2.2406818141762317, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5531222820281982, + "step": 960, + "unclipped_grad_norm": 1.583830714225769 + }, + { + "bits_per_byte": 2.231838466268866, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.546992540359497, + "step": 970, + "unclipped_grad_norm": 1.559972882270813 + }, + { + "bits_per_byte": 2.218478853949843, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5377323627471924, + "step": 980, + "unclipped_grad_norm": 1.5388225317001343 + }, + { + "bits_per_byte": 2.310613399670067, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.601595163345337, + "step": 990, + "unclipped_grad_norm": 1.6706602573394775 + }, + { + "bits_per_byte": 2.175180673751273, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.5077203512191772, + "step": 1000, + "unclipped_grad_norm": 1.5570387840270996 + }, + { + "bits_per_byte": 2.1357780725773967, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.4804085493087769, + "step": 1010, + "unclipped_grad_norm": 1.7706972360610962 + }, + { + "bits_per_byte": 2.2317376844354606, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5469226837158203, + "step": 1020, + "unclipped_grad_norm": 1.402220368385315 + }, + { + "bits_per_byte": 2.1736610350482497, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.5066670179367065, + "step": 1030, + "unclipped_grad_norm": 1.530462622642517 + }, + { + "bits_per_byte": 2.2887912090326887, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5864691734313965, + "step": 1040, + "unclipped_grad_norm": 1.7111684083938599 + }, + { + "bits_per_byte": 2.1553627689270503, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.4939836263656616, + "step": 1050, + "unclipped_grad_norm": 1.1934901475906372 + }, + { + "bits_per_byte": 2.154836330032759, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.4936187267303467, + "step": 1060, + "unclipped_grad_norm": 1.544889211654663 + }, + { + "bits_per_byte": 2.090904187331622, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4493043422698975, + "step": 1070, + "unclipped_grad_norm": 1.5413001775741577 + }, + { + "bits_per_byte": 2.194022920997152, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.5207808017730713, + "step": 1080, + "unclipped_grad_norm": 1.6356488466262817 + }, + { + "bits_per_byte": 2.1801652469212303, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.5111753940582275, + "step": 1090, + "unclipped_grad_norm": 1.4393450021743774 + }, + { + "bits_per_byte": 2.116354523971602, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.4669451713562012, + "step": 1100, + "unclipped_grad_norm": 1.4577889442443848 + }, + { + "bits_per_byte": 2.107281407242684, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.4606561660766602, + "step": 1110, + "unclipped_grad_norm": 1.4607727527618408 + }, + { + "bits_per_byte": 2.109131424618015, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.461938500404358, + "step": 1120, + "unclipped_grad_norm": 1.5313950777053833 + }, + { + "bits_per_byte": 2.1833758190475367, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.5134007930755615, + "step": 1130, + "unclipped_grad_norm": 1.5467443466186523 + }, + { + "bits_per_byte": 2.1181828715329245, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.4682124853134155, + "step": 1140, + "unclipped_grad_norm": 1.5539406538009644 + }, + { + "bits_per_byte": 2.1454147764531926, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4870882034301758, + "step": 1150, + "unclipped_grad_norm": 1.5069210529327393 + }, + { + "bits_per_byte": 2.0759895078834973, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4389662742614746, + "step": 1160, + "unclipped_grad_norm": 1.6428672075271606 + }, + { + "bits_per_byte": 2.0193105614998488, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.39967942237854, + "step": 1170, + "unclipped_grad_norm": 1.5131758451461792 + }, + { + "bits_per_byte": 2.0898077979323717, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.4485443830490112, + "step": 1180, + "unclipped_grad_norm": 1.578716516494751 + }, + { + "bits_per_byte": 2.0831603245112222, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.4439367055892944, + "step": 1190, + "unclipped_grad_norm": 1.4620931148529053 + }, + { + "bits_per_byte": 2.1669989431017767, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.502049207687378, + "step": 1200, + "unclipped_grad_norm": 1.452580213546753 + }, + { + "bits_per_byte": 2.119249679916227, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.468951940536499, + "step": 1210, + "unclipped_grad_norm": 1.6440621614456177 + }, + { + "bits_per_byte": 2.1089788760066996, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4618327617645264, + "step": 1220, + "unclipped_grad_norm": 1.3628098964691162 + }, + { + "bits_per_byte": 2.074528687247069, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4379537105560303, + "step": 1230, + "unclipped_grad_norm": 1.3671886920928955 + }, + { + "bits_per_byte": 2.076871434917122, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.439577579498291, + "step": 1240, + "unclipped_grad_norm": 1.5058611631393433 + }, + { + "bits_per_byte": 2.0583951667522795, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.426770806312561, + "step": 1250, + "unclipped_grad_norm": 1.7707449197769165 + }, + { + "bits_per_byte": 2.127417823936305, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.4746136665344238, + "step": 1260, + "unclipped_grad_norm": 1.6528021097183228 + }, + { + "bits_per_byte": 2.029645514938147, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4068430662155151, + "step": 1270, + "unclipped_grad_norm": 1.6002910137176514 + }, + { + "bits_per_byte": 2.0569790616050754, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4257892370224, + "step": 1280, + "unclipped_grad_norm": 1.4181681871414185 + }, + { + "bits_per_byte": 1.9429846610469341, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.3467743396759033, + "step": 1290, + "unclipped_grad_norm": 1.708141803741455 + }, + { + "bits_per_byte": 1.9866660185916263, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.3770519495010376, + "step": 1300, + "unclipped_grad_norm": 1.4762359857559204 + }, + { + "bits_per_byte": 2.093079079934433, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4508118629455566, + "step": 1310, + "unclipped_grad_norm": 1.4360853433609009 + }, + { + "bits_per_byte": 2.0294391357571118, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4067000150680542, + "step": 1320, + "unclipped_grad_norm": 1.570637583732605 + }, + { + "bits_per_byte": 2.0943651661975853, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.4517033100128174, + "step": 1330, + "unclipped_grad_norm": 1.582815408706665 + }, + { + "bits_per_byte": 2.102091314804946, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4570586681365967, + "step": 1340, + "unclipped_grad_norm": 1.598098874092102 + }, + { + "bits_per_byte": 2.150653024033172, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4907190799713135, + "step": 1350, + "unclipped_grad_norm": 1.8129605054855347 + }, + { + "bits_per_byte": 2.0838604658828848, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.4444220066070557, + "step": 1360, + "unclipped_grad_norm": 1.586807370185852 + }, + { + "bits_per_byte": 2.069158185008576, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.434231162071228, + "step": 1370, + "unclipped_grad_norm": 1.5549230575561523 + }, + { + "bits_per_byte": 2.0359835915703943, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.41123628616333, + "step": 1380, + "unclipped_grad_norm": 1.5326250791549683 + }, + { + "bits_per_byte": 1.9907796716176138, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.3799033164978027, + "step": 1390, + "unclipped_grad_norm": 1.4710757732391357 + }, + { + "bits_per_byte": 1.9945343968512506, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.3825058937072754, + "step": 1400, + "unclipped_grad_norm": 1.551713228225708 + }, + { + "bits_per_byte": 2.014134743622132, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3960918188095093, + "step": 1410, + "unclipped_grad_norm": 1.5555942058563232 + }, + { + "bits_per_byte": 2.0340443151992655, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.4098920822143555, + "step": 1420, + "unclipped_grad_norm": 1.4015933275222778 + }, + { + "bits_per_byte": 1.9404405216927205, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.3450108766555786, + "step": 1430, + "unclipped_grad_norm": 1.6107710599899292 + }, + { + "bits_per_byte": 1.9311199219292106, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.338550329208374, + "step": 1440, + "unclipped_grad_norm": 1.4844772815704346 + }, + { + "bits_per_byte": 1.9790831315324342, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.371795892715454, + "step": 1450, + "unclipped_grad_norm": 1.3763822317123413 + }, + { + "bits_per_byte": 2.002784060647838, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.3882241249084473, + "step": 1460, + "unclipped_grad_norm": 1.563849925994873 + }, + { + "bits_per_byte": 1.9806593525275917, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.372888445854187, + "step": 1470, + "unclipped_grad_norm": 1.495335340499878 + }, + { + "bits_per_byte": 1.9191303234069608, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.3302397727966309, + "step": 1480, + "unclipped_grad_norm": 1.4079076051712036 + }, + { + "bits_per_byte": 2.0531488359877095, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.4231343269348145, + "step": 1490, + "unclipped_grad_norm": 1.521498441696167 + }, + { + "bits_per_byte": 1.9208553093951148, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.3314354419708252, + "step": 1500, + "unclipped_grad_norm": 1.5382153987884521 + }, + { + "bits_per_byte": 2.004471210452802, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.3893935680389404, + "step": 1510, + "unclipped_grad_norm": 1.5830496549606323 + }, + { + "bits_per_byte": 1.9285491252641134, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.336768388748169, + "step": 1520, + "unclipped_grad_norm": 1.547311782836914 + }, + { + "bits_per_byte": 2.0056568588478503, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.3902153968811035, + "step": 1530, + "unclipped_grad_norm": 1.5983179807662964 + }, + { + "bits_per_byte": 2.003735468672411, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.3888835906982422, + "step": 1540, + "unclipped_grad_norm": 1.4397978782653809 + }, + { + "bits_per_byte": 1.9801272382058224, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.372519612312317, + "step": 1550, + "unclipped_grad_norm": 1.374143362045288 + }, + { + "bits_per_byte": 1.9371915974352714, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3427588939666748, + "step": 1560, + "unclipped_grad_norm": 1.4906530380249023 + }, + { + "bits_per_byte": 1.9625256738032681, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.3603191375732422, + "step": 1570, + "unclipped_grad_norm": 1.5533661842346191 + }, + { + "bits_per_byte": 1.9283473896146512, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.3366285562515259, + "step": 1580, + "unclipped_grad_norm": 1.5570321083068848 + }, + { + "bits_per_byte": 1.9149678273081283, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.3273545503616333, + "step": 1590, + "unclipped_grad_norm": 1.4556387662887573 + }, + { + "bits_per_byte": 1.9600014844365548, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.3585695028305054, + "step": 1600, + "unclipped_grad_norm": 1.4785267114639282 + }, + { + "bits_per_byte": 1.9058278093280243, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.321019172668457, + "step": 1610, + "unclipped_grad_norm": 1.447818636894226 + }, + { + "bits_per_byte": 1.9813928585335217, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.373396873474121, + "step": 1620, + "unclipped_grad_norm": 1.6257572174072266 + }, + { + "bits_per_byte": 1.9453573336982375, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.348418951034546, + "step": 1630, + "unclipped_grad_norm": 1.4857879877090454 + }, + { + "bits_per_byte": 1.9392175530624352, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.344163179397583, + "step": 1640, + "unclipped_grad_norm": 1.518738031387329 + }, + { + "bits_per_byte": 1.9240538427358613, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3336524963378906, + "step": 1650, + "unclipped_grad_norm": 1.6569823026657104 + }, + { + "bits_per_byte": 1.9508893276458905, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.3522534370422363, + "step": 1660, + "unclipped_grad_norm": 1.5780971050262451 + }, + { + "bits_per_byte": 1.9101879134926982, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3240413665771484, + "step": 1670, + "unclipped_grad_norm": 1.613952398300171 + }, + { + "bits_per_byte": 1.955958516280072, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.3557671308517456, + "step": 1680, + "unclipped_grad_norm": 1.435490369796753 + }, + { + "bits_per_byte": 1.9591266086916157, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.3579630851745605, + "step": 1690, + "unclipped_grad_norm": 1.6102374792099 + }, + { + "bits_per_byte": 1.9326743011277088, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.339627742767334, + "step": 1700, + "unclipped_grad_norm": 1.5318570137023926 + }, + { + "bits_per_byte": 1.9155991756194457, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3277921676635742, + "step": 1710, + "unclipped_grad_norm": 1.5616979598999023 + }, + { + "bits_per_byte": 1.955118381030607, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.35518479347229, + "step": 1720, + "unclipped_grad_norm": 1.4461182355880737 + }, + { + "bits_per_byte": 1.9762939169007412, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3698625564575195, + "step": 1730, + "unclipped_grad_norm": 1.6153925657272339 + }, + { + "bits_per_byte": 1.830253817024682, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.2686352729797363, + "step": 1740, + "unclipped_grad_norm": 1.398887038230896 + }, + { + "bits_per_byte": 1.9846696439804108, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3756681680679321, + "step": 1750, + "unclipped_grad_norm": 1.5006694793701172 + }, + { + "bits_per_byte": 1.9106534705285838, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3243640661239624, + "step": 1760, + "unclipped_grad_norm": 1.608572006225586 + }, + { + "bits_per_byte": 1.9044405972661649, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.3200576305389404, + "step": 1770, + "unclipped_grad_norm": 1.5702035427093506 + }, + { + "bits_per_byte": 1.922893303807839, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.332848072052002, + "step": 1780, + "unclipped_grad_norm": 1.702118992805481 + }, + { + "bits_per_byte": 1.9601645239895726, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3586825132369995, + "step": 1790, + "unclipped_grad_norm": 1.7346928119659424 + }, + { + "bits_per_byte": 1.939119866916745, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3440954685211182, + "step": 1800, + "unclipped_grad_norm": 1.4843788146972656 + }, + { + "bits_per_byte": 1.9844417669680177, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.3755102157592773, + "step": 1810, + "unclipped_grad_norm": 1.5920997858047485 + }, + { + "bits_per_byte": 1.9448633995249593, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.348076581954956, + "step": 1820, + "unclipped_grad_norm": 1.5741735696792603 + }, + { + "bits_per_byte": 1.9244160381985784, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3339035511016846, + "step": 1830, + "unclipped_grad_norm": 1.5307493209838867 + }, + { + "bits_per_byte": 1.8904549680980012, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.310363531112671, + "step": 1840, + "unclipped_grad_norm": 1.4690238237380981 + }, + { + "bits_per_byte": 1.9109368979372057, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.324560523033142, + "step": 1850, + "unclipped_grad_norm": 1.5920608043670654 + }, + { + "bits_per_byte": 1.8967994080883304, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3147611618041992, + "step": 1860, + "unclipped_grad_norm": 1.4872475862503052 + }, + { + "bits_per_byte": 1.947727254627127, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3500616550445557, + "step": 1870, + "unclipped_grad_norm": 1.5730692148208618 + }, + { + "bits_per_byte": 1.9722923965631163, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.3670889139175415, + "step": 1880, + "unclipped_grad_norm": 1.5368778705596924 + }, + { + "bits_per_byte": 2.015472940628496, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.397019386291504, + "step": 1890, + "unclipped_grad_norm": 1.6881088018417358 + }, + { + "bits_per_byte": 1.944011397472585, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3474860191345215, + "step": 1900, + "unclipped_grad_norm": 1.5687869787216187 + }, + { + "bits_per_byte": 1.9954354139591208, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.383130431175232, + "step": 1910, + "unclipped_grad_norm": 1.6694469451904297 + }, + { + "bits_per_byte": 1.958329985052819, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3574109077453613, + "step": 1920, + "unclipped_grad_norm": 1.661957025527954 + }, + { + "bits_per_byte": 1.8963151049435005, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3144254684448242, + "step": 1930, + "unclipped_grad_norm": 1.5375442504882812 + }, + { + "bits_per_byte": 1.928742777728985, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3369026184082031, + "step": 1940, + "unclipped_grad_norm": 1.5338168144226074 + }, + { + "bits_per_byte": 1.909401952778255, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3234965801239014, + "step": 1950, + "unclipped_grad_norm": 1.5769730806350708 + }, + { + "bits_per_byte": 1.8727693041791753, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.2981047630310059, + "step": 1960, + "unclipped_grad_norm": 1.5053786039352417 + }, + { + "bits_per_byte": 1.854571131960778, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.2854907512664795, + "step": 1970, + "unclipped_grad_norm": 1.487942099571228 + }, + { + "bits_per_byte": 1.9687404388748466, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.3646268844604492, + "step": 1980, + "unclipped_grad_norm": 1.7152676582336426 + }, + { + "bits_per_byte": 1.915661261356407, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.327835202217102, + "step": 1990, + "unclipped_grad_norm": 1.5617114305496216 + }, + { + "bits_per_byte": 1.9440347871131025, + "learning_rate": 3e-05, + "loss_nats": 1.3475022315979004, + "step": 2000, + "unclipped_grad_norm": 1.5631486177444458 + } + ] + }, + { + "architecture": "baseline", + "batch_size": 32, + "canonical_sha256_without_self": "5845bfeda5638b1a967709fd78fe2e64a2d01b40f0ced50d8ea707e36a038b00", + "diagnostic": { + "bits_per_byte": 2.1607138371259955, + "branch_output_rms": [ + 0.024467680603265762, + 0.03252410888671875, + 0.04544902965426445, + 0.012055210769176483, + 0.031342122703790665, + 0.007620926480740309, + 0.016936156898736954, + 0.00682956725358963, + 0.013681754469871521, + 0.00827884953469038, + 0.014085480943322182, + 0.009727690368890762, + 0.016019638627767563, + 0.010121054947376251, + 0.019276386126875877, + 0.010847908444702625, + 0.020217377692461014, + 0.011856775730848312, + 0.022621875628829002, + 0.011872673407196999, + 0.029588066041469574, + 0.014059662818908691, + 0.045646462589502335, + 0.026948343962430954, + 0.026632636785507202, + 0.03553422540426254, + 0.012291512452065945, + 0.041563451290130615, + 0.009686238132417202, + 0.06082121655344963, + 0.010164310224354267, + 0.11796895414590836 + ], + "core_parameter_grad_rms_by_block": [ + 0.0007421151672166933, + 0.0005532402766490476, + 0.00043713799901330333, + 0.00033044463575291374, + 0.0003495170932308389, + 0.00035395098520716656, + 0.00041254654039553357, + 0.0005141244473775357, + 0.0005928146240310421, + 0.0006948711477092932, + 0.0008331387604410197, + 0.00101380642253527, + 0.0008229312767183026, + 0.0006303107513205802, + 0.0005291293137127175, + 0.00039352251161280246 + ], + "depth_weights": [], + "layer_input_rms": [ + 0.053540125489234924, + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.12191740423440933, + 0.1230250671505928, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12828852236270905, + 0.13023048639297485, + 0.12875308096408844, + 0.13129456341266632, + 0.1299162209033966, + 0.12953807413578033, + 0.12750186026096344, + 0.12542401254177094, + 0.12287117540836334, + 0.12003147602081299, + 0.11691971123218536, + 0.10439829528331757, + 0.10274062305688858, + 0.08582619577646255, + 0.08917074650526047, + 0.08274785429239273, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594 + ], + "loss_nats": 1.4976927042007446, + "output_weights": null, + "stream_state_rms": [ + 0.06334399431943893, + 0.07044117897748947, + 0.09905195981264114, + 0.10195541381835938, + 0.12191740423440933, + 0.1230250671505928, + 0.12738725543022156, + 0.12774977087974548, + 0.12893876433372498, + 0.12828852236270905, + 0.13023048639297485, + 0.12875308096408844, + 0.13129456341266632, + 0.1299162209033966, + 0.12953807413578033, + 0.12750186026096344, + 0.12542401254177094, + 0.12287117540836334, + 0.12003147602081299, + 0.11691971123218536, + 0.10439829528331757, + 0.10274062305688858, + 0.08582619577646255, + 0.08917074650526047, + 0.08274785429239273, + 0.09088693559169769, + 0.09016570448875427, + 0.10361852496862411, + 0.10365726053714752, + 0.1343299299478531, + 0.1344241201877594, + 0.2156660109758377 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.032852405924473, + "cross_entropy_nats": 5.567948997020721, + "step": 0 + }, + { + "bits_per_byte": 3.864131094074297, + "cross_entropy_nats": 2.6784115731716156, + "step": 100 + }, + { + "bits_per_byte": 3.4184366785021214, + "cross_entropy_nats": 2.3694797456264496, + "step": 250 + }, + { + "bits_per_byte": 2.907037430552872, + "cross_entropy_nats": 2.015004798769951, + "step": 500 + }, + { + "bits_per_byte": 2.249317836997982, + "cross_entropy_nats": 1.559108316898346, + "step": 1000 + }, + { + "bits_per_byte": 2.061515899441342, + "cross_entropy_nats": 1.428933933377266, + "step": 1500 + }, + { + "bits_per_byte": 1.9984182165621913, + "cross_entropy_nats": 1.385197952389717, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "2990a2e96c6f0079cda2290d66d0516bcaeea84ed2d1fe92d11df7f29cf09831", + "final_mixer_parameters": null, + "initial_common_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "initial_mixer_parameters": null + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 0, + "total": 9541824 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 21.765943476063025, + "measured_steps": 1980, + "median_ms": 21.700910496292636, + "p95_ms": 22.111314281937666, + "peak_allocated_bytes": 3036005376, + "peak_reserved_bytes": 3296722944, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.035497241120767, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.569782257080078, + "step": 1, + "unclipped_grad_norm": 18.44968605041504 + }, + { + "bits_per_byte": 6.81231053683404, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.721933841705322, + "step": 10, + "unclipped_grad_norm": 5.288197994232178 + }, + { + "bits_per_byte": 6.4291448855319615, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.456343650817871, + "step": 20, + "unclipped_grad_norm": 2.9842424392700195 + }, + { + "bits_per_byte": 6.17300905981017, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.278803825378418, + "step": 30, + "unclipped_grad_norm": 2.7347075939178467 + }, + { + "bits_per_byte": 5.661636533665298, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.924347400665283, + "step": 40, + "unclipped_grad_norm": 2.4705514907836914 + }, + { + "bits_per_byte": 5.215856470732993, + "learning_rate": 0.00015, + "loss_nats": 3.615356206893921, + "step": 50, + "unclipped_grad_norm": 2.1234562397003174 + }, + { + "bits_per_byte": 4.766508455725888, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.303891897201538, + "step": 60, + "unclipped_grad_norm": 1.712656855583191 + }, + { + "bits_per_byte": 4.453768668056224, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.0871171951293945, + "step": 70, + "unclipped_grad_norm": 1.2776987552642822 + }, + { + "bits_per_byte": 4.1494068432406905, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.8761496543884277, + "step": 80, + "unclipped_grad_norm": 0.8716163635253906 + }, + { + "bits_per_byte": 3.984153873506144, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.7616050243377686, + "step": 90, + "unclipped_grad_norm": 0.6423436403274536 + }, + { + "bits_per_byte": 3.86688199957051, + "learning_rate": 0.0003, + "loss_nats": 2.6803183555603027, + "step": 100, + "unclipped_grad_norm": 0.5165872573852539 + }, + { + "bits_per_byte": 3.794172550299939, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.62992000579834, + "step": 110, + "unclipped_grad_norm": 0.4223806858062744 + }, + { + "bits_per_byte": 3.679491079051601, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.550428867340088, + "step": 120, + "unclipped_grad_norm": 0.5236887335777283 + }, + { + "bits_per_byte": 3.622362913983898, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5108306407928467, + "step": 130, + "unclipped_grad_norm": 0.49523109197616577 + }, + { + "bits_per_byte": 3.623576423568386, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.511671781539917, + "step": 140, + "unclipped_grad_norm": 0.6828150749206543 + }, + { + "bits_per_byte": 3.59349046655705, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.4908177852630615, + "step": 150, + "unclipped_grad_norm": 0.3979291319847107 + }, + { + "bits_per_byte": 3.5662504784521913, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4719364643096924, + "step": 160, + "unclipped_grad_norm": 0.3806772530078888 + }, + { + "bits_per_byte": 3.5424697493867874, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.4554529190063477, + "step": 170, + "unclipped_grad_norm": 0.5449486970901489 + }, + { + "bits_per_byte": 3.5798230052929827, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.481344223022461, + "step": 180, + "unclipped_grad_norm": 0.4826596975326538 + }, + { + "bits_per_byte": 3.471296105188421, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.4061191082000732, + "step": 190, + "unclipped_grad_norm": 0.5398088097572327 + }, + { + "bits_per_byte": 3.5515917091885507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.461775779724121, + "step": 200, + "unclipped_grad_norm": 0.5915853381156921 + }, + { + "bits_per_byte": 3.4712194009261363, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.4060659408569336, + "step": 210, + "unclipped_grad_norm": 0.651177704334259 + }, + { + "bits_per_byte": 3.4404341624563917, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.3847272396087646, + "step": 220, + "unclipped_grad_norm": 0.5972820520401001 + }, + { + "bits_per_byte": 3.4474469270279737, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.3895881175994873, + "step": 230, + "unclipped_grad_norm": 0.5432801842689514 + }, + { + "bits_per_byte": 3.408640417721988, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.36268949508667, + "step": 240, + "unclipped_grad_norm": 0.6190944314002991 + }, + { + "bits_per_byte": 3.4115892322536814, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.3647334575653076, + "step": 250, + "unclipped_grad_norm": 0.5721532702445984 + }, + { + "bits_per_byte": 3.393394499688301, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3521218299865723, + "step": 260, + "unclipped_grad_norm": 0.5345604419708252 + }, + { + "bits_per_byte": 3.3956017250294748, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.353651762008667, + "step": 270, + "unclipped_grad_norm": 0.9968402981758118 + }, + { + "bits_per_byte": 3.3753586790923187, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.339620351791382, + "step": 280, + "unclipped_grad_norm": 0.7370518445968628 + }, + { + "bits_per_byte": 3.362280774355409, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3305554389953613, + "step": 290, + "unclipped_grad_norm": 0.6113799214363098 + }, + { + "bits_per_byte": 3.3737802223227, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.3385262489318848, + "step": 300, + "unclipped_grad_norm": 0.7529512047767639 + }, + { + "bits_per_byte": 3.2761240016138746, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.270836114883423, + "step": 310, + "unclipped_grad_norm": 0.7870320677757263 + }, + { + "bits_per_byte": 3.3143196285439958, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.297311305999756, + "step": 320, + "unclipped_grad_norm": 0.8573781251907349 + }, + { + "bits_per_byte": 3.338609082290652, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.314147472381592, + "step": 330, + "unclipped_grad_norm": 0.6909317970275879 + }, + { + "bits_per_byte": 3.3247407452903768, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.304534673690796, + "step": 340, + "unclipped_grad_norm": 0.7453696131706238 + }, + { + "bits_per_byte": 3.2655198953269755, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.263485908508301, + "step": 350, + "unclipped_grad_norm": 0.6869654059410095 + }, + { + "bits_per_byte": 3.3318315899854505, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.3094496726989746, + "step": 360, + "unclipped_grad_norm": 0.7315223813056946 + }, + { + "bits_per_byte": 3.272271246269246, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.2681655883789062, + "step": 370, + "unclipped_grad_norm": 0.742798388004303 + }, + { + "bits_per_byte": 3.1709683054315256, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.1979477405548096, + "step": 380, + "unclipped_grad_norm": 1.0848703384399414 + }, + { + "bits_per_byte": 3.2111940155724317, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.225830078125, + "step": 390, + "unclipped_grad_norm": 0.9177330732345581 + }, + { + "bits_per_byte": 3.1480867016648326, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.1820874214172363, + "step": 400, + "unclipped_grad_norm": 0.6777055859565735 + }, + { + "bits_per_byte": 3.1118781622827805, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.156989574432373, + "step": 410, + "unclipped_grad_norm": 0.9509795904159546 + }, + { + "bits_per_byte": 3.092301033169765, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.1434197425842285, + "step": 420, + "unclipped_grad_norm": 0.7206020951271057 + }, + { + "bits_per_byte": 3.0938812097658928, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.144515037536621, + "step": 430, + "unclipped_grad_norm": 0.8741110563278198 + }, + { + "bits_per_byte": 3.099168300418717, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.148179769515991, + "step": 440, + "unclipped_grad_norm": 1.004489779472351 + }, + { + "bits_per_byte": 2.9747378206235697, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.0619311332702637, + "step": 450, + "unclipped_grad_norm": 1.1045528650283813 + }, + { + "bits_per_byte": 3.007708270655177, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.084784507751465, + "step": 460, + "unclipped_grad_norm": 0.981886088848114 + }, + { + "bits_per_byte": 3.020738020249845, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.093816041946411, + "step": 470, + "unclipped_grad_norm": 1.0859395265579224 + }, + { + "bits_per_byte": 2.9696443824356167, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.0584006309509277, + "step": 480, + "unclipped_grad_norm": 0.9367339015007019 + }, + { + "bits_per_byte": 2.9153040561384005, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.0207347869873047, + "step": 490, + "unclipped_grad_norm": 0.8243921995162964 + }, + { + "bits_per_byte": 2.861443561437091, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.9834015369415283, + "step": 500, + "unclipped_grad_norm": 0.9895554780960083 + }, + { + "bits_per_byte": 2.8471105273141846, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.9734666347503662, + "step": 510, + "unclipped_grad_norm": 1.3591206073760986 + }, + { + "bits_per_byte": 2.871144070876357, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.9901254177093506, + "step": 520, + "unclipped_grad_norm": 1.1574872732162476 + }, + { + "bits_per_byte": 2.8462717679259266, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.972885251045227, + "step": 530, + "unclipped_grad_norm": 1.0184993743896484 + }, + { + "bits_per_byte": 2.847254648775608, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.9735665321350098, + "step": 540, + "unclipped_grad_norm": 1.0364676713943481 + }, + { + "bits_per_byte": 2.78121743842791, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.927793025970459, + "step": 550, + "unclipped_grad_norm": 1.061570405960083 + }, + { + "bits_per_byte": 2.8474340266804576, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.9736908674240112, + "step": 560, + "unclipped_grad_norm": 0.9626652598381042 + }, + { + "bits_per_byte": 2.710518122380623, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.8787879943847656, + "step": 570, + "unclipped_grad_norm": 1.0989454984664917 + }, + { + "bits_per_byte": 2.693105910876669, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.8667187690734863, + "step": 580, + "unclipped_grad_norm": 0.9580764174461365 + }, + { + "bits_per_byte": 2.6563119425510795, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8412151336669922, + "step": 590, + "unclipped_grad_norm": 1.046923279762268 + }, + { + "bits_per_byte": 2.6415379449113607, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8309745788574219, + "step": 600, + "unclipped_grad_norm": 1.0676153898239136 + }, + { + "bits_per_byte": 2.674974123961505, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8541507720947266, + "step": 610, + "unclipped_grad_norm": 1.1263657808303833 + }, + { + "bits_per_byte": 2.6408398673315086, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.8304907083511353, + "step": 620, + "unclipped_grad_norm": 1.0577467679977417 + }, + { + "bits_per_byte": 2.6339875625731817, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8257410526275635, + "step": 630, + "unclipped_grad_norm": 1.6502782106399536 + }, + { + "bits_per_byte": 2.632331369645373, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.8245930671691895, + "step": 640, + "unclipped_grad_norm": 1.1552574634552002 + }, + { + "bits_per_byte": 2.569303167757171, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.7809052467346191, + "step": 650, + "unclipped_grad_norm": 1.0123298168182373 + }, + { + "bits_per_byte": 2.5976727379128963, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.8005695343017578, + "step": 660, + "unclipped_grad_norm": 1.0683128833770752 + }, + { + "bits_per_byte": 2.5075208081918188, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.7380809783935547, + "step": 670, + "unclipped_grad_norm": 1.4276427030563354 + }, + { + "bits_per_byte": 2.511354301479551, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7407381534576416, + "step": 680, + "unclipped_grad_norm": 1.3551582098007202 + }, + { + "bits_per_byte": 2.5641455800404462, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.7773302793502808, + "step": 690, + "unclipped_grad_norm": 1.249750018119812 + }, + { + "bits_per_byte": 2.5201479464008174, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7468334436416626, + "step": 700, + "unclipped_grad_norm": 1.1330938339233398 + }, + { + "bits_per_byte": 2.510212508660473, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.739946722984314, + "step": 710, + "unclipped_grad_norm": 1.1181308031082153 + }, + { + "bits_per_byte": 2.4879306083973383, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7245020866394043, + "step": 720, + "unclipped_grad_norm": 1.2327544689178467 + }, + { + "bits_per_byte": 2.444598031668596, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.694466233253479, + "step": 730, + "unclipped_grad_norm": 1.2720160484313965 + }, + { + "bits_per_byte": 2.4135790687763286, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6729655265808105, + "step": 740, + "unclipped_grad_norm": 1.3740355968475342 + }, + { + "bits_per_byte": 2.4610192791382786, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7058485746383667, + "step": 750, + "unclipped_grad_norm": 1.2575128078460693 + }, + { + "bits_per_byte": 2.501580011483065, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.733963131904602, + "step": 760, + "unclipped_grad_norm": 1.2599754333496094 + }, + { + "bits_per_byte": 2.4602904166639217, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.7053433656692505, + "step": 770, + "unclipped_grad_norm": 1.1989470720291138 + }, + { + "bits_per_byte": 2.499036216094153, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7321999073028564, + "step": 780, + "unclipped_grad_norm": 1.1080107688903809 + }, + { + "bits_per_byte": 2.3574134906055053, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.634034514427185, + "step": 790, + "unclipped_grad_norm": 1.203811764717102 + }, + { + "bits_per_byte": 2.355069883022198, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6324100494384766, + "step": 800, + "unclipped_grad_norm": 1.0658862590789795 + }, + { + "bits_per_byte": 2.4456361189492037, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6951857805252075, + "step": 810, + "unclipped_grad_norm": 1.6502019166946411 + }, + { + "bits_per_byte": 2.316791360454361, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.60587739944458, + "step": 820, + "unclipped_grad_norm": 1.1490819454193115 + }, + { + "bits_per_byte": 2.354223728379953, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6318235397338867, + "step": 830, + "unclipped_grad_norm": 1.1026827096939087 + }, + { + "bits_per_byte": 2.280742764937611, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.580890417098999, + "step": 840, + "unclipped_grad_norm": 1.3507121801376343 + }, + { + "bits_per_byte": 2.2850806833403237, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.5838972330093384, + "step": 850, + "unclipped_grad_norm": 1.3287672996520996 + }, + { + "bits_per_byte": 2.2906374427897007, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.5877488851547241, + "step": 860, + "unclipped_grad_norm": 1.1499532461166382 + }, + { + "bits_per_byte": 2.3225503715011526, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6098692417144775, + "step": 870, + "unclipped_grad_norm": 1.2718987464904785 + }, + { + "bits_per_byte": 2.2553460868846997, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.5632867813110352, + "step": 880, + "unclipped_grad_norm": 1.2209762334823608 + }, + { + "bits_per_byte": 2.2870478209008924, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.5852607488632202, + "step": 890, + "unclipped_grad_norm": 1.1711307764053345 + }, + { + "bits_per_byte": 2.2850450829315947, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5838725566864014, + "step": 900, + "unclipped_grad_norm": 1.2894444465637207 + }, + { + "bits_per_byte": 2.2735931021759423, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.575934648513794, + "step": 910, + "unclipped_grad_norm": 1.2043088674545288 + }, + { + "bits_per_byte": 2.3203622362342253, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.608352541923523, + "step": 920, + "unclipped_grad_norm": 1.2657806873321533 + }, + { + "bits_per_byte": 2.2179293693803364, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5373514890670776, + "step": 930, + "unclipped_grad_norm": 1.1855711936950684 + }, + { + "bits_per_byte": 2.271919710983047, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5747747421264648, + "step": 940, + "unclipped_grad_norm": 1.2835133075714111 + }, + { + "bits_per_byte": 2.1784084441426663, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.5099576711654663, + "step": 950, + "unclipped_grad_norm": 1.274409294128418 + }, + { + "bits_per_byte": 2.3027625636408304, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5961533784866333, + "step": 960, + "unclipped_grad_norm": 1.2841647863388062 + }, + { + "bits_per_byte": 2.2296800840005377, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.5454964637756348, + "step": 970, + "unclipped_grad_norm": 1.195611596107483 + }, + { + "bits_per_byte": 2.263055725157578, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5686306953430176, + "step": 980, + "unclipped_grad_norm": 1.468763828277588 + }, + { + "bits_per_byte": 2.247340638452338, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.5577378273010254, + "step": 990, + "unclipped_grad_norm": 1.217491865158081 + }, + { + "bits_per_byte": 2.3139811639392627, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.6039295196533203, + "step": 1000, + "unclipped_grad_norm": 1.142075538635254 + }, + { + "bits_per_byte": 2.172504795686499, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.5058655738830566, + "step": 1010, + "unclipped_grad_norm": 1.1583794355392456 + }, + { + "bits_per_byte": 2.2625459685804206, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.568277359008789, + "step": 1020, + "unclipped_grad_norm": 1.17495596408844 + }, + { + "bits_per_byte": 2.2648876843545684, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.5699005126953125, + "step": 1030, + "unclipped_grad_norm": 1.2168478965759277 + }, + { + "bits_per_byte": 2.2598303625232967, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5663950443267822, + "step": 1040, + "unclipped_grad_norm": 1.127879023551941 + }, + { + "bits_per_byte": 2.2624138859045577, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.568185806274414, + "step": 1050, + "unclipped_grad_norm": 1.1199740171432495 + }, + { + "bits_per_byte": 2.197401348190701, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.5231225490570068, + "step": 1060, + "unclipped_grad_norm": 1.1645094156265259 + }, + { + "bits_per_byte": 2.1289845858856653, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4756996631622314, + "step": 1070, + "unclipped_grad_norm": 1.1294811964035034 + }, + { + "bits_per_byte": 2.1784510958400802, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.509987235069275, + "step": 1080, + "unclipped_grad_norm": 1.0845705270767212 + }, + { + "bits_per_byte": 2.2503272891672212, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.5598080158233643, + "step": 1090, + "unclipped_grad_norm": 1.1829944849014282 + }, + { + "bits_per_byte": 2.1777856949638923, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.509526014328003, + "step": 1100, + "unclipped_grad_norm": 1.1637873649597168 + }, + { + "bits_per_byte": 2.2100276264864447, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.531874418258667, + "step": 1110, + "unclipped_grad_norm": 1.1930115222930908 + }, + { + "bits_per_byte": 2.2095047992278216, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.5315120220184326, + "step": 1120, + "unclipped_grad_norm": 1.087839961051941 + }, + { + "bits_per_byte": 2.1492424223307953, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.489741325378418, + "step": 1130, + "unclipped_grad_norm": 1.2925478219985962 + }, + { + "bits_per_byte": 2.096193169793606, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.4529703855514526, + "step": 1140, + "unclipped_grad_norm": 1.2214796543121338 + }, + { + "bits_per_byte": 2.103899024448165, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.458311676979065, + "step": 1150, + "unclipped_grad_norm": 1.3142298460006714 + }, + { + "bits_per_byte": 2.12195324718779, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4708259105682373, + "step": 1160, + "unclipped_grad_norm": 1.0609383583068848 + }, + { + "bits_per_byte": 2.225786052819702, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.542797327041626, + "step": 1170, + "unclipped_grad_norm": 1.1666395664215088 + }, + { + "bits_per_byte": 2.18731714545736, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.5161327123641968, + "step": 1180, + "unclipped_grad_norm": 1.151012897491455 + }, + { + "bits_per_byte": 2.2596112566260977, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.5662431716918945, + "step": 1190, + "unclipped_grad_norm": 1.2014691829681396 + }, + { + "bits_per_byte": 2.0849009609206046, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.445143222808838, + "step": 1200, + "unclipped_grad_norm": 1.1766064167022705 + }, + { + "bits_per_byte": 2.132521925048611, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.478151559829712, + "step": 1210, + "unclipped_grad_norm": 1.1839491128921509 + }, + { + "bits_per_byte": 2.1092824253854725, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4620431661605835, + "step": 1220, + "unclipped_grad_norm": 1.1294699907302856 + }, + { + "bits_per_byte": 2.1734188834758346, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.506499171257019, + "step": 1230, + "unclipped_grad_norm": 1.1228277683258057 + }, + { + "bits_per_byte": 2.0920416805844284, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4500927925109863, + "step": 1240, + "unclipped_grad_norm": 1.165982961654663 + }, + { + "bits_per_byte": 2.1490356991844584, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.489598035812378, + "step": 1250, + "unclipped_grad_norm": 1.1223030090332031 + }, + { + "bits_per_byte": 2.1172175329136316, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.467543363571167, + "step": 1260, + "unclipped_grad_norm": 1.3497945070266724 + }, + { + "bits_per_byte": 2.073896651005148, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4375156164169312, + "step": 1270, + "unclipped_grad_norm": 1.2885284423828125 + }, + { + "bits_per_byte": 2.1027761497206816, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.457533359527588, + "step": 1280, + "unclipped_grad_norm": 1.1530907154083252 + }, + { + "bits_per_byte": 2.1152459238041406, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.4661767482757568, + "step": 1290, + "unclipped_grad_norm": 1.1667933464050293 + }, + { + "bits_per_byte": 2.136353526527184, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4808074235916138, + "step": 1300, + "unclipped_grad_norm": 1.1147785186767578 + }, + { + "bits_per_byte": 2.064416107376336, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4309442043304443, + "step": 1310, + "unclipped_grad_norm": 1.2490577697753906 + }, + { + "bits_per_byte": 2.038054262686783, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.4126715660095215, + "step": 1320, + "unclipped_grad_norm": 1.109447717666626 + }, + { + "bits_per_byte": 2.0457604613066436, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.418013095855713, + "step": 1330, + "unclipped_grad_norm": 1.2294974327087402 + }, + { + "bits_per_byte": 2.121491989718176, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.470506191253662, + "step": 1340, + "unclipped_grad_norm": 1.1620794534683228 + }, + { + "bits_per_byte": 2.0912457448762356, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4495410919189453, + "step": 1350, + "unclipped_grad_norm": 1.129377007484436 + }, + { + "bits_per_byte": 2.0836024919065905, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.4442431926727295, + "step": 1360, + "unclipped_grad_norm": 1.1350606679916382 + }, + { + "bits_per_byte": 2.1361019159089714, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.480633020401001, + "step": 1370, + "unclipped_grad_norm": 1.1213557720184326 + }, + { + "bits_per_byte": 2.1429779542731175, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.4853991270065308, + "step": 1380, + "unclipped_grad_norm": 1.287025809288025 + }, + { + "bits_per_byte": 2.075487318542978, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.4386181831359863, + "step": 1390, + "unclipped_grad_norm": 1.2520166635513306 + }, + { + "bits_per_byte": 2.0633470632185724, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.4302031993865967, + "step": 1400, + "unclipped_grad_norm": 1.3125451803207397 + }, + { + "bits_per_byte": 1.9928475910115881, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3813366889953613, + "step": 1410, + "unclipped_grad_norm": 1.2774182558059692 + }, + { + "bits_per_byte": 2.009795449358213, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.3930840492248535, + "step": 1420, + "unclipped_grad_norm": 1.091497540473938 + }, + { + "bits_per_byte": 2.132539983226952, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.4781640768051147, + "step": 1430, + "unclipped_grad_norm": 1.3443310260772705 + }, + { + "bits_per_byte": 2.1201240397132133, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.4695580005645752, + "step": 1440, + "unclipped_grad_norm": 1.2321568727493286 + }, + { + "bits_per_byte": 2.0538771825141136, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.423639178276062, + "step": 1450, + "unclipped_grad_norm": 1.2397739887237549 + }, + { + "bits_per_byte": 2.0560635979545325, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.425154685974121, + "step": 1460, + "unclipped_grad_norm": 1.2537752389907837 + }, + { + "bits_per_byte": 2.0139589773529507, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.3959699869155884, + "step": 1470, + "unclipped_grad_norm": 1.2340072393417358 + }, + { + "bits_per_byte": 2.076120558663455, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4390571117401123, + "step": 1480, + "unclipped_grad_norm": 1.2547029256820679 + }, + { + "bits_per_byte": 2.023815131091247, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.402801752090454, + "step": 1490, + "unclipped_grad_norm": 1.1801016330718994 + }, + { + "bits_per_byte": 2.0640116041815064, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.430663824081421, + "step": 1500, + "unclipped_grad_norm": 1.3750890493392944 + }, + { + "bits_per_byte": 2.0712145815649423, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4356565475463867, + "step": 1510, + "unclipped_grad_norm": 1.2360960245132446 + }, + { + "bits_per_byte": 2.0839572921153207, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.4444891214370728, + "step": 1520, + "unclipped_grad_norm": 1.1663187742233276 + }, + { + "bits_per_byte": 2.067838218163204, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4333162307739258, + "step": 1530, + "unclipped_grad_norm": 1.3313146829605103 + }, + { + "bits_per_byte": 1.9819610892119723, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.3737907409667969, + "step": 1540, + "unclipped_grad_norm": 1.1131634712219238 + }, + { + "bits_per_byte": 2.0168095897910017, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3979458808898926, + "step": 1550, + "unclipped_grad_norm": 1.2821860313415527 + }, + { + "bits_per_byte": 1.9600456839778264, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.35860013961792, + "step": 1560, + "unclipped_grad_norm": 1.2062005996704102 + }, + { + "bits_per_byte": 1.9982278962611804, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.385066032409668, + "step": 1570, + "unclipped_grad_norm": 1.2085074186325073 + }, + { + "bits_per_byte": 2.071372117673133, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.4357657432556152, + "step": 1580, + "unclipped_grad_norm": 1.2473686933517456 + }, + { + "bits_per_byte": 2.0737870980565485, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.4374396800994873, + "step": 1590, + "unclipped_grad_norm": 1.264146089553833 + }, + { + "bits_per_byte": 1.9781449661719777, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.3711456060409546, + "step": 1600, + "unclipped_grad_norm": 1.1431471109390259 + }, + { + "bits_per_byte": 2.047029693270011, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.4188928604125977, + "step": 1610, + "unclipped_grad_norm": 1.1972074508666992 + }, + { + "bits_per_byte": 2.0015000381764962, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.3873341083526611, + "step": 1620, + "unclipped_grad_norm": 1.1556639671325684 + }, + { + "bits_per_byte": 2.0871023388516488, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.446669101715088, + "step": 1630, + "unclipped_grad_norm": 1.1515010595321655 + }, + { + "bits_per_byte": 2.010660522092053, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.393683671951294, + "step": 1640, + "unclipped_grad_norm": 1.2786402702331543 + }, + { + "bits_per_byte": 1.9851147350808438, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3759766817092896, + "step": 1650, + "unclipped_grad_norm": 1.2074501514434814 + }, + { + "bits_per_byte": 2.0509651722697044, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.4216207265853882, + "step": 1660, + "unclipped_grad_norm": 1.242030143737793 + }, + { + "bits_per_byte": 2.013557397863186, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3956916332244873, + "step": 1670, + "unclipped_grad_norm": 1.184446096420288 + }, + { + "bits_per_byte": 2.051535122774664, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.4220157861709595, + "step": 1680, + "unclipped_grad_norm": 1.1960315704345703 + }, + { + "bits_per_byte": 2.0243812979778877, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.4031941890716553, + "step": 1690, + "unclipped_grad_norm": 1.2300301790237427 + }, + { + "bits_per_byte": 1.9816625273300745, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.3735837936401367, + "step": 1700, + "unclipped_grad_norm": 1.255767822265625 + }, + { + "bits_per_byte": 1.9922007642616932, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3808883428573608, + "step": 1710, + "unclipped_grad_norm": 1.2163702249526978 + }, + { + "bits_per_byte": 2.0822888884193005, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.4433326721191406, + "step": 1720, + "unclipped_grad_norm": 1.276936650276184 + }, + { + "bits_per_byte": 1.950270878033388, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3518247604370117, + "step": 1730, + "unclipped_grad_norm": 1.1594082117080688 + }, + { + "bits_per_byte": 2.059790118033428, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.4277377128601074, + "step": 1740, + "unclipped_grad_norm": 1.2870954275131226 + }, + { + "bits_per_byte": 1.9446166044209714, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3479055166244507, + "step": 1750, + "unclipped_grad_norm": 1.2092489004135132 + }, + { + "bits_per_byte": 2.019376258872478, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3997249603271484, + "step": 1760, + "unclipped_grad_norm": 1.3044506311416626 + }, + { + "bits_per_byte": 1.9001516939189482, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.317084789276123, + "step": 1770, + "unclipped_grad_norm": 1.122287392616272 + }, + { + "bits_per_byte": 1.9583643815829916, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3574347496032715, + "step": 1780, + "unclipped_grad_norm": 1.2124186754226685 + }, + { + "bits_per_byte": 1.998471423694802, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3852348327636719, + "step": 1790, + "unclipped_grad_norm": 1.1893150806427002 + }, + { + "bits_per_byte": 1.9742430237892021, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3684409856796265, + "step": 1800, + "unclipped_grad_norm": 1.1270394325256348 + }, + { + "bits_per_byte": 2.061872483970075, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.4291810989379883, + "step": 1810, + "unclipped_grad_norm": 1.2006360292434692 + }, + { + "bits_per_byte": 1.9255944633222903, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.3347203731536865, + "step": 1820, + "unclipped_grad_norm": 1.2263959646224976 + }, + { + "bits_per_byte": 2.0007073701386693, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3867846727371216, + "step": 1830, + "unclipped_grad_norm": 1.2227226495742798 + }, + { + "bits_per_byte": 1.9698975381498516, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3654289245605469, + "step": 1840, + "unclipped_grad_norm": 1.2281519174575806 + }, + { + "bits_per_byte": 1.95853945992157, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.3575561046600342, + "step": 1850, + "unclipped_grad_norm": 1.2122280597686768 + }, + { + "bits_per_byte": 1.960907317058649, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3591973781585693, + "step": 1860, + "unclipped_grad_norm": 1.294276237487793 + }, + { + "bits_per_byte": 1.9533636420438538, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3539685010910034, + "step": 1870, + "unclipped_grad_norm": 1.2658793926239014 + }, + { + "bits_per_byte": 1.9780618985516107, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.3710880279541016, + "step": 1880, + "unclipped_grad_norm": 1.2895562648773193 + }, + { + "bits_per_byte": 2.030031959954636, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.4071109294891357, + "step": 1890, + "unclipped_grad_norm": 1.2255303859710693 + }, + { + "bits_per_byte": 1.9487619022547176, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3507788181304932, + "step": 1900, + "unclipped_grad_norm": 1.2463823556900024 + }, + { + "bits_per_byte": 1.9223761519766946, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.3324896097183228, + "step": 1910, + "unclipped_grad_norm": 1.1382452249526978 + }, + { + "bits_per_byte": 1.9529930194312444, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3537116050720215, + "step": 1920, + "unclipped_grad_norm": 1.2469284534454346 + }, + { + "bits_per_byte": 1.9897054679803248, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3791587352752686, + "step": 1930, + "unclipped_grad_norm": 1.1771650314331055 + }, + { + "bits_per_byte": 2.00629869810087, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.390660285949707, + "step": 1940, + "unclipped_grad_norm": 1.2291309833526611 + }, + { + "bits_per_byte": 1.948145860399327, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3503518104553223, + "step": 1950, + "unclipped_grad_norm": 1.1810752153396606 + }, + { + "bits_per_byte": 1.9711624705469475, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3663057088851929, + "step": 1960, + "unclipped_grad_norm": 1.1791642904281616 + }, + { + "bits_per_byte": 1.9627234538517604, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.3604562282562256, + "step": 1970, + "unclipped_grad_norm": 1.2636510133743286 + }, + { + "bits_per_byte": 2.0003914380090344, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.3865656852722168, + "step": 1980, + "unclipped_grad_norm": 1.3609625101089478 + }, + { + "bits_per_byte": 1.986054620267809, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3766281604766846, + "step": 1990, + "unclipped_grad_norm": 1.3484030961990356 + }, + { + "bits_per_byte": 1.9058405360441881, + "learning_rate": 3e-05, + "loss_nats": 1.3210279941558838, + "step": 2000, + "unclipped_grad_norm": 1.2427077293395996 + } + ] + }, + { + "architecture": "full", + "batch_size": 32, + "canonical_sha256_without_self": "13acc5eacf91339b0d3e995797bc5c4c454b381f6633fd11dff269d0ce645c35", + "diagnostic": { + "bits_per_byte": 2.1565845336787794, + "branch_output_rms": [ + 0.022041022777557373, + 0.05085906386375427, + 0.019325949251651764, + 0.04230710119009018, + 0.03489644080400467, + 0.0346454493701458, + 0.021648211404681206, + 0.03129609674215317, + 0.02268683910369873, + 0.027197115123271942, + 0.08029001951217651, + 0.021779853850603104, + 0.0675000324845314, + 0.035913847386837006, + 0.03414471819996834, + 0.028103213757276535, + 0.028869561851024628, + 0.020587174221873283, + 0.02986966073513031, + 0.024125492200255394, + 0.029734039679169655, + 0.03037451021373272, + 0.026668334379792213, + 0.02991577982902527, + 0.029499253258109093, + 0.02968704327940941, + 0.02333390712738037, + 0.03404445946216583, + 0.04889204725623131, + 0.041366904973983765, + 1.430643916130066, + 0.04837163910269737 + ], + "core_parameter_grad_rms_by_block": [ + 0.0005982409384922379, + 0.0003745185363114891, + 0.0002684786750324616, + 0.00021098033769564252, + 0.0002558489145159978, + 0.0004401354335484962, + 0.00044057723716284654, + 0.0003844568143161905, + 0.0008786784638329018, + 0.0009877862033168205, + 0.0014801483865081893, + 0.0009276232454397661, + 0.0008520028126550238, + 0.0007384596329355597, + 0.0006049584856703907, + 0.0005101241707709288 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6729873418807983, + "mean_weights": [ + 0.5415679216384888, + 0.45843207836151123 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0850456953048706, + "mean_weights": [ + 0.35197967290878296, + 0.34077274799346924, + 0.3072476089000702 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3302907943725586, + "mean_weights": [ + 0.32419267296791077, + 0.2350599616765976, + 0.283867746591568, + 0.15687958896160126 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5865752696990967, + "mean_weights": [ + 0.19751642644405365, + 0.24572689831256866, + 0.19522060453891754, + 0.1828836351633072, + 0.17865243554115295 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7134463787078857, + "mean_weights": [ + 0.22016389667987823, + 0.17474377155303955, + 0.2189655750989914, + 0.0946241021156311, + 0.1725701093673706, + 0.11893254518508911 + ], + "sources": 6 + }, + { + "entropy_mean": 1.9155155420303345, + "mean_weights": [ + 0.1364203840494156, + 0.19427818059921265, + 0.1398499608039856, + 0.14137552678585052, + 0.12325861304998398, + 0.16717085242271423, + 0.09764648973941803 + ], + "sources": 7 + }, + { + "entropy_mean": 2.013637065887451, + "mean_weights": [ + 0.15989838540554047, + 0.16567963361740112, + 0.17018935084342957, + 0.08326703310012817, + 0.1255929172039032, + 0.10163220763206482, + 0.08141089230775833, + 0.11232960224151611 + ], + "sources": 8 + }, + { + "entropy_mean": 2.152099132537842, + "mean_weights": [ + 0.10577912628650665, + 0.1679803729057312, + 0.11086834967136383, + 0.12190738320350647, + 0.09470722079277039, + 0.13951465487480164, + 0.07132268697023392, + 0.11843714118003845, + 0.06948307156562805 + ], + "sources": 9 + }, + { + "entropy_mean": 2.2092034816741943, + "mean_weights": [ + 0.1153113842010498, + 0.15279120206832886, + 0.1402735710144043, + 0.07219236344099045, + 0.09849530458450317, + 0.08882901817560196, + 0.054274044930934906, + 0.09643824398517609, + 0.053892385214567184, + 0.1275024712085724 + ], + "sources": 10 + }, + { + "entropy_mean": 2.354865550994873, + "mean_weights": [ + 0.09587182849645615, + 0.1432799994945526, + 0.10607591271400452, + 0.09098442643880844, + 0.08830785751342773, + 0.09942737221717834, + 0.06171825900673866, + 0.0892450362443924, + 0.05928339809179306, + 0.10438614338636398, + 0.061419762670993805 + ], + "sources": 11 + }, + { + "entropy_mean": 2.336538791656494, + "mean_weights": [ + 0.07547644525766373, + 0.13703621923923492, + 0.07857562601566315, + 0.06304672360420227, + 0.057530030608177185, + 0.10460546612739563, + 0.03513855114579201, + 0.10390961170196533, + 0.03422156721353531, + 0.1351599395275116, + 0.0357474610209465, + 0.13955236971378326 + ], + "sources": 12 + }, + { + "entropy_mean": 2.5176010131835938, + "mean_weights": [ + 0.0796700119972229, + 0.11683808267116547, + 0.08914110064506531, + 0.07646283507347107, + 0.07501859962940216, + 0.08741771429777145, + 0.052085015922784805, + 0.0768856555223465, + 0.05020909011363983, + 0.0911530926823616, + 0.05123138427734375, + 0.10421460121870041, + 0.049672819674015045 + ], + "sources": 13 + }, + { + "entropy_mean": 2.4945454597473145, + "mean_weights": [ + 0.06561537086963654, + 0.09355079382658005, + 0.06685388088226318, + 0.053076956421136856, + 0.04973440617322922, + 0.07566464692354202, + 0.03249009698629379, + 0.08296319842338562, + 0.03228827938437462, + 0.11132033169269562, + 0.03331531211733818, + 0.12028384208679199, + 0.03534143790602684, + 0.14750143885612488 + ], + "sources": 14 + }, + { + "entropy_mean": 2.6540651321411133, + "mean_weights": [ + 0.06886160373687744, + 0.09747712314128876, + 0.07814480364322662, + 0.0637136846780777, + 0.06521213799715042, + 0.07648377120494843, + 0.04355360195040703, + 0.06458049267530441, + 0.041585274040699005, + 0.07710802555084229, + 0.04251036420464516, + 0.09308374673128128, + 0.03947167471051216, + 0.09898729622364044, + 0.049226392060518265 + ], + "sources": 15 + }, + { + "entropy_mean": 2.6280198097229004, + "mean_weights": [ + 0.058306701481342316, + 0.08338350057601929, + 0.06455911695957184, + 0.048944804817438126, + 0.04782275855541229, + 0.0634540244936943, + 0.028440188616514206, + 0.06720633804798126, + 0.02799014188349247, + 0.0754932314157486, + 0.027788802981376648, + 0.0831533670425415, + 0.027261856943368912, + 0.10670867562294006, + 0.03730925917625427, + 0.15217722952365875 + ], + "sources": 16 + }, + { + "entropy_mean": 2.767390251159668, + "mean_weights": [ + 0.06178465485572815, + 0.08917313069105148, + 0.07443886995315552, + 0.056030258536338806, + 0.06048816069960594, + 0.0678013488650322, + 0.03849330544471741, + 0.05870400369167328, + 0.036133598536252975, + 0.06866978108882904, + 0.03669501096010208, + 0.07848089933395386, + 0.03052154928445816, + 0.08055596053600311, + 0.03682012856006622, + 0.08804018795490265, + 0.03716913238167763 + ], + "sources": 17 + }, + { + "entropy_mean": 2.7082083225250244, + "mean_weights": [ + 0.05415575951337814, + 0.07410329580307007, + 0.06703503429889679, + 0.04500044137239456, + 0.04791802912950516, + 0.049455322325229645, + 0.027285916730761528, + 0.050750039517879486, + 0.025858767330646515, + 0.06146513298153877, + 0.02560192346572876, + 0.05381612852215767, + 0.02093941532075405, + 0.06895509362220764, + 0.02658342570066452, + 0.09236625581979752, + 0.030610017478466034, + 0.17809998989105225 + ], + "sources": 18 + }, + { + "entropy_mean": 2.843248128890991, + "mean_weights": [ + 0.052123311907052994, + 0.0771489292383194, + 0.06580406427383423, + 0.048162005841732025, + 0.05262218043208122, + 0.056285858154296875, + 0.03198905289173126, + 0.04824855923652649, + 0.0290340818464756, + 0.0561947226524353, + 0.029458094388246536, + 0.06192345544695854, + 0.02214386686682701, + 0.06416039168834686, + 0.026201868429780006, + 0.07109721004962921, + 0.02618592232465744, + 0.1183215081691742, + 0.06289492547512054 + ], + "sources": 19 + }, + { + "entropy_mean": 2.69278621673584, + "mean_weights": [ + 0.04636082425713539, + 0.06267163902521133, + 0.06653302162885666, + 0.03618290647864342, + 0.04716038703918457, + 0.03163399174809456, + 0.025041161105036736, + 0.030430380254983902, + 0.0222464669495821, + 0.03393416851758957, + 0.022439144551753998, + 0.024175649508833885, + 0.015173064544796944, + 0.028799600899219513, + 0.015809716656804085, + 0.04313249513506889, + 0.023028947412967682, + 0.14034098386764526, + 0.09938119351863861, + 0.18552425503730774 + ], + "sources": 20 + }, + { + "entropy_mean": 2.9454524517059326, + "mean_weights": [ + 0.046522434800863266, + 0.06328508257865906, + 0.05805140361189842, + 0.0403708852827549, + 0.04625870659947395, + 0.0469144731760025, + 0.029031485319137573, + 0.040144823491573334, + 0.026257982477545738, + 0.04807794466614723, + 0.026870664209127426, + 0.05034159868955612, + 0.020034663379192352, + 0.05283484235405922, + 0.02309003844857216, + 0.05737072229385376, + 0.02263886108994484, + 0.1005539745092392, + 0.04884829372167587, + 0.0869474709033966, + 0.06555366516113281 + ], + "sources": 21 + }, + { + "entropy_mean": 2.746567964553833, + "mean_weights": [ + 0.0366671159863472, + 0.0499417781829834, + 0.053435664623975754, + 0.02685418725013733, + 0.03673935309052467, + 0.025906749069690704, + 0.01858557015657425, + 0.023889299482107162, + 0.01604510098695755, + 0.027686815708875656, + 0.016290105879306793, + 0.01955290138721466, + 0.01071642804890871, + 0.02449692226946354, + 0.01178262010216713, + 0.036614298820495605, + 0.016314543783664703, + 0.11080493032932281, + 0.07186820358037949, + 0.12521207332611084, + 0.07383045554161072, + 0.16676487028598785 + ], + "sources": 22 + }, + { + "entropy_mean": 3.0187454223632812, + "mean_weights": [ + 0.039449214935302734, + 0.05104460567235947, + 0.049436185508966446, + 0.03413715213537216, + 0.039762645959854126, + 0.038754187524318695, + 0.025393862277269363, + 0.031819939613342285, + 0.02264287881553173, + 0.03933633118867874, + 0.023263612762093544, + 0.03831089287996292, + 0.01755552366375923, + 0.040420740842819214, + 0.019530076533555984, + 0.044742144644260406, + 0.019852448254823685, + 0.08511257916688919, + 0.04219760745763779, + 0.07815174013376236, + 0.05597345530986786, + 0.08694594353437424, + 0.07616622745990753 + ], + "sources": 23 + }, + { + "entropy_mean": 2.858419179916382, + "mean_weights": [ + 0.031151654198765755, + 0.04520886391401291, + 0.04391387850046158, + 0.024464460089802742, + 0.029798392206430435, + 0.023444917052984238, + 0.015139062888920307, + 0.0211042333394289, + 0.013019464910030365, + 0.025321319699287415, + 0.013402963057160378, + 0.01772514171898365, + 0.009747758507728577, + 0.02170286700129509, + 0.010442852973937988, + 0.032705679535865784, + 0.014461012557148933, + 0.0916256383061409, + 0.05753351002931595, + 0.09848864376544952, + 0.06255853176116943, + 0.12920886278152466, + 0.11416684091091156, + 0.05366343632340431 + ], + "sources": 24 + }, + { + "entropy_mean": 3.128446340560913, + "mean_weights": [ + 0.035351768136024475, + 0.04359465092420578, + 0.04451177269220352, + 0.03406605124473572, + 0.0362301729619503, + 0.03687244653701782, + 0.023829244077205658, + 0.0316133014857769, + 0.021598801016807556, + 0.03727494180202484, + 0.0220341756939888, + 0.0375342033803463, + 0.016676215454936028, + 0.03924456238746643, + 0.01815236732363701, + 0.04310743138194084, + 0.018381308764219284, + 0.07054175436496735, + 0.03873004764318466, + 0.06452803313732147, + 0.047743722796440125, + 0.06617903709411621, + 0.06116033345460892, + 0.04463959112763405, + 0.0664040595293045 + ], + "sources": 25 + }, + { + "entropy_mean": 2.981506109237671, + "mean_weights": [ + 0.02693083882331848, + 0.04026307910680771, + 0.03759497031569481, + 0.0250069759786129, + 0.026590794324874878, + 0.021937593817710876, + 0.01446850597858429, + 0.019658321514725685, + 0.012340669520199299, + 0.023457378149032593, + 0.012700674124062061, + 0.017305653542280197, + 0.009962480515241623, + 0.020566467195749283, + 0.010754444636404514, + 0.028647713363170624, + 0.013661740347743034, + 0.07367710024118423, + 0.04099779948592186, + 0.07613380253314972, + 0.050152115523815155, + 0.09418854117393494, + 0.08853016793727875, + 0.046182356774806976, + 0.11083158105611801, + 0.05745822936296463 + ], + "sources": 26 + }, + { + "entropy_mean": 3.135222911834717, + "mean_weights": [ + 0.029316537082195282, + 0.03526735305786133, + 0.03873258829116821, + 0.026845242828130722, + 0.030799202620983124, + 0.024889115244150162, + 0.02016657032072544, + 0.021765198558568954, + 0.018228869885206223, + 0.02429221197962761, + 0.018316473811864853, + 0.02068156562745571, + 0.013477165251970291, + 0.022544220089912415, + 0.013410115614533424, + 0.027712862938642502, + 0.015440389513969421, + 0.06577210128307343, + 0.03711368143558502, + 0.06945986300706863, + 0.04708745330572128, + 0.07969659566879272, + 0.06402754783630371, + 0.0435522086918354, + 0.07197526842355728, + 0.049606479704380035, + 0.06982311606407166 + ], + "sources": 27 + }, + { + "entropy_mean": 3.0709128379821777, + "mean_weights": [ + 0.024143118411302567, + 0.03585217148065567, + 0.033878762274980545, + 0.021963510662317276, + 0.024535927921533585, + 0.021463904529809952, + 0.014758460223674774, + 0.018913857638835907, + 0.013151668943464756, + 0.020739154890179634, + 0.01308390498161316, + 0.01596016064286232, + 0.011353774927556515, + 0.018535705283284187, + 0.01241147518157959, + 0.026461055502295494, + 0.015554314479231834, + 0.060647815465927124, + 0.04129564017057419, + 0.06371809542179108, + 0.04218059778213501, + 0.07493473589420319, + 0.07294894754886627, + 0.03683961182832718, + 0.09312114119529724, + 0.0440625362098217, + 0.09894537925720215, + 0.028544548898935318 + ], + "sources": 28 + }, + { + "entropy_mean": 3.2788450717926025, + "mean_weights": [ + 0.02790890634059906, + 0.036593958735466, + 0.036926425993442535, + 0.03128509223461151, + 0.031192155554890633, + 0.030657097697257996, + 0.022549249231815338, + 0.02624376118183136, + 0.020323745906352997, + 0.026015497744083405, + 0.020450560376048088, + 0.025668887421488762, + 0.015300879254937172, + 0.02599893882870674, + 0.015674903988838196, + 0.029266130179166794, + 0.016373496502637863, + 0.05352478101849556, + 0.03226107731461525, + 0.05289877951145172, + 0.03880441188812256, + 0.06415591388940811, + 0.05095038563013077, + 0.036606382578611374, + 0.05453774333000183, + 0.038659147918224335, + 0.054248981177806854, + 0.033882662653923035, + 0.05104004219174385 + ], + "sources": 29 + }, + { + "entropy_mean": 3.1905009746551514, + "mean_weights": [ + 0.02360181324183941, + 0.03278586268424988, + 0.033213719725608826, + 0.02081158384680748, + 0.02500304952263832, + 0.020195089280605316, + 0.016563309356570244, + 0.017775937914848328, + 0.015179699286818504, + 0.02013733983039856, + 0.014823689125478268, + 0.01671958714723587, + 0.01209183968603611, + 0.019336730241775513, + 0.013511688448488712, + 0.025129228830337524, + 0.016828063875436783, + 0.053963836282491684, + 0.03764616698026657, + 0.056100994348526, + 0.038609620183706284, + 0.06754268705844879, + 0.05626947805285454, + 0.03495379537343979, + 0.0710434541106224, + 0.04068417474627495, + 0.08452381193637848, + 0.025568395853042603, + 0.0741381049156189, + 0.015247252769768238 + ], + "sources": 30 + }, + { + "entropy_mean": 3.3770389556884766, + "mean_weights": [ + 0.03834422677755356, + 0.024918753653764725, + 0.026876211166381836, + 0.03044244833290577, + 0.030749086290597916, + 0.03265486657619476, + 0.04033009707927704, + 0.03379246965050697, + 0.04460468888282776, + 0.03375828266143799, + 0.044760994613170624, + 0.03831042721867561, + 0.0547671802341938, + 0.03856009989976883, + 0.0571574941277504, + 0.031226547434926033, + 0.050398603081703186, + 0.019914820790290833, + 0.02455613762140274, + 0.01995367556810379, + 0.026365697383880615, + 0.019913647323846817, + 0.022452158853411674, + 0.02588173747062683, + 0.020718030631542206, + 0.024731352925300598, + 0.022264346480369568, + 0.033183224499225616, + 0.02518976479768753, + 0.035754814743995667, + 0.027468115091323853 + ], + "sources": 31 + }, + { + "entropy_mean": 3.1546459197998047, + "mean_weights": [ + 0.02086162194609642, + 0.029070349410176277, + 0.030426999554038048, + 0.01851825974881649, + 0.02319766953587532, + 0.018016867339611053, + 0.01649550348520279, + 0.01648765429854393, + 0.01461261697113514, + 0.017644580453634262, + 0.014277349226176739, + 0.014900920912623405, + 0.010711734183132648, + 0.016357004642486572, + 0.011678468436002731, + 0.021006643772125244, + 0.01370309665799141, + 0.04932617396116257, + 0.031208142638206482, + 0.04631024971604347, + 0.03265630453824997, + 0.05976565182209015, + 0.043623000383377075, + 0.02866201475262642, + 0.06050873547792435, + 0.031501032412052155, + 0.08868618309497833, + 0.02194211632013321, + 0.09221703559160233, + 0.01105012372136116, + 0.09231007099151611, + 0.0022658214438706636 + ], + "sources": 32 + } + ], + "layer_input_rms": [ + 0.05461336672306061, + 0.03162544220685959, + 0.02765912003815174, + 0.02553493343293667, + 0.02228466607630253, + 0.024269601330161095, + 0.020151648670434952, + 0.02089775912463665, + 0.018591508269309998, + 0.018769782036542892, + 0.017839200794696808, + 0.022477135062217712, + 0.0208187997341156, + 0.02693997137248516, + 0.02364688739180565, + 0.024786626920104027, + 0.02244819700717926, + 0.01847153715789318, + 0.01782904379069805, + 0.010763376951217651, + 0.014036872424185276, + 0.008387424051761627, + 0.010053877718746662, + 0.007874744012951851, + 0.009214057587087154, + 0.007483004592359066, + 0.00694533996284008, + 0.007451741956174374, + 0.0071115195751190186, + 0.007179815322160721, + 0.013574209995567799, + 0.008806144818663597 + ], + "loss_nats": 1.4948304891586304, + "output_weights": { + "entropy_mean": 3.2301199436187744, + "mean_weights": [ + 0.026340482756495476, + 0.025305280461907387, + 0.027144940569996834, + 0.01731031946837902, + 0.02435101754963398, + 0.01820286177098751, + 0.020118871703743935, + 0.015996068716049194, + 0.019381744787096977, + 0.01712716743350029, + 0.019458027556538582, + 0.01558986958116293, + 0.01378586981445551, + 0.01643575355410576, + 0.013341902755200863, + 0.02009432017803192, + 0.015623869374394417, + 0.037450648844242096, + 0.028013382107019424, + 0.03596869856119156, + 0.030010312795639038, + 0.04403690993785858, + 0.02777731418609619, + 0.02930103801190853, + 0.035311371088027954, + 0.02818802371621132, + 0.055513784289360046, + 0.02441634237766266, + 0.07792899012565613, + 0.01957770064473152, + 0.08567806333303452, + 0.0030412522610276937, + 0.11217780411243439 + ], + "sources": 33 + }, + "stream_state_rms": [ + 0.04164384678006172, + 0.044926099479198456, + 0.040089137852191925, + 0.04054243862628937, + 0.03965730220079422, + 0.038980793207883835, + 0.03725782036781311, + 0.03664333373308182, + 0.035495493561029434, + 0.03482290729880333, + 0.04060528427362442, + 0.039477188140153885, + 0.042101990431547165, + 0.0417180135846138, + 0.041285403072834015, + 0.04062855243682861, + 0.040065914392471313, + 0.039282262325286865, + 0.03886580839753151, + 0.038292769342660904, + 0.037945639342069626, + 0.03764813393354416, + 0.03725530579686165, + 0.03698969632387161, + 0.036729857325553894, + 0.03649326041340828, + 0.036105964332818985, + 0.036036837846040726, + 0.03653828799724579, + 0.036703966557979584, + 0.2554716467857361, + 0.25171196460723877 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.006702013904155, + "cross_entropy_nats": 5.549822926521301, + "step": 0 + }, + { + "bits_per_byte": 3.877691539133865, + "cross_entropy_nats": 2.687810957431793, + "step": 100 + }, + { + "bits_per_byte": 3.4643856702941025, + "cross_entropy_nats": 2.4013291597366333, + "step": 250 + }, + { + "bits_per_byte": 2.967935433829656, + "cross_entropy_nats": 2.057216078042984, + "step": 500 + }, + { + "bits_per_byte": 2.2438350085906444, + "cross_entropy_nats": 1.5553079098463058, + "step": 1000 + }, + { + "bits_per_byte": 2.046430935671032, + "cross_entropy_nats": 1.4184778332710266, + "step": 1500 + }, + { + "bits_per_byte": 1.9839206165402405, + "cross_entropy_nats": 1.375148981809616, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "03ece6a9c412a399a983b2bcc1bc20db17872ce4143a8813b6687c8e4542cbc7", + "final_mixer_parameters": "6e9380086b68678046261228fa6322f95d4cd358283d127c6a291df7d74b41c0", + "initial_common_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 146.85863199750506, + "measured_steps": 1980, + "median_ms": 146.81190850387793, + "p95_ms": 147.1791482414119, + "peak_allocated_bytes": 14260571648, + "peak_reserved_bytes": 14950596608, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.006538114437884, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.549709320068359, + "step": 1, + "unclipped_grad_norm": 11.608716011047363 + }, + { + "bits_per_byte": 6.899441763275363, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.7823286056518555, + "step": 10, + "unclipped_grad_norm": 5.470485687255859 + }, + { + "bits_per_byte": 6.44656604013376, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.468419075012207, + "step": 20, + "unclipped_grad_norm": 2.9642276763916016 + }, + { + "bits_per_byte": 6.212329109311632, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.306058406829834, + "step": 30, + "unclipped_grad_norm": 2.729346752166748 + }, + { + "bits_per_byte": 5.75943962755796, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.9921393394470215, + "step": 40, + "unclipped_grad_norm": 2.547452449798584 + }, + { + "bits_per_byte": 5.281263880613231, + "learning_rate": 0.00015, + "loss_nats": 3.6606931686401367, + "step": 50, + "unclipped_grad_norm": 2.1190736293792725 + }, + { + "bits_per_byte": 4.795200321369331, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.323779582977295, + "step": 60, + "unclipped_grad_norm": 1.694022297859192 + }, + { + "bits_per_byte": 4.476015999806536, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.1025378704071045, + "step": 70, + "unclipped_grad_norm": 1.266960859298706 + }, + { + "bits_per_byte": 4.170242885358022, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.890592098236084, + "step": 80, + "unclipped_grad_norm": 0.8680030703544617 + }, + { + "bits_per_byte": 4.013085138999587, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.78165864944458, + "step": 90, + "unclipped_grad_norm": 0.840398907661438 + }, + { + "bits_per_byte": 3.88356982014903, + "learning_rate": 0.0003, + "loss_nats": 2.691885471343994, + "step": 100, + "unclipped_grad_norm": 0.586207389831543 + }, + { + "bits_per_byte": 3.817232672058228, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.645904064178467, + "step": 110, + "unclipped_grad_norm": 0.48479172587394714 + }, + { + "bits_per_byte": 3.6949144831809777, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.561119556427002, + "step": 120, + "unclipped_grad_norm": 0.43896225094795227 + }, + { + "bits_per_byte": 3.6381780945919404, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.5217928886413574, + "step": 130, + "unclipped_grad_norm": 0.4176184833049774 + }, + { + "bits_per_byte": 3.634649698526839, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5193471908569336, + "step": 140, + "unclipped_grad_norm": 0.6141960024833679 + }, + { + "bits_per_byte": 3.612379665066614, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.503910779953003, + "step": 150, + "unclipped_grad_norm": 0.5053157806396484 + }, + { + "bits_per_byte": 3.5916922159596285, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4895713329315186, + "step": 160, + "unclipped_grad_norm": 0.6958001255989075 + }, + { + "bits_per_byte": 3.56339384662136, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.469956398010254, + "step": 170, + "unclipped_grad_norm": 0.7530415654182434 + }, + { + "bits_per_byte": 3.6052977634693852, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.499001979827881, + "step": 180, + "unclipped_grad_norm": 0.5429796576499939 + }, + { + "bits_per_byte": 3.504544823149124, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.4291653633117676, + "step": 190, + "unclipped_grad_norm": 0.6759348511695862 + }, + { + "bits_per_byte": 3.5856738550753353, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.4853997230529785, + "step": 200, + "unclipped_grad_norm": 0.588739275932312 + }, + { + "bits_per_byte": 3.5062230298562436, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.430328607559204, + "step": 210, + "unclipped_grad_norm": 0.8195160627365112 + }, + { + "bits_per_byte": 3.4750140261347733, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.408696174621582, + "step": 220, + "unclipped_grad_norm": 0.819287896156311 + }, + { + "bits_per_byte": 3.4739226242323977, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.4079396724700928, + "step": 230, + "unclipped_grad_norm": 0.5483809113502502 + }, + { + "bits_per_byte": 3.4382032035093992, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.383180856704712, + "step": 240, + "unclipped_grad_norm": 0.7085433006286621 + }, + { + "bits_per_byte": 3.455764351989001, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.395353317260742, + "step": 250, + "unclipped_grad_norm": 2.141403913497925 + }, + { + "bits_per_byte": 3.4287520689138846, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3766298294067383, + "step": 260, + "unclipped_grad_norm": 0.8582319021224976 + }, + { + "bits_per_byte": 3.424241307947055, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.3735032081604004, + "step": 270, + "unclipped_grad_norm": 1.4108314514160156 + }, + { + "bits_per_byte": 3.413981854892485, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.366391897201538, + "step": 280, + "unclipped_grad_norm": 0.9375557899475098 + }, + { + "bits_per_byte": 3.398295317307288, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.3555188179016113, + "step": 290, + "unclipped_grad_norm": 1.4061901569366455 + }, + { + "bits_per_byte": 3.396630525246936, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.3543648719787598, + "step": 300, + "unclipped_grad_norm": 1.0047663450241089 + }, + { + "bits_per_byte": 3.311230476169198, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.2951700687408447, + "step": 310, + "unclipped_grad_norm": 1.5728901624679565 + }, + { + "bits_per_byte": 3.344908462826455, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.318513870239258, + "step": 320, + "unclipped_grad_norm": 1.4831609725952148 + }, + { + "bits_per_byte": 3.3727070505813157, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.337782382965088, + "step": 330, + "unclipped_grad_norm": 0.968090832233429 + }, + { + "bits_per_byte": 3.3683593291675042, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.334768772125244, + "step": 340, + "unclipped_grad_norm": 1.152740716934204 + }, + { + "bits_per_byte": 3.3181596571724605, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.2999730110168457, + "step": 350, + "unclipped_grad_norm": 2.3692808151245117 + }, + { + "bits_per_byte": 3.3760208122981403, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.3400793075561523, + "step": 360, + "unclipped_grad_norm": 1.6198455095291138 + }, + { + "bits_per_byte": 3.3181259485728916, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.2999496459960938, + "step": 370, + "unclipped_grad_norm": 1.0024447441101074 + }, + { + "bits_per_byte": 3.2332325603845953, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.2411060333251953, + "step": 380, + "unclipped_grad_norm": 1.3168435096740723 + }, + { + "bits_per_byte": 3.2682465082737546, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.265375852584839, + "step": 390, + "unclipped_grad_norm": 2.0385279655456543 + }, + { + "bits_per_byte": 3.2118998323715724, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.2263193130493164, + "step": 400, + "unclipped_grad_norm": 1.3429930210113525 + }, + { + "bits_per_byte": 3.1724305019291617, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.1989612579345703, + "step": 410, + "unclipped_grad_norm": 1.7254024744033813 + }, + { + "bits_per_byte": 3.160277519888592, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.190537452697754, + "step": 420, + "unclipped_grad_norm": 1.4813185930252075 + }, + { + "bits_per_byte": 3.1506513069544986, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.1838650703430176, + "step": 430, + "unclipped_grad_norm": 1.0130081176757812 + }, + { + "bits_per_byte": 3.1718701824526505, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.198572874069214, + "step": 440, + "unclipped_grad_norm": 1.3835983276367188 + }, + { + "bits_per_byte": 3.0539337675540827, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.116825580596924, + "step": 450, + "unclipped_grad_norm": 2.7714529037475586 + }, + { + "bits_per_byte": 3.0615209541795463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.122084617614746, + "step": 460, + "unclipped_grad_norm": 1.4769980907440186 + }, + { + "bits_per_byte": 3.068287439595093, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.126774787902832, + "step": 470, + "unclipped_grad_norm": 1.316390037536621 + }, + { + "bits_per_byte": 3.03732712278677, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.1053147315979004, + "step": 480, + "unclipped_grad_norm": 1.4704573154449463 + }, + { + "bits_per_byte": 2.9950641061637433, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.0760202407836914, + "step": 490, + "unclipped_grad_norm": 1.1373145580291748 + }, + { + "bits_per_byte": 2.925527392836289, + "learning_rate": 0.0002715339687685131, + "loss_nats": 2.0278210639953613, + "step": 500, + "unclipped_grad_norm": 1.3723366260528564 + }, + { + "bits_per_byte": 2.8913826452645908, + "learning_rate": 0.00027014843263288514, + "loss_nats": 2.0041537284851074, + "step": 510, + "unclipped_grad_norm": 1.5338191986083984 + }, + { + "bits_per_byte": 2.946270564322154, + "learning_rate": 0.00026873414996567596, + "loss_nats": 2.04219913482666, + "step": 520, + "unclipped_grad_norm": 1.86524498462677 + }, + { + "bits_per_byte": 2.9128632783573556, + "learning_rate": 0.0002672915074176452, + "loss_nats": 2.01904296875, + "step": 530, + "unclipped_grad_norm": 1.9401085376739502 + }, + { + "bits_per_byte": 2.912016435784507, + "learning_rate": 0.00026582089939286075, + "loss_nats": 2.018455982208252, + "step": 540, + "unclipped_grad_norm": 1.6132173538208008 + }, + { + "bits_per_byte": 2.8614452812635998, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.9834027290344238, + "step": 550, + "unclipped_grad_norm": 1.5073490142822266 + }, + { + "bits_per_byte": 2.8991259921370376, + "learning_rate": 0.000262797402646797, + "loss_nats": 2.009521007537842, + "step": 560, + "unclipped_grad_norm": 1.4087388515472412 + }, + { + "bits_per_byte": 2.772488630966019, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.9217426776885986, + "step": 570, + "unclipped_grad_norm": 1.5618728399276733 + }, + { + "bits_per_byte": 2.7610638234692026, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.9138236045837402, + "step": 580, + "unclipped_grad_norm": 1.3366087675094604 + }, + { + "bits_per_byte": 2.7248253591059, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.8887050151824951, + "step": 590, + "unclipped_grad_norm": 2.0284080505371094 + }, + { + "bits_per_byte": 2.7092691843700574, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8779222965240479, + "step": 600, + "unclipped_grad_norm": 1.784683346748352 + }, + { + "bits_per_byte": 2.749075256842858, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.9055137634277344, + "step": 610, + "unclipped_grad_norm": 1.6987285614013672 + }, + { + "bits_per_byte": 2.703916568327254, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.8742121458053589, + "step": 620, + "unclipped_grad_norm": 1.7045960426330566 + }, + { + "bits_per_byte": 2.70965838110896, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8781920671463013, + "step": 630, + "unclipped_grad_norm": 3.2970776557922363 + }, + { + "bits_per_byte": 2.6848615785422583, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.8610042333602905, + "step": 640, + "unclipped_grad_norm": 2.0453710556030273 + }, + { + "bits_per_byte": 2.613054866206066, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.8112316131591797, + "step": 650, + "unclipped_grad_norm": 1.57058584690094 + }, + { + "bits_per_byte": 2.6428233432439097, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.8318655490875244, + "step": 660, + "unclipped_grad_norm": 1.996256947517395 + }, + { + "bits_per_byte": 2.5613101020756712, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.775364875793457, + "step": 670, + "unclipped_grad_norm": 1.7073612213134766 + }, + { + "bits_per_byte": 2.5564693064018353, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7720094919204712, + "step": 680, + "unclipped_grad_norm": 1.8557730913162231 + }, + { + "bits_per_byte": 2.593966167821501, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.7980003356933594, + "step": 690, + "unclipped_grad_norm": 1.7103161811828613 + }, + { + "bits_per_byte": 2.5405347698340948, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7609645128250122, + "step": 700, + "unclipped_grad_norm": 1.5256067514419556 + }, + { + "bits_per_byte": 2.5351795740515284, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.75725257396698, + "step": 710, + "unclipped_grad_norm": 1.4567885398864746 + }, + { + "bits_per_byte": 2.527787759717445, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.7521289587020874, + "step": 720, + "unclipped_grad_norm": 1.846672534942627 + }, + { + "bits_per_byte": 2.4773526434733704, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.717170000076294, + "step": 730, + "unclipped_grad_norm": 2.0692570209503174 + }, + { + "bits_per_byte": 2.4480091355658087, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6968306303024292, + "step": 740, + "unclipped_grad_norm": 1.6199586391448975 + }, + { + "bits_per_byte": 2.487858031718674, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.7244517803192139, + "step": 750, + "unclipped_grad_norm": 1.9068433046340942 + }, + { + "bits_per_byte": 2.5264039873086026, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.7511698007583618, + "step": 760, + "unclipped_grad_norm": 1.8140562772750854 + }, + { + "bits_per_byte": 2.477230191825956, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.7170851230621338, + "step": 770, + "unclipped_grad_norm": 1.8399772644042969 + }, + { + "bits_per_byte": 2.509324562234068, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7393312454223633, + "step": 780, + "unclipped_grad_norm": 1.9230952262878418 + }, + { + "bits_per_byte": 2.379177379124241, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.6491200923919678, + "step": 790, + "unclipped_grad_norm": 1.9386330842971802 + }, + { + "bits_per_byte": 2.389327107247561, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6561553478240967, + "step": 800, + "unclipped_grad_norm": 2.0783488750457764 + }, + { + "bits_per_byte": 2.4505555106944836, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.698595643043518, + "step": 810, + "unclipped_grad_norm": 1.9072158336639404 + }, + { + "bits_per_byte": 2.3393465411497165, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.621511459350586, + "step": 820, + "unclipped_grad_norm": 1.9094164371490479 + }, + { + "bits_per_byte": 2.368450821189928, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6416850090026855, + "step": 830, + "unclipped_grad_norm": 1.8504053354263306 + }, + { + "bits_per_byte": 2.27160739048908, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.5745582580566406, + "step": 840, + "unclipped_grad_norm": 1.7616382837295532 + }, + { + "bits_per_byte": 2.3069735588472065, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.5990722179412842, + "step": 850, + "unclipped_grad_norm": 1.8892234563827515 + }, + { + "bits_per_byte": 2.292006252707918, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.5886976718902588, + "step": 860, + "unclipped_grad_norm": 1.8934842348098755 + }, + { + "bits_per_byte": 2.3343556046216776, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.6180520057678223, + "step": 870, + "unclipped_grad_norm": 2.397369861602783 + }, + { + "bits_per_byte": 2.2624235169330063, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.568192481994629, + "step": 880, + "unclipped_grad_norm": 1.5984070301055908 + }, + { + "bits_per_byte": 2.3019181288250943, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.595568060874939, + "step": 890, + "unclipped_grad_norm": 2.054062843322754 + }, + { + "bits_per_byte": 2.2608538312785815, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.567104458808899, + "step": 900, + "unclipped_grad_norm": 1.6685210466384888 + }, + { + "bits_per_byte": 2.2727890832831585, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.575377345085144, + "step": 910, + "unclipped_grad_norm": 1.5382899045944214 + }, + { + "bits_per_byte": 2.3132535053434617, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.603425145149231, + "step": 920, + "unclipped_grad_norm": 1.6540344953536987 + }, + { + "bits_per_byte": 2.2234663508248644, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.541189432144165, + "step": 930, + "unclipped_grad_norm": 2.3204345703125 + }, + { + "bits_per_byte": 2.2575312984465628, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5648014545440674, + "step": 940, + "unclipped_grad_norm": 1.7289704084396362 + }, + { + "bits_per_byte": 2.168401289636912, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.503021240234375, + "step": 950, + "unclipped_grad_norm": 2.265657424926758 + }, + { + "bits_per_byte": 2.286551822935804, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5849169492721558, + "step": 960, + "unclipped_grad_norm": 1.6856869459152222 + }, + { + "bits_per_byte": 2.2305083524470932, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.5460705757141113, + "step": 970, + "unclipped_grad_norm": 2.097700834274292 + }, + { + "bits_per_byte": 2.246348642522161, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5570502281188965, + "step": 980, + "unclipped_grad_norm": 1.5164785385131836 + }, + { + "bits_per_byte": 2.235582012630197, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.549587368965149, + "step": 990, + "unclipped_grad_norm": 1.5587987899780273 + }, + { + "bits_per_byte": 2.308289570091609, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.5999844074249268, + "step": 1000, + "unclipped_grad_norm": 1.7392001152038574 + }, + { + "bits_per_byte": 2.158439022603033, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.4961159229278564, + "step": 1010, + "unclipped_grad_norm": 1.6276477575302124 + }, + { + "bits_per_byte": 2.2758154619903914, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5774750709533691, + "step": 1020, + "unclipped_grad_norm": 1.835997462272644 + }, + { + "bits_per_byte": 2.246595953574102, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.5572216510772705, + "step": 1030, + "unclipped_grad_norm": 1.6562429666519165 + }, + { + "bits_per_byte": 2.2524282292301616, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5612642765045166, + "step": 1040, + "unclipped_grad_norm": 1.9437512159347534 + }, + { + "bits_per_byte": 2.2464858846775497, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.557145357131958, + "step": 1050, + "unclipped_grad_norm": 1.647452712059021 + }, + { + "bits_per_byte": 2.1836625141265253, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.5135995149612427, + "step": 1060, + "unclipped_grad_norm": 1.6598215103149414 + }, + { + "bits_per_byte": 2.127502955348482, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4746726751327515, + "step": 1070, + "unclipped_grad_norm": 2.0589494705200195 + }, + { + "bits_per_byte": 2.155262675024248, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.493914246559143, + "step": 1080, + "unclipped_grad_norm": 1.7585431337356567 + }, + { + "bits_per_byte": 2.224907565439095, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.5421884059906006, + "step": 1090, + "unclipped_grad_norm": 1.6346040964126587 + }, + { + "bits_per_byte": 2.149358338637477, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.4898216724395752, + "step": 1100, + "unclipped_grad_norm": 1.4800000190734863 + }, + { + "bits_per_byte": 2.1838417200487243, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.5137237310409546, + "step": 1110, + "unclipped_grad_norm": 1.6107548475265503 + }, + { + "bits_per_byte": 2.179564683504417, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.5107591152191162, + "step": 1120, + "unclipped_grad_norm": 1.684011459350586 + }, + { + "bits_per_byte": 2.135530761525456, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.4802371263504028, + "step": 1130, + "unclipped_grad_norm": 1.8119640350341797 + }, + { + "bits_per_byte": 2.0872667542658734, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.4467830657958984, + "step": 1140, + "unclipped_grad_norm": 1.5754022598266602 + }, + { + "bits_per_byte": 2.0853348731487316, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4454439878463745, + "step": 1150, + "unclipped_grad_norm": 1.8813883066177368 + }, + { + "bits_per_byte": 2.114025190948316, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4653306007385254, + "step": 1160, + "unclipped_grad_norm": 1.5591171979904175 + }, + { + "bits_per_byte": 2.193285287407601, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.520269513130188, + "step": 1170, + "unclipped_grad_norm": 1.5820043087005615 + }, + { + "bits_per_byte": 2.166971253894988, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.5020300149917603, + "step": 1180, + "unclipped_grad_norm": 1.5079671144485474 + }, + { + "bits_per_byte": 2.254331733209911, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.5625836849212646, + "step": 1190, + "unclipped_grad_norm": 2.0068087577819824 + }, + { + "bits_per_byte": 2.0666226447869054, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.4324736595153809, + "step": 1200, + "unclipped_grad_norm": 1.7765110731124878 + }, + { + "bits_per_byte": 2.130135493785239, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.4764974117279053, + "step": 1210, + "unclipped_grad_norm": 1.6730155944824219 + }, + { + "bits_per_byte": 2.086766284751863, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4464361667633057, + "step": 1220, + "unclipped_grad_norm": 1.7207565307617188 + }, + { + "bits_per_byte": 2.1609322550925913, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4978440999984741, + "step": 1230, + "unclipped_grad_norm": 1.6892027854919434 + }, + { + "bits_per_byte": 2.0648900915621136, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4312727451324463, + "step": 1240, + "unclipped_grad_norm": 1.7397513389587402 + }, + { + "bits_per_byte": 2.123108626636286, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.4716267585754395, + "step": 1250, + "unclipped_grad_norm": 1.5211966037750244 + }, + { + "bits_per_byte": 2.0856349828744873, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.4456520080566406, + "step": 1260, + "unclipped_grad_norm": 1.6454085111618042 + }, + { + "bits_per_byte": 2.0401644898128692, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.4141342639923096, + "step": 1270, + "unclipped_grad_norm": 1.9636037349700928 + }, + { + "bits_per_byte": 2.0968991585753978, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4534597396850586, + "step": 1280, + "unclipped_grad_norm": 1.6678627729415894 + }, + { + "bits_per_byte": 2.089551199817284, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.4483665227890015, + "step": 1290, + "unclipped_grad_norm": 1.7391536235809326 + }, + { + "bits_per_byte": 2.0947576306068543, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4519753456115723, + "step": 1300, + "unclipped_grad_norm": 1.4500073194503784 + }, + { + "bits_per_byte": 2.036971803882252, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.4119212627410889, + "step": 1310, + "unclipped_grad_norm": 1.8026293516159058 + }, + { + "bits_per_byte": 2.0194985385372415, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.399809718132019, + "step": 1320, + "unclipped_grad_norm": 1.6847851276397705 + }, + { + "bits_per_byte": 2.0178315107024285, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.3986542224884033, + "step": 1330, + "unclipped_grad_norm": 1.6490837335586548 + }, + { + "bits_per_byte": 2.078929895265299, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4410043954849243, + "step": 1340, + "unclipped_grad_norm": 1.5964237451553345 + }, + { + "bits_per_byte": 2.0637579297714836, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4304879903793335, + "step": 1350, + "unclipped_grad_norm": 1.6262972354888916 + }, + { + "bits_per_byte": 2.0554370651574394, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.4247204065322876, + "step": 1360, + "unclipped_grad_norm": 1.5097728967666626 + }, + { + "bits_per_byte": 2.102097850145679, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.4570631980895996, + "step": 1370, + "unclipped_grad_norm": 1.604797124862671 + }, + { + "bits_per_byte": 2.1126727193819312, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.464393138885498, + "step": 1380, + "unclipped_grad_norm": 1.825904130935669 + }, + { + "bits_per_byte": 2.046610743532509, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.418602466583252, + "step": 1390, + "unclipped_grad_norm": 2.0515387058258057 + }, + { + "bits_per_byte": 2.0465077259246423, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.418531060218811, + "step": 1400, + "unclipped_grad_norm": 1.6702975034713745 + }, + { + "bits_per_byte": 1.961047826884404, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3592947721481323, + "step": 1410, + "unclipped_grad_norm": 1.626113772392273 + }, + { + "bits_per_byte": 1.9924620059083538, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.3810694217681885, + "step": 1420, + "unclipped_grad_norm": 1.817312479019165 + }, + { + "bits_per_byte": 2.1091584258942007, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.4619572162628174, + "step": 1430, + "unclipped_grad_norm": 1.836112380027771 + }, + { + "bits_per_byte": 2.080447642159163, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.44205641746521, + "step": 1440, + "unclipped_grad_norm": 1.520651936531067 + }, + { + "bits_per_byte": 2.0264249678180906, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.4046107530593872, + "step": 1450, + "unclipped_grad_norm": 1.729941964149475 + }, + { + "bits_per_byte": 2.033933386389459, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.4098151922225952, + "step": 1460, + "unclipped_grad_norm": 1.6458605527877808 + }, + { + "bits_per_byte": 1.9987902795295016, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.385455846786499, + "step": 1470, + "unclipped_grad_norm": 1.662819743156433 + }, + { + "bits_per_byte": 2.05313094979202, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4231219291687012, + "step": 1480, + "unclipped_grad_norm": 1.4910722970962524 + }, + { + "bits_per_byte": 1.9934261406490907, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.3817377090454102, + "step": 1490, + "unclipped_grad_norm": 1.629915475845337 + }, + { + "bits_per_byte": 2.0238037802362903, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.4027938842773438, + "step": 1500, + "unclipped_grad_norm": 1.679500699043274 + }, + { + "bits_per_byte": 2.0618504701907643, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4291658401489258, + "step": 1510, + "unclipped_grad_norm": 1.6392786502838135 + }, + { + "bits_per_byte": 2.0494945486221767, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.4206013679504395, + "step": 1520, + "unclipped_grad_norm": 1.5018537044525146 + }, + { + "bits_per_byte": 2.0390381754323688, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.4133535623550415, + "step": 1530, + "unclipped_grad_norm": 1.7712345123291016 + }, + { + "bits_per_byte": 1.9636298024218073, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.3610844612121582, + "step": 1540, + "unclipped_grad_norm": 1.53538978099823 + }, + { + "bits_per_byte": 1.996493279244578, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3838636875152588, + "step": 1550, + "unclipped_grad_norm": 1.4971659183502197 + }, + { + "bits_per_byte": 1.9336547742202777, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.340307354927063, + "step": 1560, + "unclipped_grad_norm": 1.5087686777114868 + }, + { + "bits_per_byte": 1.9822796010813704, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.374011516571045, + "step": 1570, + "unclipped_grad_norm": 1.6810358762741089 + }, + { + "bits_per_byte": 2.046016371491127, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.4181904792785645, + "step": 1580, + "unclipped_grad_norm": 1.679996371269226 + }, + { + "bits_per_byte": 2.053019161068959, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.4230444431304932, + "step": 1590, + "unclipped_grad_norm": 1.7087212800979614 + }, + { + "bits_per_byte": 1.9613551598814958, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.3595077991485596, + "step": 1600, + "unclipped_grad_norm": 1.513136386871338 + }, + { + "bits_per_byte": 2.025119103550089, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.4037055969238281, + "step": 1610, + "unclipped_grad_norm": 1.6738353967666626 + }, + { + "bits_per_byte": 1.9898034980913164, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.3792266845703125, + "step": 1620, + "unclipped_grad_norm": 1.5587013959884644 + }, + { + "bits_per_byte": 2.057287426498072, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.4260029792785645, + "step": 1630, + "unclipped_grad_norm": 1.6315467357635498 + }, + { + "bits_per_byte": 1.982021971070378, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3738329410552979, + "step": 1640, + "unclipped_grad_norm": 1.647518515586853 + }, + { + "bits_per_byte": 1.9524098262621685, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3533073663711548, + "step": 1650, + "unclipped_grad_norm": 1.6323529481887817 + }, + { + "bits_per_byte": 2.0326827285523845, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.408948302268982, + "step": 1660, + "unclipped_grad_norm": 1.6469638347625732 + }, + { + "bits_per_byte": 1.972278809933698, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.367079496383667, + "step": 1670, + "unclipped_grad_norm": 1.6755335330963135 + }, + { + "bits_per_byte": 2.0348887500150017, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.4104773998260498, + "step": 1680, + "unclipped_grad_norm": 1.7751716375350952 + }, + { + "bits_per_byte": 1.9889148637343084, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.3786107301712036, + "step": 1690, + "unclipped_grad_norm": 1.777902603149414 + }, + { + "bits_per_byte": 1.9596723096428033, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.3583413362503052, + "step": 1700, + "unclipped_grad_norm": 1.7119793891906738 + }, + { + "bits_per_byte": 1.9726306864373633, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.367323398590088, + "step": 1710, + "unclipped_grad_norm": 1.4510502815246582 + }, + { + "bits_per_byte": 2.03698934611264, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.411933422088623, + "step": 1720, + "unclipped_grad_norm": 1.5922540426254272 + }, + { + "bits_per_byte": 1.920482622990695, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3311771154403687, + "step": 1730, + "unclipped_grad_norm": 1.5325313806533813 + }, + { + "bits_per_byte": 2.0362487888180247, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.4114201068878174, + "step": 1740, + "unclipped_grad_norm": 1.616685152053833 + }, + { + "bits_per_byte": 1.9280871798638959, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.3364481925964355, + "step": 1750, + "unclipped_grad_norm": 1.510337233543396 + }, + { + "bits_per_byte": 1.998571689580255, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.38530433177948, + "step": 1760, + "unclipped_grad_norm": 1.543134093284607 + }, + { + "bits_per_byte": 1.8708183329877874, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.2967524528503418, + "step": 1770, + "unclipped_grad_norm": 1.4701957702636719 + }, + { + "bits_per_byte": 1.9268164000566705, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3355673551559448, + "step": 1780, + "unclipped_grad_norm": 1.5400384664535522 + }, + { + "bits_per_byte": 1.9692582786365944, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.3649858236312866, + "step": 1790, + "unclipped_grad_norm": 1.674070119857788 + }, + { + "bits_per_byte": 1.938927074365128, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3439618349075317, + "step": 1800, + "unclipped_grad_norm": 1.5654456615447998 + }, + { + "bits_per_byte": 2.0168247242642776, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.397956371307373, + "step": 1810, + "unclipped_grad_norm": 1.8591063022613525 + }, + { + "bits_per_byte": 1.8948121485576106, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.31338369846344, + "step": 1820, + "unclipped_grad_norm": 1.6202964782714844 + }, + { + "bits_per_byte": 1.9531240702112018, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3538024425506592, + "step": 1830, + "unclipped_grad_norm": 1.7272347211837769 + }, + { + "bits_per_byte": 1.9388921618870028, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.343937635421753, + "step": 1840, + "unclipped_grad_norm": 1.6212350130081177 + }, + { + "bits_per_byte": 1.9345912197542257, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.340956449508667, + "step": 1850, + "unclipped_grad_norm": 1.702932596206665 + }, + { + "bits_per_byte": 1.9254059703369446, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3345897197723389, + "step": 1860, + "unclipped_grad_norm": 1.63266921043396 + }, + { + "bits_per_byte": 1.9210379549703311, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3315620422363281, + "step": 1870, + "unclipped_grad_norm": 1.5814175605773926 + }, + { + "bits_per_byte": 1.9515824177288676, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.352733850479126, + "step": 1880, + "unclipped_grad_norm": 1.6361923217773438 + }, + { + "bits_per_byte": 2.012705911758764, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.3951014280319214, + "step": 1890, + "unclipped_grad_norm": 1.8325941562652588 + }, + { + "bits_per_byte": 1.9381443813210513, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3434193134307861, + "step": 1900, + "unclipped_grad_norm": 1.6236236095428467 + }, + { + "bits_per_byte": 1.8902840173430437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.3102450370788574, + "step": 1910, + "unclipped_grad_norm": 1.5828723907470703 + }, + { + "bits_per_byte": 1.927230706262599, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.3358545303344727, + "step": 1920, + "unclipped_grad_norm": 1.6751060485839844 + }, + { + "bits_per_byte": 1.9741809380522406, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.3683979511260986, + "step": 1930, + "unclipped_grad_norm": 1.6039025783538818 + }, + { + "bits_per_byte": 1.9842474265725425, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.375375509262085, + "step": 1940, + "unclipped_grad_norm": 1.5702033042907715 + }, + { + "bits_per_byte": 1.9249440249367271, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3342695236206055, + "step": 1950, + "unclipped_grad_norm": 1.5467878580093384 + }, + { + "bits_per_byte": 1.959973623247115, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3585501909255981, + "step": 1960, + "unclipped_grad_norm": 1.6715258359909058 + }, + { + "bits_per_byte": 1.9462280818595559, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.3490225076675415, + "step": 1970, + "unclipped_grad_norm": 1.6887352466583252 + }, + { + "bits_per_byte": 1.9551462422200467, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.3552041053771973, + "step": 1980, + "unclipped_grad_norm": 1.6765621900558472 + }, + { + "bits_per_byte": 1.9530907055769344, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3537793159484863, + "step": 1990, + "unclipped_grad_norm": 1.7666816711425781 + }, + { + "bits_per_byte": 1.885217752413927, + "learning_rate": 3e-05, + "loss_nats": 1.3067333698272705, + "step": 2000, + "unclipped_grad_norm": 1.611930012702942 + } + ] + }, + { + "architecture": "block", + "batch_size": 32, + "canonical_sha256_without_self": "5e2460a3ba6e54fc860f0e13325e71aec54a4d555af3ae2315641654690dded4", + "diagnostic": { + "bits_per_byte": 2.1201721948554546, + "branch_output_rms": [ + 0.028859306126832962, + 0.048244379460811615, + 0.030812542885541916, + 0.036034028977155685, + 0.04880734533071518, + 0.022128254175186157, + 0.0301019549369812, + 0.018285444006323814, + 0.05344751849770546, + 0.019084658473730087, + 0.11068977415561676, + 0.017863860353827477, + 0.06856144964694977, + 0.01103654783219099, + 0.02528632991015911, + 0.015162505209445953, + 0.022348226979374886, + 0.016425520181655884, + 0.027489805594086647, + 0.021545369178056717, + 0.025723380967974663, + 0.02158590778708458, + 0.017817942425608635, + 0.033413130789995193, + 0.016243331134319305, + 0.02716788649559021, + 0.009424413554370403, + 0.032799478620290756, + 0.015554063953459263, + 0.046544428914785385, + 0.008579479530453682, + 0.052862487733364105 + ], + "core_parameter_grad_rms_by_block": [ + 0.0006136196561721271, + 0.00042300100546983135, + 0.0002982853623100393, + 0.00023476280042759528, + 0.0002907770609625826, + 0.00041448514649562707, + 0.00042775107615344534, + 0.00039679767722893174, + 0.0006759303554106059, + 0.0009600722047358557, + 0.001431364890916451, + 0.0012053624438772336, + 0.0010361348995012233, + 0.0007327317391797293, + 0.0005136627146630692, + 0.00042053944700778587 + ], + "depth_weights": [ + { + "entropy_mean": 0.0, + "mean_weights": [ + 1.0 + ], + "sources": 1 + }, + { + "entropy_mean": 0.6638920307159424, + "mean_weights": [ + 0.5757085084915161, + 0.4242914617061615 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6758064031600952, + "mean_weights": [ + 0.5517290234565735, + 0.4482709765434265 + ], + "sources": 2 + }, + { + "entropy_mean": 0.5902198553085327, + "mean_weights": [ + 0.7145272493362427, + 0.28547269105911255 + ], + "sources": 2 + }, + { + "entropy_mean": 0.6787354350090027, + "mean_weights": [ + 0.47956913709640503, + 0.520430862903595 + ], + "sources": 2 + }, + { + "entropy_mean": 1.0216479301452637, + "mean_weights": [ + 0.4870031177997589, + 0.28748819231987, + 0.2255086898803711 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0888967514038086, + "mean_weights": [ + 0.3058485984802246, + 0.3320831060409546, + 0.3620683550834656 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0497782230377197, + "mean_weights": [ + 0.4298950135707855, + 0.2791864275932312, + 0.29091858863830566 + ], + "sources": 3 + }, + { + "entropy_mean": 1.0873868465423584, + "mean_weights": [ + 0.31240078806877136, + 0.3293955326080322, + 0.3582037091255188 + ], + "sources": 3 + }, + { + "entropy_mean": 1.3575613498687744, + "mean_weights": [ + 0.31760936975479126, + 0.22722351551055908, + 0.2226596474647522, + 0.23250743746757507 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3743479251861572, + "mean_weights": [ + 0.24557536840438843, + 0.2715907096862793, + 0.21654395759105682, + 0.26629000902175903 + ], + "sources": 4 + }, + { + "entropy_mean": 1.3570414781570435, + "mean_weights": [ + 0.29905614256858826, + 0.20215731859207153, + 0.2139909565448761, + 0.2847955524921417 + ], + "sources": 4 + }, + { + "entropy_mean": 1.372481346130371, + "mean_weights": [ + 0.23897650837898254, + 0.28815770149230957, + 0.20619644224643707, + 0.2666693329811096 + ], + "sources": 4 + }, + { + "entropy_mean": 1.5674753189086914, + "mean_weights": [ + 0.2010083794593811, + 0.16119517385959625, + 0.1479203850030899, + 0.19103804230690002, + 0.2988380193710327 + ], + "sources": 5 + }, + { + "entropy_mean": 1.577967882156372, + "mean_weights": [ + 0.1808086633682251, + 0.18757948279380798, + 0.144127756357193, + 0.1949053704738617, + 0.2925787568092346 + ], + "sources": 5 + }, + { + "entropy_mean": 1.546515941619873, + "mean_weights": [ + 0.19701287150382996, + 0.18306899070739746, + 0.1371120810508728, + 0.15036475658416748, + 0.3324413001537323 + ], + "sources": 5 + }, + { + "entropy_mean": 1.5857822895050049, + "mean_weights": [ + 0.19598084688186646, + 0.2059629261493683, + 0.1507713794708252, + 0.174381285905838, + 0.27290356159210205 + ], + "sources": 5 + }, + { + "entropy_mean": 1.7451198101043701, + "mean_weights": [ + 0.14769580960273743, + 0.13491438329219818, + 0.11895542591810226, + 0.12873513996601105, + 0.23462235927581787, + 0.235076904296875 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7161343097686768, + "mean_weights": [ + 0.15046963095664978, + 0.14864115417003632, + 0.10848429799079895, + 0.11923535168170929, + 0.1604226976633072, + 0.31274688243865967 + ], + "sources": 6 + }, + { + "entropy_mean": 1.4035348892211914, + "mean_weights": [ + 0.13220778107643127, + 0.1419733762741089, + 0.06934956461191177, + 0.048299990594387054, + 0.0861104428768158, + 0.5220588445663452 + ], + "sources": 6 + }, + { + "entropy_mean": 1.7599127292633057, + "mean_weights": [ + 0.16512201726436615, + 0.1626027673482895, + 0.12519001960754395, + 0.13426196575164795, + 0.16173651814460754, + 0.2510867118835449 + ], + "sources": 6 + }, + { + "entropy_mean": 1.5612093210220337, + "mean_weights": [ + 0.08958975225687027, + 0.08426812291145325, + 0.04905245825648308, + 0.03659921512007713, + 0.06333694607019424, + 0.24641311168670654, + 0.4307403862476349 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8658541440963745, + "mean_weights": [ + 0.12650328874588013, + 0.12594074010849, + 0.0949375256896019, + 0.10049091279506683, + 0.11246223747730255, + 0.1588260531425476, + 0.2808392643928528 + ], + "sources": 7 + }, + { + "entropy_mean": 1.5638015270233154, + "mean_weights": [ + 0.09318068623542786, + 0.08571819961071014, + 0.05113398656249046, + 0.041206810623407364, + 0.06751164048910141, + 0.20548424124717712, + 0.45576441287994385 + ], + "sources": 7 + }, + { + "entropy_mean": 1.8683671951293945, + "mean_weights": [ + 0.13295680284500122, + 0.14563557505607605, + 0.09758639335632324, + 0.08762425184249878, + 0.10102800279855728, + 0.17467939853668213, + 0.2604895830154419 + ], + "sources": 7 + }, + { + "entropy_mean": 1.7563724517822266, + "mean_weights": [ + 0.07549049705266953, + 0.0683341696858406, + 0.040880024433135986, + 0.03398597240447998, + 0.05212044343352318, + 0.14432066679000854, + 0.31587064266204834, + 0.26899757981300354 + ], + "sources": 8 + }, + { + "entropy_mean": 1.8879387378692627, + "mean_weights": [ + 0.09486456215381622, + 0.10536793619394302, + 0.06144082546234131, + 0.05093793198466301, + 0.06251554936170578, + 0.13544753193855286, + 0.19072145223617554, + 0.29870423674583435 + ], + "sources": 8 + }, + { + "entropy_mean": 1.7447065114974976, + "mean_weights": [ + 0.07460781931877136, + 0.0729113519191742, + 0.043291281908750534, + 0.0370100662112236, + 0.05375167727470398, + 0.12649331986904144, + 0.22679051756858826, + 0.36514395475387573 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9507553577423096, + "mean_weights": [ + 0.10841415077447891, + 0.11357448995113373, + 0.0694868415594101, + 0.05887201428413391, + 0.06816299259662628, + 0.15050429105758667, + 0.19657441973686218, + 0.23441082239151 + ], + "sources": 8 + }, + { + "entropy_mean": 1.9826093912124634, + "mean_weights": [ + 0.07752279192209244, + 0.07347545027732849, + 0.04389962926506996, + 0.036320604383945465, + 0.050405919551849365, + 0.11705176532268524, + 0.17562852799892426, + 0.26277750730514526, + 0.16291777789592743 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0432467460632324, + "mean_weights": [ + 0.08395138382911682, + 0.08140303194522858, + 0.051501721143722534, + 0.04335907846689224, + 0.055790141224861145, + 0.13440418243408203, + 0.1891498565673828, + 0.2104351818561554, + 0.15000542998313904 + ], + "sources": 9 + }, + { + "entropy_mean": 2.0141000747680664, + "mean_weights": [ + 0.07707478851079941, + 0.08790881931781769, + 0.04824091121554375, + 0.03912223130464554, + 0.0574696883559227, + 0.11675351858139038, + 0.17484726011753082, + 0.23693355917930603, + 0.16164922714233398 + ], + "sources": 9 + } + ], + "layer_input_rms": [ + 0.05405270308256149, + 0.03439030423760414, + 0.042556535452604294, + 0.04696016386151314, + 0.06085871905088425, + 0.04705430194735527, + 0.053277771919965744, + 0.05260215699672699, + 0.0606326088309288, + 0.04865434020757675, + 0.054401230067014694, + 0.07243207842111588, + 0.07691676914691925, + 0.06619398295879364, + 0.06854937970638275, + 0.0655755028128624, + 0.0687255784869194, + 0.05135589465498924, + 0.04421957954764366, + 0.029444050043821335, + 0.04853617027401924, + 0.017642762511968613, + 0.032592616975307465, + 0.02168644405901432, + 0.03257649391889572, + 0.019617732614278793, + 0.021423330530524254, + 0.019785262644290924, + 0.02440592646598816, + 0.019491195678710938, + 0.02242189459502697, + 0.023190947249531746 + ], + "loss_nats": 1.4695913791656494, + "output_weights": { + "entropy_mean": 1.9263067245483398, + "mean_weights": [ + 0.09944116324186325, + 0.06237940117716789, + 0.050934016704559326, + 0.04384239390492439, + 0.05164656415581703, + 0.08634476363658905, + 0.08995818346738815, + 0.21131984889507294, + 0.30413365364074707 + ], + "sources": 9 + }, + "stream_state_rms": [ + 0.028859306126832962, + 0.05888580158352852, + 0.07637989521026611, + 0.10048534721136093, + 0.04880734533071518, + 0.05540755018591881, + 0.07589049637317657, + 0.08340234309434891, + 0.05344751849770546, + 0.059183262288570404, + 0.15584944188594818, + 0.1662624329328537, + 0.06856144964694977, + 0.0732773095369339, + 0.08456306159496307, + 0.09095416963100433, + 0.022348226979374886, + 0.030708789825439453, + 0.04433037340641022, + 0.05179491639137268, + 0.025723380967974663, + 0.034459229558706284, + 0.04300807788968086, + 0.05657916143536568, + 0.016243331134319305, + 0.033271633088588715, + 0.03624724969267845, + 0.05302892625331879, + 0.015554063953459263, + 0.050526175647974014, + 0.051927778869867325, + 0.09535783529281616 + ] + }, + "environment": { + "autocast": "cuda-bfloat16", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.029935924131141, + "cross_entropy_nats": 5.565927445888519, + "step": 0 + }, + { + "bits_per_byte": 3.861173508427409, + "cross_entropy_nats": 2.676361531019211, + "step": 100 + }, + { + "bits_per_byte": 3.427078118777374, + "cross_entropy_nats": 2.375469535589218, + "step": 250 + }, + { + "bits_per_byte": 2.9073413023991153, + "cross_entropy_nats": 2.015215426683426, + "step": 500 + }, + { + "bits_per_byte": 2.2110541049381194, + "cross_entropy_nats": 1.5325859189033508, + "step": 1000 + }, + { + "bits_per_byte": 2.0265933173354536, + "cross_entropy_nats": 1.4047274440526962, + "step": 1500 + }, + { + "bits_per_byte": 1.9650305581174223, + "cross_entropy_nats": 1.362055391073227, + "step": 2000 + } + ], + "hashes": { + "final_common_parameters": "4b5bcf7f73e493bd7e2b4db7bf5fc8b35251f393c2a7c2847442360ac1a86fa3", + "final_mixer_parameters": "e2ba4a92d4c3674f82f965adabdea1144c9c754cb2859a70f10176c5323b2911", + "initial_common_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f" + }, + "manifest": { + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "path": "experiments/k3/attnres/manifest.json", + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + }, + "model": { + "blocks_for_block_attnres": 8, + "context": 256, + "d_ff": 768, + "d_head": 32, + "d_model": 192, + "heads": 6, + "layers": 16, + "parameters": { + "core": 9541824, + "embedding": 98304, + "mixer": 12672, + "total": 9554496 + }, + "sublayers": 32, + "sublayers_per_attnres_block": 4, + "vocabulary": 256 + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 100, + "weight_decay_ndim_ge_2": 0.1 + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 2000, + "target_bytes_seen": 16384000, + "timing": { + "mean_ms": 54.8763856947342, + "measured_steps": 1980, + "median_ms": 54.84093050472438, + "p95_ms": 55.21141063363757, + "peak_allocated_bytes": 6520143872, + "peak_reserved_bytes": 6788481024, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.032227506962563, + "learning_rate": 2.9999999999999997e-06, + "loss_nats": 5.567515850067139, + "step": 1, + "unclipped_grad_norm": 17.736968994140625 + }, + { + "bits_per_byte": 6.818283838263807, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.72607421875, + "step": 10, + "unclipped_grad_norm": 5.34749698638916 + }, + { + "bits_per_byte": 6.430281346888862, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 4.457131385803223, + "step": 20, + "unclipped_grad_norm": 2.99312162399292 + }, + { + "bits_per_byte": 6.174536265749832, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 4.279862403869629, + "step": 30, + "unclipped_grad_norm": 2.752336025238037 + }, + { + "bits_per_byte": 5.662918492344829, + "learning_rate": 0.00011999999999999999, + "loss_nats": 3.9252359867095947, + "step": 40, + "unclipped_grad_norm": 2.472853899002075 + }, + { + "bits_per_byte": 5.2159331749952775, + "learning_rate": 0.00015, + "loss_nats": 3.6154093742370605, + "step": 50, + "unclipped_grad_norm": 2.109605312347412 + }, + { + "bits_per_byte": 4.7634543878118665, + "learning_rate": 0.00017999999999999998, + "loss_nats": 3.3017749786376953, + "step": 60, + "unclipped_grad_norm": 1.7173339128494263 + }, + { + "bits_per_byte": 4.451996214856432, + "learning_rate": 0.00020999999999999998, + "loss_nats": 3.085888624191284, + "step": 70, + "unclipped_grad_norm": 1.273622989654541 + }, + { + "bits_per_byte": 4.148000713087237, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.8751749992370605, + "step": 80, + "unclipped_grad_norm": 0.8782929182052612 + }, + { + "bits_per_byte": 3.984823229983302, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.7620689868927, + "step": 90, + "unclipped_grad_norm": 0.6921615600585938 + }, + { + "bits_per_byte": 3.866966271069433, + "learning_rate": 0.0003, + "loss_nats": 2.6803767681121826, + "step": 100, + "unclipped_grad_norm": 0.6132790446281433 + }, + { + "bits_per_byte": 3.794286746780112, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.6299991607666016, + "step": 110, + "unclipped_grad_norm": 0.40181323885917664 + }, + { + "bits_per_byte": 3.6877008428731877, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.556119441986084, + "step": 120, + "unclipped_grad_norm": 0.7017242312431335 + }, + { + "bits_per_byte": 3.622083614158897, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.510637044906616, + "step": 130, + "unclipped_grad_norm": 0.47804898023605347 + }, + { + "bits_per_byte": 3.6105088377905283, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.5026140213012695, + "step": 140, + "unclipped_grad_norm": 0.3960910737514496 + }, + { + "bits_per_byte": 3.5965293999977956, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.492924213409424, + "step": 150, + "unclipped_grad_norm": 0.5380170941352844 + }, + { + "bits_per_byte": 3.5663381896041315, + "learning_rate": 0.00029933619184357654, + "loss_nats": 2.4719972610473633, + "step": 160, + "unclipped_grad_norm": 0.4343404471874237 + }, + { + "bits_per_byte": 3.5359543586415008, + "learning_rate": 0.00029909675095774114, + "loss_nats": 2.450936794281006, + "step": 170, + "unclipped_grad_norm": 0.7406424880027771 + }, + { + "bits_per_byte": 3.5791261315916865, + "learning_rate": 0.00029882064935914164, + "loss_nats": 2.480861186981201, + "step": 180, + "unclipped_grad_norm": 0.597319483757019 + }, + { + "bits_per_byte": 3.473137351448558, + "learning_rate": 0.00029850796253119824, + "loss_nats": 2.407395362854004, + "step": 190, + "unclipped_grad_norm": 0.7858139872550964 + }, + { + "bits_per_byte": 3.5484980852648307, + "learning_rate": 0.0002981587759593675, + "loss_nats": 2.4596314430236816, + "step": 200, + "unclipped_grad_norm": 0.606899082660675 + }, + { + "bits_per_byte": 3.46515460472611, + "learning_rate": 0.0002977731851077716, + "loss_nats": 2.401862144470215, + "step": 210, + "unclipped_grad_norm": 0.8731884360313416 + }, + { + "bits_per_byte": 3.4351532631789983, + "learning_rate": 0.00029735129539309926, + "loss_nats": 2.3810667991638184, + "step": 220, + "unclipped_grad_norm": 0.6177881956100464 + }, + { + "bits_per_byte": 3.4394576449647927, + "learning_rate": 0.0002968932221557859, + "loss_nats": 2.3840503692626953, + "step": 230, + "unclipped_grad_norm": 0.5822129845619202 + }, + { + "bits_per_byte": 3.4035057036978276, + "learning_rate": 0.0002963990906284808, + "loss_nats": 2.359130382537842, + "step": 240, + "unclipped_grad_norm": 0.6932224631309509 + }, + { + "bits_per_byte": 3.4118310398607945, + "learning_rate": 0.00029586903590180956, + "loss_nats": 2.364901065826416, + "step": 250, + "unclipped_grad_norm": 0.786601185798645 + }, + { + "bits_per_byte": 3.3949966900637394, + "learning_rate": 0.00029530320288744205, + "loss_nats": 2.3532323837280273, + "step": 260, + "unclipped_grad_norm": 0.7221279144287109 + }, + { + "bits_per_byte": 3.40052214867066, + "learning_rate": 0.0002947017462784745, + "loss_nats": 2.357062339782715, + "step": 270, + "unclipped_grad_norm": 1.3681480884552002 + }, + { + "bits_per_byte": 3.38492469809861, + "learning_rate": 0.00029406483050713824, + "loss_nats": 2.3462510108947754, + "step": 280, + "unclipped_grad_norm": 0.827241837978363 + }, + { + "bits_per_byte": 3.363309918538172, + "learning_rate": 0.0002933926296998457, + "loss_nats": 2.331268787384033, + "step": 290, + "unclipped_grad_norm": 1.1995528936386108 + }, + { + "bits_per_byte": 3.366865143896808, + "learning_rate": 0.0002926853276295856, + "loss_nats": 2.333733081817627, + "step": 300, + "unclipped_grad_norm": 0.7762910723686218 + }, + { + "bits_per_byte": 3.2713391003015695, + "learning_rate": 0.00029194311766568174, + "loss_nats": 2.267519474029541, + "step": 310, + "unclipped_grad_norm": 0.8876886367797852 + }, + { + "bits_per_byte": 3.3040130522430893, + "learning_rate": 0.0002911662027209272, + "loss_nats": 2.2901673316955566, + "step": 320, + "unclipped_grad_norm": 0.776787281036377 + }, + { + "bits_per_byte": 3.3299095118794075, + "learning_rate": 0.00029035479519611045, + "loss_nats": 2.308117389678955, + "step": 330, + "unclipped_grad_norm": 1.0567725896835327 + }, + { + "bits_per_byte": 3.3233129453229138, + "learning_rate": 0.0002895091169219469, + "loss_nats": 2.3035449981689453, + "step": 340, + "unclipped_grad_norm": 1.1909834146499634 + }, + { + "bits_per_byte": 3.256331894187281, + "learning_rate": 0.00028862939909843273, + "loss_nats": 2.25711727142334, + "step": 350, + "unclipped_grad_norm": 1.159000277519226 + }, + { + "bits_per_byte": 3.3198347681918645, + "learning_rate": 0.00028771588223163694, + "loss_nats": 2.3011341094970703, + "step": 360, + "unclipped_grad_norm": 1.0497483015060425 + }, + { + "bits_per_byte": 3.259610227478028, + "learning_rate": 0.00028676881606794945, + "loss_nats": 2.259389638900757, + "step": 370, + "unclipped_grad_norm": 1.1961599588394165 + }, + { + "bits_per_byte": 3.1566510937124987, + "learning_rate": 0.000285788459525803, + "loss_nats": 2.188023805618286, + "step": 380, + "unclipped_grad_norm": 1.0483556985855103 + }, + { + "bits_per_byte": 3.1814812609134675, + "learning_rate": 0.0002847750806248876, + "loss_nats": 2.2052347660064697, + "step": 390, + "unclipped_grad_norm": 1.0343443155288696 + }, + { + "bits_per_byte": 3.1396640233214774, + "learning_rate": 0.000283728956412876, + "loss_nats": 2.1762492656707764, + "step": 400, + "unclipped_grad_norm": 1.0970324277877808 + }, + { + "bits_per_byte": 3.105233440584045, + "learning_rate": 0.0002826503728896824, + "loss_nats": 2.152383804321289, + "step": 410, + "unclipped_grad_norm": 1.0763155221939087 + }, + { + "bits_per_byte": 3.0858767932294358, + "learning_rate": 0.0002815396249292725, + "loss_nats": 2.1389667987823486, + "step": 420, + "unclipped_grad_norm": 1.1526751518249512 + }, + { + "bits_per_byte": 3.0763400112737913, + "learning_rate": 0.00028039701619904774, + "loss_nats": 2.1323564052581787, + "step": 430, + "unclipped_grad_norm": 1.0018411874771118 + }, + { + "bits_per_byte": 3.0866675694581027, + "learning_rate": 0.00027922285907682625, + "loss_nats": 2.139514923095703, + "step": 440, + "unclipped_grad_norm": 1.421270728111267 + }, + { + "bits_per_byte": 2.9945818668107242, + "learning_rate": 0.00027801747456544134, + "loss_nats": 2.075685977935791, + "step": 450, + "unclipped_grad_norm": 3.8166518211364746 + }, + { + "bits_per_byte": 2.99475453739219, + "learning_rate": 0.0002767811922049827, + "loss_nats": 2.0758056640625, + "step": 460, + "unclipped_grad_norm": 1.3669120073318481 + }, + { + "bits_per_byte": 3.008159553131041, + "learning_rate": 0.0002755143499827035, + "loss_nats": 2.085097312927246, + "step": 470, + "unclipped_grad_norm": 1.2348326444625854 + }, + { + "bits_per_byte": 2.969454857554366, + "learning_rate": 0.00027421729424061787, + "loss_nats": 2.0582692623138428, + "step": 480, + "unclipped_grad_norm": 1.3945213556289673 + }, + { + "bits_per_byte": 2.936720023754441, + "learning_rate": 0.0002728903795808148, + "loss_nats": 2.035579204559326, + "step": 490, + "unclipped_grad_norm": 1.8353713750839233 + }, + { + "bits_per_byte": 2.8468473938583645, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.9732842445373535, + "step": 500, + "unclipped_grad_norm": 1.0700337886810303 + }, + { + "bits_per_byte": 2.8360615019095037, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.9658080339431763, + "step": 510, + "unclipped_grad_norm": 1.542275071144104 + }, + { + "bits_per_byte": 2.8669599049635157, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.9872251749038696, + "step": 520, + "unclipped_grad_norm": 1.1954354047775269 + }, + { + "bits_per_byte": 2.838187035491517, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.9672813415527344, + "step": 530, + "unclipped_grad_norm": 1.3107086420059204 + }, + { + "bits_per_byte": 2.838769024782037, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.9676847457885742, + "step": 540, + "unclipped_grad_norm": 1.5727649927139282 + }, + { + "bits_per_byte": 2.7677634076335633, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.918467402458191, + "step": 550, + "unclipped_grad_norm": 1.2783147096633911 + }, + { + "bits_per_byte": 2.8478954561327225, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.974010705947876, + "step": 560, + "unclipped_grad_norm": 1.5621379613876343 + }, + { + "bits_per_byte": 2.704984408606461, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.8749523162841797, + "step": 570, + "unclipped_grad_norm": 1.4339773654937744 + }, + { + "bits_per_byte": 2.6924681992072697, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.866276741027832, + "step": 580, + "unclipped_grad_norm": 1.4351019859313965 + }, + { + "bits_per_byte": 2.6420122730624405, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.831303358078003, + "step": 590, + "unclipped_grad_norm": 1.1825811862945557 + }, + { + "bits_per_byte": 2.638088660865656, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.8285837173461914, + "step": 600, + "unclipped_grad_norm": 1.7106136083602905 + }, + { + "bits_per_byte": 2.669559766147042, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.8503978252410889, + "step": 610, + "unclipped_grad_norm": 1.3415066003799438 + }, + { + "bits_per_byte": 2.6278629164106557, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.821495771408081, + "step": 620, + "unclipped_grad_norm": 1.4031703472137451 + }, + { + "bits_per_byte": 2.6170906110912124, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.8140289783477783, + "step": 630, + "unclipped_grad_norm": 1.603284239768982 + }, + { + "bits_per_byte": 2.6210909275502816, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.8168017864227295, + "step": 640, + "unclipped_grad_norm": 1.3607949018478394 + }, + { + "bits_per_byte": 2.558606534804108, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.7734909057617188, + "step": 650, + "unclipped_grad_norm": 1.3679509162902832 + }, + { + "bits_per_byte": 2.5799026305124966, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.7882522344589233, + "step": 660, + "unclipped_grad_norm": 1.5151586532592773 + }, + { + "bits_per_byte": 2.4887937893220187, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.7251003980636597, + "step": 670, + "unclipped_grad_norm": 1.6341558694839478 + }, + { + "bits_per_byte": 2.490447058544763, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.7262463569641113, + "step": 680, + "unclipped_grad_norm": 1.5843167304992676 + }, + { + "bits_per_byte": 2.5361060445917265, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.75789475440979, + "step": 690, + "unclipped_grad_norm": 1.2619558572769165 + }, + { + "bits_per_byte": 2.493424078231198, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.7283098697662354, + "step": 700, + "unclipped_grad_norm": 1.6411899328231812 + }, + { + "bits_per_byte": 2.4857598433781476, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.7229974269866943, + "step": 710, + "unclipped_grad_norm": 1.5300192832946777 + }, + { + "bits_per_byte": 2.451464095038992, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.6992254257202148, + "step": 720, + "unclipped_grad_norm": 1.467871904373169 + }, + { + "bits_per_byte": 2.398867672821524, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.6627683639526367, + "step": 730, + "unclipped_grad_norm": 1.3957626819610596 + }, + { + "bits_per_byte": 2.3827824794516275, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.6516189575195312, + "step": 740, + "unclipped_grad_norm": 1.4429775476455688 + }, + { + "bits_per_byte": 2.429520484650104, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.6840152740478516, + "step": 750, + "unclipped_grad_norm": 1.3560454845428467 + }, + { + "bits_per_byte": 2.474549842212259, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.7152272462844849, + "step": 760, + "unclipped_grad_norm": 1.4064133167266846 + }, + { + "bits_per_byte": 2.4221742457184994, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.67892324924469, + "step": 770, + "unclipped_grad_norm": 1.806920051574707 + }, + { + "bits_per_byte": 2.4602881808894606, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.7053418159484863, + "step": 780, + "unclipped_grad_norm": 1.7166920900344849 + }, + { + "bits_per_byte": 2.313398830683441, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.6035258769989014, + "step": 790, + "unclipped_grad_norm": 1.3211073875427246 + }, + { + "bits_per_byte": 2.3271316453548363, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.6130447387695312, + "step": 800, + "unclipped_grad_norm": 1.621514081954956 + }, + { + "bits_per_byte": 2.4012400015075257, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.6644127368927002, + "step": 810, + "unclipped_grad_norm": 1.5366694927215576 + }, + { + "bits_per_byte": 2.28462148966252, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.5835789442062378, + "step": 820, + "unclipped_grad_norm": 1.326358437538147 + }, + { + "bits_per_byte": 2.321981108926797, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.6094746589660645, + "step": 830, + "unclipped_grad_norm": 2.0325331687927246 + }, + { + "bits_per_byte": 2.2287204208087235, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.5448312759399414, + "step": 840, + "unclipped_grad_norm": 1.7673174142837524 + }, + { + "bits_per_byte": 2.243310397012019, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.5549442768096924, + "step": 850, + "unclipped_grad_norm": 1.5296648740768433 + }, + { + "bits_per_byte": 2.245876378162892, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.55672287940979, + "step": 860, + "unclipped_grad_norm": 1.5055785179138184 + }, + { + "bits_per_byte": 2.277088305589427, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.5783573389053345, + "step": 870, + "unclipped_grad_norm": 1.4940317869186401 + }, + { + "bits_per_byte": 2.222066584029492, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.5402191877365112, + "step": 880, + "unclipped_grad_norm": 1.8097378015518188 + }, + { + "bits_per_byte": 2.244500860921291, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.555769443511963, + "step": 890, + "unclipped_grad_norm": 1.5933631658554077 + }, + { + "bits_per_byte": 2.2507785716430853, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.5601208209991455, + "step": 900, + "unclipped_grad_norm": 1.4326916933059692 + }, + { + "bits_per_byte": 2.238866021348422, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.551863670349121, + "step": 910, + "unclipped_grad_norm": 1.6749801635742188 + }, + { + "bits_per_byte": 2.2762390552594667, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.5777686834335327, + "step": 920, + "unclipped_grad_norm": 1.4450469017028809 + }, + { + "bits_per_byte": 2.1771135867643205, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.5090601444244385, + "step": 930, + "unclipped_grad_norm": 1.5193647146224976 + }, + { + "bits_per_byte": 2.2318525688462367, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.5470023155212402, + "step": 940, + "unclipped_grad_norm": 1.666275143623352 + }, + { + "bits_per_byte": 2.1284672620718696, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.4753410816192627, + "step": 950, + "unclipped_grad_norm": 1.8505492210388184 + }, + { + "bits_per_byte": 2.24931843893726, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.5591087341308594, + "step": 960, + "unclipped_grad_norm": 1.5362094640731812 + }, + { + "bits_per_byte": 2.1893511842691145, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.5175426006317139, + "step": 970, + "unclipped_grad_norm": 1.6487948894500732 + }, + { + "bits_per_byte": 2.219267222421398, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.5382788181304932, + "step": 980, + "unclipped_grad_norm": 1.416129469871521 + }, + { + "bits_per_byte": 2.1964800371300286, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.5224839448928833, + "step": 990, + "unclipped_grad_norm": 1.3682113885879517 + }, + { + "bits_per_byte": 2.276387820252463, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.5778717994689941, + "step": 1000, + "unclipped_grad_norm": 1.658738374710083 + }, + { + "bits_per_byte": 2.126170605752248, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.4737491607666016, + "step": 1010, + "unclipped_grad_norm": 1.4019750356674194 + }, + { + "bits_per_byte": 2.239975653411789, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.5526328086853027, + "step": 1020, + "unclipped_grad_norm": 1.789731740951538 + }, + { + "bits_per_byte": 2.2138576801211594, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.534529209136963, + "step": 1030, + "unclipped_grad_norm": 1.5450307130813599 + }, + { + "bits_per_byte": 2.2090235917707077, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.5311784744262695, + "step": 1040, + "unclipped_grad_norm": 1.5325547456741333 + }, + { + "bits_per_byte": 2.195114666864829, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.5215375423431396, + "step": 1050, + "unclipped_grad_norm": 1.298677921295166 + }, + { + "bits_per_byte": 2.1600745776127384, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.4972496032714844, + "step": 1060, + "unclipped_grad_norm": 1.2953684329986572 + }, + { + "bits_per_byte": 2.0943803006708612, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.4517138004302979, + "step": 1070, + "unclipped_grad_norm": 1.4746257066726685 + }, + { + "bits_per_byte": 2.1314841817333052, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.4774322509765625, + "step": 1080, + "unclipped_grad_norm": 1.3626432418823242 + }, + { + "bits_per_byte": 2.2046614238142235, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.5281548500061035, + "step": 1090, + "unclipped_grad_norm": 1.3982676267623901 + }, + { + "bits_per_byte": 2.141045385225372, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.4840595722198486, + "step": 1100, + "unclipped_grad_norm": 1.582314133644104 + }, + { + "bits_per_byte": 2.1626904337323616, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.4990627765655518, + "step": 1110, + "unclipped_grad_norm": 1.5148810148239136 + }, + { + "bits_per_byte": 2.174445103953533, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.5072104930877686, + "step": 1120, + "unclipped_grad_norm": 1.6997957229614258 + }, + { + "bits_per_byte": 2.102896193610984, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.4576165676116943, + "step": 1130, + "unclipped_grad_norm": 1.6519840955734253 + }, + { + "bits_per_byte": 2.0526834509344747, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.42281174659729, + "step": 1140, + "unclipped_grad_norm": 1.6571881771087646 + }, + { + "bits_per_byte": 2.060268745750779, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.4280694723129272, + "step": 1150, + "unclipped_grad_norm": 1.3996864557266235 + }, + { + "bits_per_byte": 2.0812459856244683, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.4426097869873047, + "step": 1160, + "unclipped_grad_norm": 1.3461713790893555 + }, + { + "bits_per_byte": 2.1668613569810864, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.5019538402557373, + "step": 1170, + "unclipped_grad_norm": 1.3941727876663208 + }, + { + "bits_per_byte": 2.1390204614941135, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.4826560020446777, + "step": 1180, + "unclipped_grad_norm": 1.4307684898376465 + }, + { + "bits_per_byte": 2.2197260721339003, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.5385968685150146, + "step": 1190, + "unclipped_grad_norm": 1.3828914165496826 + }, + { + "bits_per_byte": 2.0334588862557283, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.4094862937927246, + "step": 1200, + "unclipped_grad_norm": 1.5526052713394165 + }, + { + "bits_per_byte": 2.093582129188207, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.4511605501174927, + "step": 1210, + "unclipped_grad_norm": 1.5174455642700195 + }, + { + "bits_per_byte": 2.0660626692756963, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.4320855140686035, + "step": 1220, + "unclipped_grad_norm": 1.5183188915252686 + }, + { + "bits_per_byte": 2.123279405408593, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.4717451333999634, + "step": 1230, + "unclipped_grad_norm": 1.3779727220535278 + }, + { + "bits_per_byte": 2.0532527135088308, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.4232063293457031, + "step": 1240, + "unclipped_grad_norm": 1.626110315322876 + }, + { + "bits_per_byte": 2.1024105146049474, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.457279920578003, + "step": 1250, + "unclipped_grad_norm": 1.4061928987503052 + }, + { + "bits_per_byte": 2.0698230699368114, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.4346920251846313, + "step": 1260, + "unclipped_grad_norm": 1.5369861125946045 + }, + { + "bits_per_byte": 2.0179919705156832, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.3987654447555542, + "step": 1270, + "unclipped_grad_norm": 1.466737985610962 + }, + { + "bits_per_byte": 2.0687512740566345, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.4339491128921509, + "step": 1280, + "unclipped_grad_norm": 1.4763338565826416 + }, + { + "bits_per_byte": 2.058286645699585, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.4266955852508545, + "step": 1290, + "unclipped_grad_norm": 1.5452309846878052 + }, + { + "bits_per_byte": 2.0762402585884554, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.4391400814056396, + "step": 1300, + "unclipped_grad_norm": 1.3626232147216797 + }, + { + "bits_per_byte": 2.009281221232133, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.3927276134490967, + "step": 1310, + "unclipped_grad_norm": 1.5211925506591797 + }, + { + "bits_per_byte": 1.9909953378617957, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.3800528049468994, + "step": 1320, + "unclipped_grad_norm": 1.3448001146316528 + }, + { + "bits_per_byte": 1.991420135009427, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.3803472518920898, + "step": 1330, + "unclipped_grad_norm": 1.3966525793075562 + }, + { + "bits_per_byte": 2.0691249923569592, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.4342081546783447, + "step": 1340, + "unclipped_grad_norm": 1.525726318359375 + }, + { + "bits_per_byte": 2.0419785628141702, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.4153916835784912, + "step": 1350, + "unclipped_grad_norm": 1.4942885637283325 + }, + { + "bits_per_byte": 2.035431527261125, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.410853624343872, + "step": 1360, + "unclipped_grad_norm": 1.475142002105713 + }, + { + "bits_per_byte": 2.0837153125255568, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.4443213939666748, + "step": 1370, + "unclipped_grad_norm": 1.4500157833099365 + }, + { + "bits_per_byte": 2.0817283969601386, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.4429441690444946, + "step": 1380, + "unclipped_grad_norm": 1.5744608640670776 + }, + { + "bits_per_byte": 2.0290387601459035, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.40642249584198, + "step": 1390, + "unclipped_grad_norm": 1.592787265777588 + }, + { + "bits_per_byte": 2.0140028329289206, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.3960003852844238, + "step": 1400, + "unclipped_grad_norm": 1.6389532089233398 + }, + { + "bits_per_byte": 1.9493160303557975, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.3511629104614258, + "step": 1410, + "unclipped_grad_norm": 1.4409334659576416 + }, + { + "bits_per_byte": 1.9623861958734183, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.3602224588394165, + "step": 1420, + "unclipped_grad_norm": 1.4958455562591553 + }, + { + "bits_per_byte": 2.081341607978348, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.442676067352295, + "step": 1430, + "unclipped_grad_norm": 1.7092392444610596 + }, + { + "bits_per_byte": 2.066988967833243, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.432727575302124, + "step": 1440, + "unclipped_grad_norm": 1.5831109285354614 + }, + { + "bits_per_byte": 2.007147432482878, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.3912485837936401, + "step": 1450, + "unclipped_grad_norm": 1.631679892539978 + }, + { + "bits_per_byte": 2.007508768032341, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.3914990425109863, + "step": 1460, + "unclipped_grad_norm": 1.5492783784866333 + }, + { + "bits_per_byte": 1.9700973819901542, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.3655674457550049, + "step": 1470, + "unclipped_grad_norm": 1.588803768157959 + }, + { + "bits_per_byte": 2.0405480111242933, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.4144001007080078, + "step": 1480, + "unclipped_grad_norm": 1.4634689092636108 + }, + { + "bits_per_byte": 1.9747440092511654, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.368788242340088, + "step": 1490, + "unclipped_grad_norm": 1.4901357889175415 + }, + { + "bits_per_byte": 2.00141507874697, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.387275218963623, + "step": 1500, + "unclipped_grad_norm": 1.60263991355896 + }, + { + "bits_per_byte": 2.0348663922703896, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.4104619026184082, + "step": 1510, + "unclipped_grad_norm": 1.5950591564178467 + }, + { + "bits_per_byte": 2.029029473082757, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.4064160585403442, + "step": 1520, + "unclipped_grad_norm": 1.530339002609253 + }, + { + "bits_per_byte": 2.018125773018055, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.3988581895828247, + "step": 1530, + "unclipped_grad_norm": 1.7868852615356445 + }, + { + "bits_per_byte": 1.9485610265185098, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.3506395816802979, + "step": 1540, + "unclipped_grad_norm": 1.4713661670684814 + }, + { + "bits_per_byte": 1.9769374759802698, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.3703086376190186, + "step": 1550, + "unclipped_grad_norm": 1.7392162084579468 + }, + { + "bits_per_byte": 1.9067774975260887, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.3216774463653564, + "step": 1560, + "unclipped_grad_norm": 1.513758659362793 + }, + { + "bits_per_byte": 1.954382639250216, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.3546748161315918, + "step": 1570, + "unclipped_grad_norm": 1.4868361949920654 + }, + { + "bits_per_byte": 2.017418580357707, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.398368000984192, + "step": 1580, + "unclipped_grad_norm": 1.489198923110962 + }, + { + "bits_per_byte": 2.0330165468777093, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.4091796875, + "step": 1590, + "unclipped_grad_norm": 1.6220532655715942 + }, + { + "bits_per_byte": 1.9346562291962517, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.3410015106201172, + "step": 1600, + "unclipped_grad_norm": 1.4640506505966187 + }, + { + "bits_per_byte": 1.9997465030682988, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.3861186504364014, + "step": 1610, + "unclipped_grad_norm": 1.3909966945648193 + }, + { + "bits_per_byte": 1.966872019355873, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.3633317947387695, + "step": 1620, + "unclipped_grad_norm": 1.5845268964767456 + }, + { + "bits_per_byte": 2.0337674231313763, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.4097001552581787, + "step": 1630, + "unclipped_grad_norm": 1.483353853225708 + }, + { + "bits_per_byte": 1.9743388181257326, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.3685073852539062, + "step": 1640, + "unclipped_grad_norm": 1.6287615299224854 + }, + { + "bits_per_byte": 1.927224514887168, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 1.3358502388000488, + "step": 1650, + "unclipped_grad_norm": 1.4574376344680786 + }, + { + "bits_per_byte": 2.0155130125861467, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.3970471620559692, + "step": 1660, + "unclipped_grad_norm": 1.7612671852111816 + }, + { + "bits_per_byte": 1.9576219324792168, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.3569201231002808, + "step": 1670, + "unclipped_grad_norm": 1.518265962600708 + }, + { + "bits_per_byte": 1.9872959910417367, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.377488613128662, + "step": 1680, + "unclipped_grad_norm": 1.5170477628707886 + }, + { + "bits_per_byte": 1.9597753272506702, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.358412742614746, + "step": 1690, + "unclipped_grad_norm": 1.4914941787719727 + }, + { + "bits_per_byte": 1.9412968233113665, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.345604419708252, + "step": 1700, + "unclipped_grad_norm": 1.551422119140625 + }, + { + "bits_per_byte": 1.9571986831754435, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.3566267490386963, + "step": 1710, + "unclipped_grad_norm": 1.5766102075576782 + }, + { + "bits_per_byte": 2.017097660731197, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.3981455564498901, + "step": 1720, + "unclipped_grad_norm": 1.50546395778656 + }, + { + "bits_per_byte": 1.9046273704250019, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.3201870918273926, + "step": 1730, + "unclipped_grad_norm": 1.419884204864502 + }, + { + "bits_per_byte": 2.0077653661474284, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.391676902770996, + "step": 1740, + "unclipped_grad_norm": 1.7969075441360474 + }, + { + "bits_per_byte": 1.9043783395465526, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.320014476776123, + "step": 1750, + "unclipped_grad_norm": 1.4314870834350586 + }, + { + "bits_per_byte": 1.9746284369097857, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.3687081336975098, + "step": 1760, + "unclipped_grad_norm": 1.525225043296814 + }, + { + "bits_per_byte": 1.8440977324885346, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.278231143951416, + "step": 1770, + "unclipped_grad_norm": 1.402140736579895 + }, + { + "bits_per_byte": 1.902167674552362, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.3184821605682373, + "step": 1780, + "unclipped_grad_norm": 1.476712703704834 + }, + { + "bits_per_byte": 1.9362646109471207, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.342116355895996, + "step": 1790, + "unclipped_grad_norm": 1.435732126235962 + }, + { + "bits_per_byte": 1.9158937819003736, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.3279963731765747, + "step": 1800, + "unclipped_grad_norm": 1.4851256608963013 + }, + { + "bits_per_byte": 2.006479623849578, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.3907856941223145, + "step": 1810, + "unclipped_grad_norm": 1.5186717510223389 + }, + { + "bits_per_byte": 1.8682447846002765, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.294968605041504, + "step": 1820, + "unclipped_grad_norm": 1.6630045175552368 + }, + { + "bits_per_byte": 1.9472940303296036, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.3497613668441772, + "step": 1830, + "unclipped_grad_norm": 1.5050814151763916 + }, + { + "bits_per_byte": 1.9204367036229146, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.3311452865600586, + "step": 1840, + "unclipped_grad_norm": 1.6411678791046143 + }, + { + "bits_per_byte": 1.9091611770670474, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.3233296871185303, + "step": 1850, + "unclipped_grad_norm": 1.6045939922332764 + }, + { + "bits_per_byte": 1.906594164020269, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.3215503692626953, + "step": 1860, + "unclipped_grad_norm": 1.4719187021255493 + }, + { + "bits_per_byte": 1.897684774774972, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.3153748512268066, + "step": 1870, + "unclipped_grad_norm": 1.5002301931381226 + }, + { + "bits_per_byte": 1.932823066120705, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.3397308588027954, + "step": 1880, + "unclipped_grad_norm": 1.6046397686004639 + }, + { + "bits_per_byte": 1.9954637910965132, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.3831501007080078, + "step": 1890, + "unclipped_grad_norm": 1.7177773714065552 + }, + { + "bits_per_byte": 1.9198793078514684, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.3307589292526245, + "step": 1900, + "unclipped_grad_norm": 1.6364820003509521 + }, + { + "bits_per_byte": 1.876205517543414, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 1.3004865646362305, + "step": 1910, + "unclipped_grad_norm": 1.4873437881469727 + }, + { + "bits_per_byte": 1.9124756267144754, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.325627088546753, + "step": 1920, + "unclipped_grad_norm": 1.646178960800171 + }, + { + "bits_per_byte": 1.9244617855837078, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.333935260772705, + "step": 1930, + "unclipped_grad_norm": 1.5518192052841187 + }, + { + "bits_per_byte": 1.9647428741381914, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.3618559837341309, + "step": 1940, + "unclipped_grad_norm": 1.5392659902572632 + }, + { + "bits_per_byte": 1.901729118792662, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.3181781768798828, + "step": 1950, + "unclipped_grad_norm": 1.4225507974624634 + }, + { + "bits_per_byte": 1.9255776090225059, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 1.3347086906433105, + "step": 1960, + "unclipped_grad_norm": 1.5890121459960938 + }, + { + "bits_per_byte": 1.92014674087356, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 1.330944299697876, + "step": 1970, + "unclipped_grad_norm": 1.5071648359298706 + }, + { + "bits_per_byte": 1.9363236009963667, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.342157244682312, + "step": 1980, + "unclipped_grad_norm": 1.6999623775482178 + }, + { + "bits_per_byte": 1.947599299534885, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.3499729633331299, + "step": 1990, + "unclipped_grad_norm": 1.6239629983901978 + }, + { + "bits_per_byte": 1.868521160720213, + "learning_rate": 3e-05, + "loss_nats": 1.295160174369812, + "step": 2000, + "unclipped_grad_norm": 1.5544158220291138 + } + ] + } + ], + "manifest": { + "dataset": { + "preprocessing": "parquet row order; (text or empty string) + LF; UTF-8; no normalization; vocabulary is raw bytes 0..255", + "repository": "Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "splits": { + "test": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/test.bin", + "binary_sha256": "bfe9eb16ab9987fb88bde4ea9a30a00f2a45db01dfc14bad78d05325789c4f12", + "concatenated_bytes": 1292014, + "concatenated_sha256": "bfe9eb16ab9987fb88bde4ea9a30a00f2a45db01dfc14bad78d05325789c4f12", + "parquet_bytes": 732610, + "parquet_sha256": "5f1bea067869d04849c0f975a2b29c4ff47d867f484f5010ea5e861eab246d91", + "rows": 4358, + "source_path": "wikitext-2-raw-v1/test-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/test-00000-of-00001.parquet" + }, + "train": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/train.bin", + "binary_sha256": "0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4", + "concatenated_bytes": 10951563, + "concatenated_sha256": "0ca7d3e74dbe44564ea5942b85232f1bbcb525c9cd481cd5d28a87ee90e7e9b4", + "parquet_bytes": 6357543, + "parquet_sha256": "e83889baabc497075506f91975be5fac0d45c5290b6b20582c8cd1e853d0c9f7", + "rows": 36718, + "source_path": "wikitext-2-raw-v1/train-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/train-00000-of-00001.parquet" + }, + "validation": { + "binary_path": "/home/wuyang/.cache/llm-atlas/k3-attnres-reduced-v1/validation.bin", + "binary_sha256": "a42356f6a8ff1d25daf25ec9db49e10a537c265581b61c74604bb63231dee719", + "concatenated_bytes": 1148008, + "concatenated_sha256": "a42356f6a8ff1d25daf25ec9db49e10a537c265581b61c74604bb63231dee719", + "parquet_bytes": 657209, + "parquet_sha256": "204929b7ff9d6184953f867dedb860e40aa69c078fc1e54b3baaa8fb28511c4c", + "rows": 3760, + "source_path": "wikitext-2-raw-v1/validation-00000-of-00001.parquet", + "source_url": "https://huggingface.co/datasets/Salesforce/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/validation-00000-of-00001.parquet" + } + }, + "variant": "wikitext-2-raw-v1" + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "schema_version": 1, + "status": "frozen-before-model-output", + "windows": { + "context": 256, + "diagnostic_starts": [ + 611936, + 81370, + 284950, + 884010, + 436549, + 319425, + 664210, + 2929, + 1120568, + 1130204, + 567111, + 471404, + 178798, + 773999, + 8739, + 567335 + ], + "diagnostic_tensor_sha256": "d970af9b0c656c9826f369b5fe6e3869a6f6cfeccfa5a922fe94ed1d24b86818", + "formal_batch": 32, + "formal_schedule_cells": 192000, + "formal_schedule_sha256": "81521a70ec61f3717968f160cb711e50c5f52a665a6961538d339360cb695f48", + "formal_steps": 2000, + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ], + "target_bytes_per_window": 256, + "validation_starts": [ + 990074, + 30961, + 68731, + 926982, + 644250, + 11726, + 878377, + 825572, + 874166, + 182830, + 392129, + 822765, + 583693, + 287605, + 117755, + 463343, + 1005482, + 27886, + 1107742, + 1025132, + 190867, + 563499, + 375410, + 611922, + 576979, + 959290, + 139748, + 855267, + 439281, + 268378, + 205337, + 638978, + 916468, + 1059268, + 1098648, + 863779, + 1028257, + 340396, + 547037, + 682617, + 217662, + 744004, + 288924, + 890860, + 856857, + 312424, + 12454, + 1135148, + 385677, + 954097, + 490766, + 164873, + 620587, + 329982, + 1014567, + 236028, + 356220, + 603142, + 879272, + 511832, + 472270, + 823189, + 653568, + 159430 + ], + "validation_tensor_sha256": "5f71fda757fc75010ed16e7636bc394c69f55b34a3713b3b5a7ef8e03eae3c20" + } + }, + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "provenance": { + "formal_file_sha256": { + "baseline-2026073001.json": "f5cddecace6a70ee3824f272811d2a6336ab6dc24ad1100b18292f4413692e9a", + "baseline-2026073002.json": "4c3b9e1ea2ed6fd3078f1212984327364832a1146bfcbdb3a6f2e61b589bfaa8", + "baseline-2026073003.json": "e5ce5a8944dad0c4811f8bf19b4cfec8ce4a9283858fd59cd739dbc37626707b", + "block-2026073001.json": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "block-2026073002.json": "be1f109a630e27c8469438e33b53806cdaecac2f712af4f885da83f00ef6843b", + "block-2026073003.json": "949ec51ca3a219a11f260171da01f737296241536f5e139cf07d444c27efba92", + "full-2026073001.json": "648cb25ea98da1868779733da155275e9a16aad5efbbeeced7812c19d75817d5", + "full-2026073002.json": "1fa715bbd81a3a04a5aa0ba0fc27feb8883c44a01e9f28e281a309a394447979", + "full-2026073003.json": "4e34fa35bbfec460cabc94bce7a08f44fc188d908705ae2e256500f00813b4e3" + }, + "manifest_file_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "reproduction_sha256": "89a127e625ccefbd9b749e6ffca4ba72868e217e0c7a24773024bd2fafde427a" + }, + "reproduction": { + "canonical_sha256_without_self": "89a127e625ccefbd9b749e6ffca4ba72868e217e0c7a24773024bd2fafde427a", + "common_initial_parameters": { + "2026073001": { + "exact": true, + "hashes": { + "baseline": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "block": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1", + "full": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1" + } + }, + "2026073002": { + "exact": true, + "hashes": { + "baseline": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "block": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523", + "full": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523" + } + }, + "2026073003": { + "exact": true, + "hashes": { + "baseline": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "block": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61", + "full": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61" + } + } + }, + "formal_files": { + "baseline-2026073001.json": "f5cddecace6a70ee3824f272811d2a6336ab6dc24ad1100b18292f4413692e9a", + "baseline-2026073002.json": "4c3b9e1ea2ed6fd3078f1212984327364832a1146bfcbdb3a6f2e61b589bfaa8", + "baseline-2026073003.json": "e5ce5a8944dad0c4811f8bf19b4cfec8ce4a9283858fd59cd739dbc37626707b", + "block-2026073001.json": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "block-2026073002.json": "be1f109a630e27c8469438e33b53806cdaecac2f712af4f885da83f00ef6843b", + "block-2026073003.json": "949ec51ca3a219a11f260171da01f737296241536f5e139cf07d444c27efba92", + "full-2026073001.json": "648cb25ea98da1868779733da155275e9a16aad5efbbeeced7812c19d75817d5", + "full-2026073002.json": "1fa715bbd81a3a04a5aa0ba0fc27feb8883c44a01e9f28e281a309a394447979", + "full-2026073003.json": "4e34fa35bbfec460cabc94bce7a08f44fc188d908705ae2e256500f00813b4e3" + }, + "formal_replay": { + "all_numeric_and_hash_fields_exact": true, + "architecture": "block", + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "formal_file_sha256": "5df870369d9a86ccb4ba4191fbd1d6f3642893dd47a60f8f6d1143006bdfbdaf", + "replay_file_sha256": "e74d3323e5fe31378bb8aad7a8efa2fb91995cd7224c158cf03006466cdea2a7", + "seed": 2026073001, + "timing_exact_observed": false, + "timing_exact_required": false + }, + "manifest_sha256": "9778ade5b1c9dd7676d2cdc52b4e4e7ff5ae513cb56c667974e2422702f9dc2b", + "protocol_id": "llm-atlas-k3-attnres-reduced-v1", + "schema_version": 1, + "smoke": { + "baseline": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c", + "second_sha256": "fd8b14f70a6a14978e65f9899b694164ba91753bf82eead436a40837c251e82c" + }, + "block": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc", + "second_sha256": "980a4a534e458199d95b5864b008a81f51b565e05a7b2f24a644b36d2134eccc" + }, + "full": { + "all_exact": true, + "fields": { + "diagnostic": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "training_history": true + }, + "first_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb", + "second_sha256": "c6e32b737c7bf7fdc9b647650ebd7c9d2f9f8550f66a5b84f168f16fa7d2eacb" + } + } + }, + "schema_version": 1 +}