research: audit AttnRes gradient scale study
This commit is contained in:
wuyang
@@ -0,0 +1,685 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate, aggregate, and publish K3 AttnRes Round 05 experiment data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
PROTOCOL_ID = "llm-atlas-k3-attnres-gradient-scale-v1"
|
||||
ARCHITECTURES = ("baseline", "block")
|
||||
DEPTHS = (16, 32)
|
||||
SEEDS = (2026073001, 2026073002, 2026073003)
|
||||
STEPS = (0, 100, 500, 2000, 4000, 8000)
|
||||
FORMAL_STEPS = 8000
|
||||
FORMAL_BATCH = 32
|
||||
TARGET_BYTES_PER_RUN = 65_536_000
|
||||
EXPECTED_TOTAL_TARGET_BYTES = 786_432_000
|
||||
SMOKE_COMPARE_FIELDS = (
|
||||
"protocol_id",
|
||||
"run_kind",
|
||||
"architecture",
|
||||
"depth",
|
||||
"seed",
|
||||
"steps",
|
||||
"batch_size",
|
||||
"target_bytes_seen",
|
||||
"manifest",
|
||||
"model",
|
||||
"optimizer",
|
||||
"hashes",
|
||||
"evaluations",
|
||||
"diagnostics",
|
||||
"training_history",
|
||||
"gradient_gate",
|
||||
"environment",
|
||||
)
|
||||
REPLAY_COMPARE_FIELDS = tuple(
|
||||
field for field in SMOKE_COMPARE_FIELDS if field != "run_kind"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--formal-dir", type=Path, required=True)
|
||||
parser.add_argument("--smoke-a-dir", type=Path, required=True)
|
||||
parser.add_argument("--smoke-b-dir", type=Path, required=True)
|
||||
parser.add_argument("--replay", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--raw-output-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--compact-output", type=Path, required=True)
|
||||
parser.add_argument("--reproduction-output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def canonical_sha256(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def mean(values: Iterable[float]) -> float:
|
||||
return statistics.fmean(values)
|
||||
|
||||
|
||||
def require_finite(value: Any, path: str = "root") -> None:
|
||||
if isinstance(value, float):
|
||||
if not math.isfinite(value):
|
||||
raise ValueError(f"non-finite float at {path}")
|
||||
elif isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
require_finite(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
require_finite(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def selected(run: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
|
||||
return {field: run[field] for field in fields}
|
||||
|
||||
|
||||
def final_diagnostic(run: dict[str, Any]) -> dict[str, Any]:
|
||||
diagnostic = run["diagnostics"][-1]
|
||||
if diagnostic["step"] != FORMAL_STEPS:
|
||||
raise ValueError("final diagnostic is not step 8000")
|
||||
return diagnostic
|
||||
|
||||
|
||||
def validate_run(
|
||||
run: dict[str, Any],
|
||||
*,
|
||||
path: Path,
|
||||
depth: int,
|
||||
architecture: str,
|
||||
seed: int,
|
||||
manifest: dict[str, Any],
|
||||
manifest_hash: str,
|
||||
) -> None:
|
||||
if run["protocol_id"] != PROTOCOL_ID or run["run_kind"] != "formal":
|
||||
raise ValueError(f"formal protocol/kind mismatch: {path}")
|
||||
if (
|
||||
run["depth"] != depth
|
||||
or run["architecture"] != architecture
|
||||
or run["seed"] != seed
|
||||
):
|
||||
raise ValueError(f"formal identity mismatch: {path}")
|
||||
if (
|
||||
run["steps"] != FORMAL_STEPS
|
||||
or run["batch_size"] != FORMAL_BATCH
|
||||
or run["target_bytes_seen"] != TARGET_BYTES_PER_RUN
|
||||
):
|
||||
raise ValueError(f"formal budget mismatch: {path}")
|
||||
if run["manifest"]["file_sha256"] != manifest_hash:
|
||||
raise ValueError(f"manifest file hash mismatch: {path}")
|
||||
for key in (
|
||||
"formal_schedule_sha256",
|
||||
"validation_tensor_sha256",
|
||||
"diagnostic_tensor_sha256",
|
||||
):
|
||||
if run["manifest"][key] != manifest["windows"][key]:
|
||||
raise ValueError(f"manifest {key} mismatch: {path}")
|
||||
if [row["step"] for row in run["evaluations"]] != list(STEPS):
|
||||
raise ValueError(f"evaluation steps mismatch: {path}")
|
||||
if [row["step"] for row in run["diagnostics"]] != list(STEPS):
|
||||
raise ValueError(f"diagnostic steps mismatch: {path}")
|
||||
if run["model"]["layers"] != depth:
|
||||
raise ValueError(f"model depth mismatch: {path}")
|
||||
|
||||
for diagnostic in run["diagnostics"]:
|
||||
capture = diagnostic["capture"]
|
||||
if (
|
||||
capture["count"] != depth
|
||||
or capture["shape"] != [16, 256, 192]
|
||||
or set(capture["dtypes"]) != {"torch.float32"}
|
||||
or not capture["all_gradients_finite"]
|
||||
or not capture["all_gradients_present"]
|
||||
or not capture["storage_unique"]
|
||||
):
|
||||
raise ValueError(f"activation capture gate mismatch: {path}")
|
||||
for field in (
|
||||
"activation_grad_rms_by_block",
|
||||
"activation_output_rms_by_block",
|
||||
"core_parameter_grad_rms_by_block",
|
||||
):
|
||||
if len(diagnostic[field]) != depth:
|
||||
raise ValueError(f"{field} length mismatch: {path}")
|
||||
for field in (
|
||||
"layer_input_rms_by_sublayer",
|
||||
"branch_output_rms_by_sublayer",
|
||||
"stream_state_rms_by_sublayer",
|
||||
):
|
||||
if len(diagnostic[field]) != depth * 2:
|
||||
raise ValueError(f"{field} length mismatch: {path}")
|
||||
if architecture == "baseline":
|
||||
if diagnostic["depth_weights"] or diagnostic["output_weights"] is not None:
|
||||
raise ValueError(f"unexpected Baseline mixer trace: {path}")
|
||||
else:
|
||||
if (
|
||||
len(diagnostic["depth_weights"]) != depth * 2
|
||||
or diagnostic["output_weights"]["sources"] != 9
|
||||
):
|
||||
raise ValueError(f"Block mixer trace mismatch: {path}")
|
||||
require_finite(run, path.name)
|
||||
|
||||
|
||||
def relative_reduction(baseline: float, block: float) -> float:
|
||||
return (baseline - block) / baseline
|
||||
|
||||
|
||||
def depth_verdict(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
cv_reductions = [row["relative_cv_reduction"] for row in rows]
|
||||
imbalance_reductions = [
|
||||
row["relative_imbalance_reduction"] for row in rows
|
||||
]
|
||||
mean_cv = mean(cv_reductions)
|
||||
mean_imbalance = mean(imbalance_reductions)
|
||||
support = (
|
||||
all(value > 0 for value in cv_reductions)
|
||||
and mean_cv >= 0.20
|
||||
and all(value > 0 for value in imbalance_reductions)
|
||||
and mean_imbalance >= 0.20
|
||||
)
|
||||
concern = (
|
||||
all(value < 0 for value in cv_reductions)
|
||||
and -mean_cv >= 0.20
|
||||
and all(value < 0 for value in imbalance_reductions)
|
||||
and -mean_imbalance >= 0.20
|
||||
)
|
||||
if support:
|
||||
label = "joint directional support at this depth"
|
||||
elif concern:
|
||||
label = "joint directional concern at this depth"
|
||||
else:
|
||||
label = "mixed / inconclusive at this depth"
|
||||
return {
|
||||
"label": label,
|
||||
"threshold_relative": 0.20,
|
||||
"cv_reductions": cv_reductions,
|
||||
"mean_cv_reduction": mean_cv,
|
||||
"imbalance_reductions": imbalance_reductions,
|
||||
"mean_imbalance_reduction": mean_imbalance,
|
||||
"all_cv_improve": all(value > 0 for value in cv_reductions),
|
||||
"all_imbalance_improve": all(
|
||||
value > 0 for value in imbalance_reductions
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def summarize_depth(
|
||||
depth: int, runs: dict[tuple[int, str, int], dict[str, Any]]
|
||||
) -> dict[str, Any]:
|
||||
rows = []
|
||||
for seed in SEEDS:
|
||||
baseline = runs[(depth, "baseline", seed)]
|
||||
block = runs[(depth, "block", seed)]
|
||||
baseline_diagnostic = final_diagnostic(baseline)
|
||||
block_diagnostic = final_diagnostic(block)
|
||||
baseline_activation = baseline_diagnostic[
|
||||
"activation_grad_statistics"
|
||||
]
|
||||
block_activation = block_diagnostic["activation_grad_statistics"]
|
||||
baseline_bpc = baseline["evaluations"][-1]["bits_per_byte"]
|
||||
block_bpc = block["evaluations"][-1]["bits_per_byte"]
|
||||
rows.append(
|
||||
{
|
||||
"seed": seed,
|
||||
"baseline_bpc": baseline_bpc,
|
||||
"block_bpc": block_bpc,
|
||||
"block_minus_baseline_bpc": block_bpc - baseline_bpc,
|
||||
"baseline_activation_grad_mean": baseline_activation["mean"],
|
||||
"block_activation_grad_mean": block_activation["mean"],
|
||||
"block_to_baseline_activation_grad_mean": (
|
||||
block_activation["mean"] / baseline_activation["mean"]
|
||||
),
|
||||
"baseline_cv": baseline_activation["population_cv"],
|
||||
"block_cv": block_activation["population_cv"],
|
||||
"relative_cv_reduction": relative_reduction(
|
||||
baseline_activation["population_cv"],
|
||||
block_activation["population_cv"],
|
||||
),
|
||||
"baseline_first_to_last_ratio": baseline_activation[
|
||||
"first_to_last_ratio"
|
||||
],
|
||||
"block_first_to_last_ratio": block_activation[
|
||||
"first_to_last_ratio"
|
||||
],
|
||||
"baseline_imbalance": baseline_activation[
|
||||
"imbalance_abs_log_ratio"
|
||||
],
|
||||
"block_imbalance": block_activation[
|
||||
"imbalance_abs_log_ratio"
|
||||
],
|
||||
"relative_imbalance_reduction": relative_reduction(
|
||||
baseline_activation["imbalance_abs_log_ratio"],
|
||||
block_activation["imbalance_abs_log_ratio"],
|
||||
),
|
||||
"baseline_parameter_grad_cv": baseline_diagnostic[
|
||||
"core_parameter_grad_statistics"
|
||||
]["population_cv"],
|
||||
"block_parameter_grad_cv": block_diagnostic[
|
||||
"core_parameter_grad_statistics"
|
||||
]["population_cv"],
|
||||
}
|
||||
)
|
||||
verdict = depth_verdict(rows)
|
||||
return {
|
||||
"depth": depth,
|
||||
"by_seed": rows,
|
||||
"means": {
|
||||
"baseline_bpc": mean(row["baseline_bpc"] for row in rows),
|
||||
"block_bpc": mean(row["block_bpc"] for row in rows),
|
||||
"block_minus_baseline_bpc": mean(
|
||||
row["block_minus_baseline_bpc"] for row in rows
|
||||
),
|
||||
"baseline_cv": mean(row["baseline_cv"] for row in rows),
|
||||
"block_cv": mean(row["block_cv"] for row in rows),
|
||||
"relative_cv_reduction": mean(
|
||||
row["relative_cv_reduction"] for row in rows
|
||||
),
|
||||
"baseline_imbalance": mean(
|
||||
row["baseline_imbalance"] for row in rows
|
||||
),
|
||||
"block_imbalance": mean(
|
||||
row["block_imbalance"] for row in rows
|
||||
),
|
||||
"relative_imbalance_reduction": mean(
|
||||
row["relative_imbalance_reduction"] for row in rows
|
||||
),
|
||||
"block_to_baseline_activation_grad_mean": mean(
|
||||
row["block_to_baseline_activation_grad_mean"] for row in rows
|
||||
),
|
||||
"baseline_parameter_grad_cv": mean(
|
||||
row["baseline_parameter_grad_cv"] for row in rows
|
||||
),
|
||||
"block_parameter_grad_cv": mean(
|
||||
row["block_parameter_grad_cv"] for row in rows
|
||||
),
|
||||
},
|
||||
"verdict": verdict,
|
||||
}
|
||||
|
||||
|
||||
def compact_cell(run: dict[str, Any]) -> dict[str, Any]:
|
||||
diagnostics = []
|
||||
for row in run["diagnostics"]:
|
||||
compact = {
|
||||
"step": row["step"],
|
||||
"loss_nats": row["loss_nats"],
|
||||
"bits_per_byte": row["bits_per_byte"],
|
||||
"activation_grad_rms_by_block": row[
|
||||
"activation_grad_rms_by_block"
|
||||
],
|
||||
"activation_grad_statistics": row[
|
||||
"activation_grad_statistics"
|
||||
],
|
||||
"activation_output_rms_by_block": row[
|
||||
"activation_output_rms_by_block"
|
||||
],
|
||||
"activation_output_statistics": row[
|
||||
"activation_output_statistics"
|
||||
],
|
||||
"core_parameter_grad_rms_by_block": row[
|
||||
"core_parameter_grad_rms_by_block"
|
||||
],
|
||||
"core_parameter_grad_statistics": row[
|
||||
"core_parameter_grad_statistics"
|
||||
],
|
||||
}
|
||||
if row["depth_weights"]:
|
||||
compact["depth_weights"] = row["depth_weights"]
|
||||
compact["output_weights"] = row["output_weights"]
|
||||
diagnostics.append(compact)
|
||||
return {
|
||||
"architecture": run["architecture"],
|
||||
"depth": run["depth"],
|
||||
"seed": run["seed"],
|
||||
"evaluations": run["evaluations"],
|
||||
"diagnostics": diagnostics,
|
||||
"timing": run["timing"],
|
||||
"parameters": run["model"]["parameters"],
|
||||
"hashes": run["hashes"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
manifest = read_json(args.manifest)
|
||||
manifest_hash = file_sha256(args.manifest)
|
||||
if manifest["protocol_id"] != PROTOCOL_ID:
|
||||
raise ValueError("manifest protocol mismatch")
|
||||
if (
|
||||
manifest["windows"]["formal_schedule_cells"] != 768_000
|
||||
or manifest["windows"]["formal_steps"] != FORMAL_STEPS
|
||||
or manifest["windows"]["formal_batch"] != FORMAL_BATCH
|
||||
):
|
||||
raise ValueError("manifest schedule budget mismatch")
|
||||
|
||||
runs: dict[tuple[int, str, int], dict[str, Any]] = {}
|
||||
source_paths: dict[str, Path] = {}
|
||||
formal_hashes: dict[str, str] = {}
|
||||
for depth in DEPTHS:
|
||||
for architecture in ARCHITECTURES:
|
||||
for seed in SEEDS:
|
||||
name = f"depth-{depth}-{architecture}-seed-{seed}.json"
|
||||
path = args.formal_dir / name
|
||||
run = read_json(path)
|
||||
validate_run(
|
||||
run,
|
||||
path=path,
|
||||
depth=depth,
|
||||
architecture=architecture,
|
||||
seed=seed,
|
||||
manifest=manifest,
|
||||
manifest_hash=manifest_hash,
|
||||
)
|
||||
runs[(depth, architecture, seed)] = run
|
||||
public_name = f"formal-{name}"
|
||||
source_paths[public_name] = path
|
||||
formal_hashes[public_name] = file_sha256(path)
|
||||
|
||||
if sum(run["target_bytes_seen"] for run in runs.values()) != (
|
||||
EXPECTED_TOTAL_TARGET_BYTES
|
||||
):
|
||||
raise ValueError("formal total target-byte budget mismatch")
|
||||
|
||||
common_initial_exact: dict[str, Any] = {}
|
||||
input_gate_exact: dict[str, Any] = {}
|
||||
for depth in DEPTHS:
|
||||
for seed in SEEDS:
|
||||
baseline = runs[(depth, "baseline", seed)]
|
||||
block = runs[(depth, "block", seed)]
|
||||
public_fields = (
|
||||
"initial_public_parameter_structure",
|
||||
"initial_public_parameter_tensors",
|
||||
"initial_public_parameter_elements",
|
||||
"initial_public_parameters",
|
||||
)
|
||||
exact = all(
|
||||
baseline["hashes"][field] == block["hashes"][field]
|
||||
for field in public_fields
|
||||
)
|
||||
gate_exact = (
|
||||
baseline["manifest"]["input_gate_tensor_hashes"]
|
||||
== block["manifest"]["input_gate_tensor_hashes"]
|
||||
)
|
||||
key = f"depth-{depth}-seed-{seed}"
|
||||
common_initial_exact[key] = {
|
||||
"exact": exact,
|
||||
"baseline": {
|
||||
field: baseline["hashes"][field] for field in public_fields
|
||||
},
|
||||
"block": {
|
||||
field: block["hashes"][field] for field in public_fields
|
||||
},
|
||||
}
|
||||
input_gate_exact[key] = {
|
||||
"exact": gate_exact,
|
||||
"hashes": baseline["manifest"]["input_gate_tensor_hashes"],
|
||||
}
|
||||
if not exact or not gate_exact:
|
||||
raise ValueError(f"paired equality gate failed: {key}")
|
||||
|
||||
smoke_exact: dict[str, Any] = {}
|
||||
smoke_hashes: dict[str, str] = {}
|
||||
for depth in DEPTHS:
|
||||
for architecture in ARCHITECTURES:
|
||||
name = f"depth-{depth}-{architecture}.json"
|
||||
left_path = args.smoke_a_dir / name
|
||||
right_path = args.smoke_b_dir / name
|
||||
left = read_json(left_path)
|
||||
right = read_json(right_path)
|
||||
left_selected = selected(left, SMOKE_COMPARE_FIELDS)
|
||||
right_selected = selected(right, SMOKE_COMPARE_FIELDS)
|
||||
exact = left_selected == right_selected
|
||||
if (
|
||||
not exact
|
||||
or left["run_kind"] != "smoke"
|
||||
or left["steps"] != 20
|
||||
or not left["gradient_gate"]["passed"]
|
||||
):
|
||||
raise ValueError(f"smoke gate failed: {name}")
|
||||
key = f"depth-{depth}-{architecture}"
|
||||
smoke_exact[key] = {
|
||||
"exact": exact,
|
||||
"compare_sha256": canonical_sha256(left_selected),
|
||||
"gradient_gate": left["gradient_gate"],
|
||||
}
|
||||
for label, path in (("a", left_path), ("b", right_path)):
|
||||
public_name = f"smoke-{label}-{name}"
|
||||
source_paths[public_name] = path
|
||||
smoke_hashes[public_name] = file_sha256(path)
|
||||
|
||||
replay = read_json(args.replay)
|
||||
replay_formal = runs[(32, "block", 2026073001)]
|
||||
replay_left = selected(replay_formal, REPLAY_COMPARE_FIELDS)
|
||||
replay_right = selected(replay, REPLAY_COMPARE_FIELDS)
|
||||
replay_exact = replay_left == replay_right
|
||||
if (
|
||||
replay["run_kind"] != "replay"
|
||||
or replay["depth"] != 32
|
||||
or replay["architecture"] != "block"
|
||||
or replay["seed"] != 2026073001
|
||||
or not replay_exact
|
||||
):
|
||||
raise ValueError("formal replay gate failed")
|
||||
replay_public_name = "replay-depth-32-block-seed-2026073001.json"
|
||||
source_paths[replay_public_name] = args.replay
|
||||
|
||||
depth_summaries = {
|
||||
str(depth): summarize_depth(depth, runs) for depth in DEPTHS
|
||||
}
|
||||
depth_labels = [
|
||||
depth_summaries[str(depth)]["verdict"]["label"] for depth in DEPTHS
|
||||
]
|
||||
if all(
|
||||
label == "joint directional support at this depth"
|
||||
for label in depth_labels
|
||||
):
|
||||
overall_verdict = (
|
||||
"scale-consistent directional support in this operationalization"
|
||||
)
|
||||
elif all(
|
||||
label == "joint directional concern at this depth"
|
||||
for label in depth_labels
|
||||
):
|
||||
overall_verdict = (
|
||||
"scale-consistent directional concern in this operationalization"
|
||||
)
|
||||
else:
|
||||
overall_verdict = "depth-dependent or inconclusive"
|
||||
|
||||
full = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": PROTOCOL_ID,
|
||||
"manifest": manifest,
|
||||
"study": {
|
||||
"architectures": list(ARCHITECTURES),
|
||||
"depths": list(DEPTHS),
|
||||
"seeds": list(SEEDS),
|
||||
"diagnostic_steps": list(STEPS),
|
||||
"formal_runs": len(runs),
|
||||
"formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES,
|
||||
"replay_target_bytes": TARGET_BYTES_PER_RUN,
|
||||
"gradient_object": (
|
||||
"RMS of d(mean token CE)/d(post-MLP Transformer-block output) "
|
||||
"over batch×time×channel"
|
||||
),
|
||||
},
|
||||
"depth_summaries": depth_summaries,
|
||||
"overall_verdict": overall_verdict,
|
||||
"gates": {
|
||||
"common_initial_parameters": common_initial_exact,
|
||||
"paired_input_tensors": input_gate_exact,
|
||||
"smoke_exact": smoke_exact,
|
||||
"replay": {
|
||||
"exact": replay_exact,
|
||||
"compare_fields": list(REPLAY_COMPARE_FIELDS),
|
||||
"formal_compare_sha256": canonical_sha256(replay_left),
|
||||
"replay_compare_sha256": canonical_sha256(replay_right),
|
||||
"formal_final_model_state": replay_formal["hashes"][
|
||||
"final_model_state"
|
||||
],
|
||||
"replay_final_model_state": replay["hashes"][
|
||||
"final_model_state"
|
||||
],
|
||||
"formal_final_optimizer_state": replay_formal["hashes"][
|
||||
"final_optimizer_state"
|
||||
],
|
||||
"replay_final_optimizer_state": replay["hashes"][
|
||||
"final_optimizer_state"
|
||||
],
|
||||
},
|
||||
},
|
||||
"runs": {
|
||||
f"depth-{depth}-{architecture}-seed-{seed}": run
|
||||
for (depth, architecture, seed), run in sorted(runs.items())
|
||||
},
|
||||
}
|
||||
full["canonical_sha256_without_self"] = canonical_sha256(full)
|
||||
|
||||
compact = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": PROTOCOL_ID,
|
||||
"study": full["study"],
|
||||
"manifest_summary": {
|
||||
"file_sha256": manifest_hash,
|
||||
"dataset_revision": manifest["dataset"]["revision"],
|
||||
"train_bytes_sha256": manifest["dataset"]["splits"]["train"][
|
||||
"concatenated_sha256"
|
||||
],
|
||||
"formal_schedule_sha256": manifest["windows"][
|
||||
"formal_schedule_sha256"
|
||||
],
|
||||
"validation_tensor_sha256": manifest["windows"][
|
||||
"validation_tensor_sha256"
|
||||
],
|
||||
"diagnostic_tensor_sha256": manifest["windows"][
|
||||
"diagnostic_tensor_sha256"
|
||||
],
|
||||
},
|
||||
"depth_summaries": depth_summaries,
|
||||
"overall_verdict": overall_verdict,
|
||||
"replay_exact": replay_exact,
|
||||
"cells": [
|
||||
compact_cell(runs[(depth, architecture, seed)])
|
||||
for depth in DEPTHS
|
||||
for architecture in ARCHITECTURES
|
||||
for seed in SEEDS
|
||||
],
|
||||
}
|
||||
compact["canonical_sha256_without_self"] = canonical_sha256(compact)
|
||||
|
||||
args.raw_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for public_name, source_path in sorted(source_paths.items()):
|
||||
target = args.raw_output_dir / public_name
|
||||
temporary = target.with_suffix(target.suffix + ".tmp")
|
||||
shutil.copyfile(source_path, temporary)
|
||||
os.replace(temporary, target)
|
||||
|
||||
reproduction = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": PROTOCOL_ID,
|
||||
"manifest": {
|
||||
"path": str(args.manifest),
|
||||
"sha256": manifest_hash,
|
||||
},
|
||||
"protocol_sha256": file_sha256(
|
||||
Path("research/K3_ATTNRES_GRADIENT_SCALE_PROTOCOL.md")
|
||||
),
|
||||
"definition_audit_sha256": file_sha256(
|
||||
Path("research/K3_ATTNRES_GRADIENT_DEFINITION_AUDIT.md")
|
||||
),
|
||||
"runner_sha256": file_sha256(
|
||||
Path("experiments/k3/attnres_gradient/train.py")
|
||||
),
|
||||
"analyzer_sha256": file_sha256(Path(__file__)),
|
||||
"formal_raw_sha256": formal_hashes,
|
||||
"smoke_raw_sha256": smoke_hashes,
|
||||
"replay_raw_sha256": {
|
||||
replay_public_name: file_sha256(args.replay)
|
||||
},
|
||||
"formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES,
|
||||
"replay_target_bytes": TARGET_BYTES_PER_RUN,
|
||||
"smoke_exact": smoke_exact,
|
||||
"replay_exact": {
|
||||
"exact": replay_exact,
|
||||
"compare_sha256": canonical_sha256(replay_left),
|
||||
"final_model_state": replay["hashes"]["final_model_state"],
|
||||
"final_optimizer_state": replay["hashes"][
|
||||
"final_optimizer_state"
|
||||
],
|
||||
},
|
||||
"aggregate_sha256": full["canonical_sha256_without_self"],
|
||||
"compact_sha256": compact["canonical_sha256_without_self"],
|
||||
"overall_verdict": overall_verdict,
|
||||
}
|
||||
reproduction["canonical_sha256_without_self"] = canonical_sha256(
|
||||
reproduction
|
||||
)
|
||||
|
||||
atomic_json(args.output, full)
|
||||
atomic_json(args.compact_output, compact)
|
||||
atomic_json(args.reproduction_output, reproduction)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"formal_runs": len(runs),
|
||||
"formal_target_bytes": EXPECTED_TOTAL_TARGET_BYTES,
|
||||
"smoke_exact": all(
|
||||
row["exact"] for row in smoke_exact.values()
|
||||
),
|
||||
"replay_exact": replay_exact,
|
||||
"depth_verdicts": {
|
||||
depth: depth_summaries[str(depth)]["verdict"]["label"]
|
||||
for depth in DEPTHS
|
||||
},
|
||||
"overall_verdict": overall_verdict,
|
||||
"aggregate_sha256": full[
|
||||
"canonical_sha256_without_self"
|
||||
],
|
||||
"compact_sha256": compact[
|
||||
"canonical_sha256_without_self"
|
||||
],
|
||||
"reproduction_sha256": reproduction[
|
||||
"canonical_sha256_without_self"
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
{
|
||||
"aggregate_sha256": "69be133c8f5b11f8a58e831bc03f5432e737ed21efe89bf186784ead7c7f7b51",
|
||||
"analyzer_sha256": "017d38d92bfd5bc26e10028015f9bf85d28516fb5c5bae06d7bf315b5eca2fc8",
|
||||
"canonical_sha256_without_self": "addb2e5990af4bbec4b88b4654fc72464f4df5b57fd00e46cad1febbb042f68a",
|
||||
"compact_sha256": "8cdb71808a429e5f9513c1c6c9426bdc93e30f237001a1b47a2a2fba574a044f",
|
||||
"definition_audit_sha256": "79221c5648ba280d9177386d2fcfb2fa653b3dd14405185b5997754db2961cc6",
|
||||
"formal_raw_sha256": {
|
||||
"formal-depth-16-baseline-seed-2026073001.json": "c9144ddb458010868668ab0bb1272d4e5051300936d9b21dccee75c32a9e46de",
|
||||
"formal-depth-16-baseline-seed-2026073002.json": "939cad5054960ba8a72b50b4b250b94c3d22a6b46796f669c6e4f033d59fb9de",
|
||||
"formal-depth-16-baseline-seed-2026073003.json": "afd9a60ff323e97d21b24b413d9b9fa97b8d5f960bfc88d2040502aa5ad96a1e",
|
||||
"formal-depth-16-block-seed-2026073001.json": "23e3c68e54ec55d981066ad5730316db9084a6637a8064763952ba109ea1f715",
|
||||
"formal-depth-16-block-seed-2026073002.json": "027f8786f530d6ad5d910f0aad3d0f287887d6457c670bc8b50af7248be1487d",
|
||||
"formal-depth-16-block-seed-2026073003.json": "254b637f403a4c61e8e8bb2dd083349406b6baa7a3d8a616e37a9ab6f8ebaeb9",
|
||||
"formal-depth-32-baseline-seed-2026073001.json": "232ed3dfbbbc898c42622c4a9aee8400f76cb773c576bf1c522cdbb19ead998c",
|
||||
"formal-depth-32-baseline-seed-2026073002.json": "f4fb5fca608a6623ce4c9707714a8a9ebd2c10f6e63f54e243ec3c06d18b1fff",
|
||||
"formal-depth-32-baseline-seed-2026073003.json": "1b0bd279b165c3a2431685d8c1e6bc36ff72eb55333d5de689dca4f89506bb5e",
|
||||
"formal-depth-32-block-seed-2026073001.json": "29e1d638b481619c7b32de402122523db8b17cc1fc67a8881fa1ba132a1d5c38",
|
||||
"formal-depth-32-block-seed-2026073002.json": "21199deb2199395061e51220fd8c7afd04a1135a6381e406da9b5795e3ad5032",
|
||||
"formal-depth-32-block-seed-2026073003.json": "c0d7f1bcfa9fa7f8f3134ca4571bdf23a951182d03b7de9611a6b4b89667d4ac"
|
||||
},
|
||||
"formal_target_bytes": 786432000,
|
||||
"manifest": {
|
||||
"path": "experiments/k3/attnres_gradient/manifest.json",
|
||||
"sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371"
|
||||
},
|
||||
"overall_verdict": "depth-dependent or inconclusive",
|
||||
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
||||
"protocol_sha256": "f772629b3b82975b6756721c3a3bb57cc4171dfa26ba5b1e8043dc91c9dcce22",
|
||||
"replay_exact": {
|
||||
"compare_sha256": "46300a452840a9dc6a5180efe7949cf3d81cc4471ed4a942da407e9343064817",
|
||||
"exact": true,
|
||||
"final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59",
|
||||
"final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637"
|
||||
},
|
||||
"replay_raw_sha256": {
|
||||
"replay-depth-32-block-seed-2026073001.json": "5cea76d68a642c2b4b4f189a813b2e4af9361a939ddd31a68dfdb0ab090b4aad"
|
||||
},
|
||||
"replay_target_bytes": 65536000,
|
||||
"runner_sha256": "04ae69e10c58972c9193c2d31c7e09d924a0d0e834107afa4ba128c64ac5800f",
|
||||
"schema_version": 1,
|
||||
"smoke_exact": {
|
||||
"depth-16-baseline": {
|
||||
"compare_sha256": "525cdcd9a79ec61096ecf64fedcbf93144d3168b434e2f8fd384c4374cbebd2a",
|
||||
"exact": true,
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
}
|
||||
},
|
||||
"depth-16-block": {
|
||||
"compare_sha256": "e4330a0d878d10b474b9aeb0be58131f801ebe2e5dd9553d4713c3d16b6590cc",
|
||||
"exact": true,
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
}
|
||||
},
|
||||
"depth-32-baseline": {
|
||||
"compare_sha256": "8ee0dc37e88742b6705968f5f2845a3a1df1250f5e0fd77be6b909236ea28e56",
|
||||
"exact": true,
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
}
|
||||
},
|
||||
"depth-32-block": {
|
||||
"compare_sha256": "289073b1940d4747b7e049f8d767f9c22a49867d37403ff6ef7746a55bd1734c",
|
||||
"exact": true,
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"smoke_raw_sha256": {
|
||||
"smoke-a-depth-16-baseline.json": "fe0ff3ed21bf9b86d238d7a22e99b6e07ba99135b58d29eb6b205f2602e64556",
|
||||
"smoke-a-depth-16-block.json": "faa06a2bd02135858d3a2503e1ecfb34187b4ee3b4ce3428ed473a55b15b6705",
|
||||
"smoke-a-depth-32-baseline.json": "e327a6f5db87515c1b29b2808205e23b81228b08ce356b98804717e2e55eef1c",
|
||||
"smoke-a-depth-32-block.json": "2d9a97c2d6272c928d5b5bc01febd142d4b2f2ad3080330e849f2731a8d080c2",
|
||||
"smoke-b-depth-16-baseline.json": "fe0ff3ed21bf9b86d238d7a22e99b6e07ba99135b58d29eb6b205f2602e64556",
|
||||
"smoke-b-depth-16-block.json": "faa06a2bd02135858d3a2503e1ecfb34187b4ee3b4ce3428ed473a55b15b6705",
|
||||
"smoke-b-depth-32-baseline.json": "e327a6f5db87515c1b29b2808205e23b81228b08ce356b98804717e2e55eef1c",
|
||||
"smoke-b-depth-32-block.json": "2d9a97c2d6272c928d5b5bc01febd142d4b2f2ad3080330e849f2731a8d080c2"
|
||||
}
|
||||
}
|
||||
+7366
File diff suppressed because it is too large
Load Diff
+7366
File diff suppressed because it is too large
Load Diff
+7366
File diff suppressed because it is too large
Load Diff
+9616
File diff suppressed because it is too large
Load Diff
+9616
File diff suppressed because it is too large
Load Diff
+9616
File diff suppressed because it is too large
Load Diff
+8614
File diff suppressed because it is too large
Load Diff
+8614
File diff suppressed because it is too large
Load Diff
+8614
File diff suppressed because it is too large
Load Diff
+13072
File diff suppressed because it is too large
Load Diff
+13072
File diff suppressed because it is too large
Load Diff
+13072
File diff suppressed because it is too large
Load Diff
+13072
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,700 @@
|
||||
{
|
||||
"architecture": "baseline",
|
||||
"batch_size": 32,
|
||||
"canonical_sha256_without_self": "0ae9ad1697ca15ec4c84270ad82b9fa98056f14f20fc7372c4b9886c29845415",
|
||||
"depth": 16,
|
||||
"diagnostics": [
|
||||
{
|
||||
"activation_grad_rms_by_block": [
|
||||
0.0002865509013645351,
|
||||
0.00024785567075014114,
|
||||
0.00021551651298068464,
|
||||
0.00019683866412378848,
|
||||
0.0001823508064262569,
|
||||
0.0001679604029050097,
|
||||
0.00015584587526973337,
|
||||
0.00014737028686795384,
|
||||
0.00014130656199995428,
|
||||
0.00013612695329356939,
|
||||
0.00013129493163432926,
|
||||
0.00012807638267986476,
|
||||
0.00012550120300147682,
|
||||
0.00012324050476308912,
|
||||
0.00012108208466088399,
|
||||
0.00011888353037647903
|
||||
],
|
||||
"activation_grad_statistics": {
|
||||
"first_quartile_mean": 0.00023669043730478734,
|
||||
"first_to_last_ratio": 1.9372775995887173,
|
||||
"imbalance_abs_log_ratio": 0.6612836883477485,
|
||||
"last_quartile_mean": 0.00012217683070048224,
|
||||
"mean": 0.00016411257956860936,
|
||||
"normalized": [
|
||||
1.7460629899168627,
|
||||
1.5102783187106137,
|
||||
1.313223602646409,
|
||||
1.1994124072706904,
|
||||
1.1111324123086055,
|
||||
1.0234462424910682,
|
||||
0.9496278449793059,
|
||||
0.8979828801383491,
|
||||
0.8610343117596252,
|
||||
0.8294729974472175,
|
||||
0.8000296624393728,
|
||||
0.7804178266926869,
|
||||
0.7647262832098098,
|
||||
0.7509509940495869,
|
||||
0.7377989242455608,
|
||||
0.7244023016942358
|
||||
],
|
||||
"population_cv": 0.29361872380036635
|
||||
},
|
||||
"activation_output_rms_by_block": [
|
||||
0.02866268903017044,
|
||||
0.029099803417921066,
|
||||
0.02953805774450302,
|
||||
0.030088091269135475,
|
||||
0.030644793063402176,
|
||||
0.03131929785013199,
|
||||
0.03213750571012497,
|
||||
0.03286394104361534,
|
||||
0.033869802951812744,
|
||||
0.03499744459986687,
|
||||
0.03588006645441055,
|
||||
0.037365153431892395,
|
||||
0.03789033368229866,
|
||||
0.039445169270038605,
|
||||
0.04050002992153168,
|
||||
0.04121527820825577
|
||||
],
|
||||
"activation_output_statistics": {
|
||||
"first_quartile_mean": 0.0293471603654325,
|
||||
"first_to_last_ratio": 0.7380574840395957,
|
||||
"imbalance_abs_log_ratio": 0.3037335657624931,
|
||||
"last_quartile_mean": 0.03976270277053118,
|
||||
"mean": 0.034094841103069484,
|
||||
"normalized": [
|
||||
0.8406752488894867,
|
||||
0.8534957922212247,
|
||||
0.8663497699023965,
|
||||
0.8824822259232266,
|
||||
0.8988102619619861,
|
||||
0.9185934539320196,
|
||||
0.942591449919727,
|
||||
0.9638977622528551,
|
||||
0.9933996421752943,
|
||||
1.0264733158329964,
|
||||
1.0523605710888722,
|
||||
1.0959180985456622,
|
||||
1.1113216092650298,
|
||||
1.1569248600043318,
|
||||
1.1878638706395246,
|
||||
1.2088420674453664
|
||||
],
|
||||
"population_cv": 0.11952987076282337
|
||||
},
|
||||
"bits_per_byte": 8.096463027059821,
|
||||
"branch_output_rms_by_sublayer": [
|
||||
0.0033055038657039404,
|
||||
0.0037518907338380814,
|
||||
0.0033751516602933407,
|
||||
0.003911525942385197,
|
||||
0.004229962360113859,
|
||||
0.0038461871445178986,
|
||||
0.004243654198944569,
|
||||
0.0038366704247891903,
|
||||
0.004372371360659599,
|
||||
0.003851204412057996,
|
||||
0.005078981164842844,
|
||||
0.003818925702944398,
|
||||
0.0054668826051056385,
|
||||
0.0038749484810978174,
|
||||
0.005823117680847645,
|
||||
0.0037566579412668943,
|
||||
0.0065947300754487514,
|
||||
0.003876061411574483,
|
||||
0.006763988174498081,
|
||||
0.003795720636844635,
|
||||
0.006863070651888847,
|
||||
0.0038706199266016483,
|
||||
0.007856340147554874,
|
||||
0.003839300014078617,
|
||||
0.00830968376249075,
|
||||
0.003927029203623533,
|
||||
0.009655521251261234,
|
||||
0.0038513424806296825,
|
||||
0.009004893712699413,
|
||||
0.0039484030567109585,
|
||||
0.008608299307525158,
|
||||
0.003791053779423237
|
||||
],
|
||||
"capture": {
|
||||
"all_gradients_finite": true,
|
||||
"all_gradients_present": true,
|
||||
"count": 16,
|
||||
"dtypes": [
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32"
|
||||
],
|
||||
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
||||
"shape": [
|
||||
16,
|
||||
256,
|
||||
192
|
||||
],
|
||||
"storage_unique": true
|
||||
},
|
||||
"core_parameter_grad_rms_by_block": [
|
||||
0.00809059897248305,
|
||||
0.007313812062277777,
|
||||
0.007090710224412361,
|
||||
0.0067033738367094624,
|
||||
0.006535407867454384,
|
||||
0.006785021795736036,
|
||||
0.006295993569919972,
|
||||
0.005509300326230647,
|
||||
0.006231736048910802,
|
||||
0.006047390980174007,
|
||||
0.005629405359532731,
|
||||
0.005784042737956458,
|
||||
0.0059591736113607996,
|
||||
0.0061983173180850636,
|
||||
0.006571690039252182,
|
||||
0.005060205437758869
|
||||
],
|
||||
"core_parameter_grad_statistics": {
|
||||
"first_quartile_mean": 0.007299623773970663,
|
||||
"first_to_last_ratio": 1.227374872012571,
|
||||
"imbalance_abs_log_ratio": 0.2048776382299483,
|
||||
"last_quartile_mean": 0.005947346601614228,
|
||||
"mean": 0.006362886261765913,
|
||||
"normalized": [
|
||||
1.271529717747562,
|
||||
1.1494488132258316,
|
||||
1.1143858200043408,
|
||||
1.05351149791715,
|
||||
1.0271137340180259,
|
||||
1.066343404015675,
|
||||
0.9894870520870547,
|
||||
0.8658492545019393,
|
||||
0.9793882512652816,
|
||||
0.9504163254515969,
|
||||
0.8847251275509024,
|
||||
0.909028151691524,
|
||||
0.9365519618304368,
|
||||
0.9741361173356609,
|
||||
1.0328158902888074,
|
||||
0.7952688810682109
|
||||
],
|
||||
"population_cv": 0.11384977604868386
|
||||
},
|
||||
"depth_weights": [],
|
||||
"layer_input_rms_by_sublayer": [
|
||||
0.02823694422841072,
|
||||
0.028404271230101585,
|
||||
0.02866268903017044,
|
||||
0.02886480651795864,
|
||||
0.029099803417921066,
|
||||
0.029365191236138344,
|
||||
0.02953805774450302,
|
||||
0.029881296679377556,
|
||||
0.030088091269135475,
|
||||
0.030441828072071075,
|
||||
0.030644793063402176,
|
||||
0.03106885403394699,
|
||||
0.03131929785013199,
|
||||
0.031892918050289154,
|
||||
0.03213750571012497,
|
||||
0.03266792371869087,
|
||||
0.03286394104361534,
|
||||
0.03364725783467293,
|
||||
0.033869802951812744,
|
||||
0.03485054895281792,
|
||||
0.03499744459986687,
|
||||
0.03562505170702934,
|
||||
0.03588006645441055,
|
||||
0.03719272464513779,
|
||||
0.037365153431892395,
|
||||
0.03768136352300644,
|
||||
0.03789033368229866,
|
||||
0.03927159309387207,
|
||||
0.039445169270038605,
|
||||
0.04041972756385803,
|
||||
0.04050002992153168,
|
||||
0.04110744968056679
|
||||
],
|
||||
"loss_nats": 5.6120405197143555,
|
||||
"loss_scale": 1.0,
|
||||
"output_weights": null,
|
||||
"step": 0,
|
||||
"stream_state_rms_by_sublayer": [
|
||||
0.028404271230101585,
|
||||
0.02866268903017044,
|
||||
0.02886480651795864,
|
||||
0.029099803417921066,
|
||||
0.029365191236138344,
|
||||
0.02953805774450302,
|
||||
0.029881296679377556,
|
||||
0.030088091269135475,
|
||||
0.030441828072071075,
|
||||
0.030644793063402176,
|
||||
0.03106885403394699,
|
||||
0.03131929785013199,
|
||||
0.031892918050289154,
|
||||
0.03213750571012497,
|
||||
0.03266792371869087,
|
||||
0.03286394104361534,
|
||||
0.03364725783467293,
|
||||
0.033869802951812744,
|
||||
0.03485054895281792,
|
||||
0.03499744459986687,
|
||||
0.03562505170702934,
|
||||
0.03588006645441055,
|
||||
0.03719272464513779,
|
||||
0.037365153431892395,
|
||||
0.03768136352300644,
|
||||
0.03789033368229866,
|
||||
0.03927159309387207,
|
||||
0.039445169270038605,
|
||||
0.04041972756385803,
|
||||
0.04050002992153168,
|
||||
0.04110744968056679,
|
||||
0.04121527820825577
|
||||
]
|
||||
},
|
||||
{
|
||||
"activation_grad_rms_by_block": [
|
||||
5.69471885683015e-05,
|
||||
5.527245593839325e-05,
|
||||
5.393734318204224e-05,
|
||||
5.292302375892177e-05,
|
||||
5.23102717124857e-05,
|
||||
5.178990977583453e-05,
|
||||
5.1564093155320734e-05,
|
||||
5.129844430484809e-05,
|
||||
5.121947833686136e-05,
|
||||
5.116340616950765e-05,
|
||||
5.109804988023825e-05,
|
||||
5.104695082991384e-05,
|
||||
5.09646451973822e-05,
|
||||
5.100828275317326e-05,
|
||||
5.106439857627265e-05,
|
||||
5.11144389747642e-05
|
||||
],
|
||||
"activation_grad_statistics": {
|
||||
"first_quartile_mean": 5.477000286191469e-05,
|
||||
"first_to_last_ratio": 1.0731232762518041,
|
||||
"imbalance_abs_log_ratio": 0.07057334637995329,
|
||||
"last_quartile_mean": 5.103794137539808e-05,
|
||||
"mean": 5.2170148819641327e-05,
|
||||
"normalized": [
|
||||
1.0915665348238701,
|
||||
1.0594651767139285,
|
||||
1.033873669184083,
|
||||
1.0144311441756324,
|
||||
1.002685882559561,
|
||||
0.9927115591500164,
|
||||
0.988383094968431,
|
||||
0.9832911246274795,
|
||||
0.9817775010367221,
|
||||
0.9807027069519371,
|
||||
0.9794499543578176,
|
||||
0.9784704852268963,
|
||||
0.9768928467805085,
|
||||
0.9777292936141551,
|
||||
0.9788049244944386,
|
||||
0.9797641013345227
|
||||
],
|
||||
"population_cv": 0.032815404485150704
|
||||
},
|
||||
"activation_output_rms_by_block": [
|
||||
0.028743742033839226,
|
||||
0.02942933700978756,
|
||||
0.030596865341067314,
|
||||
0.03249936178326607,
|
||||
0.03542664647102356,
|
||||
0.03894684836268425,
|
||||
0.04318666458129883,
|
||||
0.04936420917510986,
|
||||
0.05370106175541878,
|
||||
0.06048284471035004,
|
||||
0.0664696991443634,
|
||||
0.07231792062520981,
|
||||
0.07627613097429276,
|
||||
0.07997097074985504,
|
||||
0.08494995534420013,
|
||||
0.08972473442554474
|
||||
],
|
||||
"activation_output_statistics": {
|
||||
"first_quartile_mean": 0.03031732654199004,
|
||||
"first_to_last_ratio": 0.3664591129538783,
|
||||
"imbalance_abs_log_ratio": 1.0038683247140607,
|
||||
"last_quartile_mean": 0.08273044787347317,
|
||||
"mean": 0.05450543703045696,
|
||||
"normalized": [
|
||||
0.5273555006590916,
|
||||
0.5399339701348108,
|
||||
0.5613543713807885,
|
||||
0.5962590808162097,
|
||||
0.6499653686150171,
|
||||
0.7145497859400932,
|
||||
0.7923368187501488,
|
||||
0.9056749539963465,
|
||||
0.9852422929002714,
|
||||
1.1096662646068314,
|
||||
1.21950584686113,
|
||||
1.326801958945847,
|
||||
1.399422427007981,
|
||||
1.4672108895332452,
|
||||
1.5585592919240543,
|
||||
1.6461611779281335
|
||||
],
|
||||
"population_cv": 0.3808757530834043
|
||||
},
|
||||
"bits_per_byte": 6.7509657011324835,
|
||||
"branch_output_rms_by_sublayer": [
|
||||
0.003436450148001313,
|
||||
0.003791899885982275,
|
||||
0.003964710980653763,
|
||||
0.003938332665711641,
|
||||
0.0056790695525705814,
|
||||
0.003919641952961683,
|
||||
0.006960175931453705,
|
||||
0.0038887569680809975,
|
||||
0.008646421134471893,
|
||||
0.003928068559616804,
|
||||
0.009785857982933521,
|
||||
0.0038193853106349707,
|
||||
0.010625048540532589,
|
||||
0.003996006678789854,
|
||||
0.012892307713627815,
|
||||
0.0039014811627566814,
|
||||
0.012757784686982632,
|
||||
0.003861474571749568,
|
||||
0.014726920053362846,
|
||||
0.003962590359151363,
|
||||
0.012820222415030003,
|
||||
0.004551138263195753,
|
||||
0.01401793584227562,
|
||||
0.004161422606557608,
|
||||
0.01284075528383255,
|
||||
0.004107494372874498,
|
||||
0.012963366694748402,
|
||||
0.0034911984112113714,
|
||||
0.014392351731657982,
|
||||
0.004168955609202385,
|
||||
0.013408888131380081,
|
||||
0.003909120801836252
|
||||
],
|
||||
"capture": {
|
||||
"all_gradients_finite": true,
|
||||
"all_gradients_present": true,
|
||||
"count": 16,
|
||||
"dtypes": [
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32"
|
||||
],
|
||||
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
||||
"shape": [
|
||||
16,
|
||||
256,
|
||||
192
|
||||
],
|
||||
"storage_unique": true
|
||||
},
|
||||
"core_parameter_grad_rms_by_block": [
|
||||
0.0004507690637370199,
|
||||
0.0004530669153600251,
|
||||
0.0005174622333717066,
|
||||
0.0005579942111201043,
|
||||
0.0006494724560479704,
|
||||
0.0007477190266084469,
|
||||
0.0008433132451421535,
|
||||
0.0008983201732199723,
|
||||
0.001008715304343127,
|
||||
0.0011143118429534733,
|
||||
0.0010428854825857738,
|
||||
0.0010916116075341876,
|
||||
0.0011085001103672855,
|
||||
0.0012404769719650786,
|
||||
0.0013545740271648265,
|
||||
0.0014243271893498689
|
||||
],
|
||||
"core_parameter_grad_statistics": {
|
||||
"first_quartile_mean": 0.000494823105897214,
|
||||
"first_to_last_ratio": 0.385986622193018,
|
||||
"imbalance_abs_log_ratio": 0.9519525676489338,
|
||||
"last_quartile_mean": 0.001281969574711765,
|
||||
"mean": 0.0009064699913044387,
|
||||
"normalized": [
|
||||
0.49727963204644987,
|
||||
0.4998145771025995,
|
||||
0.5708542349284638,
|
||||
0.6155683215912455,
|
||||
0.7164853357289404,
|
||||
0.824869034585972,
|
||||
0.930326710461313,
|
||||
0.99100927977468,
|
||||
1.112795033503044,
|
||||
1.2292870736404011,
|
||||
1.1504909071341995,
|
||||
1.2042446170372658,
|
||||
1.2228756836970622,
|
||||
1.3684699812069823,
|
||||
1.494339625314625,
|
||||
1.571289952246756
|
||||
],
|
||||
"population_cv": 0.338812252534038
|
||||
},
|
||||
"depth_weights": [],
|
||||
"layer_input_rms_by_sublayer": [
|
||||
0.028241552412509918,
|
||||
0.028533460572361946,
|
||||
0.028743742033839226,
|
||||
0.029236938804388046,
|
||||
0.02942933700978756,
|
||||
0.03035305254161358,
|
||||
0.030596865341067314,
|
||||
0.032195623964071274,
|
||||
0.03249936178326607,
|
||||
0.03506496921181679,
|
||||
0.03542664647102356,
|
||||
0.03852483630180359,
|
||||
0.03894684836268425,
|
||||
0.04251723736524582,
|
||||
0.04318666458129883,
|
||||
0.04861506074666977,
|
||||
0.04936420917510986,
|
||||
0.052915990352630615,
|
||||
0.05370106175541878,
|
||||
0.0598941408097744,
|
||||
0.06048284471035004,
|
||||
0.06538087129592896,
|
||||
0.0664696991443634,
|
||||
0.07142146676778793,
|
||||
0.07231792062520981,
|
||||
0.07521561533212662,
|
||||
0.07627613097429276,
|
||||
0.07935076206922531,
|
||||
0.07997097074985504,
|
||||
0.08368266373872757,
|
||||
0.08494995534420013,
|
||||
0.08860929310321808
|
||||
],
|
||||
"loss_nats": 4.679412841796875,
|
||||
"loss_scale": 1.0,
|
||||
"output_weights": null,
|
||||
"step": 20,
|
||||
"stream_state_rms_by_sublayer": [
|
||||
0.028533460572361946,
|
||||
0.028743742033839226,
|
||||
0.029236938804388046,
|
||||
0.02942933700978756,
|
||||
0.03035305254161358,
|
||||
0.030596865341067314,
|
||||
0.032195623964071274,
|
||||
0.03249936178326607,
|
||||
0.03506496921181679,
|
||||
0.03542664647102356,
|
||||
0.03852483630180359,
|
||||
0.03894684836268425,
|
||||
0.04251723736524582,
|
||||
0.04318666458129883,
|
||||
0.04861506074666977,
|
||||
0.04936420917510986,
|
||||
0.052915990352630615,
|
||||
0.05370106175541878,
|
||||
0.0598941408097744,
|
||||
0.06048284471035004,
|
||||
0.06538087129592896,
|
||||
0.0664696991443634,
|
||||
0.07142146676778793,
|
||||
0.07231792062520981,
|
||||
0.07521561533212662,
|
||||
0.07627613097429276,
|
||||
0.07935076206922531,
|
||||
0.07997097074985504,
|
||||
0.08368266373872757,
|
||||
0.08494995534420013,
|
||||
0.08860929310321808,
|
||||
0.08972473442554474
|
||||
]
|
||||
}
|
||||
],
|
||||
"environment": {
|
||||
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
||||
"compile": false,
|
||||
"compute_capability": [
|
||||
12,
|
||||
0
|
||||
],
|
||||
"cublas_workspace_config": ":4096:8",
|
||||
"cuda": "12.8",
|
||||
"deterministic_algorithms": true,
|
||||
"gpu": "NVIDIA GeForce RTX 5090",
|
||||
"python": "3.10.14",
|
||||
"torch": "2.11.0+cu128"
|
||||
},
|
||||
"evaluations": [
|
||||
{
|
||||
"bits_per_byte": 8.076786148009306,
|
||||
"cross_entropy_nats": 5.5984015464782715,
|
||||
"step": 0
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 6.752401670275862,
|
||||
"cross_entropy_nats": 4.680408179759979,
|
||||
"step": 20
|
||||
}
|
||||
],
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
},
|
||||
"hashes": {
|
||||
"final_mixer_parameters": null,
|
||||
"final_model_state": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
||||
"final_optimizer_state": "a7bce44db1478ce53933758aa5033bbb1e0aa21296e9615c33146920f30a2057",
|
||||
"final_public_parameters": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
||||
"initial_mixer_parameters": null,
|
||||
"initial_public_parameter_elements": 9541824,
|
||||
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
||||
"initial_public_parameter_tensors": 115,
|
||||
"initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
|
||||
},
|
||||
"manifest": {
|
||||
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
||||
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
||||
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
||||
"input_gate_tensor_hashes": {
|
||||
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
||||
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
||||
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
||||
},
|
||||
"path": "experiments/k3/attnres_gradient/manifest.json",
|
||||
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
||||
},
|
||||
"model": {
|
||||
"attnres_aggregation_groups": 8,
|
||||
"context": 256,
|
||||
"d_ff": 768,
|
||||
"d_head": 32,
|
||||
"d_model": 192,
|
||||
"heads": 6,
|
||||
"layers": 16,
|
||||
"parameters": {
|
||||
"core": 9541824,
|
||||
"embedding": 98304,
|
||||
"mixer": 0,
|
||||
"total": 9541824
|
||||
},
|
||||
"sublayers": 32,
|
||||
"sublayers_per_attnres_group": 4,
|
||||
"transformer_blocks_per_attnres_group": 2,
|
||||
"vocabulary": 256
|
||||
},
|
||||
"optimizer": {
|
||||
"betas": [
|
||||
0.9,
|
||||
0.95
|
||||
],
|
||||
"epsilon": 1e-08,
|
||||
"grad_clip": 1.0,
|
||||
"min_lr": 3e-05,
|
||||
"name": "AdamW",
|
||||
"peak_lr": 0.0003,
|
||||
"warmup_steps": 400,
|
||||
"weight_decay_ndim_ge_2": 0.1
|
||||
},
|
||||
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
||||
"run_kind": "smoke",
|
||||
"schema_version": 1,
|
||||
"seed": 2026073001,
|
||||
"steps": 20,
|
||||
"target_bytes_seen": 163840,
|
||||
"timing": {
|
||||
"mean_ms": null,
|
||||
"measured_steps": 0,
|
||||
"median_ms": null,
|
||||
"p95_ms": null,
|
||||
"peak_allocated_bytes": 1648265728,
|
||||
"peak_reserved_bytes": 3282042880,
|
||||
"warmup_steps_excluded": 20
|
||||
},
|
||||
"training_history": [
|
||||
{
|
||||
"bits_per_byte": 8.088097790921855,
|
||||
"learning_rate": 7.499999999999999e-07,
|
||||
"loss_nats": 5.6062421798706055,
|
||||
"step": 1,
|
||||
"unclipped_grad_norm": 19.475919723510742
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 7.474302716882146,
|
||||
"learning_rate": 7.499999999999999e-06,
|
||||
"loss_nats": 5.180791854858398,
|
||||
"step": 10,
|
||||
"unclipped_grad_norm": 12.938376426696777
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 6.789615018295581,
|
||||
"learning_rate": 1.4999999999999999e-05,
|
||||
"loss_nats": 4.706202507019043,
|
||||
"step": 20,
|
||||
"unclipped_grad_norm": 4.482712745666504
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,700 @@
|
||||
{
|
||||
"architecture": "baseline",
|
||||
"batch_size": 32,
|
||||
"canonical_sha256_without_self": "0ae9ad1697ca15ec4c84270ad82b9fa98056f14f20fc7372c4b9886c29845415",
|
||||
"depth": 16,
|
||||
"diagnostics": [
|
||||
{
|
||||
"activation_grad_rms_by_block": [
|
||||
0.0002865509013645351,
|
||||
0.00024785567075014114,
|
||||
0.00021551651298068464,
|
||||
0.00019683866412378848,
|
||||
0.0001823508064262569,
|
||||
0.0001679604029050097,
|
||||
0.00015584587526973337,
|
||||
0.00014737028686795384,
|
||||
0.00014130656199995428,
|
||||
0.00013612695329356939,
|
||||
0.00013129493163432926,
|
||||
0.00012807638267986476,
|
||||
0.00012550120300147682,
|
||||
0.00012324050476308912,
|
||||
0.00012108208466088399,
|
||||
0.00011888353037647903
|
||||
],
|
||||
"activation_grad_statistics": {
|
||||
"first_quartile_mean": 0.00023669043730478734,
|
||||
"first_to_last_ratio": 1.9372775995887173,
|
||||
"imbalance_abs_log_ratio": 0.6612836883477485,
|
||||
"last_quartile_mean": 0.00012217683070048224,
|
||||
"mean": 0.00016411257956860936,
|
||||
"normalized": [
|
||||
1.7460629899168627,
|
||||
1.5102783187106137,
|
||||
1.313223602646409,
|
||||
1.1994124072706904,
|
||||
1.1111324123086055,
|
||||
1.0234462424910682,
|
||||
0.9496278449793059,
|
||||
0.8979828801383491,
|
||||
0.8610343117596252,
|
||||
0.8294729974472175,
|
||||
0.8000296624393728,
|
||||
0.7804178266926869,
|
||||
0.7647262832098098,
|
||||
0.7509509940495869,
|
||||
0.7377989242455608,
|
||||
0.7244023016942358
|
||||
],
|
||||
"population_cv": 0.29361872380036635
|
||||
},
|
||||
"activation_output_rms_by_block": [
|
||||
0.02866268903017044,
|
||||
0.029099803417921066,
|
||||
0.02953805774450302,
|
||||
0.030088091269135475,
|
||||
0.030644793063402176,
|
||||
0.03131929785013199,
|
||||
0.03213750571012497,
|
||||
0.03286394104361534,
|
||||
0.033869802951812744,
|
||||
0.03499744459986687,
|
||||
0.03588006645441055,
|
||||
0.037365153431892395,
|
||||
0.03789033368229866,
|
||||
0.039445169270038605,
|
||||
0.04050002992153168,
|
||||
0.04121527820825577
|
||||
],
|
||||
"activation_output_statistics": {
|
||||
"first_quartile_mean": 0.0293471603654325,
|
||||
"first_to_last_ratio": 0.7380574840395957,
|
||||
"imbalance_abs_log_ratio": 0.3037335657624931,
|
||||
"last_quartile_mean": 0.03976270277053118,
|
||||
"mean": 0.034094841103069484,
|
||||
"normalized": [
|
||||
0.8406752488894867,
|
||||
0.8534957922212247,
|
||||
0.8663497699023965,
|
||||
0.8824822259232266,
|
||||
0.8988102619619861,
|
||||
0.9185934539320196,
|
||||
0.942591449919727,
|
||||
0.9638977622528551,
|
||||
0.9933996421752943,
|
||||
1.0264733158329964,
|
||||
1.0523605710888722,
|
||||
1.0959180985456622,
|
||||
1.1113216092650298,
|
||||
1.1569248600043318,
|
||||
1.1878638706395246,
|
||||
1.2088420674453664
|
||||
],
|
||||
"population_cv": 0.11952987076282337
|
||||
},
|
||||
"bits_per_byte": 8.096463027059821,
|
||||
"branch_output_rms_by_sublayer": [
|
||||
0.0033055038657039404,
|
||||
0.0037518907338380814,
|
||||
0.0033751516602933407,
|
||||
0.003911525942385197,
|
||||
0.004229962360113859,
|
||||
0.0038461871445178986,
|
||||
0.004243654198944569,
|
||||
0.0038366704247891903,
|
||||
0.004372371360659599,
|
||||
0.003851204412057996,
|
||||
0.005078981164842844,
|
||||
0.003818925702944398,
|
||||
0.0054668826051056385,
|
||||
0.0038749484810978174,
|
||||
0.005823117680847645,
|
||||
0.0037566579412668943,
|
||||
0.0065947300754487514,
|
||||
0.003876061411574483,
|
||||
0.006763988174498081,
|
||||
0.003795720636844635,
|
||||
0.006863070651888847,
|
||||
0.0038706199266016483,
|
||||
0.007856340147554874,
|
||||
0.003839300014078617,
|
||||
0.00830968376249075,
|
||||
0.003927029203623533,
|
||||
0.009655521251261234,
|
||||
0.0038513424806296825,
|
||||
0.009004893712699413,
|
||||
0.0039484030567109585,
|
||||
0.008608299307525158,
|
||||
0.003791053779423237
|
||||
],
|
||||
"capture": {
|
||||
"all_gradients_finite": true,
|
||||
"all_gradients_present": true,
|
||||
"count": 16,
|
||||
"dtypes": [
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32"
|
||||
],
|
||||
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
||||
"shape": [
|
||||
16,
|
||||
256,
|
||||
192
|
||||
],
|
||||
"storage_unique": true
|
||||
},
|
||||
"core_parameter_grad_rms_by_block": [
|
||||
0.00809059897248305,
|
||||
0.007313812062277777,
|
||||
0.007090710224412361,
|
||||
0.0067033738367094624,
|
||||
0.006535407867454384,
|
||||
0.006785021795736036,
|
||||
0.006295993569919972,
|
||||
0.005509300326230647,
|
||||
0.006231736048910802,
|
||||
0.006047390980174007,
|
||||
0.005629405359532731,
|
||||
0.005784042737956458,
|
||||
0.0059591736113607996,
|
||||
0.0061983173180850636,
|
||||
0.006571690039252182,
|
||||
0.005060205437758869
|
||||
],
|
||||
"core_parameter_grad_statistics": {
|
||||
"first_quartile_mean": 0.007299623773970663,
|
||||
"first_to_last_ratio": 1.227374872012571,
|
||||
"imbalance_abs_log_ratio": 0.2048776382299483,
|
||||
"last_quartile_mean": 0.005947346601614228,
|
||||
"mean": 0.006362886261765913,
|
||||
"normalized": [
|
||||
1.271529717747562,
|
||||
1.1494488132258316,
|
||||
1.1143858200043408,
|
||||
1.05351149791715,
|
||||
1.0271137340180259,
|
||||
1.066343404015675,
|
||||
0.9894870520870547,
|
||||
0.8658492545019393,
|
||||
0.9793882512652816,
|
||||
0.9504163254515969,
|
||||
0.8847251275509024,
|
||||
0.909028151691524,
|
||||
0.9365519618304368,
|
||||
0.9741361173356609,
|
||||
1.0328158902888074,
|
||||
0.7952688810682109
|
||||
],
|
||||
"population_cv": 0.11384977604868386
|
||||
},
|
||||
"depth_weights": [],
|
||||
"layer_input_rms_by_sublayer": [
|
||||
0.02823694422841072,
|
||||
0.028404271230101585,
|
||||
0.02866268903017044,
|
||||
0.02886480651795864,
|
||||
0.029099803417921066,
|
||||
0.029365191236138344,
|
||||
0.02953805774450302,
|
||||
0.029881296679377556,
|
||||
0.030088091269135475,
|
||||
0.030441828072071075,
|
||||
0.030644793063402176,
|
||||
0.03106885403394699,
|
||||
0.03131929785013199,
|
||||
0.031892918050289154,
|
||||
0.03213750571012497,
|
||||
0.03266792371869087,
|
||||
0.03286394104361534,
|
||||
0.03364725783467293,
|
||||
0.033869802951812744,
|
||||
0.03485054895281792,
|
||||
0.03499744459986687,
|
||||
0.03562505170702934,
|
||||
0.03588006645441055,
|
||||
0.03719272464513779,
|
||||
0.037365153431892395,
|
||||
0.03768136352300644,
|
||||
0.03789033368229866,
|
||||
0.03927159309387207,
|
||||
0.039445169270038605,
|
||||
0.04041972756385803,
|
||||
0.04050002992153168,
|
||||
0.04110744968056679
|
||||
],
|
||||
"loss_nats": 5.6120405197143555,
|
||||
"loss_scale": 1.0,
|
||||
"output_weights": null,
|
||||
"step": 0,
|
||||
"stream_state_rms_by_sublayer": [
|
||||
0.028404271230101585,
|
||||
0.02866268903017044,
|
||||
0.02886480651795864,
|
||||
0.029099803417921066,
|
||||
0.029365191236138344,
|
||||
0.02953805774450302,
|
||||
0.029881296679377556,
|
||||
0.030088091269135475,
|
||||
0.030441828072071075,
|
||||
0.030644793063402176,
|
||||
0.03106885403394699,
|
||||
0.03131929785013199,
|
||||
0.031892918050289154,
|
||||
0.03213750571012497,
|
||||
0.03266792371869087,
|
||||
0.03286394104361534,
|
||||
0.03364725783467293,
|
||||
0.033869802951812744,
|
||||
0.03485054895281792,
|
||||
0.03499744459986687,
|
||||
0.03562505170702934,
|
||||
0.03588006645441055,
|
||||
0.03719272464513779,
|
||||
0.037365153431892395,
|
||||
0.03768136352300644,
|
||||
0.03789033368229866,
|
||||
0.03927159309387207,
|
||||
0.039445169270038605,
|
||||
0.04041972756385803,
|
||||
0.04050002992153168,
|
||||
0.04110744968056679,
|
||||
0.04121527820825577
|
||||
]
|
||||
},
|
||||
{
|
||||
"activation_grad_rms_by_block": [
|
||||
5.69471885683015e-05,
|
||||
5.527245593839325e-05,
|
||||
5.393734318204224e-05,
|
||||
5.292302375892177e-05,
|
||||
5.23102717124857e-05,
|
||||
5.178990977583453e-05,
|
||||
5.1564093155320734e-05,
|
||||
5.129844430484809e-05,
|
||||
5.121947833686136e-05,
|
||||
5.116340616950765e-05,
|
||||
5.109804988023825e-05,
|
||||
5.104695082991384e-05,
|
||||
5.09646451973822e-05,
|
||||
5.100828275317326e-05,
|
||||
5.106439857627265e-05,
|
||||
5.11144389747642e-05
|
||||
],
|
||||
"activation_grad_statistics": {
|
||||
"first_quartile_mean": 5.477000286191469e-05,
|
||||
"first_to_last_ratio": 1.0731232762518041,
|
||||
"imbalance_abs_log_ratio": 0.07057334637995329,
|
||||
"last_quartile_mean": 5.103794137539808e-05,
|
||||
"mean": 5.2170148819641327e-05,
|
||||
"normalized": [
|
||||
1.0915665348238701,
|
||||
1.0594651767139285,
|
||||
1.033873669184083,
|
||||
1.0144311441756324,
|
||||
1.002685882559561,
|
||||
0.9927115591500164,
|
||||
0.988383094968431,
|
||||
0.9832911246274795,
|
||||
0.9817775010367221,
|
||||
0.9807027069519371,
|
||||
0.9794499543578176,
|
||||
0.9784704852268963,
|
||||
0.9768928467805085,
|
||||
0.9777292936141551,
|
||||
0.9788049244944386,
|
||||
0.9797641013345227
|
||||
],
|
||||
"population_cv": 0.032815404485150704
|
||||
},
|
||||
"activation_output_rms_by_block": [
|
||||
0.028743742033839226,
|
||||
0.02942933700978756,
|
||||
0.030596865341067314,
|
||||
0.03249936178326607,
|
||||
0.03542664647102356,
|
||||
0.03894684836268425,
|
||||
0.04318666458129883,
|
||||
0.04936420917510986,
|
||||
0.05370106175541878,
|
||||
0.06048284471035004,
|
||||
0.0664696991443634,
|
||||
0.07231792062520981,
|
||||
0.07627613097429276,
|
||||
0.07997097074985504,
|
||||
0.08494995534420013,
|
||||
0.08972473442554474
|
||||
],
|
||||
"activation_output_statistics": {
|
||||
"first_quartile_mean": 0.03031732654199004,
|
||||
"first_to_last_ratio": 0.3664591129538783,
|
||||
"imbalance_abs_log_ratio": 1.0038683247140607,
|
||||
"last_quartile_mean": 0.08273044787347317,
|
||||
"mean": 0.05450543703045696,
|
||||
"normalized": [
|
||||
0.5273555006590916,
|
||||
0.5399339701348108,
|
||||
0.5613543713807885,
|
||||
0.5962590808162097,
|
||||
0.6499653686150171,
|
||||
0.7145497859400932,
|
||||
0.7923368187501488,
|
||||
0.9056749539963465,
|
||||
0.9852422929002714,
|
||||
1.1096662646068314,
|
||||
1.21950584686113,
|
||||
1.326801958945847,
|
||||
1.399422427007981,
|
||||
1.4672108895332452,
|
||||
1.5585592919240543,
|
||||
1.6461611779281335
|
||||
],
|
||||
"population_cv": 0.3808757530834043
|
||||
},
|
||||
"bits_per_byte": 6.7509657011324835,
|
||||
"branch_output_rms_by_sublayer": [
|
||||
0.003436450148001313,
|
||||
0.003791899885982275,
|
||||
0.003964710980653763,
|
||||
0.003938332665711641,
|
||||
0.0056790695525705814,
|
||||
0.003919641952961683,
|
||||
0.006960175931453705,
|
||||
0.0038887569680809975,
|
||||
0.008646421134471893,
|
||||
0.003928068559616804,
|
||||
0.009785857982933521,
|
||||
0.0038193853106349707,
|
||||
0.010625048540532589,
|
||||
0.003996006678789854,
|
||||
0.012892307713627815,
|
||||
0.0039014811627566814,
|
||||
0.012757784686982632,
|
||||
0.003861474571749568,
|
||||
0.014726920053362846,
|
||||
0.003962590359151363,
|
||||
0.012820222415030003,
|
||||
0.004551138263195753,
|
||||
0.01401793584227562,
|
||||
0.004161422606557608,
|
||||
0.01284075528383255,
|
||||
0.004107494372874498,
|
||||
0.012963366694748402,
|
||||
0.0034911984112113714,
|
||||
0.014392351731657982,
|
||||
0.004168955609202385,
|
||||
0.013408888131380081,
|
||||
0.003909120801836252
|
||||
],
|
||||
"capture": {
|
||||
"all_gradients_finite": true,
|
||||
"all_gradients_present": true,
|
||||
"count": 16,
|
||||
"dtypes": [
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32",
|
||||
"torch.float32"
|
||||
],
|
||||
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
||||
"shape": [
|
||||
16,
|
||||
256,
|
||||
192
|
||||
],
|
||||
"storage_unique": true
|
||||
},
|
||||
"core_parameter_grad_rms_by_block": [
|
||||
0.0004507690637370199,
|
||||
0.0004530669153600251,
|
||||
0.0005174622333717066,
|
||||
0.0005579942111201043,
|
||||
0.0006494724560479704,
|
||||
0.0007477190266084469,
|
||||
0.0008433132451421535,
|
||||
0.0008983201732199723,
|
||||
0.001008715304343127,
|
||||
0.0011143118429534733,
|
||||
0.0010428854825857738,
|
||||
0.0010916116075341876,
|
||||
0.0011085001103672855,
|
||||
0.0012404769719650786,
|
||||
0.0013545740271648265,
|
||||
0.0014243271893498689
|
||||
],
|
||||
"core_parameter_grad_statistics": {
|
||||
"first_quartile_mean": 0.000494823105897214,
|
||||
"first_to_last_ratio": 0.385986622193018,
|
||||
"imbalance_abs_log_ratio": 0.9519525676489338,
|
||||
"last_quartile_mean": 0.001281969574711765,
|
||||
"mean": 0.0009064699913044387,
|
||||
"normalized": [
|
||||
0.49727963204644987,
|
||||
0.4998145771025995,
|
||||
0.5708542349284638,
|
||||
0.6155683215912455,
|
||||
0.7164853357289404,
|
||||
0.824869034585972,
|
||||
0.930326710461313,
|
||||
0.99100927977468,
|
||||
1.112795033503044,
|
||||
1.2292870736404011,
|
||||
1.1504909071341995,
|
||||
1.2042446170372658,
|
||||
1.2228756836970622,
|
||||
1.3684699812069823,
|
||||
1.494339625314625,
|
||||
1.571289952246756
|
||||
],
|
||||
"population_cv": 0.338812252534038
|
||||
},
|
||||
"depth_weights": [],
|
||||
"layer_input_rms_by_sublayer": [
|
||||
0.028241552412509918,
|
||||
0.028533460572361946,
|
||||
0.028743742033839226,
|
||||
0.029236938804388046,
|
||||
0.02942933700978756,
|
||||
0.03035305254161358,
|
||||
0.030596865341067314,
|
||||
0.032195623964071274,
|
||||
0.03249936178326607,
|
||||
0.03506496921181679,
|
||||
0.03542664647102356,
|
||||
0.03852483630180359,
|
||||
0.03894684836268425,
|
||||
0.04251723736524582,
|
||||
0.04318666458129883,
|
||||
0.04861506074666977,
|
||||
0.04936420917510986,
|
||||
0.052915990352630615,
|
||||
0.05370106175541878,
|
||||
0.0598941408097744,
|
||||
0.06048284471035004,
|
||||
0.06538087129592896,
|
||||
0.0664696991443634,
|
||||
0.07142146676778793,
|
||||
0.07231792062520981,
|
||||
0.07521561533212662,
|
||||
0.07627613097429276,
|
||||
0.07935076206922531,
|
||||
0.07997097074985504,
|
||||
0.08368266373872757,
|
||||
0.08494995534420013,
|
||||
0.08860929310321808
|
||||
],
|
||||
"loss_nats": 4.679412841796875,
|
||||
"loss_scale": 1.0,
|
||||
"output_weights": null,
|
||||
"step": 20,
|
||||
"stream_state_rms_by_sublayer": [
|
||||
0.028533460572361946,
|
||||
0.028743742033839226,
|
||||
0.029236938804388046,
|
||||
0.02942933700978756,
|
||||
0.03035305254161358,
|
||||
0.030596865341067314,
|
||||
0.032195623964071274,
|
||||
0.03249936178326607,
|
||||
0.03506496921181679,
|
||||
0.03542664647102356,
|
||||
0.03852483630180359,
|
||||
0.03894684836268425,
|
||||
0.04251723736524582,
|
||||
0.04318666458129883,
|
||||
0.04861506074666977,
|
||||
0.04936420917510986,
|
||||
0.052915990352630615,
|
||||
0.05370106175541878,
|
||||
0.0598941408097744,
|
||||
0.06048284471035004,
|
||||
0.06538087129592896,
|
||||
0.0664696991443634,
|
||||
0.07142146676778793,
|
||||
0.07231792062520981,
|
||||
0.07521561533212662,
|
||||
0.07627613097429276,
|
||||
0.07935076206922531,
|
||||
0.07997097074985504,
|
||||
0.08368266373872757,
|
||||
0.08494995534420013,
|
||||
0.08860929310321808,
|
||||
0.08972473442554474
|
||||
]
|
||||
}
|
||||
],
|
||||
"environment": {
|
||||
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
||||
"compile": false,
|
||||
"compute_capability": [
|
||||
12,
|
||||
0
|
||||
],
|
||||
"cublas_workspace_config": ":4096:8",
|
||||
"cuda": "12.8",
|
||||
"deterministic_algorithms": true,
|
||||
"gpu": "NVIDIA GeForce RTX 5090",
|
||||
"python": "3.10.14",
|
||||
"torch": "2.11.0+cu128"
|
||||
},
|
||||
"evaluations": [
|
||||
{
|
||||
"bits_per_byte": 8.076786148009306,
|
||||
"cross_entropy_nats": 5.5984015464782715,
|
||||
"step": 0
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 6.752401670275862,
|
||||
"cross_entropy_nats": 4.680408179759979,
|
||||
"step": 20
|
||||
}
|
||||
],
|
||||
"gradient_gate": {
|
||||
"first_to_last_ratio_abs_delta": 0.0,
|
||||
"max_abs_scale_ratio_error": 0.0,
|
||||
"normalized_spectrum_max_abs_delta": 0.0,
|
||||
"passed": true,
|
||||
"per_block_scale_ratios": [
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0,
|
||||
2.0
|
||||
],
|
||||
"population_cv_abs_delta": 0.0,
|
||||
"thresholds": {
|
||||
"scale_ratio_abs": 1e-05,
|
||||
"shape_abs": 1e-06
|
||||
}
|
||||
},
|
||||
"hashes": {
|
||||
"final_mixer_parameters": null,
|
||||
"final_model_state": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
||||
"final_optimizer_state": "a7bce44db1478ce53933758aa5033bbb1e0aa21296e9615c33146920f30a2057",
|
||||
"final_public_parameters": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
||||
"initial_mixer_parameters": null,
|
||||
"initial_public_parameter_elements": 9541824,
|
||||
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
||||
"initial_public_parameter_tensors": 115,
|
||||
"initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
|
||||
},
|
||||
"manifest": {
|
||||
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
||||
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
||||
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
||||
"input_gate_tensor_hashes": {
|
||||
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
||||
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
||||
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
||||
},
|
||||
"path": "experiments/k3/attnres_gradient/manifest.json",
|
||||
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
||||
},
|
||||
"model": {
|
||||
"attnres_aggregation_groups": 8,
|
||||
"context": 256,
|
||||
"d_ff": 768,
|
||||
"d_head": 32,
|
||||
"d_model": 192,
|
||||
"heads": 6,
|
||||
"layers": 16,
|
||||
"parameters": {
|
||||
"core": 9541824,
|
||||
"embedding": 98304,
|
||||
"mixer": 0,
|
||||
"total": 9541824
|
||||
},
|
||||
"sublayers": 32,
|
||||
"sublayers_per_attnres_group": 4,
|
||||
"transformer_blocks_per_attnres_group": 2,
|
||||
"vocabulary": 256
|
||||
},
|
||||
"optimizer": {
|
||||
"betas": [
|
||||
0.9,
|
||||
0.95
|
||||
],
|
||||
"epsilon": 1e-08,
|
||||
"grad_clip": 1.0,
|
||||
"min_lr": 3e-05,
|
||||
"name": "AdamW",
|
||||
"peak_lr": 0.0003,
|
||||
"warmup_steps": 400,
|
||||
"weight_decay_ndim_ge_2": 0.1
|
||||
},
|
||||
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
||||
"run_kind": "smoke",
|
||||
"schema_version": 1,
|
||||
"seed": 2026073001,
|
||||
"steps": 20,
|
||||
"target_bytes_seen": 163840,
|
||||
"timing": {
|
||||
"mean_ms": null,
|
||||
"measured_steps": 0,
|
||||
"median_ms": null,
|
||||
"p95_ms": null,
|
||||
"peak_allocated_bytes": 1648265728,
|
||||
"peak_reserved_bytes": 3282042880,
|
||||
"warmup_steps_excluded": 20
|
||||
},
|
||||
"training_history": [
|
||||
{
|
||||
"bits_per_byte": 8.088097790921855,
|
||||
"learning_rate": 7.499999999999999e-07,
|
||||
"loss_nats": 5.6062421798706055,
|
||||
"step": 1,
|
||||
"unclipped_grad_norm": 19.475919723510742
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 7.474302716882146,
|
||||
"learning_rate": 7.499999999999999e-06,
|
||||
"loss_nats": 5.180791854858398,
|
||||
"step": 10,
|
||||
"unclipped_grad_norm": 12.938376426696777
|
||||
},
|
||||
{
|
||||
"bits_per_byte": 6.789615018295581,
|
||||
"learning_rate": 1.4999999999999999e-05,
|
||||
"loss_nats": 4.706202507019043,
|
||||
"step": 20,
|
||||
"unclipped_grad_norm": 4.482712745666504
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user