research: publish AttnRes forward training study

experiments/k3/attnres_forward/run_matrix.py

@@ -84,6 +84,34 @@ def validate_manifest(args: argparse.Namespace) -> None:
         raise ValueError("the frozen protocol permits one or two processes")
 
 
+def stop_processes(items: list[dict[str, Any]]) -> None:
+    for item in items:
+        if item["process"].poll() is None:
+            item["process"].terminate()
+    for item in items:
+        process = item["process"]
+        if process.poll() is not None:
+            continue
+        try:
+            process.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.wait()
+
+
+def quarantine_failed_output(
+    output_dir: Path, variant: str, seed: int, run_kind: str
+) -> str | None:
+    output = cell_output(output_dir, variant, seed, run_kind)
+    if not output.exists():
+        return None
+    failed = output.with_suffix(".failed.json")
+    if failed.exists():
+        failed = output.with_suffix(f".failed-{time.time_ns()}.json")
+    output.replace(failed)
+    return str(failed)
+
+
 def run_cells(
     args: argparse.Namespace,
     cells: list[tuple[str, int, str]],
@@ -138,13 +166,15 @@ def run_cells(
             variant, seed, run_kind = item["identity"]
             elapsed = time.monotonic() - item["started"]
             if return_code != 0:
-                for survivor in survivors:
-                    survivor["process"].terminate()
-                for survivor in running:
-                    if survivor is not item and survivor not in survivors:
-                        survivor["process"].terminate()
+                stop_processes(
+                    [candidate for candidate in running if candidate is not item]
+                )
+                quarantined = quarantine_failed_output(
+                    args.output_dir, variant, seed, run_kind
+                )
                 raise RuntimeError(
-                    f"cell failed: {variant}/{seed}/{run_kind}: {return_code}"
+                    f"cell failed: {variant}/{seed}/{run_kind}: {return_code}; "
+                    f"quarantined_output={quarantined}"
                 )
             completed += 1
             print(