wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: publish AttnRes forward training study

Browse Source
This commit is contained in:
wuyang
2026-07-30 18:41:18 +08:00
parent 7ea91caabb
commit f177fa676d
47 changed files with 192164 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/k3/attnres_forward/results/gates/wrapper-smoke.json
+259 -131
View File
@@ -1,7 +1,7 @@
{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "a467cc1c05e1b860abbf44086273071bcd2a891db4354cc6e79b319e44d1cce5",
"canonical_sha256_without_self": "81b011e2ab02c49abb6b2686b3e0701d71ab6e7f46cbb7c426eec91702836b7f",
"depth": 32,
"diagnostics": [
{
@@ -2479,74 +2479,74 @@
],
"forward_intervention": {
"depth_visit_counts": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39
],
"forward_calls": 0,
"forward_calls": 39,
"output_mixer_selected": false,
"output_visit_count": 0,
"output_visit_count": 39,
"passed": true,
"selected_depth_indices": [],
"selected_parameter_reachability_gate": true,
@@ -2555,70 +2555,198 @@
"selector_gate": true,
"semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel",
"source_counts_by_depth_index": {
"0": [],
"1": [],
"10": [],
"11": [],
"12": [],
"13": [],
"14": [],
"15": [],
"16": [],
"17": [],
"18": [],
"19": [],
"2": [],
"20": [],
"21": [],
"22": [],
"23": [],
"24": [],
"25": [],
"26": [],
"27": [],
"28": [],
"29": [],
"3": [],
"30": [],
"31": [],
"32": [],
"33": [],
"34": [],
"35": [],
"36": [],
"37": [],
"38": [],
"39": [],
"4": [],
"40": [],
"41": [],
"42": [],
"43": [],
"44": [],
"45": [],
"46": [],
"47": [],
"48": [],
"49": [],
"5": [],
"50": [],
"51": [],
"52": [],
"53": [],
"54": [],
"55": [],
"56": [],
"57": [],
"58": [],
"59": [],
"6": [],
"60": [],
"61": [],
"62": [],
"63": [],
"7": [],
"8": [],
"9": []
"0": [
1
],
"1": [
2
],
"10": [
3
],
"11": [
3
],
"12": [
3
],
"13": [
3
],
"14": [
3
],
"15": [
3
],
"16": [
3
],
"17": [
4
],
"18": [
4
],
"19": [
4
],
"2": [
2
],
"20": [
4
],
"21": [
4
],
"22": [
4
],
"23": [
4
],
"24": [
4
],
"25": [
5
],
"26": [
5
],
"27": [
5
],
"28": [
5
],
"29": [
5
],
"3": [
2
],
"30": [
5
],
"31": [
5
],
"32": [
5
],
"33": [
6
],
"34": [
6
],
"35": [
6
],
"36": [
6
],
"37": [
6
],
"38": [
6
],
"39": [
6
],
"4": [
2
],
"40": [
6
],
"41": [
7
],
"42": [
7
],
"43": [
7
],
"44": [
7
],
"45": [
7
],
"46": [
7
],
"47": [
7
],
"48": [
7
],
"49": [
8
],
"5": [
2
],
"50": [
8
],
"51": [
8
],
"52": [
8
],
"53": [
8
],
"54": [
8
],
"55": [
8
],
"56": [
8
],
"57": [
9
],
"58": [
9
],
"59": [
9
],
"6": [
2
],
"60": [
9
],
"61": [
9
],
"62": [
9
],
"63": [
9
],
"7": [
2
],
"8": [
2
],
"9": [
3
]
},
"uniform_weight_gate": true,
"uniform_weight_max_abs_error": 0.0,