wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: audit DeepSeek Chat across sources

Browse Source
This commit is contained in:
wuyang
2026-07-30 03:56:55 +08:00
parent 29ceae4e1b
commit 9211333234
24 changed files with 123017 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/deepseek/v2_lite_chat_sampling_probe.py
+20 -6
View File
@@ -965,13 +965,17 @@ def main() -> None:
"use_cache": True,
"official_generation_config": official_generation,
"official_parameters_explicitly_passed_to_generate": True,
"all_eight_conditions_same_source_batch": True,
"all_conditions_same_source_batch": True,
"batch_conditions": len(EXPECTED_CONDITIONS),
"all_eight_conditions_same_source_batch": (
len(EXPECTED_CONDITIONS) == 8
),
"batch_padding": (
"left padding with PAD=EOS and attention_mask=0; "
"generation prefix ends at the same batch column"
),
"counterfactual_boundary": (
"BOS/x/period cells edit one pre-target token ID after "
"Non-EOS cells edit one pre-target token ID after "
"official rendering and are not valid official chats"
),
},
@@ -1018,12 +1022,22 @@ def main() -> None:
"sources": generated_sources,
"summary": summarize(generated_sources, greedy_rows),
"claim_boundary": [
"This is a four-source, eight-seed mechanism probe, not a benchmark.",
"Eight trajectories do not estimate the full generation distribution.",
(
f"This is a {len(source_rows)}-source, "
f"{len(args.base_seeds)}-seed mechanism probe, "
"not a benchmark."
),
(
f"{len(args.base_seeds)} trajectories per cell do not "
"estimate the full generation distribution."
),
"Batch-seed-aligned rows are not common-random-number pairs.",
"Counterfactual BOS/x/period token sequences are not official-valid chats.",
"Counterfactual non-EOS token sequences are not official-valid chats.",
"Unique token sequences are not semantic-diversity measurements.",
"One GSM8K and one HumanEval source do not estimate task ability.",
(
f"{args.per_domain} GSM8K and {args.per_domain} HumanEval "
"sources do not estimate full benchmark ability."
),
"CPU-offloaded eager latency is not serving throughput.",
"Sampling differences do not identify a hidden-state or router mediator.",
],