wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: audit DeepSeek Chat across sources

Browse Source
This commit is contained in:
wuyang
2026-07-30 03:56:55 +08:00
parent 29ceae4e1b
commit 9211333234
24 changed files with 123017 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/deepseek/v2_lite_chat_sampling_evaluator.py
+31 -9
View File
@@ -153,10 +153,18 @@ def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
code_rows = [
row for row in rows if row["domain"] == "code"
]
answers = Counter(
row["task_evaluation"]["predicted_final"]
for row in math_rows
if row["task_evaluation"]["predicted_final"] is not None
math_source_ids = sorted(
{row["source_id"] for row in math_rows}
)
single_math_source = len(math_source_ids) == 1
answers = (
Counter(
row["task_evaluation"]["predicted_final"]
for row in math_rows
if row["task_evaluation"]["predicted_final"] is not None
)
if single_math_source
else Counter()
)
max_count = max(answers.values(), default=0)
modes = sorted(
@@ -166,7 +174,7 @@ def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
)
gold = (
math_rows[0]["task_evaluation"]["gold_final"]
if math_rows
if single_math_source
else None
)
return {
@@ -185,6 +193,13 @@ def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
),
"math": {
"outputs": len(math_rows),
"sources": len(math_source_ids),
"source_ids": math_source_ids,
"answer_aggregation_scope": (
"single_source"
if single_math_source
else "disabled_across_distinct_gold_answers"
),
"evaluator_covered": sum(
row["task_evaluation"]["evaluator_covered"]
for row in math_rows
@@ -206,13 +221,13 @@ def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
"modal_count": max_count,
"absolute_majority_exists": (
max_count > len(math_rows) / 2
if math_rows
if single_math_source
else None
),
"unique_absolute_majority": (
modes[0]
if (
math_rows
single_math_source
and len(modes) == 1
and max_count > len(math_rows) / 2
)
@@ -223,12 +238,15 @@ def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
len(modes) == 1
and max_count > len(math_rows) / 2
and modes[0] == gold
if math_rows
if single_math_source
else None
),
},
"code": {
"outputs": len(code_rows),
"sources": len(
{row["source_id"] for row in code_rows}
),
"ast_parse": sum(
row["task_evaluation"]["python_ast_parse"]
for row in code_rows
@@ -516,7 +534,11 @@ def main() -> None:
"by_source_edge": edge_summary(rows),
},
"claim_boundary": [
"Four sources and eight seeds are not benchmark estimates.",
(
f"{len(sampling['sources'])} sources and "
f"{len(sampling['seed_contract']['executed_base_seeds'])} "
"seeds are not full benchmark estimates."
),
"Completion-conditioned metrics are selection-biased diagnostics.",
"A passing HumanEval test is functional evidence, not code-safety evidence.",
"A modal sampled math answer is not standard self-consistency.",