Present completion research as web document

This commit is contained in:
wuyang
2026-07-28 00:57:15 +08:00
parent 3b55bfaad2
commit 4c88faa19e
9 changed files with 3698 additions and 285 deletions
+4
View File
@@ -2,9 +2,13 @@ FROM nginx:1.29.5-alpine
COPY --chmod=0644 nginx.conf /etc/nginx/conf.d/default.conf
COPY --chmod=0644 index.html /usr/share/nginx/html/index.html
COPY --chmod=0644 evaluation-v1.html /usr/share/nginx/html/evaluation-v1.html
RUN install -d -m 0755 /usr/share/nginx/html/assets
COPY --chmod=0644 assets/styles.css /usr/share/nginx/html/assets/styles.css
COPY --chmod=0644 assets/app.js /usr/share/nginx/html/assets/app.js
COPY --chmod=0644 assets/research.css /usr/share/nginx/html/assets/research.css
COPY --chmod=0644 assets/evidence-data.js /usr/share/nginx/html/assets/evidence-data.js
COPY --chmod=0644 assets/research.js /usr/share/nginx/html/assets/research.js
EXPOSE 8080
+33 -9
View File
@@ -1,6 +1,30 @@
# Agent Evaluation Site
# Agent Completion Research Site
中文交互式报告,用于展示 K1412 Agent Evaluation v1 的结论、任务契约、发布门禁和相关产物。
中文研究文档,用网页呈现“Agent 如何证明完成、验证器怎样误判、失败后如何恢复”的
全文审计结论。
首页是当前研究结论,旧的 K1412 Agent Evaluation v1 说明页保留在:
```text
/evaluation-v1.html
```
网页是阅读层,事实源仍是:
- `research/completion-verification/findings.md`
- `research/completion-verification/evidence-ledger.md`
## Build Evidence Data
证据浏览器由 Markdown 账本确定性生成:
```bash
python3 evaluation-site/build_research_data.py
python3 evaluation-site/build_research_data.py --check
```
`tools/check_project.py` 会检查生成数据的 SHA-256、40 篇全文池、37 篇审计记录和
26 篇核心证据,防止网页与研究源漂移。
## Local Preview
@@ -8,7 +32,12 @@
python3 -m http.server 4173 --directory evaluation-site
```
打开 `http://127.0.0.1:4173/`
打开
```text
research document: http://127.0.0.1:4173/
archived v1 report: http://127.0.0.1:4173/evaluation-v1.html
```
## Production
@@ -18,10 +47,5 @@ python3 -m http.server 4173 --directory evaluation-site
- public URL: `https://agent-eval.k1412.top/`
- deployment: k1412 Unraid Compose Manager
- NAS host port: `12005`
- immutable image: `docker.k1412.top/wuyang/agent-eval:20260727T084425Z-cd1eacb`
网页是阅读入口,事实源仍是:
- `research/evaluation/k1412-agent-evaluation-v1.md`
- `data/evaluation/k1412-agent-eval-task-contracts-v1.json`
- `tools/evaluation/score_agent_runs.py`
生产部署必须使用不可变镜像,并在这里和项目状态文档中记录最终 tag。
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large Load Diff
+296
View File
@@ -0,0 +1,296 @@
(() => {
"use strict";
const evidenceData = window.K1412_EVIDENCE;
const evidenceList = document.querySelector("#evidence-list");
const evidenceCount = document.querySelector("#evidence-count");
const emptyEvidence = document.querySelector("#empty-evidence");
const searchInput = document.querySelector("#evidence-search");
const expandButton = document.querySelector("#expand-evidence");
const depthButtons = [...document.querySelectorAll("[data-depth]")];
const themeButtons = [...document.querySelectorAll("[data-theme]")];
const depthLabels = {
core: "核心证据",
support: "旁证",
context: "历史背景",
};
const themeLabels = {
verification: "完成证明",
protocol: "协议审计",
recovery: "失败恢复",
reliability: "风险可靠性",
context: "历史背景",
};
const state = {
query: "",
depth: "all",
theme: "all",
expanded: false,
};
function normalize(value) {
return String(value || "")
.toLocaleLowerCase("zh-CN")
.replace(/\s+/g, " ")
.trim();
}
function searchableText(record) {
return normalize(
[
record.id,
record.title,
record.depth,
record.theme,
record.evidence,
record.supports,
record.boundary,
].join(" ")
);
}
function filteredRecords() {
if (!evidenceData || !Array.isArray(evidenceData.records)) {
return [];
}
const terms = normalize(state.query).split(" ").filter(Boolean);
return evidenceData.records.filter((record) => {
if (state.depth !== "all" && record.depth !== state.depth) {
return false;
}
if (state.theme !== "all" && record.theme !== state.theme) {
return false;
}
const haystack = searchableText(record);
return terms.every((term) => haystack.includes(term));
});
}
function tag(label, className) {
const element = document.createElement("span");
element.className = `evidence-tag ${className || ""}`.trim();
element.textContent = label;
return element;
}
function detailColumn(title, text, className = "") {
const wrapper = document.createElement("div");
wrapper.className = className;
const heading = document.createElement("h4");
heading.textContent = title;
const paragraph = document.createElement("p");
paragraph.textContent = text || "本轮未将该论文用于直接论证。";
wrapper.append(heading, paragraph);
return wrapper;
}
function evidenceItem(record) {
const details = document.createElement("details");
details.className = "evidence-item";
details.dataset.depth = record.depth;
details.open = state.expanded;
const summary = document.createElement("summary");
const id = document.createElement("span");
id.className = "evidence-id";
id.textContent = record.id;
const heading = document.createElement("div");
heading.className = "evidence-heading";
const title = document.createElement("h3");
title.textContent = record.title;
const tags = document.createElement("div");
tags.className = "evidence-tags";
tags.append(
tag(depthLabels[record.depth] || record.depth, record.depth),
tag(themeLabels[record.theme] || record.theme)
);
heading.append(title, tags);
const openLabel = document.createElement("span");
openLabel.className = "evidence-open";
openLabel.textContent = details.open ? "收起" : "展开";
summary.append(id, heading, openLabel);
details.append(summary);
if (record.depth === "context") {
const detail = document.createElement("div");
detail.className = "evidence-detail context-detail";
detail.append(
detailColumn("本轮位置", record.boundary, "evidence-boundary")
);
details.append(detail);
} else {
const detail = document.createElement("div");
detail.className = "evidence-detail";
detail.append(
detailColumn("核过的直接证据", record.evidence),
detailColumn("可支持的判断", record.supports),
detailColumn("不能外推", record.boundary, "evidence-boundary")
);
details.append(detail);
}
const source = document.createElement("div");
source.className = "evidence-source";
const sourceLink = document.createElement("a");
sourceLink.href = record.url;
sourceLink.target = "_blank";
sourceLink.rel = "noreferrer";
sourceLink.textContent = "打开 arXiv 原文";
source.append(sourceLink);
details.append(source);
details.addEventListener("toggle", () => {
openLabel.textContent = details.open ? "收起" : "展开";
});
return details;
}
function updateFilterButtons(buttons, value, attribute) {
buttons.forEach((button) => {
const active = button.dataset[attribute] === value;
button.classList.toggle("is-active", active);
button.setAttribute("aria-pressed", String(active));
});
}
function renderEvidence() {
if (!evidenceData) {
evidenceCount.textContent = "证据数据未载入。";
emptyEvidence.hidden = false;
return;
}
const records = filteredRecords();
const fragment = document.createDocumentFragment();
records.forEach((record) => fragment.append(evidenceItem(record)));
evidenceList.replaceChildren(fragment);
const counts = evidenceData.counts;
evidenceCount.textContent =
`显示 ${records.length} / ${counts.pool}` +
` · 核心 ${counts.core} · 旁证 ${counts.support} · 背景 ${counts.context}`;
emptyEvidence.hidden = records.length !== 0;
expandButton.disabled = records.length === 0;
expandButton.textContent = state.expanded ? "收起当前结果" : "展开当前结果";
}
depthButtons.forEach((button) => {
button.addEventListener("click", () => {
state.depth = button.dataset.depth;
state.expanded = false;
updateFilterButtons(depthButtons, state.depth, "depth");
renderEvidence();
});
});
themeButtons.forEach((button) => {
button.addEventListener("click", () => {
state.theme = button.dataset.theme;
state.expanded = false;
updateFilterButtons(themeButtons, state.theme, "theme");
renderEvidence();
});
});
searchInput?.addEventListener("input", (event) => {
state.query = event.target.value;
state.expanded = false;
renderEvidence();
});
expandButton?.addEventListener("click", () => {
state.expanded = !state.expanded;
document.querySelectorAll(".evidence-item").forEach((item) => {
item.open = state.expanded;
const label = item.querySelector(".evidence-open");
if (label) {
label.textContent = state.expanded ? "收起" : "展开";
}
});
expandButton.textContent = state.expanded ? "收起当前结果" : "展开当前结果";
});
function updateReadingProgress() {
const documentHeight =
document.documentElement.scrollHeight - window.innerHeight;
const progress = documentHeight > 0 ? window.scrollY / documentHeight : 0;
const bar = document.querySelector("#reading-progress");
if (bar) {
bar.style.width = `${Math.min(1, Math.max(0, progress)) * 100}%`;
}
}
let progressFrame = 0;
window.addEventListener(
"scroll",
() => {
if (progressFrame) {
return;
}
progressFrame = window.requestAnimationFrame(() => {
updateReadingProgress();
progressFrame = 0;
});
},
{ passive: true }
);
updateReadingProgress();
const navAliases = {
conclusions: "overview",
synthesis: "model",
};
const navLinks = [...document.querySelectorAll(".primary-nav a")];
const sections = [...document.querySelectorAll("[data-section]")];
const sectionObserver = new IntersectionObserver(
(entries) => {
const visible = entries
.filter((entry) => entry.isIntersecting)
.sort((a, b) => b.intersectionRatio - a.intersectionRatio)[0];
if (!visible) {
return;
}
const targetId = navAliases[visible.target.id] || visible.target.id;
navLinks.forEach((link) => {
const active = link.getAttribute("href") === `#${targetId}`;
link.classList.toggle("is-active", active);
if (active) {
const nav = link.closest(".primary-nav");
if (nav && nav.scrollWidth > nav.clientWidth) {
nav.scrollTo({
left: Math.max(0, link.offsetLeft - nav.clientWidth / 2),
behavior: "smooth",
});
}
}
});
},
{
rootMargin: "-18% 0px -62% 0px",
threshold: [0, 0.08, 0.2],
}
);
sections.forEach((section) => sectionObserver.observe(section));
renderEvidence();
if (window.location.hash) {
window.requestAnimationFrame(() => {
window.requestAnimationFrame(() => {
document.querySelector(window.location.hash)?.scrollIntoView();
});
});
}
})();
+136
View File
@@ -0,0 +1,136 @@
#!/usr/bin/env python3
"""Build the browser evidence index from the committed Markdown ledger."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
LEDGER = ROOT / "research" / "completion-verification" / "evidence-ledger.md"
OUTPUT = ROOT / "evaluation-site" / "assets" / "evidence-data.js"
SECTION_SLUGS = {
"完成证明与 evaluator 误差": "verification",
"测试、benchmark 和协议会不会错误认证": "protocol",
"故障发现、定位和恢复": "recovery",
"变更风险、协作和重复可靠性": "reliability",
"全文池与排除": "context",
}
def parse_link(value: str) -> tuple[str, str]:
match = re.fullmatch(r"\[([^\]]+)\]\((https?://[^)]+)\)", value.strip())
if not match:
raise ValueError(f"Invalid paper link: {value}")
return match.group(1), match.group(2)
def parse_ledger(text: str) -> list[dict[str, str]]:
section = ""
records: list[dict[str, str]] = []
for raw_line in text.splitlines():
heading = re.match(r"^## \d+\.\s+(.+)$", raw_line)
if heading:
title = heading.group(1).strip()
section = SECTION_SLUGS.get(title, "")
continue
if not raw_line.startswith("| ["):
continue
columns = [column.strip() for column in raw_line.strip().strip("|").split("|")]
if len(columns) not in {3, 5}:
raise ValueError(f"Unexpected table row with {len(columns)} columns: {raw_line}")
title, url = parse_link(columns[0])
arxiv_match = re.search(r"(\d{4}\.\d{5})", url)
if not arxiv_match:
raise ValueError(f"Missing arXiv id: {url}")
arxiv_id = arxiv_match.group(1)
title = re.sub(rf",\s*{re.escape(arxiv_id)}$", "", title)
record = {
"id": arxiv_id,
"title": title,
"url": url,
"depth": columns[1],
"theme": section,
}
if len(columns) == 5:
record.update(
{
"evidence": columns[2],
"supports": columns[3],
"boundary": columns[4],
}
)
else:
record.update(
{
"evidence": "",
"supports": "",
"boundary": columns[2],
}
)
records.append(record)
return records
def build_payload(text: str) -> dict[str, object]:
records = parse_ledger(text)
counts = {
"pool": len(records),
"audited": sum(record["depth"] in {"core", "support"} for record in records),
"core": sum(record["depth"] == "core" for record in records),
"support": sum(record["depth"] == "support" for record in records),
"context": sum(record["depth"] == "context" for record in records),
}
return {
"source": "research/completion-verification/evidence-ledger.md",
"source_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
"counts": counts,
"records": records,
}
def render(payload: dict[str, object]) -> str:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
return (
"/* Generated by evaluation-site/build_research_data.py. */\n"
f"window.K1412_EVIDENCE={body};\n"
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--check",
action="store_true",
help="Fail if the generated browser data is stale.",
)
args = parser.parse_args()
expected = render(build_payload(LEDGER.read_text(encoding="utf-8")))
if args.check:
if not OUTPUT.exists() or OUTPUT.read_text(encoding="utf-8") != expected:
print(f"stale: {OUTPUT.relative_to(ROOT)}")
return 1
print(f"ok: {OUTPUT.relative_to(ROOT)}")
return 0
OUTPUT.write_text(expected, encoding="utf-8")
print(f"wrote: {OUTPUT.relative_to(ROOT)}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+353
View File
@@ -0,0 +1,353 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#f3f4ef">
<meta
name="description"
content="K1412 Agent 评估方案:用任务契约、独立判分和配对回归决定一次 Agent 改动是否应该上线。"
>
<title>K1412 Agent 评估</title>
<link
rel="icon"
href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' fill='%23121713'/%3E%3Ctext x='16' y='22' text-anchor='middle' font-family='serif' font-size='18' fill='white'%3EK%3C/text%3E%3C/svg%3E"
>
<link rel="stylesheet" href="/assets/styles.css">
<script src="/assets/app.js" defer></script>
</head>
<body>
<a class="skip-link" href="#main">跳到正文</a>
<header class="topbar">
<a class="brand" href="#conclusion" aria-label="K1412 Agent 评估首页">
<span class="brand-mark" aria-hidden="true">K</span>
<span>
<strong>Agent 评估</strong>
<small>K1412 Research</small>
</span>
</a>
<nav class="topnav" aria-label="页面导航">
<a class="is-active" href="#conclusion">结论</a>
<a href="#decision">决策</a>
<a href="#layers">层次</a>
<a href="#tasks">任务</a>
<a href="#artifacts">产物</a>
</nav>
<a
class="source-link"
href="https://git.k1412.top/wuyang/agent"
target="_blank"
rel="noreferrer"
>源仓库</a>
</header>
<main id="main">
<section class="report-intro" id="conclusion" data-section>
<div class="section-number">01 / 结论</div>
<div class="intro-layout">
<div class="intro-copy">
<p class="eyebrow">K1412 Agent Evaluation v1 · 2026.07.27</p>
<h1>不评总分,只评一次改动是否该上线。</h1>
<p class="lead">
固定模型、任务、工作区和预算,只改变一个 Agent 行为假设。
用独立验证回答它改善了什么、破坏了什么,以及证据是否足以进入生产。
</p>
</div>
<aside class="evidence-boundary" aria-label="证据边界">
<p class="boundary-label">当前证据边界</p>
<p>
方案来自对 <code>zk-data-agent@e2e7a7b</code> 真实运行事件、
隔离工作区和完成门禁的检查。
</p>
<p class="boundary-warning">
尚未运行真实模型对比。最新论文只是摘要级线索。
</p>
</aside>
</div>
<dl class="headline-stats" aria-label="方案摘要">
<div>
<dt>任务契约</dt>
<dd>10</dd>
<small>跨产物、恢复、上下文、安全与委派</small>
</div>
<div>
<dt>评测层次</dt>
<dd>6</dd>
<small>从确定性单测到生产价值抽样</small>
</div>
<div>
<dt>当前可跑</dt>
<dd>2</dd>
<small>其余任务明确标注 runner 缺口</small>
</div>
<div>
<dt>真实对比</dt>
<dd>0</dd>
<small>不把设计契约伪装成实验结论</small>
</div>
</dl>
<div class="method-rail" aria-label="评估闭环">
<div><span>01</span><strong>问题</strong><small>一次只改变一个假设</small></div>
<div><span>02</span><strong>任务</strong><small>固定夹具、预算和模型</small></div>
<div><span>03</span><strong>运行</strong><small>配对、重复、随机顺序</small></div>
<div><span>04</span><strong>判分</strong><small>独立 Validator 与轨迹规则</small></div>
<div><span>05</span><strong>决定</strong><small>推广、继续或拒绝</small></div>
</div>
</section>
<section class="decision-section" id="decision" data-section>
<div class="section-heading">
<div>
<div class="section-number">02 / 发布决策</div>
<h2>先看回归,再谈平均提升</h2>
</div>
<p>
下面是方法演算,不是真实跑分。它使用与仓库评分器一致的配对逻辑和统计口径。
</p>
</div>
<div class="decision-workbench">
<form class="simulation-controls" id="simulation-form">
<fieldset>
<legend>配对运行结果</legend>
<label class="range-row" for="gains">
<span><strong>新增成功</strong><small>基线失败,候选成功</small></span>
<input id="gains" name="gains" type="range" min="0" max="20" value="6">
<output for="gains">6</output>
</label>
<label class="range-row" for="regressions">
<span><strong>回归失败</strong><small>基线成功,候选失败</small></span>
<input id="regressions" name="regressions" type="range" min="0" max="20" value="1">
<output for="regressions">1</output>
</label>
<label class="range-row" for="both-pass">
<span><strong>双方成功</strong><small>两种版本都通过</small></span>
<input id="both-pass" name="bothPass" type="range" min="0" max="30" value="18">
<output for="both-pass">18</output>
</label>
<label class="range-row" for="both-fail">
<span><strong>双方失败</strong><small>两种版本都未通过</small></span>
<input id="both-fail" name="bothFail" type="range" min="0" max="30" value="5">
<output for="both-fail">5</output>
</label>
</fieldset>
<fieldset class="gate-controls">
<legend>硬门禁与效率</legend>
<label>
<span>安全违规</span>
<input id="safety" name="safety" type="number" min="0" max="99" value="0">
</label>
<label>
<span>新增假完成</span>
<input id="false-completion" name="falseCompletion" type="number" min="0" max="99" value="0">
</label>
<label class="latency-control" for="latency">
<span>候选延迟变化</span>
<input id="latency" name="latency" type="range" min="-50" max="50" value="-12">
<output for="latency">-12%</output>
</label>
</fieldset>
</form>
<div class="decision-output" aria-live="polite">
<div class="decision-status">
<span class="status-dot" aria-hidden="true"></span>
<div>
<small>机械建议</small>
<h3 id="decision-label">发布候选</h3>
</div>
<span class="sample-count" id="sample-count">30 次配对</span>
</div>
<p class="decision-reason" id="decision-reason">
新增成功多于回归,且没有安全或假完成阻断项。进入轨迹复核,而不是直接发布。
</p>
<div class="outcome-matrix" aria-label="基线与候选结果矩阵">
<div class="matrix-corner">基线 \ 候选</div>
<div class="matrix-axis">失败</div>
<div class="matrix-axis">成功</div>
<div class="matrix-axis matrix-row-label">失败</div>
<div class="matrix-cell neutral"><span>双方失败</span><strong id="matrix-both-fail">5</strong></div>
<div class="matrix-cell positive"><span>新增成功</span><strong id="matrix-gain">6</strong></div>
<div class="matrix-axis matrix-row-label">成功</div>
<div class="matrix-cell negative"><span>回归失败</span><strong id="matrix-regression">1</strong></div>
<div class="matrix-cell stable"><span>双方成功</span><strong id="matrix-both-pass">18</strong></div>
</div>
<div class="rate-comparison">
<div>
<span>基线成功率</span>
<strong id="baseline-rate">63.3%</strong>
<small id="baseline-ci">95% CI 45.5%78.1%</small>
</div>
<div>
<span>候选成功率</span>
<strong id="variant-rate">80.0%</strong>
<small id="variant-ci">95% CI 62.7%90.5%</small>
</div>
<div>
<span>配对差异</span>
<strong id="net-gain">+5</strong>
<small id="mcnemar-p">McNemar p = 0.125</small>
</div>
</div>
<div class="blocker-line" id="blocker-line">
<strong>硬门禁通过</strong>
<span>安全违规 0 · 新增假完成 0</span>
</div>
</div>
</div>
</section>
<section class="layer-section" id="layers" data-section>
<div class="section-heading">
<div>
<div class="section-number">03 / 评测层次</div>
<h2>高层成功不能掩盖底层回归</h2>
</div>
<p>真实模型评测建立在单测、隔离和伪模型链路之上,不替代它们。</p>
</div>
<div class="layer-tabs" role="tablist" aria-label="评测层次">
<button class="is-active" role="tab" aria-selected="true" data-layer="0">L0 单元</button>
<button role="tab" aria-selected="false" data-layer="1">L1 Docker</button>
<button role="tab" aria-selected="false" data-layer="2">L2 E2E</button>
<button role="tab" aria-selected="false" data-layer="3">L3 行为</button>
<button role="tab" aria-selected="false" data-layer="4">L4 压力</button>
<button role="tab" aria-selected="false" data-layer="5">L5 生产</button>
</div>
<article class="layer-detail" id="layer-detail" tabindex="0">
<div class="layer-code" id="layer-code">L0</div>
<div>
<p class="layer-purpose" id="layer-purpose">单元与策略门禁</p>
<h3 id="layer-title">先证明规则本身没有坏</h3>
<p id="layer-description">
检查参数规范化、完成证据、权限和事件顺序。每次提交运行,不调用真实模型。
</p>
</div>
<dl>
<div><dt>模型</dt><dd id="layer-model">不调用</dd></div>
<div><dt>节奏</dt><dd id="layer-cadence">每次提交</dd></div>
<div><dt>失败动作</dt><dd id="layer-action">阻止进入高层评测</dd></div>
</dl>
</article>
</section>
<section class="task-section" id="tasks" data-section>
<div class="section-heading">
<div>
<div class="section-number">04 / 任务契约</div>
<h2>任务描述不是真值</h2>
</div>
<p>最终状态、命令结果和副作用 Validator 才决定任务是否真的完成。</p>
</div>
<div class="task-toolbar">
<label class="search-field" for="task-search">
<span class="visually-hidden">搜索任务</span>
<input id="task-search" type="search" placeholder="搜索任务、能力或场景">
</label>
<div class="segmented" role="group" aria-label="任务状态筛选">
<button class="is-active" data-filter="all">全部 <span>10</span></button>
<button data-filter="current">当前可跑 <span>2</span></button>
<button data-filter="needed">需补能力 <span>8</span></button>
</div>
</div>
<div class="task-list-header" aria-hidden="true">
<span>任务</span>
<span>能力家族</span>
<span>Runner 状态</span>
<span></span>
</div>
<div class="task-list" id="task-list"></div>
<p class="empty-state" id="task-empty" hidden>没有匹配的任务。</p>
</section>
<section class="artifact-section" id="artifacts" data-section>
<div class="section-heading">
<div>
<div class="section-number">05 / 相关产物</div>
<h2>结论、契约、工具与边界均可追溯</h2>
</div>
<p>网页是阅读入口,Git 仓库仍是事实源。</p>
</div>
<div class="artifact-list">
<a href="https://git.k1412.top/wuyang/agent/src/branch/main/research/evaluation/k1412-agent-evaluation-v1.md" target="_blank" rel="noreferrer">
<span class="artifact-index">A1</span>
<span><strong>中文评估方案</strong><small>问题、任务、层次、判分、指标、协议和发布门禁</small></span>
<em>设计契约</em>
</a>
<a href="https://git.k1412.top/wuyang/agent/src/branch/main/data/evaluation/k1412-agent-eval-task-contracts-v1.json" target="_blank" rel="noreferrer">
<span class="artifact-index">A2</span>
<span><strong>10 个任务契约</strong><small>机器可读的目标、完成条件、禁区、预算和 Runner 缺口</small></span>
<em>JSON</em>
</a>
<a href="https://git.k1412.top/wuyang/agent/src/branch/main/tools/evaluation/score_agent_runs.py" target="_blank" rel="noreferrer">
<span class="artifact-index">A3</span>
<span><strong>运行结果评分器</strong><small>汇总成功、假完成、回归、区间、配对检验、延迟与 Token</small></span>
<em>Python</em>
</a>
<a href="https://git.k1412.top/wuyang/agent/src/branch/main/experiments/agent-evaluation/2026-07-27-k1412-agent-evaluation-design.md" target="_blank" rel="noreferrer">
<span class="artifact-index">A4</span>
<span><strong>实验记录</strong><small>已经确认的事实、设计假设、验证结果和不可外推边界</small></span>
<em>记录</em>
</a>
<a href="https://git.k1412.top/wuyang/agent/src/branch/main/papers/corpus-summary-2026-07-27.md" target="_blank" rel="noreferrer">
<span class="artifact-index">A5</span>
<span><strong>论文增量摘要</strong><small>249 个候选、157 个高相关、151 篇新增及评测研究信号</small></span>
<em>语料</em>
</a>
<a href="https://lab.k1412.top/" target="_blank" rel="noreferrer">
<span class="artifact-index">A6</span>
<span><strong>Agent 论文库</strong><small>1125 篇论文的搜索、主题探索和原始条目入口</small></span>
<em>在线</em>
</a>
</div>
<div class="command-line">
<code>python3 tools/evaluation/score_agent_runs.py results.jsonl --baseline v3 --variant v4 --format markdown</code>
<button id="copy-command" type="button">复制命令</button>
</div>
</section>
<section class="implementation-section" id="implementation">
<div class="section-number">06 / 下一步</div>
<div class="implementation-layout">
<div>
<p class="eyebrow">Implementation order</p>
<h2>先让两项任务真实可判,再扩大任务集。</h2>
</div>
<ol>
<li><span>01</span><p><strong>抽出 LiveEvalRunner</strong>从现有单次脚本提取可复用运行器。</p></li>
<li><span>02</span><p><strong>接入独立 Validator</strong>先覆盖脚本报告与数据分析两个任务。</p></li>
<li><span>03</span><p><strong>导出 Result JSONL</strong>固定模型、Prompt、工具、镜像、预算与版本。</p></li>
<li><span>04</span><p><strong>运行配对 Smoke</strong>确定性层全部通过后才调用真实模型。</p></li>
<li><span>05</span><p><strong>复核回归轨迹</strong>检查 gains、regressions、both-fail 与假完成。</p></li>
</ol>
</div>
</section>
</main>
<footer>
<span>K1412 Agent Evaluation</span>
<span>方案基线 <code>df475e8</code></span>
<span>数据更新 2026.07.27</span>
</footer>
</body>
</html>
+710 -271
View File
File diff suppressed because it is too large Load Diff
+53 -5
View File
@@ -4,6 +4,7 @@
from __future__ import annotations
import ast
import hashlib
import json
from collections import Counter
from pathlib import Path
@@ -21,12 +22,19 @@ REQUIRED_FILES = (
"data/summary.json",
"research/memory/findings.md",
"research/memory/evidence-ledger.md",
"research/completion-verification/findings.md",
"research/completion-verification/evidence-ledger.md",
"research/evaluation/k1412-agent-evaluation-v1.md",
"data/evaluation/k1412-agent-eval-task-contracts-v1.json",
"tools/evaluation/score_agent_runs.py",
"evaluation-site/index.html",
"evaluation-site/evaluation-v1.html",
"evaluation-site/assets/styles.css",
"evaluation-site/assets/app.js",
"evaluation-site/assets/research.css",
"evaluation-site/assets/research.js",
"evaluation-site/assets/evidence-data.js",
"evaluation-site/build_research_data.py",
"evaluation-site/Dockerfile",
"evaluation-site/nginx.conf",
"web/app.py",
@@ -96,7 +104,10 @@ def check_research_data() -> dict[str, int]:
def check_python_syntax() -> int:
paths = sorted((ROOT / "tools").rglob("*.py")) + [ROOT / "web" / "app.py"]
paths = sorted((ROOT / "tools").rglob("*.py")) + [
ROOT / "web" / "app.py",
ROOT / "evaluation-site" / "build_research_data.py",
]
for path in paths:
ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
return len(paths)
@@ -143,16 +154,52 @@ def check_evaluation_contract() -> int:
return len(tasks)
def check_evaluation_site() -> None:
def check_evaluation_site() -> int:
html = (ROOT / "evaluation-site/index.html").read_text(encoding="utf-8")
archived_html = (ROOT / "evaluation-site/evaluation-v1.html").read_text(encoding="utf-8")
javascript = (ROOT / "evaluation-site/assets/app.js").read_text(encoding="utf-8")
research_javascript = (ROOT / "evaluation-site/assets/research.js").read_text(encoding="utf-8")
evidence_javascript = (ROOT / "evaluation-site/assets/evidence-data.js").read_text(
encoding="utf-8"
)
ledger = (ROOT / "research/completion-verification/evidence-ledger.md").read_text(
encoding="utf-8"
)
dockerfile = (ROOT / "evaluation-site/Dockerfile").read_text(encoding="utf-8")
nginx = (ROOT / "evaluation-site/nginx.conf").read_text(encoding="utf-8")
require("不是真实跑分" in html, "evaluation simulation must keep its evidence boundary visible")
require("尚未运行真实模型对比" in html, "evaluation report overstates current evidence")
require("Agent 到底凭什么说" in html, "completion research document is missing")
require("自有跑分" in html and ">0<" in html, "research page overstates benchmark evidence")
require("completion certificate" in html.lower(), "research synthesis boundary is missing")
require(
"不是真实跑分" in archived_html,
"archived evaluation simulation must keep its evidence boundary visible",
)
require(
"尚未运行真实模型对比" in archived_html,
"archived evaluation report overstates current evidence",
)
require("const TASKS" in javascript, "evaluation task browser is missing")
require("K1412_EVIDENCE" in research_javascript, "research evidence browser is missing")
prefix = "window.K1412_EVIDENCE="
require(evidence_javascript.startswith("/* Generated") and prefix in evidence_javascript, "bad evidence data")
payload_text = evidence_javascript.split(prefix, 1)[1].strip()
require(payload_text.endswith(";"), "bad evidence data terminator")
payload = json.loads(payload_text[:-1])
counts = payload.get("counts") or {}
records = payload.get("records") or []
require(counts.get("pool") == len(records) == 40, "completion evidence pool count mismatch")
require(counts.get("audited") == 37, "completion audited count mismatch")
require(counts.get("core") == 26, "completion core count mismatch")
require(
payload.get("source_sha256") == hashlib.sha256(ledger.encode("utf-8")).hexdigest(),
"completion evidence browser data is stale",
)
require("install -d -m 0755" in dockerfile, "static asset directory must remain traversable")
require("evaluation-v1.html" in dockerfile, "archived evaluation page is not packaged")
require('location = /health' in nginx, "evaluation site health endpoint is missing")
return int(counts["audited"])
def main() -> int:
@@ -163,7 +210,7 @@ def main() -> int:
check_ollama_contract()
check_experiment_state()
evaluation_tasks = check_evaluation_contract()
check_evaluation_site()
completion_evidence = check_evaluation_site()
print(
json.dumps(
{
@@ -171,6 +218,7 @@ def main() -> int:
"collections": collections,
"research": research,
"evaluation_tasks": evaluation_tasks,
"completion_evidence": completion_evidence,
"evaluation_site": True,
"python_files_checked": python_files,
},