zk-data-agent/frontend/app/lib/annotation-pages.ts

import { createHash, randomUUID } from "node:crypto";
import {
	mkdir,
	open,
	readdir,
	readFile,
	rename,
	stat,
	writeFile,
} from "node:fs/promises";
import path from "node:path";

export type AnnotationPageInfo = {
	name: string;
	title: string;
	size: number;
	modifiedAt: string;
	href: string;
};

export type AnnotationValue = {
	requestId: string;
	result: "G" | "S" | "B" | "";
	note: string;
};

export type AnnotationSubmission = {
	id: string;
	version: number;
	pageName: string;
	submittedAt: string;
	submittedBy: { id: string; username: string };
	summary: {
		total: number;
		completed: number;
		G: number;
		S: number;
		B: number;
	};
	annotations: AnnotationValue[];
};

export type AnnotationSubmissionSummary = Omit<
	AnnotationSubmission,
	"annotations"
>;

const annotationPagesRoot = path.resolve(
	/* turbopackIgnore: true */
	process.env.ANNOTATION_PAGES_DIR?.trim() || "/home/mi/annotation-pages",
);
const annotationSubmissionsRoot = path.resolve(
	/* turbopackIgnore: true */
	process.env.ANNOTATION_SUBMISSIONS_DIR?.trim() ||
		path.join(annotationPagesRoot, ".submissions"),
);
const submissionWriteQueues = new Map<string, Promise<unknown>>();

export async function listAnnotationPages(): Promise<AnnotationPageInfo[]> {
	await mkdir(annotationPagesRoot, { recursive: true });
	const entries = await readdir(annotationPagesRoot, { withFileTypes: true });
	const pages = await Promise.all(
		entries
			.filter((entry) => entry.isFile() && isSafeHtmlName(entry.name))
			.map(async (entry) => {
				const filePath = path.join(annotationPagesRoot, entry.name);
				const [fileStat, title] = await Promise.all([
					stat(filePath),
					readHtmlTitle(filePath, entry.name),
				]);
				return {
					name: entry.name,
					title,
					size: fileStat.size,
					modifiedAt: fileStat.mtime.toISOString(),
					href: `/annotations/${encodeURIComponent(entry.name)}`,
				};
			}),
	);

	return pages.sort(
		(left, right) =>
			Date.parse(right.modifiedAt) - Date.parse(left.modifiedAt) ||
			left.name.localeCompare(right.name, "zh-CN"),
	);
}

export function resolveAnnotationPage(name: string): string | null {
	if (!isSafeHtmlName(name)) return null;
	const resolved = path.resolve(annotationPagesRoot, name);
	if (path.dirname(resolved) !== annotationPagesRoot) return null;
	return resolved;
}

export async function readAnnotationSubmissions(pageName: string) {
	const pagePath = resolveAnnotationPage(pageName);
	if (!pagePath) throw new Error("无效的标注页面");
	await stat(pagePath);

	const pageRoot = submissionPageRoot(pageName);
	const latest = await readSubmissionFile(path.join(pageRoot, "latest.json"));
	const revisionsRoot = path.join(pageRoot, "revisions");
	let names: string[] = [];
	try {
		names = await readdir(revisionsRoot);
	} catch {
		return { latest, history: [] as AnnotationSubmissionSummary[] };
	}

	const history = (
		await Promise.all(
			names
				.filter((name) => name.endsWith(".json"))
				.map((name) => readSubmissionFile(path.join(revisionsRoot, name))),
		)
	)
		.filter((item): item is AnnotationSubmission => item !== null)
		.sort((left, right) => right.version - left.version)
		.slice(0, 100)
		.map(withoutAnnotations);

	return { latest, history };
}

export async function readAnnotationSubmission(
	pageName: string,
	submissionId: string,
) {
	if (!/^[a-zA-Z0-9_-]{8,120}$/.test(submissionId)) return null;
	const pagePath = resolveAnnotationPage(pageName);
	if (!pagePath) throw new Error("无效的标注页面");
	await stat(pagePath);
	return readSubmissionFile(
		path.join(
			submissionPageRoot(pageName),
			"revisions",
			`${submissionId}.json`,
		),
	);
}

export async function createAnnotationSubmission(
	pageName: string,
	account: { id: string; username: string },
	input: unknown,
) {
	const pagePath = resolveAnnotationPage(pageName);
	if (!pagePath) throw new Error("无效的标注页面");
	await stat(pagePath);
	const annotations = normalizeAnnotations(input);

	return enqueueSubmissionWrite(pageName, async () => {
		const pageRoot = submissionPageRoot(pageName);
		const revisionsRoot = path.join(pageRoot, "revisions");
		await mkdir(revisionsRoot, { recursive: true });
		const latest = await readSubmissionFile(path.join(pageRoot, "latest.json"));
		const submittedAt = new Date().toISOString();
		const id = `${submittedAt.replace(/\D/g, "").slice(0, 17)}-${randomUUID().slice(0, 8)}`;
		const submission: AnnotationSubmission = {
			id,
			version: (latest?.version ?? 0) + 1,
			pageName,
			submittedAt,
			submittedBy: account,
			summary: summarizeAnnotations(annotations),
			annotations,
		};
		await atomicWriteJson(path.join(revisionsRoot, `${id}.json`), submission);
		await atomicWriteJson(path.join(pageRoot, "latest.json"), submission);
		return submission;
	});
}

function isSafeHtmlName(name: string): boolean {
	return (
		name === path.basename(name) &&
		!name.startsWith(".") &&
		/\.html?$/i.test(name)
	);
}

function submissionPageRoot(pageName: string) {
	const digest = createHash("sha256")
		.update(pageName)
		.digest("hex")
		.slice(0, 24);
	return path.join(annotationSubmissionsRoot, digest);
}

function normalizeAnnotations(input: unknown): AnnotationValue[] {
	if (!Array.isArray(input)) throw new Error("annotations 必须是数组");
	if (input.length > 20_000) throw new Error("单次提交最多包含 20000 条标注");
	const seen = new Set<string>();
	return input.map((raw, index) => {
		if (!raw || typeof raw !== "object") {
			throw new Error(`第 ${index + 1} 条标注格式错误`);
		}
		const value = raw as Record<string, unknown>;
		const requestId = String(value.requestId ?? value.request_id ?? "").trim();
		const result = String(value.result ?? value.GSB ?? "").trim();
		const note = String(value.note ?? value.人工备注 ?? "").trim();
		if (!requestId || requestId.length > 200) {
			throw new Error(`第 ${index + 1} 条缺少有效 request_id`);
		}
		if (seen.has(requestId)) throw new Error(`request_id 重复：${requestId}`);
		if (!["", "G", "S", "B"].includes(result)) {
			throw new Error(`第 ${index + 1} 条 GSB 只能是 G、S、B 或空`);
		}
		if (note.length > 5000) throw new Error(`第 ${index + 1} 条备注过长`);
		seen.add(requestId);
		return {
			requestId,
			result: result as AnnotationValue["result"],
			note,
		};
	});
}

function summarizeAnnotations(annotations: AnnotationValue[]) {
	const summary = {
		total: annotations.length,
		completed: 0,
		G: 0,
		S: 0,
		B: 0,
	};
	for (const annotation of annotations) {
		if (!annotation.result) continue;
		summary.completed += 1;
		summary[annotation.result] += 1;
	}
	return summary;
}

async function readSubmissionFile(filePath: string) {
	try {
		return JSON.parse(await readFile(filePath, "utf8")) as AnnotationSubmission;
	} catch {
		return null;
	}
}

function withoutAnnotations(
	submission: AnnotationSubmission,
): AnnotationSubmissionSummary {
	const { annotations: _annotations, ...summary } = submission;
	return summary;
}

async function atomicWriteJson(filePath: string, value: unknown) {
	const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
	await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
	await rename(temporaryPath, filePath);
}

function enqueueSubmissionWrite<T>(pageName: string, task: () => Promise<T>) {
	const previous = submissionWriteQueues.get(pageName) ?? Promise.resolve();
	const current = previous.then(task, task);
	submissionWriteQueues.set(pageName, current);
	const cleanup = () => {
		if (submissionWriteQueues.get(pageName) === current) {
			submissionWriteQueues.delete(pageName);
		}
	};
	void current.then(cleanup, cleanup);
	return current;
}

async function readHtmlTitle(filePath: string, fileName: string) {
	const handle = await open(filePath, "r");
	try {
		const buffer = Buffer.alloc(16 * 1024);
		const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
		const head = buffer.subarray(0, bytesRead).toString("utf8");
		const match = head.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
		const title = match?.[1]
			?.replace(/<[^>]+>/g, " ")
			.replace(/\s+/g, " ")
			.trim();
		return title ? decodeHtmlEntities(title) : stripHtmlExtension(fileName);
	} finally {
		await handle.close();
	}
}

function stripHtmlExtension(fileName: string) {
	return fileName.replace(/\.html?$/i, "");
}

function decodeHtmlEntities(value: string) {
	return value
		.replaceAll("&amp;", "&")
		.replaceAll("&lt;", "<")
		.replaceAll("&gt;", ">")
		.replaceAll("&quot;", '"')
		.replaceAll("&#39;", "'");
}