Add versioned annotation submissions

2026-07-20 17:28:47 +08:00
parent 7cd6e5e464
commit 97e1e1dd48
4 changed files with 549 additions and 3 deletions
@@ -1,4 +1,13 @@
-import { mkdir, open, readdir, stat } from "node:fs/promises";
+import { createHash, randomUUID } from "node:crypto";
+import {
+	mkdir,
+	open,
+	readdir,
+	readFile,
+	rename,
+	stat,
+	writeFile,
+} from "node:fs/promises";
 import path from "node:path";

 export type AnnotationPageInfo = {
@@ -9,9 +18,43 @@ export type AnnotationPageInfo = {
 	href: string;
 };

+export type AnnotationValue = {
+	requestId: string;
+	result: "G" | "S" | "B" | "";
+	note: string;
+};
+
+export type AnnotationSubmission = {
+	id: string;
+	version: number;
+	pageName: string;
+	submittedAt: string;
+	submittedBy: { id: string; username: string };
+	summary: {
+		total: number;
+		completed: number;
+		G: number;
+		S: number;
+		B: number;
+	};
+	annotations: AnnotationValue[];
+};
+
+export type AnnotationSubmissionSummary = Omit<
+	AnnotationSubmission,
+	"annotations"
+>;
+
 const annotationPagesRoot = path.resolve(
+	/* turbopackIgnore: true */
 	process.env.ANNOTATION_PAGES_DIR?.trim() || "/home/mi/annotation-pages",
 );
+const annotationSubmissionsRoot = path.resolve(
+	/* turbopackIgnore: true */
+	process.env.ANNOTATION_SUBMISSIONS_DIR?.trim() ||
+		path.join(annotationPagesRoot, ".submissions"),
+);
+const submissionWriteQueues = new Map<string, Promise<unknown>>();

 export async function listAnnotationPages(): Promise<AnnotationPageInfo[]> {
 	await mkdir(annotationPagesRoot, { recursive: true });
@@ -49,6 +92,85 @@ export function resolveAnnotationPage(name: string): string | null {
 	return resolved;
 }

+export async function readAnnotationSubmissions(pageName: string) {
+	const pagePath = resolveAnnotationPage(pageName);
+	if (!pagePath) throw new Error("无效的标注页面");
+	await stat(pagePath);
+
+	const pageRoot = submissionPageRoot(pageName);
+	const latest = await readSubmissionFile(path.join(pageRoot, "latest.json"));
+	const revisionsRoot = path.join(pageRoot, "revisions");
+	let names: string[] = [];
+	try {
+		names = await readdir(revisionsRoot);
+	} catch {
+		return { latest, history: [] as AnnotationSubmissionSummary[] };
+	}
+
+	const history = (
+		await Promise.all(
+			names
+				.filter((name) => name.endsWith(".json"))
+				.map((name) => readSubmissionFile(path.join(revisionsRoot, name))),
+		)
+	)
+		.filter((item): item is AnnotationSubmission => item !== null)
+		.sort((left, right) => right.version - left.version)
+		.slice(0, 100)
+		.map(withoutAnnotations);
+
+	return { latest, history };
+}
+
+export async function readAnnotationSubmission(
+	pageName: string,
+	submissionId: string,
+) {
+	if (!/^[a-zA-Z0-9_-]{8,120}$/.test(submissionId)) return null;
+	const pagePath = resolveAnnotationPage(pageName);
+	if (!pagePath) throw new Error("无效的标注页面");
+	await stat(pagePath);
+	return readSubmissionFile(
+		path.join(
+			submissionPageRoot(pageName),
+			"revisions",
+			`${submissionId}.json`,
+		),
+	);
+}
+
+export async function createAnnotationSubmission(
+	pageName: string,
+	account: { id: string; username: string },
+	input: unknown,
+) {
+	const pagePath = resolveAnnotationPage(pageName);
+	if (!pagePath) throw new Error("无效的标注页面");
+	await stat(pagePath);
+	const annotations = normalizeAnnotations(input);
+
+	return enqueueSubmissionWrite(pageName, async () => {
+		const pageRoot = submissionPageRoot(pageName);
+		const revisionsRoot = path.join(pageRoot, "revisions");
+		await mkdir(revisionsRoot, { recursive: true });
+		const latest = await readSubmissionFile(path.join(pageRoot, "latest.json"));
+		const submittedAt = new Date().toISOString();
+		const id = `${submittedAt.replace(/\D/g, "").slice(0, 17)}-${randomUUID().slice(0, 8)}`;
+		const submission: AnnotationSubmission = {
+			id,
+			version: (latest?.version ?? 0) + 1,
+			pageName,
+			submittedAt,
+			submittedBy: account,
+			summary: summarizeAnnotations(annotations),
+			annotations,
+		};
+		await atomicWriteJson(path.join(revisionsRoot, `${id}.json`), submission);
+		await atomicWriteJson(path.join(pageRoot, "latest.json"), submission);
+		return submission;
+	});
+}
+
 function isSafeHtmlName(name: string): boolean {
 	return (
 		name === path.basename(name) &&
@@ -57,6 +179,93 @@ function isSafeHtmlName(name: string): boolean {
 	);
 }

+function submissionPageRoot(pageName: string) {
+	const digest = createHash("sha256")
+		.update(pageName)
+		.digest("hex")
+		.slice(0, 24);
+	return path.join(annotationSubmissionsRoot, digest);
+}
+
+function normalizeAnnotations(input: unknown): AnnotationValue[] {
+	if (!Array.isArray(input)) throw new Error("annotations 必须是数组");
+	if (input.length > 20_000) throw new Error("单次提交最多包含 20000 条标注");
+	const seen = new Set<string>();
+	return input.map((raw, index) => {
+		if (!raw || typeof raw !== "object") {
+			throw new Error(`第 ${index + 1} 条标注格式错误`);
+		}
+		const value = raw as Record<string, unknown>;
+		const requestId = String(value.requestId ?? value.request_id ?? "").trim();
+		const result = String(value.result ?? value.GSB ?? "").trim();
+		const note = String(value.note ?? value.人工备注 ?? "").trim();
+		if (!requestId || requestId.length > 200) {
+			throw new Error(`第 ${index + 1} 条缺少有效 request_id`);
+		}
+		if (seen.has(requestId)) throw new Error(`request_id 重复：${requestId}`);
+		if (!["", "G", "S", "B"].includes(result)) {
+			throw new Error(`第 ${index + 1} 条 GSB 只能是 G、S、B 或空`);
+		}
+		if (note.length > 5000) throw new Error(`第 ${index + 1} 条备注过长`);
+		seen.add(requestId);
+		return {
+			requestId,
+			result: result as AnnotationValue["result"],
+			note,
+		};
+	});
+}
+
+function summarizeAnnotations(annotations: AnnotationValue[]) {
+	const summary = {
+		total: annotations.length,
+		completed: 0,
+		G: 0,
+		S: 0,
+		B: 0,
+	};
+	for (const annotation of annotations) {
+		if (!annotation.result) continue;
+		summary.completed += 1;
+		summary[annotation.result] += 1;
+	}
+	return summary;
+}
+
+async function readSubmissionFile(filePath: string) {
+	try {
+		return JSON.parse(await readFile(filePath, "utf8")) as AnnotationSubmission;
+	} catch {
+		return null;
+	}
+}
+
+function withoutAnnotations(
+	submission: AnnotationSubmission,
+): AnnotationSubmissionSummary {
+	const { annotations: _annotations, ...summary } = submission;
+	return summary;
+}
+
+async function atomicWriteJson(filePath: string, value: unknown) {
+	const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
+	await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+	await rename(temporaryPath, filePath);
+}
+
+function enqueueSubmissionWrite<T>(pageName: string, task: () => Promise<T>) {
+	const previous = submissionWriteQueues.get(pageName) ?? Promise.resolve();
+	const current = previous.then(task, task);
+	submissionWriteQueues.set(pageName, current);
+	const cleanup = () => {
+		if (submissionWriteQueues.get(pageName) === current) {
+			submissionWriteQueues.delete(pageName);
+		}
+	};
+	void current.then(cleanup, cleanup);
+	return current;
+}
+
 async function readHtmlTitle(filePath: string, fileName: string) {
 	const handle = await open(filePath, "r");
 	try {