import { createHash, randomUUID } from "node:crypto"; import { mkdir, open, readdir, readFile, rename, stat, writeFile, } from "node:fs/promises"; import path from "node:path"; export type AnnotationPageInfo = { name: string; title: string; size: number; modifiedAt: string; href: string; }; export type AnnotationValue = { requestId: string; result: "G" | "S" | "B" | ""; note: string; }; export type AnnotationSubmission = { id: string; version: number; pageName: string; submittedAt: string; submittedBy: { id: string; username: string }; summary: { total: number; completed: number; G: number; S: number; B: number; }; annotations: AnnotationValue[]; }; export type AnnotationSubmissionSummary = Omit< AnnotationSubmission, "annotations" >; const annotationPagesRoot = path.resolve( /* turbopackIgnore: true */ process.env.ANNOTATION_PAGES_DIR?.trim() || "/home/mi/annotation-pages", ); const annotationSubmissionsRoot = path.resolve( /* turbopackIgnore: true */ process.env.ANNOTATION_SUBMISSIONS_DIR?.trim() || path.join(annotationPagesRoot, ".submissions"), ); const submissionWriteQueues = new Map>(); export async function listAnnotationPages(): Promise { await mkdir(annotationPagesRoot, { recursive: true }); const entries = await readdir(annotationPagesRoot, { withFileTypes: true }); const pages = await Promise.all( entries .filter((entry) => entry.isFile() && isSafeHtmlName(entry.name)) .map(async (entry) => { const filePath = path.join(annotationPagesRoot, entry.name); const [fileStat, title] = await Promise.all([ stat(filePath), readHtmlTitle(filePath, entry.name), ]); return { name: entry.name, title, size: fileStat.size, modifiedAt: fileStat.mtime.toISOString(), href: `/annotations/${encodeURIComponent(entry.name)}`, }; }), ); return pages.sort( (left, right) => Date.parse(right.modifiedAt) - Date.parse(left.modifiedAt) || left.name.localeCompare(right.name, "zh-CN"), ); } export function resolveAnnotationPage(name: string): string | null { if (!isSafeHtmlName(name)) return null; const resolved = path.resolve(annotationPagesRoot, name); if (path.dirname(resolved) !== annotationPagesRoot) return null; return resolved; } export async function readAnnotationSubmissions(pageName: string) { const pagePath = resolveAnnotationPage(pageName); if (!pagePath) throw new Error("无效的标注页面"); await stat(pagePath); const pageRoot = submissionPageRoot(pageName); const latest = await readSubmissionFile(path.join(pageRoot, "latest.json")); const revisionsRoot = path.join(pageRoot, "revisions"); let names: string[] = []; try { names = await readdir(revisionsRoot); } catch { return { latest, history: [] as AnnotationSubmissionSummary[] }; } const history = ( await Promise.all( names .filter((name) => name.endsWith(".json")) .map((name) => readSubmissionFile(path.join(revisionsRoot, name))), ) ) .filter((item): item is AnnotationSubmission => item !== null) .sort((left, right) => right.version - left.version) .slice(0, 100) .map(withoutAnnotations); return { latest, history }; } export async function readAnnotationSubmission( pageName: string, submissionId: string, ) { if (!/^[a-zA-Z0-9_-]{8,120}$/.test(submissionId)) return null; const pagePath = resolveAnnotationPage(pageName); if (!pagePath) throw new Error("无效的标注页面"); await stat(pagePath); return readSubmissionFile( path.join( submissionPageRoot(pageName), "revisions", `${submissionId}.json`, ), ); } export async function createAnnotationSubmission( pageName: string, account: { id: string; username: string }, input: unknown, ) { const pagePath = resolveAnnotationPage(pageName); if (!pagePath) throw new Error("无效的标注页面"); await stat(pagePath); const annotations = normalizeAnnotations(input); return enqueueSubmissionWrite(pageName, async () => { const pageRoot = submissionPageRoot(pageName); const revisionsRoot = path.join(pageRoot, "revisions"); await mkdir(revisionsRoot, { recursive: true }); const latest = await readSubmissionFile(path.join(pageRoot, "latest.json")); const submittedAt = new Date().toISOString(); const id = `${submittedAt.replace(/\D/g, "").slice(0, 17)}-${randomUUID().slice(0, 8)}`; const submission: AnnotationSubmission = { id, version: (latest?.version ?? 0) + 1, pageName, submittedAt, submittedBy: account, summary: summarizeAnnotations(annotations), annotations, }; await atomicWriteJson(path.join(revisionsRoot, `${id}.json`), submission); await atomicWriteJson(path.join(pageRoot, "latest.json"), submission); return submission; }); } function isSafeHtmlName(name: string): boolean { return ( name === path.basename(name) && !name.startsWith(".") && /\.html?$/i.test(name) ); } function submissionPageRoot(pageName: string) { const digest = createHash("sha256") .update(pageName) .digest("hex") .slice(0, 24); return path.join(annotationSubmissionsRoot, digest); } function normalizeAnnotations(input: unknown): AnnotationValue[] { if (!Array.isArray(input)) throw new Error("annotations 必须是数组"); if (input.length > 20_000) throw new Error("单次提交最多包含 20000 条标注"); const seen = new Set(); return input.map((raw, index) => { if (!raw || typeof raw !== "object") { throw new Error(`第 ${index + 1} 条标注格式错误`); } const value = raw as Record; const requestId = String(value.requestId ?? value.request_id ?? "").trim(); const result = String(value.result ?? value.GSB ?? "").trim(); const note = String(value.note ?? value.人工备注 ?? "").trim(); if (!requestId || requestId.length > 200) { throw new Error(`第 ${index + 1} 条缺少有效 request_id`); } if (seen.has(requestId)) throw new Error(`request_id 重复:${requestId}`); if (!["", "G", "S", "B"].includes(result)) { throw new Error(`第 ${index + 1} 条 GSB 只能是 G、S、B 或空`); } if (note.length > 5000) throw new Error(`第 ${index + 1} 条备注过长`); seen.add(requestId); return { requestId, result: result as AnnotationValue["result"], note, }; }); } function summarizeAnnotations(annotations: AnnotationValue[]) { const summary = { total: annotations.length, completed: 0, G: 0, S: 0, B: 0, }; for (const annotation of annotations) { if (!annotation.result) continue; summary.completed += 1; summary[annotation.result] += 1; } return summary; } async function readSubmissionFile(filePath: string) { try { return JSON.parse(await readFile(filePath, "utf8")) as AnnotationSubmission; } catch { return null; } } function withoutAnnotations( submission: AnnotationSubmission, ): AnnotationSubmissionSummary { const { annotations: _annotations, ...summary } = submission; return summary; } async function atomicWriteJson(filePath: string, value: unknown) { const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); await rename(temporaryPath, filePath); } function enqueueSubmissionWrite(pageName: string, task: () => Promise) { const previous = submissionWriteQueues.get(pageName) ?? Promise.resolve(); const current = previous.then(task, task); submissionWriteQueues.set(pageName, current); const cleanup = () => { if (submissionWriteQueues.get(pageName) === current) { submissionWriteQueues.delete(pageName); } }; void current.then(cleanup, cleanup); return current; } async function readHtmlTitle(filePath: string, fileName: string) { const handle = await open(filePath, "r"); try { const buffer = Buffer.alloc(16 * 1024); const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); const head = buffer.subarray(0, bytesRead).toString("utf8"); const match = head.match(/]*>([\s\S]*?)<\/title>/i); const title = match?.[1] ?.replace(/<[^>]+>/g, " ") .replace(/\s+/g, " ") .trim(); return title ? decodeHtmlEntities(title) : stripHtmlExtension(fileName); } finally { await handle.close(); } } function stripHtmlExtension(fileName: string) { return fileName.replace(/\.html?$/i, ""); } function decodeHtmlEntities(value: string) { return value .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll(""", '"') .replaceAll("'", "'"); }