Add label master skill
This commit is contained in:
wuyang6
+331
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从 Markdown 标签知识库生成机器可读索引。
|
||||
|
||||
这个脚本只做确定性整理，不做语义判断：
|
||||
- 人继续维护 knowledge/ 下的中文 Markdown。
|
||||
- 脚本把标签、intent、function、object、边界文件整理成 JSON manifest。
|
||||
- Agent 和校验脚本优先读取 manifest，避免每次全量扫描知识库。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_PATH = Path(__file__).resolve()
|
||||
SKILL_ROOT = SCRIPT_PATH.parents[1]
|
||||
KNOWLEDGE_ROOT = SKILL_ROOT / "knowledge"
|
||||
DEFAULT_OUTPUT = KNOWLEDGE_ROOT / "索引" / "label_manifest.json"
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path) -> str:
|
||||
return path.relative_to(SKILL_ROOT).as_posix()
|
||||
|
||||
|
||||
def first_heading(text: str, fallback: str) -> str:
|
||||
match = re.search(r"^#\s+(.+?)\s*$", text, re.MULTILINE)
|
||||
return match.group(1).strip() if match else fallback
|
||||
|
||||
|
||||
def extract_section(text: str, heading: str) -> str:
|
||||
pattern = rf"^##\s+{re.escape(heading)}\s*$"
|
||||
match = re.search(pattern, text, re.MULTILINE)
|
||||
if not match:
|
||||
return ""
|
||||
start = match.end()
|
||||
next_match = re.search(r"^##\s+", text[start:], re.MULTILINE)
|
||||
end = start + next_match.start() if next_match else len(text)
|
||||
return text[start:end].strip()
|
||||
|
||||
|
||||
def extract_bullet_value(text: str, label: str) -> str:
|
||||
match = re.search(rf"^-\s*{re.escape(label)}[：:]\s*(.+?)\s*$", text, re.MULTILINE)
|
||||
return match.group(1).strip() if match else ""
|
||||
|
||||
|
||||
def parse_markdown_table(text: str) -> list[dict[str, str]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
table_lines = [line.strip() for line in text.splitlines() if line.strip().startswith("|")]
|
||||
if len(table_lines) < 2:
|
||||
return rows
|
||||
header = [cell.strip() for cell in table_lines[0].strip("|").split("|")]
|
||||
for line in table_lines[2:]:
|
||||
cells = [cell.strip() for cell in line.strip("|").split("|")]
|
||||
if len(cells) != len(header):
|
||||
continue
|
||||
rows.append(dict(zip(header, cells)))
|
||||
return rows
|
||||
|
||||
|
||||
def parse_enum_values(description: str) -> list[str]:
|
||||
if "枚举" not in description:
|
||||
return []
|
||||
desc = re.sub(r"[，。；;].*$", "", description.split("枚举", 1)[-1])
|
||||
desc = desc.replace("：", ":")
|
||||
if ":" in desc:
|
||||
desc = desc.split(":", 1)[1]
|
||||
parts = re.split(r"[、,，/| 或 和]+", desc)
|
||||
values: list[str] = []
|
||||
for part in parts:
|
||||
cleaned = part.strip().strip("`").strip()
|
||||
cleaned = re.sub(r"[（(].*?[）)]", "", cleaned).strip()
|
||||
if cleaned and re.fullmatch(r"[A-Za-z0-9_]+", cleaned):
|
||||
values.append(cleaned)
|
||||
return values
|
||||
|
||||
|
||||
def parse_label_cards() -> list[dict[str, Any]]:
|
||||
labels_root = KNOWLEDGE_ROOT / "标签"
|
||||
labels: list[dict[str, Any]] = []
|
||||
if not labels_root.exists():
|
||||
return labels
|
||||
|
||||
for path in sorted(labels_root.rglob("*.md")):
|
||||
text = read_text(path)
|
||||
name = first_heading(text, path.stem)
|
||||
domain = path.parent.name
|
||||
label = {
|
||||
"name": name,
|
||||
"domain": domain,
|
||||
"path": rel(path),
|
||||
"old_tag": extract_bullet_value(text, "旧 tag 标签"),
|
||||
"agent_candidate": extract_bullet_value(text, "Agent 包装候选（不代表最终）"),
|
||||
"function_candidate": extract_bullet_value(text, "Function 输出候选"),
|
||||
"recommended_output_shape": extract_bullet_value(text, "推荐输出形态"),
|
||||
"scope": extract_section(text, "适用范围"),
|
||||
"examples": extract_section(text, "典型 Query"),
|
||||
"confusing_labels": extract_section(text, "易混淆标签"),
|
||||
"principles": extract_section(text, "划分原则"),
|
||||
}
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
def parse_function_catalog() -> list[dict[str, Any]]:
|
||||
path = KNOWLEDGE_ROOT / "输出能力" / "函数目录.md"
|
||||
if not path.exists():
|
||||
return []
|
||||
text = read_text(path)
|
||||
functions: list[dict[str, Any]] = []
|
||||
sections = list(re.finditer(r"^##\s+(.+?)\s*$", text, re.MULTILINE))
|
||||
for index, match in enumerate(sections):
|
||||
name = match.group(1).strip()
|
||||
start = match.end()
|
||||
end = sections[index + 1].start() if index + 1 < len(sections) else len(text)
|
||||
body = text[start:end].strip()
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
param_block = ""
|
||||
if "参数：" in body:
|
||||
param_block = body.split("参数：", 1)[1]
|
||||
if "示例：" in param_block:
|
||||
param_block = param_block.split("示例：", 1)[0]
|
||||
for row in parse_markdown_table(param_block):
|
||||
param_name = row.get("参数", "").strip()
|
||||
description = row.get("说明", "").strip()
|
||||
if not param_name:
|
||||
continue
|
||||
params[param_name] = {
|
||||
"description": description,
|
||||
"enum_values": parse_enum_values(description),
|
||||
}
|
||||
functions.append(
|
||||
{
|
||||
"name": name,
|
||||
"path": rel(path),
|
||||
"params": params,
|
||||
"allow_unknown_params": False,
|
||||
"summary": re.sub(r"\s+", " ", extract_summary(body)).strip(),
|
||||
}
|
||||
)
|
||||
return functions
|
||||
|
||||
|
||||
def parse_object_catalog() -> list[dict[str, Any]]:
|
||||
path = KNOWLEDGE_ROOT / "输出能力" / "对象目录.md"
|
||||
if not path.exists():
|
||||
return []
|
||||
text = read_text(path)
|
||||
objects: list[dict[str, Any]] = []
|
||||
|
||||
sections = list(re.finditer(r"^##\s+(.+?)\s*$", text, re.MULTILINE))
|
||||
for index, match in enumerate(sections):
|
||||
name = match.group(1).strip()
|
||||
start = match.end()
|
||||
end = sections[index + 1].start() if index + 1 < len(sections) else len(text)
|
||||
body = text[start:end].strip()
|
||||
|
||||
if name in {"VisionQA 辅助对象", "生成类辅助对象"}:
|
||||
for row in parse_markdown_table(body):
|
||||
object_name = row.get("对象", "").strip()
|
||||
if object_name:
|
||||
objects.append(
|
||||
{
|
||||
"name": object_name,
|
||||
"path": rel(path),
|
||||
"params": {},
|
||||
"allow_unknown_params": True,
|
||||
"summary": row.get("说明", "").strip(),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
param_block = body.split("参数：", 1)[1] if "参数：" in body else ""
|
||||
if "示例：" in param_block:
|
||||
param_block = param_block.split("示例：", 1)[0]
|
||||
for row in parse_markdown_table(param_block):
|
||||
param_name = row.get("参数", "").strip()
|
||||
description = row.get("说明", "").strip()
|
||||
if not param_name:
|
||||
continue
|
||||
params[param_name] = {
|
||||
"description": description,
|
||||
"enum_values": parse_enum_values(description),
|
||||
}
|
||||
objects.append(
|
||||
{
|
||||
"name": name,
|
||||
"path": rel(path),
|
||||
"params": params,
|
||||
"allow_unknown_params": False,
|
||||
"summary": re.sub(r"\s+", " ", extract_summary(body)).strip(),
|
||||
}
|
||||
)
|
||||
return objects
|
||||
|
||||
|
||||
def parse_intent_catalog() -> list[dict[str, Any]]:
|
||||
path = KNOWLEDGE_ROOT / "输出能力" / "意图目录.md"
|
||||
if not path.exists():
|
||||
return []
|
||||
text = read_text(path)
|
||||
intents: list[dict[str, Any]] = []
|
||||
for row in parse_markdown_table(text):
|
||||
name = row.get("intent", "").replace("\\|", "|").strip()
|
||||
if not name:
|
||||
continue
|
||||
intents.append(
|
||||
{
|
||||
"name": name,
|
||||
"agent": row.get("Agent", "").strip(),
|
||||
"instruction_type": row.get("指令类型", "").strip(),
|
||||
"scope": row.get("简要范围", "").strip(),
|
||||
"path": rel(path),
|
||||
}
|
||||
)
|
||||
return intents
|
||||
|
||||
|
||||
def extract_summary(body: str) -> str:
|
||||
match = re.search(r"功能范围[：:]\s*(.+?)(?:\n\n|$)", body, re.DOTALL)
|
||||
return match.group(1).strip() if match else ""
|
||||
|
||||
|
||||
def parse_boundary_files() -> list[dict[str, str]]:
|
||||
boundary_root = KNOWLEDGE_ROOT / "边界"
|
||||
if not boundary_root.exists():
|
||||
return []
|
||||
files: list[dict[str, str]] = []
|
||||
for path in sorted(boundary_root.rglob("*.md")):
|
||||
if path.name == "README.md":
|
||||
continue
|
||||
text = read_text(path)
|
||||
parts = path.relative_to(boundary_root).parts
|
||||
category = parts[0] if len(parts) > 1 else "root"
|
||||
files.append({"name": first_heading(text, path.stem), "path": rel(path), "category": category})
|
||||
return files
|
||||
|
||||
|
||||
def parse_dimension_files() -> list[dict[str, str]]:
|
||||
root = KNOWLEDGE_ROOT / "判断维度"
|
||||
if not root.exists():
|
||||
return []
|
||||
files: list[dict[str, str]] = []
|
||||
for path in sorted(root.glob("*.md")):
|
||||
if path.name == "README.md":
|
||||
continue
|
||||
text = read_text(path)
|
||||
files.append({"name": first_heading(text, path.stem), "path": rel(path)})
|
||||
return files
|
||||
|
||||
|
||||
def build_manifest() -> dict[str, Any]:
|
||||
labels = parse_label_cards()
|
||||
functions = parse_function_catalog()
|
||||
objects = parse_object_catalog()
|
||||
intents = parse_intent_catalog()
|
||||
|
||||
old_tags = sorted({item["old_tag"] for item in labels if item.get("old_tag") and item["old_tag"] != "待确认。"})
|
||||
label_names = sorted({item["name"] for item in labels})
|
||||
agent_tags = sorted(set(old_tags) | set(label_names))
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"skill": "label-master",
|
||||
"display_name": "标签大师",
|
||||
"description": "标签大师的中控标签知识库机器索引；由 scripts/build_label_manifest.py 从 Markdown 生成。",
|
||||
"counts": {
|
||||
"labels": len(labels),
|
||||
"agent_tags": len(agent_tags),
|
||||
"functions": len(functions),
|
||||
"objects": len(objects),
|
||||
"intents": len(intents),
|
||||
},
|
||||
"agent_tags": agent_tags,
|
||||
"labels": labels,
|
||||
"functions": functions,
|
||||
"objects": objects,
|
||||
"intents": intents,
|
||||
"boundaries": parse_boundary_files(),
|
||||
"dimensions": parse_dimension_files(),
|
||||
}
|
||||
|
||||
|
||||
def write_manifest(path: Path, manifest: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="生成 label-master 标签大师机器索引")
|
||||
parser.add_argument("--output", default=str(DEFAULT_OUTPUT), help="输出 JSON 路径")
|
||||
parser.add_argument("--check", action="store_true", help="只检查输出文件是否为最新，不写入")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
output = Path(args.output)
|
||||
if not output.is_absolute():
|
||||
output = SKILL_ROOT / output
|
||||
|
||||
manifest = build_manifest()
|
||||
new_text = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
if args.check:
|
||||
if not output.exists():
|
||||
print(f"ERROR: manifest 不存在：{output}", file=sys.stderr)
|
||||
return 1
|
||||
old_text = output.read_text(encoding="utf-8")
|
||||
if old_text != new_text:
|
||||
print(f"ERROR: manifest 不是最新：{output}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"OK: manifest 已是最新：{output}")
|
||||
return 0
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(new_text, encoding="utf-8")
|
||||
print(f"Wrote {output}")
|
||||
print(json.dumps(manifest["counts"], ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""校验标签输出格式是否合法。
|
||||
|
||||
这个脚本只做“格式、存在性、引用关系”的确定性校验，不判断 query 语义。
|
||||
适合在 Agent 给出 target 后、写入训练/评测数据前调用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from build_label_manifest import DEFAULT_OUTPUT, SKILL_ROOT, build_manifest
|
||||
except ImportError: # pragma: no cover - 兼容从其他 cwd 直接执行
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from build_label_manifest import DEFAULT_OUTPUT, SKILL_ROOT, build_manifest
|
||||
|
||||
|
||||
AGENT_RE = re.compile(r'^Agent\(\s*tag\s*=\s*["\'](.+?)["\']\s*\)$')
|
||||
ASSIGNMENT_RE = re.compile(r"^(x\d+)\s*=\s*(.+)$")
|
||||
|
||||
|
||||
def normalize_quotes(text: str) -> str:
|
||||
return (
|
||||
text.replace("“", '"')
|
||||
.replace("”", '"')
|
||||
.replace("‘", "'")
|
||||
.replace("’", "'")
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict[str, Any]:
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return build_manifest()
|
||||
|
||||
|
||||
def names_by_key(items: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {item["name"]: item for item in items if item.get("name")}
|
||||
|
||||
|
||||
class LabelOutputValidator:
|
||||
def __init__(self, manifest: dict[str, Any]):
|
||||
self.manifest = manifest
|
||||
self.agent_tags = set(manifest.get("agent_tags", []))
|
||||
self.functions = names_by_key(manifest.get("functions", []))
|
||||
self.objects = names_by_key(manifest.get("objects", []))
|
||||
self.intents = names_by_key(manifest.get("intents", []))
|
||||
|
||||
def validate(self, raw_target: str) -> dict[str, Any]:
|
||||
target = normalize_quotes(raw_target)
|
||||
result: dict[str, Any] = {
|
||||
"valid": True,
|
||||
"detected_type": "unknown",
|
||||
"normalized_output": target,
|
||||
"errors": [],
|
||||
"warnings": [],
|
||||
"references": [],
|
||||
}
|
||||
|
||||
if not target:
|
||||
self.error(result, "输出为空")
|
||||
return result
|
||||
|
||||
dimensions, target_body = self.split_dimension_prefix(target)
|
||||
if dimensions:
|
||||
result["dimensions"] = dimensions
|
||||
target_body = target_body.strip()
|
||||
|
||||
if target_body.startswith("["):
|
||||
self.validate_group_json(target_body, result)
|
||||
elif AGENT_RE.match(target_body):
|
||||
self.validate_agent_target(target_body, result)
|
||||
elif self.looks_like_call_program(target_body):
|
||||
self.validate_call_program(target_body, result)
|
||||
elif target_body in self.intents:
|
||||
result["detected_type"] = "intent"
|
||||
result["references"].append({"type": "intent", "name": target_body})
|
||||
elif target_body in self.agent_tags:
|
||||
result["detected_type"] = "bare_label"
|
||||
result["warnings"].append("只输出了标签名；如果当前任务需要可消费 target，建议明确 intent、function 或 Agent 包装形式")
|
||||
result["references"].append({"type": "label", "name": target_body})
|
||||
else:
|
||||
self.error(result, f"无法识别的输出形式：{target_body}")
|
||||
|
||||
if dimensions:
|
||||
result["normalized_output"] = "\n".join([*dimensions, result["normalized_output"]])
|
||||
result["valid"] = not result["errors"]
|
||||
return result
|
||||
|
||||
def split_dimension_prefix(self, target: str) -> tuple[list[str], str]:
|
||||
dimensions: list[str] = []
|
||||
lines = [line.strip() for line in target.splitlines() if line.strip()]
|
||||
while lines and re.fullmatch(r"(complex|multi_instruction|auto_task)\s*=\s*(true|false)", lines[0], re.I):
|
||||
key, value = [part.strip().lower() for part in lines.pop(0).split("=", 1)]
|
||||
dimensions.append(f"{key}={value}")
|
||||
return dimensions, "\n".join(lines)
|
||||
|
||||
def validate_agent_target(self, target: str, result: dict[str, Any]) -> None:
|
||||
result["detected_type"] = "agent"
|
||||
match = AGENT_RE.match(target)
|
||||
assert match is not None
|
||||
tag = match.group(1)
|
||||
result["normalized_output"] = f'Agent(tag="{tag}")'
|
||||
if tag not in self.agent_tags:
|
||||
self.error(result, f'Agent tag 不在标签知识库中：{tag}')
|
||||
return
|
||||
result["references"].append({"type": "agent_tag", "name": tag})
|
||||
|
||||
def looks_like_call_program(self, target: str) -> bool:
|
||||
lines = [line.strip() for line in target.splitlines() if line.strip()]
|
||||
return bool(lines) and all("(" in line and line.endswith(")") for line in lines)
|
||||
|
||||
def validate_call_program(self, target: str, result: dict[str, Any]) -> None:
|
||||
result["detected_type"] = "function_program"
|
||||
variables: dict[str, str] = {}
|
||||
lines = [line.strip() for line in target.splitlines() if line.strip()]
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
assignment = ASSIGNMENT_RE.match(line)
|
||||
if assignment:
|
||||
var_name, expr = assignment.group(1), assignment.group(2).strip()
|
||||
call_info = self.parse_call(expr, result, line)
|
||||
if not call_info:
|
||||
continue
|
||||
object_name = call_info["name"]
|
||||
if object_name not in self.objects:
|
||||
self.error(result, f"变量 {var_name} 只能绑定 object，但 {object_name} 不在对象目录中")
|
||||
continue
|
||||
self.validate_call_kwargs(call_info, self.objects[object_name], result)
|
||||
variables[var_name] = object_name
|
||||
result["references"].append({"type": "object", "name": object_name, "variable": var_name})
|
||||
continue
|
||||
|
||||
call_info = self.parse_call(line, result, line)
|
||||
if not call_info:
|
||||
continue
|
||||
function_name = call_info["name"]
|
||||
if function_name not in self.functions:
|
||||
self.error(result, f"最终调用必须是已定义 function，但 {function_name} 不在函数目录中")
|
||||
continue
|
||||
self.validate_call_kwargs(call_info, self.functions[function_name], result)
|
||||
self.validate_variable_refs(call_info, variables, result)
|
||||
result["references"].append({"type": "function", "name": function_name})
|
||||
|
||||
if index != len(lines) - 1:
|
||||
self.error(result, f"非最后一行不能直接调用 function：{line}")
|
||||
|
||||
def parse_call(self, expr: str, result: dict[str, Any], raw_line: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
node = ast.parse(expr, mode="eval").body
|
||||
except SyntaxError as exc:
|
||||
self.error(result, f"调用语法无法解析：{raw_line} ({exc.msg})")
|
||||
return None
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name):
|
||||
self.error(result, f"不是合法的函数调用：{raw_line}")
|
||||
return None
|
||||
return {
|
||||
"name": node.func.id,
|
||||
"keywords": node.keywords,
|
||||
"positional_count": len(node.args),
|
||||
"raw": raw_line,
|
||||
}
|
||||
|
||||
def validate_call_kwargs(self, call_info: dict[str, Any], spec: dict[str, Any], result: dict[str, Any]) -> None:
|
||||
if call_info["positional_count"]:
|
||||
self.error(result, f"暂不支持位置参数，请使用命名参数：{call_info['raw']}")
|
||||
|
||||
params = spec.get("params", {})
|
||||
allow_unknown = bool(spec.get("allow_unknown_params"))
|
||||
for keyword in call_info["keywords"]:
|
||||
if keyword.arg is None:
|
||||
self.error(result, f"暂不支持 **kwargs：{call_info['raw']}")
|
||||
continue
|
||||
if not allow_unknown and keyword.arg not in params:
|
||||
self.error(result, f"{call_info['name']} 未定义参数：{keyword.arg}")
|
||||
continue
|
||||
enum_values = params.get(keyword.arg, {}).get("enum_values", [])
|
||||
if enum_values:
|
||||
self.validate_enum(keyword, enum_values, call_info, result)
|
||||
|
||||
def validate_enum(self, keyword: ast.keyword, enum_values: list[str], call_info: dict[str, Any], result: dict[str, Any]) -> None:
|
||||
values = self.literal_string_values(keyword.value)
|
||||
for value in values:
|
||||
if value not in enum_values:
|
||||
self.error(result, f"{call_info['name']}.{keyword.arg} 枚举值不合法：{value}，可选：{', '.join(enum_values)}")
|
||||
|
||||
def literal_string_values(self, node: ast.AST) -> list[str]:
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
return [node.value]
|
||||
if isinstance(node, ast.List):
|
||||
values: list[str] = []
|
||||
for item in node.elts:
|
||||
values.extend(self.literal_string_values(item))
|
||||
return values
|
||||
return []
|
||||
|
||||
def validate_variable_refs(self, call_info: dict[str, Any], variables: dict[str, str], result: dict[str, Any]) -> None:
|
||||
referenced = sorted({node.id for keyword in call_info["keywords"] for node in ast.walk(keyword.value) if isinstance(node, ast.Name)})
|
||||
for name in referenced:
|
||||
if name not in variables:
|
||||
self.error(result, f"引用了未定义 object 变量：{name}")
|
||||
|
||||
def validate_group_json(self, target: str, result: dict[str, Any]) -> None:
|
||||
result["detected_type"] = "group_json"
|
||||
try:
|
||||
groups = json.loads(target)
|
||||
except json.JSONDecodeError as exc:
|
||||
self.error(result, f"多指令/自动任务 JSON 解析失败：{exc.msg}")
|
||||
return
|
||||
if not isinstance(groups, list):
|
||||
self.error(result, "多指令/自动任务输出必须是 JSON list")
|
||||
return
|
||||
|
||||
for group_index, group in enumerate(groups):
|
||||
if not isinstance(group, dict):
|
||||
self.error(result, f"第 {group_index} 个 group 不是 object")
|
||||
continue
|
||||
if "condition" not in group:
|
||||
self.error(result, f"第 {group_index} 个 group 缺少 condition")
|
||||
querys = group.get("querys")
|
||||
if not isinstance(querys, list):
|
||||
self.error(result, f"第 {group_index} 个 group 的 querys 必须是 list")
|
||||
continue
|
||||
for query_index, item in enumerate(querys):
|
||||
self.validate_group_item(item, group_index, query_index, result)
|
||||
|
||||
def validate_group_item(self, item: Any, group_index: int, query_index: int, result: dict[str, Any]) -> None:
|
||||
prefix = f"第 {group_index} 个 group 的第 {query_index} 条 query"
|
||||
if not isinstance(item, dict):
|
||||
self.error(result, f"{prefix} 不是 object")
|
||||
return
|
||||
if not item.get("subquery"):
|
||||
self.error(result, f"{prefix} 缺少 subquery")
|
||||
|
||||
value_fields = [field for field in ("function", "intent", "target", "label") if item.get(field)]
|
||||
if not value_fields:
|
||||
self.error(result, f"{prefix} 缺少 function、intent、target 或 label")
|
||||
return
|
||||
if len(value_fields) > 1:
|
||||
self.error(result, f"{prefix} 同时包含多个输出字段：{', '.join(value_fields)}")
|
||||
return
|
||||
|
||||
field = value_fields[0]
|
||||
value = str(item[field])
|
||||
if field == "function":
|
||||
nested = self.validate(value)
|
||||
self.merge_nested(result, nested, prefix)
|
||||
elif field == "intent":
|
||||
if value not in self.intents and value not in self.agent_tags:
|
||||
self.error(result, f"{prefix} 的 intent 不在知识库中：{value}")
|
||||
else:
|
||||
result["references"].append({"type": "intent", "name": value})
|
||||
else:
|
||||
nested = self.validate(value)
|
||||
self.merge_nested(result, nested, prefix)
|
||||
|
||||
def merge_nested(self, result: dict[str, Any], nested: dict[str, Any], prefix: str) -> None:
|
||||
for error in nested.get("errors", []):
|
||||
self.error(result, f"{prefix}: {error}")
|
||||
for warning in nested.get("warnings", []):
|
||||
result["warnings"].append(f"{prefix}: {warning}")
|
||||
result["references"].extend(nested.get("references", []))
|
||||
|
||||
def error(self, result: dict[str, Any], message: str) -> None:
|
||||
result["errors"].append(message)
|
||||
|
||||
|
||||
def load_targets_from_file(path: Path, field: str) -> list[str]:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
if not text:
|
||||
return []
|
||||
if path.suffix == ".jsonl":
|
||||
targets: list[str] = []
|
||||
for line_no, line in enumerate(text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
item = json.loads(line)
|
||||
if field not in item:
|
||||
raise ValueError(f"{path}:{line_no} 缺少字段 {field}")
|
||||
targets.append(str(item[field]))
|
||||
return targets
|
||||
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict):
|
||||
if field in data:
|
||||
return [str(data[field])]
|
||||
if "records" in data and isinstance(data["records"], list):
|
||||
return [str(item[field]) for item in data["records"] if isinstance(item, dict) and field in item]
|
||||
if isinstance(data, list):
|
||||
return [str(item[field]) for item in data if isinstance(item, dict) and field in item]
|
||||
raise ValueError(f"无法从 {path} 读取字段 {field}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="校验标签输出格式")
|
||||
parser.add_argument("--target", help="直接校验一个 target 字符串")
|
||||
parser.add_argument("--file", help="从 JSON/JSONL 文件中批量读取 target")
|
||||
parser.add_argument("--field", default="target", help="批量文件中的字段名，默认 target")
|
||||
parser.add_argument("--manifest", default=str(DEFAULT_OUTPUT), help="manifest JSON 路径")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.target and not args.file:
|
||||
parser.error("必须提供 --target 或 --file")
|
||||
|
||||
manifest_path = Path(args.manifest)
|
||||
if not manifest_path.is_absolute():
|
||||
manifest_path = SKILL_ROOT / manifest_path
|
||||
validator = LabelOutputValidator(load_manifest(manifest_path))
|
||||
|
||||
targets = [args.target] if args.target else load_targets_from_file(Path(args.file), args.field)
|
||||
results = [validator.validate(target or "") for target in targets]
|
||||
payload: dict[str, Any]
|
||||
if len(results) == 1:
|
||||
payload = results[0]
|
||||
else:
|
||||
payload = {
|
||||
"valid": all(item["valid"] for item in results),
|
||||
"total": len(results),
|
||||
"invalid": sum(1 for item in results if not item["valid"]),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if payload["valid"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""把旧 docx 标签表迁移为中文 Markdown 标签知识卡片。
|
||||
|
||||
这个脚本是一次性/阶段性迁移辅助工具，不是运行时分类器。
|
||||
长期维护入口应该是 skills/label-master/knowledge 下的 Markdown 文件。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
|
||||
NS = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
|
||||
|
||||
|
||||
@dataclass
|
||||
class LabelRow:
|
||||
domain: str
|
||||
name: str
|
||||
raw_tag: str
|
||||
agent_candidate: str
|
||||
abstraction: str
|
||||
scope: str
|
||||
examples: str
|
||||
semantic_points: str
|
||||
function_def: str
|
||||
boundary: str
|
||||
confusing: str
|
||||
principle: str
|
||||
notes: str
|
||||
|
||||
|
||||
def _cell_text(cell: ET.Element) -> str:
|
||||
parts: list[str] = []
|
||||
for text_node in cell.findall('.//w:t', NS):
|
||||
if text_node.text:
|
||||
parts.append(text_node.text)
|
||||
return re.sub(r'\s+', ' ', ''.join(parts)).strip()
|
||||
|
||||
|
||||
def _docx_tables(path: Path) -> list[list[list[str]]]:
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
xml = zf.read('word/document.xml')
|
||||
root = ET.fromstring(xml)
|
||||
tables: list[list[list[str]]] = []
|
||||
for table in root.findall('.//w:tbl', NS):
|
||||
rows: list[list[str]] = []
|
||||
for tr in table.findall('./w:tr', NS):
|
||||
row = [_cell_text(tc) for tc in tr.findall('./w:tc', NS)]
|
||||
if any(cell.strip() for cell in row):
|
||||
rows.append(row)
|
||||
if rows:
|
||||
tables.append(rows)
|
||||
return tables
|
||||
|
||||
|
||||
def _norm_header(value: str) -> str:
|
||||
return re.sub(r'[\s/（）()]+', '', value)
|
||||
|
||||
|
||||
def _find_index(headers: list[str], candidates: Iterable[str]) -> int | None:
|
||||
normalized = [_norm_header(h) for h in headers]
|
||||
for candidate in candidates:
|
||||
needle = _norm_header(candidate)
|
||||
for idx, header in enumerate(normalized):
|
||||
if needle and needle in header:
|
||||
return idx
|
||||
return None
|
||||
|
||||
|
||||
def _safe_filename(value: str) -> str:
|
||||
value = re.sub(r'[a-f0-9]{24,}$', '', value, flags=re.IGNORECASE)
|
||||
value = re.sub(r'[\\/:*?"<>|]+', '、', value).strip()
|
||||
value = re.sub(r'\s+', '', value)
|
||||
return value[:48] or '未命名标签'
|
||||
|
||||
|
||||
def _domain_name(path: Path) -> str:
|
||||
stem = path.stem
|
||||
return stem.removeprefix('Label定义-')
|
||||
|
||||
|
||||
def _read_cell(row: list[str], idx: int | None) -> str:
|
||||
if idx is None or idx >= len(row):
|
||||
return ''
|
||||
return row[idx].strip()
|
||||
|
||||
|
||||
def _compact_label_name(tag: str, abstraction: str) -> str:
|
||||
"""从旧表格的 tag 列中提取更适合人工维护的中文标签名。
|
||||
|
||||
部分旧表格把标签名和长描述写进了同一个单元格，例如
|
||||
“闹钟闹钟/小憩定时闹钟的增删改查...”。这里尽量保留短标签，
|
||||
不把整段说明变成文件名或 Agent(tag=...)。
|
||||
"""
|
||||
|
||||
tag = re.sub(r'[a-f0-9]{24,}$', '', tag.strip(), flags=re.IGNORECASE)
|
||||
abstraction = abstraction.strip()
|
||||
if tag.startswith('应该属于'):
|
||||
return abstraction or tag.removeprefix('应该属于').strip()
|
||||
if not tag:
|
||||
stop_match = re.search(r'(关于|回答|打开|查询|增|删|改|查|的|功能|垂域|/|、|，|。)', abstraction)
|
||||
if stop_match and stop_match.start() >= 2:
|
||||
return abstraction[:stop_match.start()]
|
||||
return abstraction
|
||||
if abstraction and tag.startswith(abstraction) and len(abstraction) <= 12:
|
||||
return abstraction
|
||||
repeated = re.match(r'^(.{2,8})\1', tag)
|
||||
if repeated:
|
||||
return repeated.group(1)
|
||||
if len(tag) <= 24:
|
||||
return tag
|
||||
stop_match = re.search(r'(关于|回答|打开|查询|增|删|改|查|的|功能|垂域|/|、|，|。)', tag)
|
||||
if stop_match and stop_match.start() >= 2:
|
||||
return tag[:stop_match.start()]
|
||||
return abstraction or tag[:12]
|
||||
|
||||
|
||||
def _agent_candidate_for_label(name: str, raw_tag: str) -> str:
|
||||
if not name:
|
||||
return ''
|
||||
if raw_tag.startswith('应该属于'):
|
||||
target = raw_tag.removeprefix('应该属于').strip()
|
||||
if target:
|
||||
return f'Agent(tag="{target}")'
|
||||
if raw_tag and len(raw_tag) <= 24:
|
||||
return f'Agent(tag="{name}")'
|
||||
if raw_tag.startswith(name):
|
||||
return f'Agent(tag="{name}")'
|
||||
return ''
|
||||
|
||||
|
||||
def _function_candidate(value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return '待确认。'
|
||||
# 保留旧表中的函数/参数定义原文，避免在迁移阶段误判最终输出格式。
|
||||
return value
|
||||
|
||||
|
||||
def _looks_like_header(row: list[str]) -> bool:
|
||||
joined = ''.join(row)
|
||||
return 'tag标签' in joined or ('功能抽象' in joined and '功能和范围定义' in joined)
|
||||
|
||||
|
||||
def extract_label_rows(source_dir: Path) -> list[LabelRow]:
|
||||
rows: list[LabelRow] = []
|
||||
for docx_path in sorted(source_dir.glob('Label定义-*.docx')):
|
||||
domain = _domain_name(docx_path)
|
||||
for table in _docx_tables(docx_path):
|
||||
if not table:
|
||||
continue
|
||||
header_pos = next((idx for idx, row in enumerate(table) if _looks_like_header(row)), None)
|
||||
if header_pos is None:
|
||||
continue
|
||||
headers = table[header_pos]
|
||||
idx_abstraction = _find_index(headers, ['功能抽象'])
|
||||
idx_tag = _find_index(headers, ['tag标签'])
|
||||
idx_scope = _find_index(headers, ['功能和范围定义'])
|
||||
idx_examples = _find_index(headers, ['示例query'])
|
||||
idx_semantic = _find_index(headers, ['三级语义功能点举例'])
|
||||
idx_function = _find_index(headers, ['Function及参数定义', 'Function及参数定义飘黄部分未共识'])
|
||||
idx_boundary = _find_index(headers, ['满足边界问题'])
|
||||
idx_confusing = _find_index(headers, ['易混淆tagfunctionAgent', '易混淆tagfunction', '易混淆'])
|
||||
idx_principle = _find_index(headers, ['划分原则'])
|
||||
idx_notes = _find_index(headers, ['问题备注', '未解决'])
|
||||
|
||||
for row in table[header_pos + 1:]:
|
||||
abstraction = _read_cell(row, idx_abstraction)
|
||||
tag = _read_cell(row, idx_tag)
|
||||
name = _compact_label_name(tag, abstraction)
|
||||
if not name or name in {'/', '-', '待定'}:
|
||||
continue
|
||||
if len(name) > 80 and not tag:
|
||||
continue
|
||||
agent_candidate = _agent_candidate_for_label(name, tag)
|
||||
rows.append(
|
||||
LabelRow(
|
||||
domain=domain,
|
||||
name=name,
|
||||
raw_tag=tag,
|
||||
agent_candidate=agent_candidate,
|
||||
abstraction=abstraction,
|
||||
scope=_read_cell(row, idx_scope),
|
||||
examples=_read_cell(row, idx_examples),
|
||||
semantic_points=_read_cell(row, idx_semantic),
|
||||
function_def=_read_cell(row, idx_function),
|
||||
boundary=_read_cell(row, idx_boundary),
|
||||
confusing=_read_cell(row, idx_confusing),
|
||||
principle=_read_cell(row, idx_principle),
|
||||
notes=_read_cell(row, idx_notes),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _section(title: str, body: str) -> str:
|
||||
body = body.strip()
|
||||
if not body:
|
||||
body = '待补充。'
|
||||
return f'## {title}\n\n{body}\n'
|
||||
|
||||
|
||||
def _table_cell(value: str, *, limit: int = 80) -> str:
|
||||
value = re.sub(r'\s+', ' ', value.strip()).replace('|', '、')
|
||||
if not value:
|
||||
return '待确认'
|
||||
return value[:limit] + ('...' if len(value) > limit else '')
|
||||
|
||||
|
||||
def _output_section(row: LabelRow) -> str:
|
||||
raw_tag = row.raw_tag or '待确认。'
|
||||
agent_candidate = row.agent_candidate or '待确认。'
|
||||
function_candidate = _function_candidate(row.function_def)
|
||||
return '\n'.join(
|
||||
[
|
||||
'## 标注输出',
|
||||
'',
|
||||
'> 注意：旧标签资料中同时存在 tag 标签和 Function 定义。本卡片不直接声明最终训练标签；生成数据前必须结合 `判断维度/标注输出形态.md` 确认当前任务使用 Agent 形式还是 Function 形式。',
|
||||
'',
|
||||
f'- 旧 tag 标签：{raw_tag}',
|
||||
f'- Agent 包装候选（不代表最终）：{agent_candidate}',
|
||||
f'- Function 输出候选：{function_candidate}',
|
||||
'- 推荐输出形态：待确认。',
|
||||
'',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _card_text(row: LabelRow) -> str:
|
||||
title = row.name
|
||||
return '\n'.join(
|
||||
[
|
||||
f'# {title}',
|
||||
'',
|
||||
_output_section(row),
|
||||
_section('功能抽象', row.abstraction),
|
||||
_section('适用范围', row.scope),
|
||||
_section('典型 Query', row.examples),
|
||||
_section('三级语义功能点', row.semantic_points),
|
||||
_section('Function / Agent 说明', row.function_def),
|
||||
_section('满足边界问题', row.boundary),
|
||||
_section('易混淆标签', row.confusing),
|
||||
_section('划分原则', row.principle),
|
||||
_section('未解决问题', row.notes),
|
||||
]
|
||||
).rstrip() + '\n'
|
||||
|
||||
|
||||
def write_knowledge(rows: list[LabelRow], output_dir: Path) -> None:
|
||||
label_root = output_dir / '标签'
|
||||
boundary_root = output_dir / '边界'
|
||||
index_root = output_dir / '索引'
|
||||
label_root.mkdir(parents=True, exist_ok=True)
|
||||
boundary_root.mkdir(parents=True, exist_ok=True)
|
||||
index_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
domains: dict[str, list[LabelRow]] = {}
|
||||
for row in rows:
|
||||
domains.setdefault(row.domain, []).append(row)
|
||||
domain_dir = label_root / _safe_filename(row.domain)
|
||||
domain_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = domain_dir / f'{_safe_filename(row.name)}.md'
|
||||
path.write_text(_card_text(row), encoding='utf-8')
|
||||
|
||||
for domain, domain_rows in sorted(domains.items()):
|
||||
lines = [f'# {domain}边界', '']
|
||||
for row in domain_rows:
|
||||
if not row.confusing and not row.principle and not row.boundary:
|
||||
continue
|
||||
lines.append(f'## {row.name}')
|
||||
if row.confusing:
|
||||
lines.extend(['', f'- 易混淆：{row.confusing}'])
|
||||
if row.principle:
|
||||
lines.extend(['', f'- 划分原则：{row.principle}'])
|
||||
if row.boundary:
|
||||
lines.extend(['', f'- 满足边界问题：{row.boundary}'])
|
||||
lines.append('')
|
||||
if len(lines) > 2:
|
||||
(boundary_root / f'{_safe_filename(domain)}边界.md').write_text('\n'.join(lines).rstrip() + '\n', encoding='utf-8')
|
||||
|
||||
overview_lines = ['# 标签迁移索引', '']
|
||||
label_index_lines = [
|
||||
'# 标签索引',
|
||||
'',
|
||||
'这个索引用于 Agent 第一阶段快速判断候选标签和输出形态。详细边界仍需读取具体标签卡片和边界文件。',
|
||||
'',
|
||||
'| 领域 | 标签 | 旧 tag 标签 | Agent 形式候选 | Function 形式候选 | 知识卡片 |',
|
||||
'| --- | --- | --- | --- | --- | --- |',
|
||||
]
|
||||
for domain, domain_rows in sorted(domains.items()):
|
||||
overview_lines.append(f'## {domain}')
|
||||
overview_lines.append('')
|
||||
seen_links: set[str] = set()
|
||||
for row in sorted(domain_rows, key=lambda item: item.name):
|
||||
rel = Path('标签') / _safe_filename(domain) / f'{_safe_filename(row.name)}.md'
|
||||
rel_text = rel.as_posix()
|
||||
if rel_text in seen_links:
|
||||
continue
|
||||
seen_links.add(rel_text)
|
||||
overview_lines.append(f'- [{row.name}]({rel.as_posix()})')
|
||||
rel_from_index = Path('..') / rel
|
||||
function_hint = _table_cell(row.function_def, limit=60) if row.function_def else '待确认'
|
||||
label_index_lines.append(
|
||||
'| {domain} | {name} | {raw_tag} | {agent} | {function} | [{name}]({path}) |'.format(
|
||||
domain=_table_cell(domain, limit=20),
|
||||
name=_table_cell(row.name, limit=24),
|
||||
raw_tag=_table_cell(row.raw_tag, limit=32),
|
||||
agent=_table_cell(row.agent_candidate, limit=36),
|
||||
function=function_hint,
|
||||
path=rel_from_index.as_posix(),
|
||||
)
|
||||
)
|
||||
overview_lines.append('')
|
||||
(output_dir / '标签迁移索引.md').write_text('\n'.join(overview_lines).rstrip() + '\n', encoding='utf-8')
|
||||
(index_root / '标签索引.md').write_text('\n'.join(label_index_lines).rstrip() + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description='迁移 docx 标签定义表为 Markdown 知识卡片')
|
||||
parser.add_argument('--source-dir', default='标签定义', help='旧标签定义目录')
|
||||
parser.add_argument('--output-dir', default='skills/label-master/knowledge', help='知识库输出目录')
|
||||
args = parser.parse_args()
|
||||
|
||||
source_dir = Path(args.source_dir)
|
||||
output_dir = Path(args.output_dir)
|
||||
rows = extract_label_rows(source_dir)
|
||||
write_knowledge(rows, output_dir)
|
||||
print(f'已迁移 {len(rows)} 条标签知识到 {output_dir}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user