Collect first agent research and market sources

2026-07-08 12:08:24 +08:00
parent 36c2499d3c
commit 1cbc98e432
34 changed files with 3417 additions and 58 deletions
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Build a JSON index from Markdown frontmatter."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+COLLECTIONS = ("jobs", "papers", "industry")
+
+
+def parse_scalar(value: str):
+    value = value.strip()
+    if value == "":
+        return None
+    return value
+
+
+def parse_frontmatter(text: str) -> dict:
+    start = text.find("\n---\n")
+    if text.startswith("---\n"):
+        start = -1
+        body_start = 4
+    elif start != -1:
+        body_start = start + 5
+    else:
+        return {}
+    end = text.find("\n---", body_start)
+    if end == -1:
+        return {}
+    lines = text[body_start:end].splitlines()
+    data: dict[str, object] = {}
+    current_key: str | None = None
+    for raw in lines:
+        line = raw.rstrip()
+        if not line:
+            continue
+        if line.startswith("  -"):
+            if current_key:
+                item = line[3:].strip()
+                if item:
+                    data.setdefault(current_key, [])
+                    assert isinstance(data[current_key], list)
+                    data[current_key].append(item)
+            continue
+        if ":" in line:
+            key, value = line.split(":", 1)
+            key = key.strip()
+            value = value.strip()
+            current_key = key
+            data[key] = [] if value == "" else parse_scalar(value)
+    return data
+
+
+def iter_items():
+    for collection in COLLECTIONS:
+        item_dir = ROOT / collection / "items"
+        for path in sorted(item_dir.glob("*.md")):
+            if path.name == "README.md":
+                continue
+            text = path.read_text(encoding="utf-8")
+            meta = parse_frontmatter(text)
+            yield {
+                "collection": collection,
+                "path": str(path.relative_to(ROOT)),
+                "title": meta.get("title") or meta.get("role") or path.stem,
+                "type": meta.get("type") or collection.rstrip("s"),
+                "meta": meta,
+            }
+
+
+def main() -> int:
+    items = list(iter_items())
+    data_dir = ROOT / "data"
+    data_dir.mkdir(exist_ok=True)
+    (data_dir / "index.json").write_text(
+        json.dumps(items, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    by_collection = Counter(item["collection"] for item in items)
+    by_status = defaultdict(Counter)
+    topics = Counter()
+    skills = Counter()
+    companies = Counter()
+    for item in items:
+        meta = item["meta"]
+        by_status[item["collection"]][meta.get("status") or "unknown"] += 1
+        for topic in meta.get("topics") or []:
+            topics[topic] += 1
+        for skill in meta.get("skills") or []:
+            skills[skill] += 1
+        company = meta.get("company")
+        if company:
+            companies[str(company)] += 1
+
+    summary = {
+        "total": len(items),
+        "by_collection": dict(by_collection),
+        "by_status": {key: dict(value) for key, value in by_status.items()},
+        "top_topics": topics.most_common(20),
+        "top_skills": skills.most_common(20),
+        "top_companies": companies.most_common(20),
+    }
+    (data_dir / "summary.json").write_text(
+        json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())