wuyang
118 lines
3.4 KiB
Python
Executable File
118 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Build a JSON index from Markdown frontmatter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
COLLECTIONS = ("jobs", "papers", "industry")
|
|
|
|
|
|
def parse_scalar(value: str):
|
|
value = value.strip()
|
|
if value == "":
|
|
return None
|
|
return value
|
|
|
|
|
|
def parse_frontmatter(text: str) -> dict:
|
|
start = text.find("\n---\n")
|
|
if text.startswith("---\n"):
|
|
start = -1
|
|
body_start = 4
|
|
elif start != -1:
|
|
body_start = start + 5
|
|
else:
|
|
return {}
|
|
end = text.find("\n---", body_start)
|
|
if end == -1:
|
|
return {}
|
|
lines = text[body_start:end].splitlines()
|
|
data: dict[str, object] = {}
|
|
current_key: str | None = None
|
|
for raw in lines:
|
|
line = raw.rstrip()
|
|
if not line:
|
|
continue
|
|
if line.startswith(" -"):
|
|
if current_key:
|
|
item = line[3:].strip()
|
|
if item:
|
|
data.setdefault(current_key, [])
|
|
assert isinstance(data[current_key], list)
|
|
data[current_key].append(item)
|
|
continue
|
|
if ":" in line:
|
|
key, value = line.split(":", 1)
|
|
key = key.strip()
|
|
value = value.strip()
|
|
current_key = key
|
|
data[key] = [] if value == "" else parse_scalar(value)
|
|
return data
|
|
|
|
|
|
def iter_items():
|
|
for collection in COLLECTIONS:
|
|
item_dir = ROOT / collection / "items"
|
|
for path in sorted(item_dir.glob("*.md")):
|
|
if path.name == "README.md":
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
meta = parse_frontmatter(text)
|
|
yield {
|
|
"collection": collection,
|
|
"path": str(path.relative_to(ROOT)),
|
|
"title": meta.get("title") or meta.get("role") or path.stem,
|
|
"type": meta.get("type") or collection.rstrip("s"),
|
|
"meta": meta,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
items = list(iter_items())
|
|
data_dir = ROOT / "data"
|
|
data_dir.mkdir(exist_ok=True)
|
|
(data_dir / "index.json").write_text(
|
|
json.dumps(items, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
by_collection = Counter(item["collection"] for item in items)
|
|
by_status = defaultdict(Counter)
|
|
topics = Counter()
|
|
skills = Counter()
|
|
companies = Counter()
|
|
for item in items:
|
|
meta = item["meta"]
|
|
by_status[item["collection"]][meta.get("status") or "unknown"] += 1
|
|
for topic in meta.get("topics") or []:
|
|
topics[topic] += 1
|
|
for skill in meta.get("skills") or []:
|
|
skills[skill] += 1
|
|
company = meta.get("company")
|
|
if company:
|
|
companies[str(company)] += 1
|
|
|
|
summary = {
|
|
"total": len(items),
|
|
"by_collection": dict(by_collection),
|
|
"by_status": {key: dict(value) for key, value in by_status.items()},
|
|
"top_topics": topics.most_common(20),
|
|
"top_skills": skills.most_common(20),
|
|
"top_companies": companies.most_common(20),
|
|
}
|
|
(data_dir / "summary.json").write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|