Collect first agent research and market sources
This commit is contained in:
wuyang
@@ -0,0 +1,53 @@
|
||||
# Collection Tools
|
||||
|
||||
这些脚本用于把 JD、论文和行业资料从 Markdown 笔记变成可检索、可检查、可视化的数据。
|
||||
|
||||
## Scripts
|
||||
|
||||
### `new_item.py`
|
||||
|
||||
生成一个带 frontmatter 的新条目草稿。
|
||||
|
||||
```bash
|
||||
python3 tools/collection/new_item.py jobs "baidu aidu agent algorithm engineer beijing" \
|
||||
--title "Baidu AIDU Agent Algorithm Engineer" \
|
||||
--url "https://talent.baidu.com/jobs/list?projectType=3&recruitType=GRADUATE"
|
||||
```
|
||||
|
||||
支持的 collection:
|
||||
|
||||
- `jobs`
|
||||
- `papers`
|
||||
- `industry`
|
||||
|
||||
### `build_index.py`
|
||||
|
||||
扫描 `jobs/items/`、`papers/items/`、`industry/items/` 的 frontmatter，生成结构化索引。
|
||||
|
||||
```bash
|
||||
python3 tools/collection/build_index.py
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
- `data/index.json`
|
||||
- `data/summary.json`
|
||||
|
||||
### `check_urls.py`
|
||||
|
||||
检查条目中的 `url`、`source_url`、`code_url` 是否可访问。
|
||||
|
||||
```bash
|
||||
python3 tools/collection/check_urls.py
|
||||
```
|
||||
|
||||
可加 `--json` 输出机器可读结果。
|
||||
|
||||
## Method
|
||||
|
||||
1. 搜索或导入资料。
|
||||
2. 用 `new_item.py` 生成草稿。
|
||||
3. 人工整理摘要、标签、判断和关联。
|
||||
4. 运行 `build_index.py` 更新结构化数据。
|
||||
5. 运行 `check_urls.py` 检查来源可达性。
|
||||
6. 更新对应 insight 文件。
|
||||
Binary file not shown.
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a JSON index from Markdown frontmatter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
COLLECTIONS = ("jobs", "papers", "industry")
|
||||
|
||||
|
||||
def parse_scalar(value: str):
|
||||
value = value.strip()
|
||||
if value == "":
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> dict:
|
||||
start = text.find("\n---\n")
|
||||
if text.startswith("---\n"):
|
||||
start = -1
|
||||
body_start = 4
|
||||
elif start != -1:
|
||||
body_start = start + 5
|
||||
else:
|
||||
return {}
|
||||
end = text.find("\n---", body_start)
|
||||
if end == -1:
|
||||
return {}
|
||||
lines = text[body_start:end].splitlines()
|
||||
data: dict[str, object] = {}
|
||||
current_key: str | None = None
|
||||
for raw in lines:
|
||||
line = raw.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith(" -"):
|
||||
if current_key:
|
||||
item = line[3:].strip()
|
||||
if item:
|
||||
data.setdefault(current_key, [])
|
||||
assert isinstance(data[current_key], list)
|
||||
data[current_key].append(item)
|
||||
continue
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
current_key = key
|
||||
data[key] = [] if value == "" else parse_scalar(value)
|
||||
return data
|
||||
|
||||
|
||||
def iter_items():
|
||||
for collection in COLLECTIONS:
|
||||
item_dir = ROOT / collection / "items"
|
||||
for path in sorted(item_dir.glob("*.md")):
|
||||
if path.name == "README.md":
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8")
|
||||
meta = parse_frontmatter(text)
|
||||
yield {
|
||||
"collection": collection,
|
||||
"path": str(path.relative_to(ROOT)),
|
||||
"title": meta.get("title") or meta.get("role") or path.stem,
|
||||
"type": meta.get("type") or collection.rstrip("s"),
|
||||
"meta": meta,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
items = list(iter_items())
|
||||
data_dir = ROOT / "data"
|
||||
data_dir.mkdir(exist_ok=True)
|
||||
(data_dir / "index.json").write_text(
|
||||
json.dumps(items, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
by_collection = Counter(item["collection"] for item in items)
|
||||
by_status = defaultdict(Counter)
|
||||
topics = Counter()
|
||||
skills = Counter()
|
||||
companies = Counter()
|
||||
for item in items:
|
||||
meta = item["meta"]
|
||||
by_status[item["collection"]][meta.get("status") or "unknown"] += 1
|
||||
for topic in meta.get("topics") or []:
|
||||
topics[topic] += 1
|
||||
for skill in meta.get("skills") or []:
|
||||
skills[skill] += 1
|
||||
company = meta.get("company")
|
||||
if company:
|
||||
companies[str(company)] += 1
|
||||
|
||||
summary = {
|
||||
"total": len(items),
|
||||
"by_collection": dict(by_collection),
|
||||
"by_status": {key: dict(value) for key, value in by_status.items()},
|
||||
"top_topics": topics.most_common(20),
|
||||
"top_skills": skills.most_common(20),
|
||||
"top_companies": companies.most_common(20),
|
||||
}
|
||||
(data_dir / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check source URLs stored in collection item frontmatter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from build_index import parse_frontmatter
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
URL_KEYS = ("url", "source_url", "code_url")
|
||||
|
||||
|
||||
def check_url(url: str, timeout: int) -> dict:
|
||||
request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "agent-kb-url-check/1.0"})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return {"url": url, "ok": True, "status": response.status}
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (403, 405):
|
||||
return check_url_get(url, timeout, exc.code)
|
||||
return {"url": url, "ok": False, "status": exc.code, "error": str(exc)}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"url": url, "ok": False, "status": None, "error": str(exc)}
|
||||
|
||||
|
||||
def check_url_get(url: str, timeout: int, head_status: int) -> dict:
|
||||
request = urllib.request.Request(url, method="GET", headers={"User-Agent": "agent-kb-url-check/1.0"})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return {"url": url, "ok": True, "status": response.status, "head_status": head_status}
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 403:
|
||||
return {
|
||||
"url": url,
|
||||
"ok": True,
|
||||
"status": exc.code,
|
||||
"head_status": head_status,
|
||||
"blocked": True,
|
||||
"error": "blocked automated request; verify in browser",
|
||||
}
|
||||
return {"url": url, "ok": False, "status": exc.code, "head_status": head_status, "error": str(exc)}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"url": url, "ok": False, "status": head_status, "error": str(exc)}
|
||||
|
||||
|
||||
def iter_urls():
|
||||
for collection in ("jobs", "papers", "industry"):
|
||||
for path in sorted((ROOT / collection / "items").glob("*.md")):
|
||||
if path.name == "README.md":
|
||||
continue
|
||||
meta = parse_frontmatter(path.read_text(encoding="utf-8"))
|
||||
for key in URL_KEYS:
|
||||
value = meta.get(key)
|
||||
if value:
|
||||
yield path.relative_to(ROOT).as_posix(), key, str(value)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", action="store_true")
|
||||
parser.add_argument("--timeout", type=int, default=12)
|
||||
args = parser.parse_args()
|
||||
|
||||
results = []
|
||||
for path, key, url in iter_urls():
|
||||
result = check_url(url, args.timeout)
|
||||
result.update({"path": path, "key": key})
|
||||
results.append(result)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(results, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
for result in results:
|
||||
marker = "BLOCKED" if result.get("blocked") else ("OK" if result["ok"] else "FAIL")
|
||||
print(f"{marker} {result.get('status')} {result['path']} {result['key']} {result['url']}")
|
||||
return 1 if any(not result["ok"] for result in results) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+242
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a new collection item draft."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
TEMPLATES = {
|
||||
"jobs": """# Job: {title}
|
||||
|
||||
---
|
||||
type: job
|
||||
company:
|
||||
role: {title}
|
||||
location:
|
||||
source_url: {url}
|
||||
source_name:
|
||||
source_quality: unknown
|
||||
collected_at: {today}
|
||||
posted_at:
|
||||
first_seen_at: {today}
|
||||
last_checked_at: {today}
|
||||
snapshot_type: new
|
||||
job_code:
|
||||
previous_snapshot:
|
||||
status: unknown
|
||||
level:
|
||||
team:
|
||||
business_area:
|
||||
employment_type:
|
||||
salary:
|
||||
skills:
|
||||
-
|
||||
topics:
|
||||
-
|
||||
models:
|
||||
-
|
||||
related_papers:
|
||||
-
|
||||
related_experiments:
|
||||
-
|
||||
related_projects:
|
||||
-
|
||||
relevance: medium
|
||||
---
|
||||
|
||||
## Source Snapshot
|
||||
|
||||
- source:
|
||||
- collected_at: {today}
|
||||
- availability:
|
||||
|
||||
## Change Snapshot
|
||||
|
||||
- first_seen_at: {today}
|
||||
- last_checked_at: {today}
|
||||
- snapshot_type: new
|
||||
- previous_snapshot:
|
||||
- changed_fields:
|
||||
|
||||
## Raw JD Summary
|
||||
|
||||
-
|
||||
|
||||
## Extracted Signals
|
||||
|
||||
- hard requirements:
|
||||
- preferred skills:
|
||||
- hidden signals:
|
||||
- business direction:
|
||||
|
||||
## Knowledge Links
|
||||
|
||||
- concepts:
|
||||
- papers:
|
||||
- experiments:
|
||||
- projects:
|
||||
|
||||
## Gaps for Us
|
||||
|
||||
-
|
||||
""",
|
||||
"papers": """# Paper: {title}
|
||||
|
||||
---
|
||||
type: paper
|
||||
title: {title}
|
||||
authors:
|
||||
year:
|
||||
venue:
|
||||
url: {url}
|
||||
code_url:
|
||||
source:
|
||||
collected_at: {today}
|
||||
status: queued
|
||||
relevance: medium
|
||||
topics:
|
||||
-
|
||||
methods:
|
||||
-
|
||||
benchmarks:
|
||||
-
|
||||
models:
|
||||
-
|
||||
datasets:
|
||||
-
|
||||
related_concepts:
|
||||
-
|
||||
related_jobs:
|
||||
-
|
||||
related_experiments:
|
||||
-
|
||||
related_projects:
|
||||
-
|
||||
---
|
||||
|
||||
## One-line Takeaway
|
||||
|
||||
-
|
||||
|
||||
## Problem
|
||||
|
||||
-
|
||||
|
||||
## Core Idea
|
||||
|
||||
-
|
||||
|
||||
## Evidence
|
||||
|
||||
-
|
||||
|
||||
## Useful For Us
|
||||
|
||||
-
|
||||
|
||||
## Follow-up Experiments
|
||||
|
||||
-
|
||||
""",
|
||||
"industry": """# Industry: {title}
|
||||
|
||||
---
|
||||
type: industry
|
||||
company:
|
||||
team:
|
||||
title: {title}
|
||||
url: {url}
|
||||
source_name:
|
||||
source_type: other
|
||||
source_quality: unknown
|
||||
published_at:
|
||||
collected_at: {today}
|
||||
status: queued
|
||||
topics:
|
||||
-
|
||||
implementation_signals:
|
||||
-
|
||||
product_area:
|
||||
-
|
||||
models:
|
||||
-
|
||||
tools:
|
||||
-
|
||||
benchmarks:
|
||||
-
|
||||
related_papers:
|
||||
-
|
||||
related_jobs:
|
||||
-
|
||||
related_experiments:
|
||||
-
|
||||
related_projects:
|
||||
-
|
||||
evidence_level: medium
|
||||
relevance: medium
|
||||
---
|
||||
|
||||
## One-line Takeaway
|
||||
|
||||
-
|
||||
|
||||
## What They Built or Claimed
|
||||
|
||||
-
|
||||
|
||||
## Technical Signals
|
||||
|
||||
- architecture:
|
||||
- tool use:
|
||||
- memory:
|
||||
- evaluation:
|
||||
- safety:
|
||||
- deployment:
|
||||
- data:
|
||||
|
||||
## Evidence Quality
|
||||
|
||||
-
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
text = text.lower().strip()
|
||||
text = re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "-", text)
|
||||
text = re.sub(r"-+", "-", text).strip("-")
|
||||
return text or "untitled"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("collection", choices=sorted(TEMPLATES))
|
||||
parser.add_argument("slug")
|
||||
parser.add_argument("--title", default=None)
|
||||
parser.add_argument("--url", default="")
|
||||
parser.add_argument("--date", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
title = args.title or args.slug.replace("-", " ").title()
|
||||
filename = f"{args.date}-{slugify(args.slug)}.md"
|
||||
path = ROOT / args.collection / "items" / filename
|
||||
if path.exists():
|
||||
raise SystemExit(f"refusing to overwrite existing file: {path}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
TEMPLATES[args.collection].format(title=title, url=args.url, today=args.date),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(path.relative_to(ROOT))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user