wuyang/agent
2
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate knowledge atlas to llama.cpp

Browse Source
This commit is contained in:
wuyang
2026-07-28 17:16:34 +08:00
parent 88fb212985
commit efbd82532f
9 changed files with 152 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/app.py
+98 -70
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Serve a local paper browser with lightweight Ollama-backed actions."""
"""Serve a local paper browser with lightweight llama.cpp-backed actions."""
from __future__ import annotations
@@ -30,8 +30,13 @@ INDEX_PATH = ROOT / "data" / "index.json"
CACHE_ROOT = WEB_ROOT / "cache"
ARXIV_CACHE = CACHE_ROOT / "arxiv"
AI_CACHE = CACHE_ROOT / "ai"
OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://192.168.1.10:11434").rstrip("/")
ASSET_VERSION = "atlas-20260712-01"
MODEL_API_LUNA_URL = os.environ.get(
"MODEL_API_LUNA_URL", "http://192.168.1.10:11435/v1"
).rstrip("/")
MODEL_API_SOL_URL = os.environ.get(
"MODEL_API_SOL_URL", "http://192.168.1.10:11436/v1"
).rstrip("/")
ASSET_VERSION = "atlas-20260728-01"
def detect_app_version() -> str:
@@ -51,9 +56,9 @@ def detect_app_version() -> str:
APP_VERSION = detect_app_version()
MODEL_LIGHT = os.environ.get("OLLAMA_MODEL_LIGHT", "ChatGPT-5.6:Luna")
MODEL_FAST = os.environ.get("OLLAMA_MODEL_FAST", "ChatGPT-5.6:Terra")
MODEL_LARGE = os.environ.get("OLLAMA_MODEL_LARGE", "ChatGPT-5.6:Sol")
MODEL_LIGHT = os.environ.get("MODEL_LIGHT", "ChatGPT-5.6:Luna")
MODEL_FAST = os.environ.get("MODEL_FAST", "ChatGPT-5.6:Terra")
MODEL_LARGE = os.environ.get("MODEL_LARGE", "ChatGPT-5.6:Sol")
DEFAULT_MODELS = {
"translate": MODEL_LIGHT,
@@ -75,7 +80,7 @@ NS = {
}
INDEX_LOCK = threading.Lock()
OLLAMA_LOCK = threading.Lock()
MODEL_API_LOCK = threading.Lock()
SEARCH_LOCK = threading.Lock()
INDEX_CACHE: dict[str, Any] = {"mtime": 0.0, "papers": [], "by_id": {}}
ATLAS_CACHE: dict[str, Any] = {"mtime": 0.0, "atlas": None}
@@ -512,8 +517,8 @@ def plan_search_query(query: str, scope_topic: str = "") -> tuple[dict[str, Any]
model = DEFAULT_MODELS["search_plan"]
result: dict[str, Any] | None = None
try:
with OLLAMA_LOCK:
result = call_ollama(model, search_plan_prompt(query, scope_topic), "search_plan")
with MODEL_API_LOCK:
result = call_model(model, search_plan_prompt(query, scope_topic), "search_plan")
plan = extract_json_object(result["response"])
merged = {
**fallback,
@@ -676,8 +681,8 @@ def fallback_search_summary(query: str, plan: dict[str, Any], aggregation: dict[
def generate_search_summary(query: str, plan: dict[str, Any], items: list[dict[str, Any]], aggregation: dict[str, Any]) -> dict[str, Any]:
model = DEFAULT_MODELS["search_summary"]
try:
with OLLAMA_LOCK:
result = call_ollama(model, search_summary_prompt(query, plan, items, aggregation), "search_summary")
with MODEL_API_LOCK:
result = call_model(model, search_summary_prompt(query, plan, items, aggregation), "search_summary")
return {
"text": result["response"],
"model": result["model"],
@@ -1240,45 +1245,84 @@ def clean_model_response(value: str) -> str:
return text or value.strip()
def call_ollama(model: str, prompt: str, mode: str) -> dict[str, Any]:
def model_api_url(model: str) -> str:
if model in {MODEL_LIGHT, "ChatGPT-5.6:auto", "gpt-4o"}:
return MODEL_API_LUNA_URL
return MODEL_API_SOL_URL
def model_api_request(url: str, *, data: bytes | None = None) -> urllib.request.Request:
headers = {"content-type": "application/json"} if data is not None else {}
return urllib.request.Request(url, data=data, headers=headers)
def available_model_inventory(timeout: float = 5) -> list[dict[str, Any]]:
aliases: set[str] = set()
for base_url in (MODEL_API_LUNA_URL, MODEL_API_SOL_URL):
request = model_api_request(f"{base_url}/models")
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
for item in payload.get("data", []):
if item.get("id"):
aliases.add(str(item["id"]))
aliases.update(str(alias) for alias in item.get("aliases", []) if alias)
context_lengths = {
MODEL_LIGHT: 8192,
MODEL_FAST: 32768,
MODEL_LARGE: 32768,
}
return [
{
"name": model,
"parameter_size": None,
"context_length": context_lengths.get(model),
}
for model in sorted(aliases)
]
def call_model(model: str, prompt: str, mode: str) -> dict[str, Any]:
options = {
"temperature": 0.2,
"num_predict": 520,
"max_tokens": 520,
}
if mode == "translate":
options.update({"num_predict": 700, "temperature": 0.1})
options.update({"max_tokens": 700, "temperature": 0.1})
elif mode == "deep":
options.update({"num_predict": 900, "temperature": 0.25})
options.update({"max_tokens": 900, "temperature": 0.25})
elif mode in {"atlas", "topic", "path", "compare"}:
options.update({"num_predict": 950, "temperature": 0.22})
options.update({"max_tokens": 950, "temperature": 0.22})
elif mode == "search_plan":
options.update({"num_predict": 420, "temperature": 0.1})
options.update({"max_tokens": 420, "temperature": 0.1})
elif mode in {"search_summary", "search_followup", "paper_chat"}:
options.update({"num_predict": 780, "temperature": 0.18})
options.update({"max_tokens": 780, "temperature": 0.18})
payload = {
"model": model,
"prompt": prompt,
"messages": [{"role": "user", "content": prompt}],
"stream": False,
"options": options,
"chat_template_kwargs": {"enable_thinking": model == MODEL_LARGE},
**options,
}
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(
f"{OLLAMA_URL}/api/generate",
request = model_api_request(
f"{model_api_url(model)}/chat/completions",
data=data,
headers={"content-type": "application/json"},
method="POST",
)
started = time.time()
with urllib.request.urlopen(request, timeout=240) as response:
result = json.loads(response.read().decode("utf-8"))
choice = (result.get("choices") or [{}])[0]
message = choice.get("message") or {}
usage = result.get("usage") or {}
return {
"model": model,
"mode": mode,
"response": clean_model_response(str(result.get("response") or "")),
"response": clean_model_response(str(message.get("content") or "")),
"duration_seconds": round(time.time() - started, 2),
"eval_count": result.get("eval_count"),
"prompt_eval_count": result.get("prompt_eval_count"),
"eval_count": usage.get("completion_tokens"),
"prompt_eval_count": usage.get("prompt_tokens"),
}
@@ -1379,20 +1423,14 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
def handle_health(self) -> None:
papers, _ = load_papers()
ollama_ok = False
model_api_ok = False
available_models: list[str] = []
try:
request = urllib.request.Request(f"{OLLAMA_URL}/api/tags")
with urllib.request.urlopen(request, timeout=3) as response:
ollama_ok = response.status == 200
payload = json.loads(response.read().decode("utf-8"))
available_models = sorted(
str(item.get("name"))
for item in payload.get("models", [])
if item.get("name")
)
inventory = available_model_inventory(timeout=3)
available_models = [str(item["name"]) for item in inventory]
model_api_ok = True
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
ollama_ok = False
model_api_ok = False
missing_default_models = sorted(set(DEFAULT_MODELS.values()) - set(available_models))
write_json(
self,
@@ -1401,9 +1439,9 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
"paper_count": len(papers),
"app_version": APP_VERSION,
"asset_version": ASSET_VERSION,
"ollama_url": OLLAMA_URL,
"ollama_ok": ollama_ok,
"ollama_ready": ollama_ok and not missing_default_models,
"model_api_urls": [MODEL_API_LUNA_URL, MODEL_API_SOL_URL],
"model_api_ok": model_api_ok,
"model_api_ready": model_api_ok and not missing_default_models,
"available_models": available_models,
"missing_default_models": missing_default_models,
"default_models": DEFAULT_MODELS,
@@ -1412,20 +1450,10 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
def handle_models(self) -> None:
try:
request = urllib.request.Request(f"{OLLAMA_URL}/api/tags")
with urllib.request.urlopen(request, timeout=5) as response:
payload = json.loads(response.read().decode("utf-8"))
models = [
{
"name": item.get("name"),
"parameter_size": (item.get("details") or {}).get("parameter_size"),
"context_length": (item.get("details") or {}).get("context_length"),
}
for item in payload.get("models", [])
]
models = available_model_inventory()
write_json(self, {"models": models, "default_models": DEFAULT_MODELS})
except Exception as exc: # noqa: BLE001
write_error(self, HTTPStatus.BAD_GATEWAY, f"ollama unavailable: {exc}")
write_error(self, HTTPStatus.BAD_GATEWAY, f"model API unavailable: {exc}")
def handle_paper_search(self, query: dict[str, list[str]]) -> None:
papers, _ = load_papers()
@@ -1508,11 +1536,11 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
if not job or job.get("status") != "done":
write_error(self, HTTPStatus.NOT_FOUND, "completed search job not found")
return
if not OLLAMA_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "ollama is busy; try again later")
if not MODEL_API_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "model API is busy; try again later")
return
try:
result = call_ollama(model, search_followup_prompt(job, question), "search_followup")
result = call_model(model, search_followup_prompt(job, question), "search_followup")
write_json(
self,
{
@@ -1524,7 +1552,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
},
)
finally:
OLLAMA_LOCK.release()
MODEL_API_LOCK.release()
except Exception as exc: # noqa: BLE001
write_error(self, HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
@@ -1581,8 +1609,8 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
if not paper:
write_error(self, HTTPStatus.NOT_FOUND, "paper not found")
return
if not OLLAMA_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "ollama is busy; try again later")
if not MODEL_API_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "model API is busy; try again later")
return
try:
abstract = None
@@ -1592,7 +1620,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
abstract = fetch_arxiv(arxiv_id)
except Exception:
abstract = None
result = call_ollama(model, paper_chat_prompt(paper, abstract, question, history), "paper_chat")
result = call_model(model, paper_chat_prompt(paper, abstract, question, history), "paper_chat")
write_json(
self,
{
@@ -1604,7 +1632,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
},
)
finally:
OLLAMA_LOCK.release()
MODEL_API_LOCK.release()
except Exception as exc: # noqa: BLE001
write_error(self, HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
@@ -1633,8 +1661,8 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
write_json(self, result)
return
if not OLLAMA_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "ollama is busy; try again later")
if not MODEL_API_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "model API is busy; try again later")
return
try:
abstract = None
@@ -1642,7 +1670,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
if arxiv_id:
abstract = fetch_arxiv(arxiv_id)
context = paper_context(paper, abstract)
result = call_ollama(model, prompt_for(mode, context), mode)
result = call_model(model, prompt_for(mode, context), mode)
result.update(
{
"id": paper_id,
@@ -1654,7 +1682,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
cache_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
write_json(self, result)
finally:
OLLAMA_LOCK.release()
MODEL_API_LOCK.release()
except Exception as exc: # noqa: BLE001
write_error(self, HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
@@ -1685,12 +1713,12 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
write_json(self, result)
return
if not OLLAMA_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "ollama is busy; try again later")
if not MODEL_API_LOCK.acquire(blocking=False):
write_error(self, HTTPStatus.TOO_MANY_REQUESTS, "model API is busy; try again later")
return
try:
context = atlas_context(topic, topic_b)
result = call_ollama(model, atlas_prompt_for(mode, context), mode)
result = call_model(model, atlas_prompt_for(mode, context), mode)
result.update(
{
"scope": cache_key,
@@ -1701,7 +1729,7 @@ class PaperBrowserHandler(SimpleHTTPRequestHandler):
cache_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
write_json(self, result)
finally:
OLLAMA_LOCK.release()
MODEL_API_LOCK.release()
except Exception as exc: # noqa: BLE001
write_error(self, HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
@@ -1719,7 +1747,7 @@ def main() -> int:
load_papers()
server = ThreadingHTTPServer((args.host, args.port), PaperBrowserHandler)
print(f"Paper browser: http://{args.host}:{args.port}")
print(f"Ollama: {OLLAMA_URL}")
print(f"Model APIs: Luna={MODEL_API_LUNA_URL}, Sol/Terra={MODEL_API_SOL_URL}")
try:
server.serve_forever()
except KeyboardInterrupt: