Add Feishu online document conversion

This commit is contained in:
wuyang6
2026-05-11 17:18:40 +08:00
parent 64ad7aa97a
commit 2984cc44e8
7 changed files with 1045 additions and 8 deletions
+536
View File
@@ -38,6 +38,8 @@ from src.agent_types import (
ModelConfig,
)
from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
from src.data_agent_inputs import DataAgentInputError, load_input_sources
from src.mcp_runtime import MCPRuntime, MCPServerProfile
from src.openai_compat import OpenAICompatClient, OpenAICompatError
from src.session_store import (
DEFAULT_AGENT_SESSION_DIR,
@@ -53,6 +55,27 @@ from src.token_budget import calculate_token_budget
STATIC_DIR = Path(__file__).resolve().parents[2] / 'frontend' / 'legacy-static'
API_TOOL_CONTENT_MAX_CHARS = 20000
FEISHU_NPM_REGISTRY = 'https://pkgs.d.xiaomi.net/artifactory/api/npm/mi-npm/'
FEISHU_MCP_PACKAGE = '@mi/feishu-mcp-pro@latest'
FEISHU_MCP_SERVER_NAME = 'feishu-mcp-pro'
FEISHU_SUPPORTED_DOCUMENT_SUFFIXES = {'.csv', '.docx', '.md', '.txt', '.xlsx'}
FEISHU_UNSUPPORTED_DOCUMENT_SUFFIXES = {'.json', '.jsonl'}
FEISHU_DOC_MARKDOWN_MAX_CHARS = 100_000
FEISHU_ONLINE_DOCS_FILENAME = 'online-docs.json'
@dataclass
class FeishuLoginProcess:
process: subprocess.Popen[str]
started_at: float
output: list[str] = field(default_factory=list)
login_url: str | None = None
user_code: str | None = None
_FEISHU_LOGIN_PROCESSES: dict[str, FeishuLoginProcess] = {}
_FEISHU_LOGIN_LOCK = threading.Lock()
_FEISHU_DOC_MAP_LOCK = threading.Lock()
VALIDATED_CHAT_MODEL_PROVIDERS = {
# 这些 provider 已验证可以在当前 WebUI 中使用。
@@ -892,6 +915,17 @@ class SessionTitleUpdate(BaseModel):
title: str = Field(min_length=1, max_length=80)
class FeishuAccountRequest(BaseModel):
account_id: str | None = None
class FeishuOnlineDocRequest(BaseModel):
path: str = Field(min_length=1)
title: str | None = None
folder_token: str | None = None
account_id: str | None = None
# ---------------------------------------------------------------------------
# App factory
# ---------------------------------------------------------------------------
@@ -1000,6 +1034,90 @@ def create_app(state: AgentState) -> FastAPI:
payload.api_key or config.api_key,
)
# ------------- Feishu integration ---------------------------------------
@app.get('/api/integrations/feishu/status')
async def feishu_status(account_id: str | None = None) -> dict[str, Any]:
return _feishu_status_payload(state, account_id)
@app.post('/api/integrations/feishu/login')
async def feishu_login(payload: FeishuAccountRequest) -> dict[str, Any]:
return _start_feishu_login(state, payload.account_id)
@app.post('/api/integrations/feishu/logout')
async def feishu_logout(payload: FeishuAccountRequest) -> dict[str, Any]:
_stop_feishu_login(payload.account_id)
paths = _feishu_paths(state, payload.account_id)
result = _run_feishu_cli(paths, ['logout'], timeout_seconds=30)
status = _feishu_status_payload(state, payload.account_id)
status['logout_output'] = result['output']
return status
@app.post('/api/files/online-doc')
async def create_feishu_online_doc(payload: FeishuOnlineDocRequest) -> dict[str, Any]:
file_path = _resolve_account_file(state, payload.account_id, payload.path)
suffix = file_path.suffix.lower()
if suffix in FEISHU_UNSUPPORTED_DOCUMENT_SUFFIXES:
raise HTTPException(
status_code=400,
detail='当前先不支持 json/jsonl 转在线文档。',
)
if suffix not in FEISHU_SUPPORTED_DOCUMENT_SUFFIXES:
raise HTTPException(
status_code=400,
detail='当前仅支持 md、txt、csv、docx、xlsx 转在线文档。',
)
status = _feishu_status_payload(state, payload.account_id)
if not status.get('logged_in'):
raise HTTPException(
status_code=409,
detail={
'code': 'feishu_not_logged_in',
'message': '当前账号还没有完成飞书授权,请先授权后再转换。',
'status': status,
},
)
title = _clean_feishu_doc_title(payload.title or file_path.stem)
try:
markdown = _convert_file_to_feishu_markdown(file_path, title=title)
except DataAgentInputError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except OSError as exc:
raise HTTPException(status_code=400, detail=f'读取文件失败: {exc}')
arguments: dict[str, Any] = {'title': title, 'markdown': markdown}
if payload.folder_token and payload.folder_token.strip():
arguments['folder_token'] = payload.folder_token.strip()
paths = _feishu_paths(state, payload.account_id)
runtime = MCPRuntime(servers=(_feishu_server_profile(paths),))
try:
rendered, metadata = runtime.call_tool(
'doc_create',
arguments=arguments,
server_name=FEISHU_MCP_SERVER_NAME,
max_chars=API_TOOL_CONTENT_MAX_CHARS,
timeout_seconds=60.0,
)
except Exception as exc:
raise HTTPException(status_code=502, detail=f'飞书文档创建失败: {exc}')
url = _extract_first_url(rendered)
if url:
_record_feishu_online_doc(
state,
payload.account_id,
file_path=file_path,
title=title,
url=url,
)
return {
'ok': True,
'title': title,
'url': url,
'file_path': str(file_path),
'file_type': suffix.lstrip('.'),
'result': rendered,
'metadata': metadata,
}
# ------------- sessions --------------------------------------------------
@app.get('/api/sessions')
async def list_sessions(account_id: str | None = None) -> list[dict[str, Any]]:
@@ -2150,6 +2268,424 @@ def _session_json_path(directory: Path, session_id: str) -> Path:
return directory / f'{session_id}.json'
def _feishu_paths(state: AgentState, account_id: str | None) -> dict[str, Path]:
base = state.account_paths(account_id)['base'] / 'integrations' / 'feishu'
paths = {
'root': base,
'home': base / 'home',
'config': base / 'config',
'npm_cache': base / 'npm-cache',
}
for path in paths.values():
path.mkdir(parents=True, exist_ok=True)
return paths
def _feishu_command() -> list[str]:
return [
'npx',
'-y',
f'--registry={FEISHU_NPM_REGISTRY}',
FEISHU_MCP_PACKAGE,
]
def _feishu_env(paths: dict[str, Path]) -> dict[str, str]:
return {
'HOME': str(paths['home']),
'XDG_CONFIG_HOME': str(paths['config']),
'npm_config_cache': str(paths['npm_cache']),
'npm_config_update_notifier': 'false',
'NO_UPDATE_NOTIFIER': 'true',
'FEISHU_LOGIN_MODE': 'devicecode',
}
def _run_feishu_cli(
paths: dict[str, Path],
args: list[str],
*,
timeout_seconds: float,
) -> dict[str, Any]:
env = os.environ.copy()
env.update(_feishu_env(paths))
try:
completed = subprocess.run(
[*_feishu_command(), *args],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
cwd=str(paths['root']),
env=env,
timeout=timeout_seconds,
check=False,
)
return {
'returncode': completed.returncode,
'output': completed.stdout.strip(),
}
except FileNotFoundError as exc:
return {
'returncode': 127,
'output': f'缺少命令: {exc.filename}',
}
except subprocess.TimeoutExpired as exc:
output = exc.stdout or ''
return {
'returncode': 124,
'output': str(output).strip() or '飞书状态检查超时',
}
def _feishu_status_payload(state: AgentState, account_id: str | None) -> dict[str, Any]:
paths = _feishu_paths(state, account_id)
_reap_feishu_login(account_id)
result = _run_feishu_cli(paths, ['status'], timeout_seconds=30)
output = str(result.get('output') or '')
lowered = output.lower()
logged_in = result.get('returncode') == 0 and not any(
marker in lowered
for marker in ('not logged in', '未登录', 'no credentials')
)
pending = _feishu_login_snapshot(account_id)
payload: dict[str, Any] = {
'logged_in': logged_in,
'status': 'logged_in' if logged_in else 'not_logged_in',
'output': output,
}
if pending:
payload['login'] = pending
if not logged_in:
payload['status'] = 'login_pending'
if not logged_in and result.get('returncode') not in (0, 1):
payload['status'] = 'error'
payload['error'] = output or '无法获取飞书登录状态'
return payload
def _start_feishu_login(state: AgentState, account_id: str | None) -> dict[str, Any]:
status = _feishu_status_payload(state, account_id)
if status.get('logged_in'):
return status
key = _feishu_login_key(account_id)
with _FEISHU_LOGIN_LOCK:
existing = _FEISHU_LOGIN_PROCESSES.get(key)
if existing is not None and existing.process.poll() is None:
return {
'logged_in': False,
'status': 'login_pending',
'login': _feishu_login_snapshot_unlocked(existing),
}
_FEISHU_LOGIN_PROCESSES.pop(key, None)
paths = _feishu_paths(state, account_id)
env = os.environ.copy()
env.update(_feishu_env(paths))
try:
process = subprocess.Popen(
[*_feishu_command(), 'login', '--device'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
cwd=str(paths['root']),
env=env,
)
except FileNotFoundError as exc:
raise HTTPException(status_code=500, detail=f'缺少命令: {exc.filename}')
entry = FeishuLoginProcess(process=process, started_at=time.time())
with _FEISHU_LOGIN_LOCK:
_FEISHU_LOGIN_PROCESSES[key] = entry
reader = threading.Thread(
target=_read_feishu_login_output,
args=(key, entry),
name=f'feishu-login-{key}',
daemon=True,
)
reader.start()
deadline = time.time() + 5.0
while time.time() < deadline:
snapshot = _feishu_login_snapshot(account_id)
if snapshot and snapshot.get('login_url'):
return {
'logged_in': False,
'status': 'login_pending',
'login': snapshot,
}
if process.poll() is not None:
break
time.sleep(0.1)
snapshot = _feishu_login_snapshot(account_id)
return {
'logged_in': False,
'status': 'login_pending' if process.poll() is None else 'login_failed',
'login': snapshot,
}
def _stop_feishu_login(account_id: str | None) -> None:
key = _feishu_login_key(account_id)
with _FEISHU_LOGIN_LOCK:
entry = _FEISHU_LOGIN_PROCESSES.pop(key, None)
if entry is None:
return
process = entry.process
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=2.0)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=2.0)
def _reap_feishu_login(account_id: str | None) -> None:
key = _feishu_login_key(account_id)
with _FEISHU_LOGIN_LOCK:
entry = _FEISHU_LOGIN_PROCESSES.get(key)
if entry is not None and entry.process.poll() is not None:
_FEISHU_LOGIN_PROCESSES.pop(key, None)
def _read_feishu_login_output(key: str, entry: FeishuLoginProcess) -> None:
stream = entry.process.stdout
if stream is None:
return
for line in stream:
with _FEISHU_LOGIN_LOCK:
current = _FEISHU_LOGIN_PROCESSES.get(key)
if current is not entry:
return
entry.output.append(line.rstrip())
text = '\n'.join(entry.output)
login_url, user_code = _parse_feishu_login_details(text)
if login_url:
entry.login_url = login_url
if user_code:
entry.user_code = user_code
def _feishu_login_snapshot(account_id: str | None) -> dict[str, Any] | None:
key = _feishu_login_key(account_id)
with _FEISHU_LOGIN_LOCK:
entry = _FEISHU_LOGIN_PROCESSES.get(key)
if entry is None:
return None
return _feishu_login_snapshot_unlocked(entry)
def _feishu_login_snapshot_unlocked(entry: FeishuLoginProcess) -> dict[str, Any]:
output = '\n'.join(entry.output[-20:])
if not entry.login_url or not entry.user_code:
login_url, user_code = _parse_feishu_login_details(output)
entry.login_url = entry.login_url or login_url
entry.user_code = entry.user_code or user_code
return {
'running': entry.process.poll() is None,
'started_at': entry.started_at,
'elapsed_seconds': max(0, int(time.time() - entry.started_at)),
'login_url': entry.login_url,
'user_code': entry.user_code,
'output': output,
}
def _parse_feishu_login_details(text: str) -> tuple[str | None, str | None]:
urls = re.findall(r'https?://[^\s]+', text)
login_url = next((url.rstrip('.,;') for url in urls if 'feishu' in url.lower()), None)
user_code = None
match = re.search(r'user_code=([A-Za-z0-9_-]+)', login_url or '')
if match:
user_code = match.group(1)
if not user_code:
match = re.search(r'\b([A-Z0-9]{4}-[A-Z0-9]{4})\b', text)
if match:
user_code = match.group(1)
return login_url, user_code
def _feishu_login_key(account_id: str | None) -> str:
return _safe_account_id(account_id)
def _feishu_server_profile(paths: dict[str, Path]) -> MCPServerProfile:
return MCPServerProfile(
name=FEISHU_MCP_SERVER_NAME,
source_manifest='builtin:feishu-mcp-pro',
transport='stdio',
command='npx',
args=(
'-y',
f'--registry={FEISHU_NPM_REGISTRY}',
FEISHU_MCP_PACKAGE,
),
env=_feishu_env(paths),
cwd=str(paths['root']),
description='账号隔离的飞书 MCP Pro。',
)
def _record_feishu_online_doc(
state: AgentState,
account_id: str | None,
*,
file_path: Path,
title: str,
url: str,
) -> None:
paths = _feishu_paths(state, account_id)
map_path = paths['root'] / FEISHU_ONLINE_DOCS_FILENAME
now = int(time.time())
clean_url = _clean_url(url)
with _FEISHU_DOC_MAP_LOCK:
try:
payload = json.loads(map_path.read_text(encoding='utf-8'))
except (OSError, json.JSONDecodeError):
payload = {}
files = payload.get('files') if isinstance(payload, dict) else None
if not isinstance(files, dict):
files = {}
previous = files.get(str(file_path))
created_at = (
previous.get('created_at')
if isinstance(previous, dict) and isinstance(previous.get('created_at'), int)
else now
)
files[str(file_path)] = {
'url': clean_url,
'title': title,
'file_path': str(file_path),
'created_at': created_at,
'updated_at': now,
}
map_path.write_text(
json.dumps({'files': files}, ensure_ascii=False, indent=2),
encoding='utf-8',
)
def _resolve_account_file(state: AgentState, account_id: str | None, raw_path: str) -> Path:
path = Path(raw_path).expanduser().resolve()
account_base = state.account_paths(account_id)['base'].resolve()
try:
path.relative_to(account_base)
except ValueError:
raise HTTPException(status_code=403, detail='文件不在当前账号目录内')
if not path.exists():
raise HTTPException(status_code=404, detail='文件不存在')
if not path.is_file():
raise HTTPException(status_code=400, detail='目标不是文件')
return path
def _clean_feishu_doc_title(value: str) -> str:
title = re.sub(r'[\r\n\t/\\]+', ' ', value).strip()
return title[:80].strip() or 'Claw 生成文档'
def _convert_file_to_feishu_markdown(file_path: Path, *, title: str) -> str:
suffix = file_path.suffix.lower()
if suffix == '.md':
body = file_path.read_text(encoding='utf-8', errors='replace')
return _limit_feishu_markdown(f'# {title}\n\n> 来源文件:{file_path.name}\n\n{body}')
if suffix == '.txt':
body = file_path.read_text(encoding='utf-8', errors='replace')
return _limit_feishu_markdown(f'# {title}\n\n> 来源文件:{file_path.name}\n\n{body}')
loaded = load_input_sources(
file_path.parent,
[file_path.name],
max_files=1,
max_paragraphs_per_file=200,
max_tables_per_file=30,
max_rows_per_table=120,
max_cell_chars=500,
)
sources = loaded.get('sources')
if not isinstance(sources, list) or not sources:
raise DataAgentInputError('没有解析到可转成在线文档的内容')
source = sources[0]
lines = [
f'# {title}',
'',
f'> 来源文件:{file_path.name}',
f'> 文件类型:{suffix.lstrip(".")}',
]
warnings = source.get('warnings')
if isinstance(warnings, list) and warnings:
lines.append(f'> 解析提示:{"".join(str(item) for item in warnings)}')
for paragraph in source.get('paragraphs', []):
if not isinstance(paragraph, dict):
continue
text = str(paragraph.get('text') or '').strip()
if text:
lines.extend(['', text])
for table in source.get('tables', []):
if not isinstance(table, dict):
continue
rows = table.get('rows')
if not isinstance(rows, list) or not rows:
continue
title_text = str(table.get('title') or '表格').strip() or '表格'
lines.extend(['', f'## {title_text}', ''])
lines.append(_render_markdown_table(rows))
row_count = table.get('row_count')
if isinstance(row_count, int) and row_count > len(rows):
lines.append(f'\n> 仅展示前 {len(rows)} 行,原表约 {row_count} 行。')
markdown = '\n'.join(lines).strip()
if not markdown:
raise DataAgentInputError('没有解析到可转成在线文档的内容')
return _limit_feishu_markdown(markdown)
def _render_markdown_table(raw_rows: list[Any]) -> str:
rows = [
[str(cell) for cell in row]
for row in raw_rows
if isinstance(row, list)
]
if not rows:
return ''
width = max(len(row) for row in rows)
normalized = [row + [''] * (width - len(row)) for row in rows]
header = normalized[0] if any(cell.strip() for cell in normalized[0]) else [
f'{index + 1}' for index in range(width)
]
body = normalized[1:] if header is normalized[0] else normalized
lines = [
'| ' + ' | '.join(_escape_markdown_table_cell(cell) for cell in header) + ' |',
'| ' + ' | '.join('---' for _ in range(width)) + ' |',
]
for row in body:
lines.append('| ' + ' | '.join(_escape_markdown_table_cell(cell) for cell in row) + ' |')
return '\n'.join(lines)
def _escape_markdown_table_cell(value: str) -> str:
return value.replace('\\', '\\\\').replace('|', '\\|').replace('\n', ' ').strip()
def _limit_feishu_markdown(markdown: str) -> str:
if len(markdown) <= FEISHU_DOC_MARKDOWN_MAX_CHARS:
return markdown
suffix = '\n\n> 内容较长,已截断后写入在线文档。'
return markdown[: FEISHU_DOC_MARKDOWN_MAX_CHARS - len(suffix)].rstrip() + suffix
def _extract_first_url(text: str) -> str | None:
match = re.search(r'https?://[^\s<>\]\)\"\'“”‘’]+', text)
return _clean_url(match.group(0)) if match else None
def _clean_url(value: str) -> str:
return value.strip().rstrip('.,;,。;、"\'“”‘’')
def _safe_account_id(account_id: str | None) -> str:
normalized = re.sub(r'[^a-zA-Z0-9._-]+', '_', (account_id or '').strip())
return normalized[:80] or 'default'