refactor!: 删除 Web/CLI 应用层与内置 LLM 抽取,出库单功能下线(2b)

入口只剩 MCP 服务器;编排由宿主 Agent 负责,本项目只留工具与确定性内核。

删除:
- src/web/(Flask 路由、SSE 轮询、自研聊天前端、模板与静态资源,约 3400 行)
- src/main.py、src/pipeline.py、src/pipeline_core.py(CLI 全流程入口与两套重复管道)
- core/extraction/{extractor,llm_extractor}.py(内置 LLM 抽取、llm_stream.log、file_events.log、并行抽取)
- infra/documents/consumable.py 与 infra/documents/pdf.py:出库单功能下线(Word 模板 4fbb4c3 起已不在仓库,
  该路径此前一直静默失败),PDF 渲染由 core/extraction/render.py 取代
- mcp__arb__doc_consumable 工具;exceptions.ExtractionError;config 的 consumable_storage / get_llm_config /
  load_session_config / SESSION_CONFIG_KEYS(Web 专属,已无引用)
- 针对被删代码的 tests/test_extractor.py、tests/test_llm_extractor.py 与 scripts/ 下的 LLM 调试脚本
- 依赖:llama-index、llama-index-llms-openai-like、python-dotenv(连同其 50+ 传递依赖)
- 提交进来的调试截图 src/images/、根目录 images/ 与 .coverage(并补 .gitignore)

保留与新增:
- 材料入口改为独立进程 src/upload(手机扫码/PC 上传 → cases/<id>/inbox/)
- runner.py 自带 save_invoice_groups,不再依赖 pipeline_core
- scripts/mcp_smoke.py:stdio 自检(拉起自身、列工具、调 case_list、验证 instructions 下发)
- make mcp / mcp-smoke / upload 取代 make run;check 的 mypy 目标从 src/main.py 扩到 src

验证:mcp_smoke 通过(17 个工具、instructions 1449 字节、case_list 正常);申购单实机生成此前已验证。
门禁:ruff check / format --check(43) / mypy src(36 文件) / deptry / pytest 107 passed。
注:仍用 --no-verify 提交并手动跑等价检查(原因见 2f08bbd 提交说明)。
This commit is contained in:
wandering
2026-10-19 22:03:55 +08:00
parent b2d271c5ed
commit 87b4080b1b
59 changed files with 116 additions and 8298 deletions
-69
View File
@@ -1,69 +0,0 @@
"""诊断脚本:检查 stream_chat 返回对象的字段结构"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
# 加载 .env
from dotenv import load_dotenv # noqa: E402
from llama_index.core.llms import ChatMessage # noqa: E402
from src.config import get_llm_config # noqa: E402
from src.core.extraction import _create_llm # noqa: E402
load_dotenv(Path(__file__).parent / ".env")
llm_config = get_llm_config()
print(f"LLM config: model={llm_config['model']}, api_base={llm_config['api_base']}")
llm = _create_llm()
messages = [
ChatMessage(role="system", content="你是一个助手"),
ChatMessage(role="user", content="1+1 等于几?"),
]
print("\n=== 检查 stream_chat 返回对象的字段 ===")
count = 0
try:
for resp in llm.stream_chat(messages, temperature=0.1):
count += 1
if count <= 3:
print(f"\n--- chunk #{count} ---")
print(f" type: {type(resp).__name__}")
print(f" delta: {repr(resp.delta)[:200]}")
if hasattr(resp, "additional_kwargs") and resp.additional_kwargs:
print(f" additional_kwargs keys: {list(resp.additional_kwargs.keys())}")
for k, v in resp.additional_kwargs.items():
val_preview = str(v)[:200]
print(f" additional_kwargs['{k}']: {val_preview}")
if hasattr(resp, "raw"):
raw = resp.raw
if isinstance(raw, dict):
print(f" raw keys: {list(raw.keys())}")
for k, v in raw.items():
val_preview = str(v)[:200]
print(f" raw['{k}']: {val_preview}")
else:
print(f" raw type: {type(raw)}")
if hasattr(resp, "message") and resp.message:
msg = resp.message
print(f" message type: {type(msg).__name__}")
if hasattr(msg, "additional_kwargs") and msg.additional_kwargs:
print(f" message.additional_kwargs keys: {list(msg.additional_kwargs.keys())}")
for k, v in msg.additional_kwargs.items():
val_preview = str(v)[:200]
print(f" message.additional_kwargs['{k}']: {val_preview}")
if hasattr(msg, "reasoning_content"):
rc = msg.reasoning_content
print(f" message.reasoning_content: {repr(rc)[:200]}")
elif count == 4:
print("\n... (更多 chunk 省略)")
if count >= 10:
break
print(f"\n=== 共收到 {count} 个 chunk ===")
except Exception as e:
print(f"\n错误: {e}")
import traceback
traceback.print_exc()
+47
View File
@@ -0,0 +1,47 @@
"""MCP 服务器自检:以 stdio 拉起自身,列出工具并调一次 case_list。
用法:uv run python scripts/mcp_smoke.py
验证要点:工具数量与名称、structured_content 可解析、instructions 已下发。
"""
from __future__ import annotations
import asyncio
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from mcp.client.stdio import StdioServerParameters # noqa: E402
from mcp import Client # noqa: E402
async def main() -> int:
root = str(Path(__file__).resolve().parents[1])
params = StdioServerParameters(command=sys.executable, args=["-m", "src.mcp.server"], cwd=root)
async with Client(params) as client:
listed = await client.list_tools()
tools = list(getattr(listed, "tools", listed))
names = sorted(t.name for t in tools)
print("instructions bytes:", len(getattr(client, "instructions", "") or ""))
print("tools:", len(names))
for name in names:
print(" -", name)
result = await client.call_tool("case_list", {})
payload = getattr(result, "structured_content", None)
if not isinstance(payload, dict) or not payload.get("ok"):
print("case_list 调用异常:", payload)
return 1
print("case_list ok, cases:", len(payload.get("cases", [])))
missing = [
n
for n in ("case_status", "match_payments", "info_skeleton", "doc_requisition", "case_patch_extraction")
if n not in names
]
print("关键工具缺失:", missing or "无")
return 1 if missing else 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
-208
View File
@@ -1,208 +0,0 @@
#!/usr/bin/env python3
"""单独测试事前申请单的信息提取功能
用于调试 LLM 对事前申请单的提取准确率。
用法:
# 测试项目根目录的事前申请单.pdf
python scripts/test_application_extract.py
# 测试指定文件
python scripts/test_application_extract.py --file path/to/file.pdf
# 测试 scripts/data 目录下的事前申请单
python scripts/test_application_extract.py --dir scripts/data
"""
import argparse
import io
import json
import sys
from pathlib import Path
# 加载 .env 环境变量(在导入 src 模块之前)
from dotenv import load_dotenv
ROOT = Path(__file__).resolve().parent.parent
load_dotenv(ROOT / ".env")
# Windows 终端强制 UTF-8
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
sys.path.insert(0, str(ROOT)) # noqa: E402
from src.core.extraction import extract_document # noqa: E402
def test_single_file(file_path: Path) -> None:
"""测试单个文件的提取效果"""
print("=" * 60)
print(f"测试文件: {file_path.name}")
print("=" * 60)
if not file_path.exists():
print(f"文件不存在: {file_path}")
return
try:
# 调用统一提取接口
result = extract_document(file_path)
if not result:
print("[ERROR] 提取返回空结果")
return
# 检查类型判断
inv_type = result.get("invoice_type", "")
print(f"\n[类型判断] invoice_type = {inv_type}")
if inv_type != "application":
print(f"[WARNING] 类型判断错误!期望 'application',实际得到 '{inv_type}'")
else:
print("[OK] 类型判断正确")
# 展示提取结果
print("\n[提取结果]")
print(json.dumps(result, ensure_ascii=False, indent=2))
# 字段完整性检查
print("\n[字段检查]")
expected_fields = {
"invoice_type": "类型标识",
"project_name": "项目名称",
"purpose": "出差事由",
"start_date": "开始日期",
"end_date": "结束日期",
"person_info": "人员信息",
}
for field, desc in expected_fields.items():
value = result.get(field)
if value is None:
print(f" [MISSING] {desc} ({field}) - 字段缺失")
elif value == "" or value == []:
print(f" [EMPTY] {desc} ({field}) - 字段为空")
else:
print(f" [OK] {desc} ({field})")
# 日期格式检查
for date_field in ["start_date", "end_date"]:
date_value = result.get(date_field, "")
if date_value and len(date_value) == 10:
try:
parts = date_value.split("-")
if len(parts) == 3:
int(parts[0]) # year
int(parts[1]) # month
int(parts[2]) # day
print(f" [OK] {date_field} 格式正确 (YYYY-MM-DD)")
except (ValueError, IndexError):
print(f" [ERROR] {date_field} 格式错误: {date_value}")
elif date_value:
print(f" [ERROR] {date_field} 格式错误: {date_value}")
# 人员信息结构检查
person_info = result.get("person_info")
if person_info:
if isinstance(person_info, list):
print(f"\n[人员信息] 共 {len(person_info)} 人")
for i, person in enumerate(person_info, 1):
pid = person.get("person_id", "")
pname = person.get("person_name", "")
print(f" 人员 {i}: {pname} ({pid})")
elif isinstance(person_info, dict):
print("\n[人员信息] 单人格式")
print(f" 姓名: {person_info.get('person_name', '')}")
print(f" 编号: {person_info.get('person_id', '')}")
except Exception as e:
print(f"\n[EXCEPTION] 提取失败: {e}")
import traceback
traceback.print_exc()
def test_cache_comparison(file_path: Path) -> None:
"""对比原始提取和缓存结果"""
cache_dir = file_path.parent / ".invoice_cache"
cache_file = cache_dir / f"{file_path.stem}{file_path.suffix}.json"
if not cache_file.exists():
print(f"\n[INFO] 无缓存文件对比: {cache_file}")
return
print("\n" + "=" * 60)
print("[缓存对比]")
print("=" * 60)
try:
with open(cache_file, encoding="utf-8") as f:
cache_data = json.load(f)
cached_result = cache_data.get("extracted_data", {})
print("\n[缓存数据]")
print(json.dumps(cached_result, ensure_ascii=False, indent=2))
# 对比关键字段
print("\n[字段对比]")
for key in ["invoice_type", "project_name", "purpose", "start_date", "end_date"]:
cached_value = cached_result.get(key, "<缺失>")
print(f" {key}: {cached_value}")
except Exception as e:
print(f"[ERROR] 读取缓存失败: {e}")
def main() -> None:
parser = argparse.ArgumentParser(description="测试事前申请单信息提取")
parser.add_argument(
"--file",
type=str,
default=None,
help="要测试的文件路径 (默认: 扫描 scripts/data 目录)",
)
parser.add_argument(
"--dir",
type=str,
default=str(ROOT / "scripts" / "data"),
help="扫描目录下所有事前申请单文件 (默认: scripts/data)",
)
parser.add_argument(
"--no-cache-compare",
action="store_true",
help="不执行缓存对比",
)
args = parser.parse_args()
if args.dir:
# 目录模式:扫描所有PDF文件
dir_path = Path(args.dir)
if not dir_path.exists():
print(f"目录不存在: {dir_path}")
sys.exit(1)
pdf_files = sorted(dir_path.glob("*.pdf"))
if not pdf_files:
print(f"目录下没有找到PDF文件: {dir_path}")
sys.exit(1)
print(f"发现 {len(pdf_files)} 个PDF文件,开始逐个测试...\n")
for pdf in pdf_files:
if "事前申请" in pdf.name or "申请单" in pdf.name:
test_single_file(pdf)
if not args.no_cache_compare:
test_cache_comparison(pdf)
print()
else:
# 单文件模式
file_path = Path(args.file)
test_single_file(file_path)
if not args.no_cache_compare:
test_cache_comparison(file_path)
print("\n测试完成!")
if __name__ == "__main__":
main()
-81
View File
@@ -1,81 +0,0 @@
#!/usr/bin/env python3
"""测试 PDF 多模态提取完整链路
用法:
python scripts/test_multimodal.py
"""
import io
import sys
from pathlib import Path
# Windows 终端强制 UTF-8
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT)) # noqa: E402
from src.core.extraction import extract_document # noqa: E402
from src.infra.documents.pdf import render_pdf_to_images # noqa: E402
def test_render() -> None:
"""测试 PDF 渲染"""
pdf_path = ROOT / "事前申请单.pdf"
if not pdf_path.exists():
print(f"跳过: {pdf_path.name} 不存在")
return
print("=" * 60)
print("测试 PDF 渲染")
print("=" * 60)
try:
images = render_pdf_to_images(pdf_path, dpi=150)
if images:
print(f"成功渲染 {len(images)} 页")
for i, img_b64 in enumerate(images):
print(f" 第 {i + 1} 页: base64 长度 {len(img_b64)} 字符")
else:
print("渲染返回空列表!")
except ImportError as e:
print(f"导入失败: {e}")
except Exception as e:
print(f"渲染异常: {e}")
def test_multimodal_extract() -> None:
"""测试完整的多模态提取链路"""
pdf_path = ROOT / "事前申请单.pdf"
if not pdf_path.exists():
print(f"跳过: {pdf_path.name} 不存在")
return
print()
print("=" * 60)
print("测试多模态提取")
print("=" * 60)
try:
result = extract_document(pdf_path)
if result:
print("提取成功:")
import json
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
print("提取返回空字典!")
except Exception as e:
print(f"提取异常: {e}")
import traceback
traceback.print_exc()
def main() -> None:
test_render()
test_multimodal_extract()
print()
print("测试完成!")
if __name__ == "__main__":
main()
-54
View File
@@ -1,54 +0,0 @@
#!/usr/bin/env python3
"""直接测试 extract_travel_info 函数
所有数据从 .invoice_cache 缓存中自动加载,无需手动构造样本数据。
用法:
python scripts/test_travel_info.py
"""
import json
import sys
from pathlib import Path
# 项目根目录
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT)) # noqa: E402
from src.core.extraction import extract_travel_info # noqa: E402
def main() -> None:
source_dir = ROOT / "scripts" / "data"
if not source_dir.exists():
print(f"源文件目录不存在: {source_dir}")
sys.exit(1)
print("=" * 60)
print("测试 extract_travel_info(数据来自缓存)")
print("=" * 60)
print(f"源文件目录: {source_dir}")
print()
try:
result = extract_travel_info(
source_dir=source_dir,
)
print()
print("=" * 60)
print("提取结果:")
print("=" * 60)
print(json.dumps(result, ensure_ascii=False, indent=2))
print()
print("测试通过!")
except Exception as e:
print(f"测试失败: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
if __name__ == "__main__":
main()