Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix: complete doclib cleanup across lifespan failures
myhloli73424fa72d23
diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.ymlindex 775c0829..79dbef47 100644--- a/.github/workflows/cli.yml+++ b/.github/workflows/cli.yml@@ -121,6 +121,7 @@ jobs:tests/unittest/test_doclib_ingest_service.py \tests/unittest/test_doclib_parse_services.py \tests/unittest/test_doclib_instance_dependencies.py \+ tests/unittest/test_doclib_lifespan.py \tests/unittest/test_doclib_architecture.py- name: Retain test reportsif: ${{ always() }}diff --git a/docs/next/architecture.md b/docs/next/architecture.mdindex 9e1ca930..c3ef6c9e 100644--- a/docs/next/architecture.md+++ b/docs/next/architecture.md@@ -112,6 +112,9 @@ Routes| `TelemetryFlushLoop` | 1 | 按固定间隔批量上报 telemetry |所有后台组件都是 asyncio task,随 server 启停。+启动阶段也处于生命周期清理保护内:中途失败会清理已经创建的组件和托管进程。+退出时先取消并等待后台编排循环,防止在途健康探测于收尾阶段重启进程,再停止工作池和各组件,最后清理剩余托管资源、数据库与发现信息。+单个停止错误不会跳过后续资源;退出取消会等待既有清理完成,启动或请求的原始异常优先保留。内容读取由 `DoclibServer` 委托给 `ContentReadService`。读取服务接收数据库、产物目录与公式配置快照,不持有 HTTP 请求、应用全局状态或解析调度器;接口层保留参数绑定、响应和遥测。diff --git a/docs/next/implementation-plan.md b/docs/next/implementation-plan.mdindex c9478990..3ed2ba28 100644--- a/docs/next/implementation-plan.md+++ b/docs/next/implementation-plan.md@@ -5,6 +5,12 @@范围: 将 `docs/next/` 中的设计文档拆成可落地、可验证、可并行领取的工程任务非目标: 替代专题设计文档;一次性重写现有实现;要求所有任务在同一个 PR 完成+结构迁移说明:本文任务卡片保留早期实现位置,部分 `services/parse_svc.py` 引用已失效。+当前文件刷新与入库归属 `services/ingest_svc.py`,解析请求/队列与缓存失效归属 `services/parse_queue_svc.py`,+执行/产物与索引更新归属 `services/parse_execution_svc.py`,缓存读取和导出归属 `services/content_read_svc.py`。+共享产物路径及覆盖计算位于 `core/parse_artifacts.py`;短标识登记由 `ingest_svc.ensure_doc_record()` 维护。+领取任务前请对照 [当前架构](architecture.md) 和 [代码地图](code-map.md) 追踪实际调用,不创建旧模块转发或重建旧组合服务。+## 1. 使用方式本文件是给编程 Agent 使用的任务手册。Agent 领取任务时,应只做一个任务卡片内的内容,并以该任务的“完成边界”和“验证方式”为准。diff --git a/mineru/doclib/app.py b/mineru/doclib/app.pyindex 1cc17adf..917e2974 100644--- a/mineru/doclib/app.py+++ b/mineru/doclib/app.py@@ -21,20 +21,21 @@ from loguru import logger as loguru_loggerfrom ..config import Config, ConfigSource, LogConfig, LogLevel, _mineru_home, get_config, get_config_sourcefrom ..errors import MineruError, error_response, http_status_for+from ..utils.async_utils import drain_futurefrom ..utils.logger import configure_global_log_levelfrom ..utils.stdio import configure_standard_streamsfrom .endpoint import EndpointTransport, read_endpoint_file, remove_endpoint_file, uds_available, write_endpoint_filefrom .instance_lock import DoclibLockUnavailable, build_doclib_home_owned_message, doclib_home_lock-from .server import DoclibServerfrom .parse_server_state import ParseServerHealth+from .server import DoclibServerfrom .types import PARSE_STATUS_FAILED, PARSE_STATUS_PARSING, SCAN_STATUS_FAILED, SCAN_STATUS_RUNNINGif TYPE_CHECKING:from loguru import Message as LoguruMessagefrom .services.ingest_svc import IngestService- from .services.parse_queue_svc import ParseQueueServicefrom .services.parse_execution_svc import ParseExecutionService+ from .services.parse_queue_svc import ParseQueueServiceelse:LoguruMessage = Any@@ -70,8 +71,8 @@ def create_app(cfg: Config | None = None) -> FastAPI:from .services.cleanup_svc import CleanupServicefrom .services.config_svc import ConfigServicefrom .services.ingest_svc import IngestService- from .services.parse_queue_svc import ParseQueueServicefrom .services.parse_execution_svc import ParseExecutionService+ from .services.parse_queue_svc import ParseQueueServicefrom .services.scan_svc import ScanServicefrom .services.search_svc import SearchServicefrom .telemetry import TelemetryService, TelemetryStore@@ -221,47 +222,91 @@ def create_app(cfg: Config | None = None) -> FastAPI:state.tcp_host = cfg.doclib.tcp.hostasync def shutdown() -> None:+ """先结束后台编排,再清理全部已拥有资源,单项失败不截断后续清理。"""from .background.parse_server_health import stop_managed_parse_server+ errors: list[Exception] = []+ # 探测循环可能正在网络等待,必须先取消并等待它退出,禁止收尾时再次重启进程。+ try:+ await _cancel_background_tasks(state)+ except Exception as exc:+ errors.append(exc)+ logger.exception("Failed to finish Doclib background tasks")++ for name, component in (+ ("watch", state.watch),+ ("scan_workers", state.scan_workers),+ ("ingest_workers", state.ingest_workers),+ ("parse_workers", state.parse_workers),+ ("device_monitor", state.device_monitor),+ ("compaction", state.compaction),+ ("telemetry_flush", state.telemetry_flush),+ ("health_check", state.health_check),+ ):+ if component is None:+ continue+ try:+ await component.stop()+ except Exception as exc:+ errors.append(exc)+ logger.exception("Failed to stop Doclib component %s", name)++ # 启动可能在创建健康检查器之前失败;停止失败也可能留下仍归属应用的进程。health = state.health- if health.managed_proc:- stop_managed_parse_server(- health.managed_proc,- control=health.managed_control,- timeout_sec=cfg.doclib.parse_server_stop_timeout_sec,- reason="doclib shutdown",- startup_in_progress=health.local_starting,- )- health.managed_proc = None- health.managed_control = None-- for comp in [- "watch",- "scan_workers",- "ingest_workers",- "parse_workers",- "device_monitor",- "compaction",- "health_check",- "telemetry_flush",- ]:- c = getattr(state, comp, None)- if c and hasattr(c, "stop"):- await c.stop()-- await _cancel_background_tasks(state)-- if state.db:- await state.db.close()+ if health.managed_proc is not None or health.managed_control is not None:+ try:+ stop_managed_parse_server(+ health.managed_proc,+ control=health.managed_control,+ timeout_sec=cfg.doclib.parse_server_stop_timeout_sec,+ reason="doclib shutdown",+ startup_in_progress=health.local_starting,+ )+ health.managed_proc = None+ health.managed_control = None+ except Exception as exc:+ errors.append(exc)+ logger.exception("Failed to stop Doclib managed parse-server")++ if state.db is not None:+ try:+ await state.db.close()+ except Exception as exc:+ errors.append(exc)+ logger.exception("Failed to close Doclib database")+ try:+ _run_discovery_cleanup(state)+ except Exception as exc:+ errors.append(exc)+ logger.exception("Failed to clean Doclib discovery information")+ if errors:+ raise errors[0]@asynccontextmanagerasync def lifespan(_app: FastAPI) -> AsyncIterator[None]:- await startup()+ """启动失败也清理已拥有资源,退出取消先排空清理且保留最初异常。"""+ failed = Falsetry:+ await startup()yield+ except BaseException:+ failed = True+ raisefinally:try:- await shutdown()+ cleanup = asyncio.create_task(shutdown(), name="doclib:shutdown")+ try:+ await asyncio.shield(cleanup)+ except asyncio.CancelledError:+ try:+ await drain_future(cleanup)+ except Exception:+ logger.exception("Doclib cleanup failed while teardown was canceled")+ raise+ except BaseException:+ if not failed:+ raise+ logger.exception("Doclib cleanup failed; preserving the original lifespan error")finally:_run_discovery_cleanup(state)diff --git a/tests/unittest/test_doclib_lifespan.py b/tests/unittest/test_doclib_lifespan.pynew file mode 100644index 00000000..34aa5759--- /dev/null+++ b/tests/unittest/test_doclib_lifespan.py@@ -0,0 +1,243 @@+"""验证启动失败、退出重启竞态、组件错误和取消时的应用资源清理。"""++from __future__ import annotations++import asyncio+from pathlib import Path+from types import SimpleNamespace+from typing import Any++import pytest++from mineru.config import Config, LogConfig+from mineru.doclib import app as doclib_app+from mineru.doclib.background import parse_server_health as probes+from mineru.doclib.background import watch as watch_module+from mineru.doclib.core.db import DatabaseManager+from mineru.doclib.parse_server_state import ProbeResult+++def _configuration(root: Path) -> Config:+ """将本用例的数据库、日志和产物全部放入临时目录。"""+ cfg = Config()+ cfg.doclib.data_dir = str(root / "data")+ cfg.doclib.sqlite.path = str(root / "library.sqlite")+ cfg.doclib.log = LogConfig(dir=str(root / "logs"))+ return cfg+++async def _enable_managed(cfg: Config) -> None:+ """使用真实配置表启用托管模式,不启动模型或真实子进程。"""+ db = DatabaseManager(cfg.doclib.sqlite.path)+ await db.initialize()+ await db.execute("INSERT OR REPLACE INTO config(key,value) VALUES ('parse_server.local.mode','managed')")++[email protected]("cleanup_fails", [False, True])+def test_failed_startup_releases_managed_process_and_discovery(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cleanup_fails: bool+) -> None:+ """托管进程创建后发生初始化错误,仍清理已拥有资源并保留最初错误。"""+ events: list[str] = []+ process = SimpleNamespace(pid=101)++ def start(**kwargs: object) -> tuple[Any, str, None]:+ """记录托管进程已经属于当前应用。"""+ events.append("start")+ return process, "http://127.0.0.1:16580", None++ def stop(proc: object, **kwargs: object) -> None:+ """仅记录真实拥有的非空进程。"""+ if proc is not None:+ assert proc is process+ events.append("stop")++ def fail_watch(*args: object, **kwargs: object) -> None:+ """在进程已创建但后台组件尚未完整组装时抛出主错误。"""+ raise RuntimeError("watch setup failed")++ async def close_db(self: DatabaseManager) -> None:+ """验证失败启动也进入数据库清理,附加错误不能覆盖主错误。"""+ events.append("db.close")+ if cleanup_fails:+ raise ValueError("database close failed")++ monkeypatch.setattr(probes, "start_managed_parse_server", start)+ monkeypatch.setattr(probes, "stop_managed_parse_server", stop)+ monkeypatch.setattr(watch_module, "WatchLoop", fail_watch)+ monkeypatch.setattr(DatabaseManager, "close", close_db)++ async def scenario() -> None:+ """在同一事件循环内检查清理,不依赖测试框架退出时兜底取消。"""+ cfg = _configuration(tmp_path)+ await _enable_managed(cfg)+ app = doclib_app.create_app(cfg)+ state = app.state.doclib_state+ state.discovery_cleanup = lambda: events.append("discovery")+ with pytest.raises(RuntimeError, match="watch setup failed"):+ async with app.router.lifespan_context(app):+ raise AssertionError("startup unexpectedly succeeded")+ assert events == ["start", "stop", "db.close", "discovery"]+ assert state.health.managed_proc is None and not state.background_tasks++ asyncio.run(scenario())+++def test_shutdown_does_not_restart_managed_process_after_inflight_probe(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """停止应用时先结束在途探测,避免组件收尾期间再次拉起托管进程。"""++ async def scenario() -> None:+ """用事件控制真实探测循环,确定性触发旧关闭顺序的重启窗口。"""+ cfg = _configuration(tmp_path)+ await _enable_managed(cfg)+ probe_entered, release_probe, restarted = asyncio.Event(), asyncio.Event(), asyncio.Event()+ starts: list[object] = []+ stopped: list[object] = []+ tasks: list[asyncio.Task[Any]] = []+ create = doclib_app._create_background_task++ def start(**kwargs: object) -> tuple[Any, str, None]:+ """第二次创建代表退出过程中发生了不应有的重启。"""+ proc = SimpleNamespace(pid=len(starts) + 1)+ starts.append(proc)+ if len(starts) > 1:+ restarted.set()+ return proc, "http://127.0.0.1:16580", None++ def stop(proc: object, **kwargs: object) -> None:+ """收集最终被应用清理的进程。"""+ if proc is not None:+ stopped.append(proc)++ def schedule(state: Any, name: str, runner: Any) -> Any:+ """只运行本用例需要的真实健康循环,其余组件保留真实停止接口。"""+ if name == "health_check":+ task = create(state, name, runner)+ tasks.append(task)+ return task+ return None++ async def probe(self: Any, url: str, *, api_key: str | None = None) -> ProbeResult:+ """本地探测挂起到应用收尾,远端探测立即返回。"""+ if url == "http://127.0.0.1:16580":+ probe_entered.set()+ await release_probe.wait()+ return ProbeResult(error_code="parse_server_unavailable")+ return ProbeResult(healthy=True, tiers=["standard"])++ monkeypatch.setattr(probes, "start_managed_parse_server", start)+ monkeypatch.setattr(probes, "stop_managed_parse_server", stop)+ monkeypatch.setattr(probes.ParseServerHealthCheck, "_probe", probe)+ monkeypatch.setattr(doclib_app, "_create_background_task", schedule)+ app = doclib_app.create_app(cfg)+ manager = app.router.lifespan_context(app)+ await manager.__aenter__()+ state = app.state.doclib_state+ await asyncio.wait_for(probe_entered.wait(), timeout=2)++ async def stop_watch() -> None:+ """模拟其它组件让出执行权,暴露在途探测返回后的重启机会。"""+ release_probe.set()+ if not tasks[0].done():+ await asyncio.wait_for(restarted.wait(), timeout=2)++ monkeypatch.setattr(state.watch, "stop", stop_watch)+ try:+ await manager.__aexit__(None, None, None)+ assert len(starts) == 1 and stopped == starts+ assert all(task.done() for task in tasks)+ finally:+ release_probe.set()+ for task in tasks:+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)++ asyncio.run(scenario())+++def test_component_stop_failure_does_not_skip_remaining_resources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """一个组件停止失败仍应清理其它工作池、数据库及发现信息,然后报告最初错误。"""+ events: list[str] = []++ def skip_background(*args: object, **kwargs: object) -> None:+ """固定本用例的组件状态,保留应用真实生命周期组装。"""++ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)++ async def scenario() -> None:+ """对实际组件实例注入停止错误,观察后续资源仍被处理。"""+ app = doclib_app.create_app(_configuration(tmp_path))+ manager = app.router.lifespan_context(app)+ await manager.__aenter__()+ state = app.state.doclib_state++ async def bad_stop() -> None:+ """模拟一个组件的意外停止错误。"""+ events.append("watch")+ raise ValueError("watch stop failed")++ async def pool_stop() -> None:+ """确认解析工作池不因前一组件错误而被跳过。"""+ events.append("parse_workers")++ async def close() -> None:+ """确认数据库仍进入最终清理。"""+ events.append("db.close")++ monkeypatch.setattr(state.watch, "stop", bad_stop)+ monkeypatch.setattr(state.parse_workers, "stop", pool_stop)+ monkeypatch.setattr(state.db, "close", close)+ state.discovery_cleanup = lambda: events.append("discovery")+ with pytest.raises(ValueError, match="watch stop failed"):+ await manager.__aexit__(None, None, None)+ assert events == ["watch", "parse_workers", "db.close", "discovery"]++ asyncio.run(scenario())+++def test_teardown_cancellation_waits_for_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """退出请求被取消时先排空已有清理任务,再向调用方传播取消。"""+ events: list[str] = []++ def skip_background(*args: object, **kwargs: object) -> None:+ """仅隔离周期后台任务,不替代实际关闭编排。"""++ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)++ async def scenario() -> None:+ """挂起一个关闭阶段并取消外部等待者,检查数据库最终仍关闭。"""+ app = doclib_app.create_app(_configuration(tmp_path))+ manager = app.router.lifespan_context(app)+ await manager.__aenter__()+ state = app.state.doclib_state+ entered, release = asyncio.Event(), asyncio.Event()++ async def stop() -> None:+ """受控关闭阶段必须完成,不能被外部取消截断。"""+ entered.set()+ await release.wait()+ events.append("watch.done")++ async def close() -> None:+ """记录清理任务最终关闭数据库。"""+ events.append("db.close")++ monkeypatch.setattr(state.watch, "stop", stop)+ monkeypatch.setattr(state.db, "close", close)+ shutdown = asyncio.create_task(manager.__aexit__(None, None, None))+ await asyncio.wait_for(entered.wait(), timeout=2)+ shutdown.cancel()+ await asyncio.sleep(0)+ shutdown.cancel()+ await asyncio.sleep(0)+ try:+ assert not shutdown.done()+ finally:+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await shutdown+ assert events == ["watch.done", "db.close"]++ asyncio.run(scenario())