Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix: complete doclib cleanup across lifespan failures

myhloli73424fa72d23
diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml
index 775c0829..79dbef47 100644
--- a/.github/workflows/cli.yml
+++ b/.github/workflows/cli.yml
@@ -121,6 +121,7 @@ jobs:
tests/unittest/test_doclib_ingest_service.py \
tests/unittest/test_doclib_parse_services.py \
tests/unittest/test_doclib_instance_dependencies.py \
+ tests/unittest/test_doclib_lifespan.py \
tests/unittest/test_doclib_architecture.py
- name: Retain test reports
if: ${{ always() }}
diff --git a/docs/next/architecture.md b/docs/next/architecture.md
index 9e1ca930..c3ef6c9e 100644
--- a/docs/next/architecture.md
+++ b/docs/next/architecture.md
@@ -112,6 +112,9 @@ Routes
| `TelemetryFlushLoop` | 1 | 按固定间隔批量上报 telemetry |
所有后台组件都是 asyncio task,随 server 启停。
+启动阶段也处于生命周期清理保护内:中途失败会清理已经创建的组件和托管进程。
+退出时先取消并等待后台编排循环,防止在途健康探测于收尾阶段重启进程,再停止工作池和各组件,最后清理剩余托管资源、数据库与发现信息。
+单个停止错误不会跳过后续资源;退出取消会等待既有清理完成,启动或请求的原始异常优先保留。
内容读取由 `DoclibServer` 委托给 `ContentReadService`。读取服务接收数据库、产物目录与公式配置快照,
不持有 HTTP 请求、应用全局状态或解析调度器;接口层保留参数绑定、响应和遥测。
diff --git a/docs/next/implementation-plan.md b/docs/next/implementation-plan.md
index c9478990..3ed2ba28 100644
--- a/docs/next/implementation-plan.md
+++ b/docs/next/implementation-plan.md
@@ -5,6 +5,12 @@
范围: 将 `docs/next/` 中的设计文档拆成可落地、可验证、可并行领取的工程任务
非目标: 替代专题设计文档;一次性重写现有实现;要求所有任务在同一个 PR 完成
+结构迁移说明:本文任务卡片保留早期实现位置,部分 `services/parse_svc.py` 引用已失效。
+当前文件刷新与入库归属 `services/ingest_svc.py`,解析请求/队列与缓存失效归属 `services/parse_queue_svc.py`,
+执行/产物与索引更新归属 `services/parse_execution_svc.py`,缓存读取和导出归属 `services/content_read_svc.py`。
+共享产物路径及覆盖计算位于 `core/parse_artifacts.py`;短标识登记由 `ingest_svc.ensure_doc_record()` 维护。
+领取任务前请对照 [当前架构](architecture.md) 和 [代码地图](code-map.md) 追踪实际调用,不创建旧模块转发或重建旧组合服务。
+
## 1. 使用方式
本文件是给编程 Agent 使用的任务手册。Agent 领取任务时,应只做一个任务卡片内的内容,并以该任务的“完成边界”和“验证方式”为准。
diff --git a/mineru/doclib/app.py b/mineru/doclib/app.py
index 1cc17adf..917e2974 100644
--- a/mineru/doclib/app.py
+++ b/mineru/doclib/app.py
@@ -21,20 +21,21 @@ from loguru import logger as loguru_logger
from ..config import Config, ConfigSource, LogConfig, LogLevel, _mineru_home, get_config, get_config_source
from ..errors import MineruError, error_response, http_status_for
+from ..utils.async_utils import drain_future
from ..utils.logger import configure_global_log_level
from ..utils.stdio import configure_standard_streams
from .endpoint import EndpointTransport, read_endpoint_file, remove_endpoint_file, uds_available, write_endpoint_file
from .instance_lock import DoclibLockUnavailable, build_doclib_home_owned_message, doclib_home_lock
-from .server import DoclibServer
from .parse_server_state import ParseServerHealth
+from .server import DoclibServer
from .types import PARSE_STATUS_FAILED, PARSE_STATUS_PARSING, SCAN_STATUS_FAILED, SCAN_STATUS_RUNNING
if TYPE_CHECKING:
from loguru import Message as LoguruMessage
from .services.ingest_svc import IngestService
- from .services.parse_queue_svc import ParseQueueService
from .services.parse_execution_svc import ParseExecutionService
+ from .services.parse_queue_svc import ParseQueueService
else:
LoguruMessage = Any
@@ -70,8 +71,8 @@ def create_app(cfg: Config | None = None) -> FastAPI:
from .services.cleanup_svc import CleanupService
from .services.config_svc import ConfigService
from .services.ingest_svc import IngestService
- from .services.parse_queue_svc import ParseQueueService
from .services.parse_execution_svc import ParseExecutionService
+ from .services.parse_queue_svc import ParseQueueService
from .services.scan_svc import ScanService
from .services.search_svc import SearchService
from .telemetry import TelemetryService, TelemetryStore
@@ -221,47 +222,91 @@ def create_app(cfg: Config | None = None) -> FastAPI:
state.tcp_host = cfg.doclib.tcp.host
async def shutdown() -> None:
+ """先结束后台编排,再清理全部已拥有资源,单项失败不截断后续清理。"""
from .background.parse_server_health import stop_managed_parse_server
+ errors: list[Exception] = []
+ # 探测循环可能正在网络等待,必须先取消并等待它退出,禁止收尾时再次重启进程。
+ try:
+ await _cancel_background_tasks(state)
+ except Exception as exc:
+ errors.append(exc)
+ logger.exception("Failed to finish Doclib background tasks")
+
+ for name, component in (
+ ("watch", state.watch),
+ ("scan_workers", state.scan_workers),
+ ("ingest_workers", state.ingest_workers),
+ ("parse_workers", state.parse_workers),
+ ("device_monitor", state.device_monitor),
+ ("compaction", state.compaction),
+ ("telemetry_flush", state.telemetry_flush),
+ ("health_check", state.health_check),
+ ):
+ if component is None:
+ continue
+ try:
+ await component.stop()
+ except Exception as exc:
+ errors.append(exc)
+ logger.exception("Failed to stop Doclib component %s", name)
+
+ # 启动可能在创建健康检查器之前失败;停止失败也可能留下仍归属应用的进程。
health = state.health
- if health.managed_proc:
- stop_managed_parse_server(
- health.managed_proc,
- control=health.managed_control,
- timeout_sec=cfg.doclib.parse_server_stop_timeout_sec,
- reason="doclib shutdown",
- startup_in_progress=health.local_starting,
- )
- health.managed_proc = None
- health.managed_control = None
-
- for comp in [
- "watch",
- "scan_workers",
- "ingest_workers",
- "parse_workers",
- "device_monitor",
- "compaction",
- "health_check",
- "telemetry_flush",
- ]:
- c = getattr(state, comp, None)
- if c and hasattr(c, "stop"):
- await c.stop()
-
- await _cancel_background_tasks(state)
-
- if state.db:
- await state.db.close()
+ if health.managed_proc is not None or health.managed_control is not None:
+ try:
+ stop_managed_parse_server(
+ health.managed_proc,
+ control=health.managed_control,
+ timeout_sec=cfg.doclib.parse_server_stop_timeout_sec,
+ reason="doclib shutdown",
+ startup_in_progress=health.local_starting,
+ )
+ health.managed_proc = None
+ health.managed_control = None
+ except Exception as exc:
+ errors.append(exc)
+ logger.exception("Failed to stop Doclib managed parse-server")
+
+ if state.db is not None:
+ try:
+ await state.db.close()
+ except Exception as exc:
+ errors.append(exc)
+ logger.exception("Failed to close Doclib database")
+ try:
+ _run_discovery_cleanup(state)
+ except Exception as exc:
+ errors.append(exc)
+ logger.exception("Failed to clean Doclib discovery information")
+ if errors:
+ raise errors[0]
@asynccontextmanager
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
- await startup()
+ """启动失败也清理已拥有资源,退出取消先排空清理且保留最初异常。"""
+ failed = False
try:
+ await startup()
yield
+ except BaseException:
+ failed = True
+ raise
finally:
try:
- await shutdown()
+ cleanup = asyncio.create_task(shutdown(), name="doclib:shutdown")
+ try:
+ await asyncio.shield(cleanup)
+ except asyncio.CancelledError:
+ try:
+ await drain_future(cleanup)
+ except Exception:
+ logger.exception("Doclib cleanup failed while teardown was canceled")
+ raise
+ except BaseException:
+ if not failed:
+ raise
+ logger.exception("Doclib cleanup failed; preserving the original lifespan error")
finally:
_run_discovery_cleanup(state)
diff --git a/tests/unittest/test_doclib_lifespan.py b/tests/unittest/test_doclib_lifespan.py
new file mode 100644
index 00000000..34aa5759
--- /dev/null
+++ b/tests/unittest/test_doclib_lifespan.py
@@ -0,0 +1,243 @@
+"""验证启动失败、退出重启竞态、组件错误和取消时的应用资源清理。"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from mineru.config import Config, LogConfig
+from mineru.doclib import app as doclib_app
+from mineru.doclib.background import parse_server_health as probes
+from mineru.doclib.background import watch as watch_module
+from mineru.doclib.core.db import DatabaseManager
+from mineru.doclib.parse_server_state import ProbeResult
+
+
+def _configuration(root: Path) -> Config:
+ """将本用例的数据库、日志和产物全部放入临时目录。"""
+ cfg = Config()
+ cfg.doclib.data_dir = str(root / "data")
+ cfg.doclib.sqlite.path = str(root / "library.sqlite")
+ cfg.doclib.log = LogConfig(dir=str(root / "logs"))
+ return cfg
+
+
+async def _enable_managed(cfg: Config) -> None:
+ """使用真实配置表启用托管模式,不启动模型或真实子进程。"""
+ db = DatabaseManager(cfg.doclib.sqlite.path)
+ await db.initialize()
+ await db.execute("INSERT OR REPLACE INTO config(key,value) VALUES ('parse_server.local.mode','managed')")
+
+
[email protected]("cleanup_fails", [False, True])
+def test_failed_startup_releases_managed_process_and_discovery(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cleanup_fails: bool
+) -> None:
+ """托管进程创建后发生初始化错误,仍清理已拥有资源并保留最初错误。"""
+ events: list[str] = []
+ process = SimpleNamespace(pid=101)
+
+ def start(**kwargs: object) -> tuple[Any, str, None]:
+ """记录托管进程已经属于当前应用。"""
+ events.append("start")
+ return process, "http://127.0.0.1:16580", None
+
+ def stop(proc: object, **kwargs: object) -> None:
+ """仅记录真实拥有的非空进程。"""
+ if proc is not None:
+ assert proc is process
+ events.append("stop")
+
+ def fail_watch(*args: object, **kwargs: object) -> None:
+ """在进程已创建但后台组件尚未完整组装时抛出主错误。"""
+ raise RuntimeError("watch setup failed")
+
+ async def close_db(self: DatabaseManager) -> None:
+ """验证失败启动也进入数据库清理,附加错误不能覆盖主错误。"""
+ events.append("db.close")
+ if cleanup_fails:
+ raise ValueError("database close failed")
+
+ monkeypatch.setattr(probes, "start_managed_parse_server", start)
+ monkeypatch.setattr(probes, "stop_managed_parse_server", stop)
+ monkeypatch.setattr(watch_module, "WatchLoop", fail_watch)
+ monkeypatch.setattr(DatabaseManager, "close", close_db)
+
+ async def scenario() -> None:
+ """在同一事件循环内检查清理,不依赖测试框架退出时兜底取消。"""
+ cfg = _configuration(tmp_path)
+ await _enable_managed(cfg)
+ app = doclib_app.create_app(cfg)
+ state = app.state.doclib_state
+ state.discovery_cleanup = lambda: events.append("discovery")
+ with pytest.raises(RuntimeError, match="watch setup failed"):
+ async with app.router.lifespan_context(app):
+ raise AssertionError("startup unexpectedly succeeded")
+ assert events == ["start", "stop", "db.close", "discovery"]
+ assert state.health.managed_proc is None and not state.background_tasks
+
+ asyncio.run(scenario())
+
+
+def test_shutdown_does_not_restart_managed_process_after_inflight_probe(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """停止应用时先结束在途探测,避免组件收尾期间再次拉起托管进程。"""
+
+ async def scenario() -> None:
+ """用事件控制真实探测循环,确定性触发旧关闭顺序的重启窗口。"""
+ cfg = _configuration(tmp_path)
+ await _enable_managed(cfg)
+ probe_entered, release_probe, restarted = asyncio.Event(), asyncio.Event(), asyncio.Event()
+ starts: list[object] = []
+ stopped: list[object] = []
+ tasks: list[asyncio.Task[Any]] = []
+ create = doclib_app._create_background_task
+
+ def start(**kwargs: object) -> tuple[Any, str, None]:
+ """第二次创建代表退出过程中发生了不应有的重启。"""
+ proc = SimpleNamespace(pid=len(starts) + 1)
+ starts.append(proc)
+ if len(starts) > 1:
+ restarted.set()
+ return proc, "http://127.0.0.1:16580", None
+
+ def stop(proc: object, **kwargs: object) -> None:
+ """收集最终被应用清理的进程。"""
+ if proc is not None:
+ stopped.append(proc)
+
+ def schedule(state: Any, name: str, runner: Any) -> Any:
+ """只运行本用例需要的真实健康循环,其余组件保留真实停止接口。"""
+ if name == "health_check":
+ task = create(state, name, runner)
+ tasks.append(task)
+ return task
+ return None
+
+ async def probe(self: Any, url: str, *, api_key: str | None = None) -> ProbeResult:
+ """本地探测挂起到应用收尾,远端探测立即返回。"""
+ if url == "http://127.0.0.1:16580":
+ probe_entered.set()
+ await release_probe.wait()
+ return ProbeResult(error_code="parse_server_unavailable")
+ return ProbeResult(healthy=True, tiers=["standard"])
+
+ monkeypatch.setattr(probes, "start_managed_parse_server", start)
+ monkeypatch.setattr(probes, "stop_managed_parse_server", stop)
+ monkeypatch.setattr(probes.ParseServerHealthCheck, "_probe", probe)
+ monkeypatch.setattr(doclib_app, "_create_background_task", schedule)
+ app = doclib_app.create_app(cfg)
+ manager = app.router.lifespan_context(app)
+ await manager.__aenter__()
+ state = app.state.doclib_state
+ await asyncio.wait_for(probe_entered.wait(), timeout=2)
+
+ async def stop_watch() -> None:
+ """模拟其它组件让出执行权,暴露在途探测返回后的重启机会。"""
+ release_probe.set()
+ if not tasks[0].done():
+ await asyncio.wait_for(restarted.wait(), timeout=2)
+
+ monkeypatch.setattr(state.watch, "stop", stop_watch)
+ try:
+ await manager.__aexit__(None, None, None)
+ assert len(starts) == 1 and stopped == starts
+ assert all(task.done() for task in tasks)
+ finally:
+ release_probe.set()
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ asyncio.run(scenario())
+
+
+def test_component_stop_failure_does_not_skip_remaining_resources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """一个组件停止失败仍应清理其它工作池、数据库及发现信息,然后报告最初错误。"""
+ events: list[str] = []
+
+ def skip_background(*args: object, **kwargs: object) -> None:
+ """固定本用例的组件状态,保留应用真实生命周期组装。"""
+
+ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)
+
+ async def scenario() -> None:
+ """对实际组件实例注入停止错误,观察后续资源仍被处理。"""
+ app = doclib_app.create_app(_configuration(tmp_path))
+ manager = app.router.lifespan_context(app)
+ await manager.__aenter__()
+ state = app.state.doclib_state
+
+ async def bad_stop() -> None:
+ """模拟一个组件的意外停止错误。"""
+ events.append("watch")
+ raise ValueError("watch stop failed")
+
+ async def pool_stop() -> None:
+ """确认解析工作池不因前一组件错误而被跳过。"""
+ events.append("parse_workers")
+
+ async def close() -> None:
+ """确认数据库仍进入最终清理。"""
+ events.append("db.close")
+
+ monkeypatch.setattr(state.watch, "stop", bad_stop)
+ monkeypatch.setattr(state.parse_workers, "stop", pool_stop)
+ monkeypatch.setattr(state.db, "close", close)
+ state.discovery_cleanup = lambda: events.append("discovery")
+ with pytest.raises(ValueError, match="watch stop failed"):
+ await manager.__aexit__(None, None, None)
+ assert events == ["watch", "parse_workers", "db.close", "discovery"]
+
+ asyncio.run(scenario())
+
+
+def test_teardown_cancellation_waits_for_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """退出请求被取消时先排空已有清理任务,再向调用方传播取消。"""
+ events: list[str] = []
+
+ def skip_background(*args: object, **kwargs: object) -> None:
+ """仅隔离周期后台任务,不替代实际关闭编排。"""
+
+ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)
+
+ async def scenario() -> None:
+ """挂起一个关闭阶段并取消外部等待者,检查数据库最终仍关闭。"""
+ app = doclib_app.create_app(_configuration(tmp_path))
+ manager = app.router.lifespan_context(app)
+ await manager.__aenter__()
+ state = app.state.doclib_state
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def stop() -> None:
+ """受控关闭阶段必须完成,不能被外部取消截断。"""
+ entered.set()
+ await release.wait()
+ events.append("watch.done")
+
+ async def close() -> None:
+ """记录清理任务最终关闭数据库。"""
+ events.append("db.close")
+
+ monkeypatch.setattr(state.watch, "stop", stop)
+ monkeypatch.setattr(state.db, "close", close)
+ shutdown = asyncio.create_task(manager.__aexit__(None, None, None))
+ await asyncio.wait_for(entered.wait(), timeout=2)
+ shutdown.cancel()
+ await asyncio.sleep(0)
+ shutdown.cancel()
+ await asyncio.sleep(0)
+ try:
+ assert not shutdown.done()
+ finally:
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await shutdown
+ assert events == ["watch.done", "db.close"]
+
+ asyncio.run(scenario())