Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
refactor: separate doclib file ingestion from parsing
myhlolif18db4a3f7a7
diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.ymlindex 31a5a981..67b02fe8 100644--- a/.github/workflows/cli.yml+++ b/.github/workflows/cli.yml@@ -117,7 +117,9 @@ jobs:tests/unittest/test_docvortex_routing.py \tests/unittest/test_parser_api_contract.py \tests/unittest/test_gradio_lifecycle.py \- tests/unittest/test_router_route_state.py+ tests/unittest/test_router_route_state.py \+ tests/unittest/test_doclib_ingest_service.py \+ tests/unittest/test_doclib_architecture.py- name: Retain test reportsif: ${{ always() }}uses: actions/upload-artifact@v6diff --git a/docs/next/telemetry-minimum.md b/docs/next/telemetry-minimum.mdindex ab249f0a..cdc5e270 100644--- a/docs/next/telemetry-minimum.md+++ b/docs/next/telemetry-minimum.md@@ -73,16 +73,18 @@ startup 初始化顺序:DatabaseManager.initialize()-> TelemetryStore(db)-> TelemetryService(store, context collector)- -> ParseService(..., telemetry_svc=state.telemetry_svc)+ -> IngestService(..., telemetry_svc=state.telemetry_svc)+ -> ParseService(..., ingest_svc=state.ingest_svc, telemetry_svc=state.telemetry_svc)-> ScanService(..., telemetry_svc=state.telemetry_svc)- -> SearchService(..., telemetry_svc=state.telemetry_svc)+ -> SearchService(db, fts)-> background workers-> TelemetryFlushWorker(...)```传递规则:-- `ParseService`、`ScanService`、`SearchService` 构造函数显式接收 `telemetry_svc`。+- `IngestService`、`ParseService`、`ScanService` 构造函数显式接收 `telemetry_svc`。+- 文件入库及初始解析任务创建指标由 `IngestService` 记录;扫描、watch 和入库工作池共享应用的同一入库服务。- `DoclibServer` route 通过 `self.state.telemetry_svc` 记录 route-level metric。- background worker 不创建 telemetry service,只使用已注入到 service 的 telemetry。- telemetry service 所有 record 方法内部吞掉异常,最多记录 debug / warning log,不影响主流程。diff --git a/mineru/doclib/app.py b/mineru/doclib/app.pyindex 5565146d..0cda4e0a 100644--- a/mineru/doclib/app.py+++ b/mineru/doclib/app.py@@ -30,6 +30,8 @@ from .types import PARSE_STATUS_FAILED, PARSE_STATUS_PARSING, SCAN_STATUS_FAILEDif TYPE_CHECKING:from loguru import Message as LoguruMessage++ from .services.ingest_svc import IngestServiceelse:LoguruMessage = Any@@ -65,6 +67,7 @@ def create_app(cfg: Config | None = None) -> FastAPI:from .core.fts import FTSManagerfrom .services.cleanup_svc import CleanupServicefrom .services.config_svc import ConfigService+ from .services.ingest_svc import IngestServicefrom .services.parse_svc import ParseServicefrom .services.scan_svc import ScanServicefrom .services.search_svc import SearchService@@ -85,18 +88,20 @@ def create_app(cfg: Config | None = None) -> FastAPI:state.config_svc = ConfigService(state.db)state.telemetry_svc = TelemetryService(TelemetryStore(state.db))await state.telemetry_svc.initialize()+ state.ingest_svc = IngestService(state.db, state.fts, state.config_svc, telemetry_svc=state.telemetry_svc)state.parse_svc = ParseService(state.db,state.fts,state.config_svc,data_dir,+ ingest_svc=state.ingest_svc,parse_lock_timeout_sec=cfg.doclib.parse_lock_timeout_sec,telemetry_svc=state.telemetry_svc,)state.scan_svc = ScanService(state.db,state.config_svc,- state.parse_svc,+ state.ingest_svc,scan_lock_timeout_sec=cfg.doclib.scan_lock_timeout_sec,telemetry_svc=state.telemetry_svc,)@@ -149,13 +154,13 @@ def create_app(cfg: Config | None = None) -> FastAPI:state.watch = WatchLoop(state.db,state.config_svc,- state.parse_svc,+ state.ingest_svc,scan_interval_sec=cfg.doclib.scan_interval_sec,scan_svc=state.scan_svc,)state.scan_workers = ScanWorkerPool(state.scan_svc, num_workers=1)state.ingest_workers = IngestWorkerPool(- state.parse_svc,+ state.ingest_svc,num_workers=cfg.doclib.ingest_workers,lock_timeout_sec=cfg.doclib.ingest_lock_timeout_sec,)@@ -295,6 +300,7 @@ class AppState:self.db: Any = Noneself.fts: Any = Noneself.config_svc: Any = None+ self.ingest_svc: IngestService | None = Noneself.parse_svc: Any = Noneself.scan_svc: Any = Noneself.search_svc: Any = Nonediff --git a/mineru/doclib/background/ingest.py b/mineru/doclib/background/ingest.pyindex bed7e9fd..7bad3810 100644--- a/mineru/doclib/background/ingest.py+++ b/mineru/doclib/background/ingest.py@@ -9,15 +9,16 @@ from typing import castfrom ...errors import MineruErrorfrom ..rows import CountRow, FileRow, IngestTaskRow-from ..services.parse_svc import ParseService+from ..services.ingest_svc import IngestServicefrom ..types import FILE_STATUS_ACTIVElogger = logging.getLogger("mineru.ingest")class IngestWorkerPool:- def __init__(self, parse_svc: ParseService, *, num_workers: int, lock_timeout_sec: int) -> None:- self.parse_svc = parse_svc+ def __init__(self, ingest_svc: IngestService, *, num_workers: int, lock_timeout_sec: int) -> None:+ """保存入库服务与原有工作池数量、抢占超时设置。"""+ self.ingest_svc = ingest_svcself.num_workers = num_workersself.lock_timeout_ms = lock_timeout_sec * 1000self.running = False@@ -42,7 +43,7 @@ class IngestWorkerPool:if no_task_count == 1 or no_task_count % 20 == 0:q = cast(CountRow | None,- await self.parse_svc.db.fetchone(+ await self.ingest_svc.db.fetchone("SELECT COUNT(*) as cnt FROM files WHERE sha256 IS NULL AND status=? AND error_code IS NULL",(FILE_STATUS_ACTIVE,),),@@ -63,7 +64,7 @@ class IngestWorkerPool:no_task_count = 0try:- await self.parse_svc.ingest_file(task["path"], watch_id=task.get("watch_id"), trigger="background")+ await self.ingest_svc.ingest_file(task["path"], watch_id=task.get("watch_id"), trigger="background")processed += 1if processed % 100 == 0:logger.info(f"Ingest worker {worker_id} ingested {processed} files")@@ -84,7 +85,7 @@ class IngestWorkerPool:timeout = now - self.lock_timeout_msreturn cast(FileRow | None,- await self.parse_svc.db.fetchone_write(+ await self.ingest_svc.db.fetchone_write("UPDATE files SET locked_at=? ""WHERE id = ("" SELECT id FROM files "@@ -101,7 +102,7 @@ class IngestWorkerPool:try:if isinstance(exc, FileNotFoundError):watch_id = task.get("watch_id")- await self.parse_svc.refresh_file(+ await self.ingest_svc.refresh_file(str(task["path"]),watch_id=watch_id if isinstance(watch_id, int) else None,)@@ -118,7 +119,7 @@ class IngestWorkerPool:error_code = "stat_failed"now = int(time.time() * 1000)- await self.parse_svc.db.execute(+ await self.ingest_svc.db.execute("UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",(error_code, error_msg, now, task["id"]),)diff --git a/mineru/doclib/background/watch.py b/mineru/doclib/background/watch.pyindex fc2f01fb..8ab62df3 100644--- a/mineru/doclib/background/watch.py+++ b/mineru/doclib/background/watch.py@@ -9,10 +9,10 @@ from pathlib import Pathfrom watchfiles import awatch-from ..core.db import DatabaseManagerfrom ...filetypes import DISCOVERABLE_EXTENSIONS, is_office_temp_lock_file+from ..core.db import DatabaseManagerfrom ..services.config_svc import ConfigService-from ..services.parse_svc import ParseService+from ..services.ingest_svc import IngestServicefrom ..services.scan_svc import ScanServicefrom ..types import FILE_STATUS_ACTIVE, WATCH_STATUS_ACTIVE, WATCH_STATUS_UNREACHABLEfrom ..utils.path_utils import normalize_doclib_path, rebase_watch_event_path@@ -25,14 +25,15 @@ class WatchLoop:self,db: DatabaseManager,config_svc: ConfigService,- parse_svc: ParseService,+ ingest_svc: IngestService,*,scan_interval_sec: int,scan_svc: ScanService | None = None,) -> None:+ """保存目录监控和入库依赖,文件事件交给同一刷新入口。"""self.db = dbself.config_svc = config_svc- self.parse_svc = parse_svc+ self.ingest_svc = ingest_svcself.scan_svc = scan_svcself.scan_interval_sec = scan_interval_secself.running = False@@ -131,11 +132,7 @@ class WatchLoop:)async def _handle_event(self, filepath: str, watch_id: int, *, watch_root: str | None = None) -> None:- filepath = (- rebase_watch_event_path(filepath, watch_root)- if watch_root is not None- else normalize_doclib_path(filepath)- )+ filepath = rebase_watch_event_path(filepath, watch_root) if watch_root is not None else normalize_doclib_path(filepath)ext = Path(filepath).suffix.lstrip(".").lower()if ext not in DISCOVERABLE_EXTENSIONS or is_office_temp_lock_file(filepath):return@@ -144,7 +141,7 @@ class WatchLoop:await self._refresh_file(filepath, watch_id)async def _refresh_file(self, filepath: str, watch_id: int) -> None:- await self.parse_svc.refresh_file(filepath, watch_id=watch_id)+ await self.ingest_svc.refresh_file(filepath, watch_id=watch_id)async def stop(self) -> None:self.running = Falsediff --git a/mineru/doclib/core/fts.py b/mineru/doclib/core/fts.pyindex 35e2c201..1e9ec098 100644--- a/mineru/doclib/core/fts.py+++ b/mineru/doclib/core/fts.py@@ -148,3 +148,14 @@ def _sanitize_query_tokens(tokenized: str) -> list[str]:if t and t.upper() not in ("AND", "OR", "NOT", "NEAR"):safe.append(t)return safe+++MAX_FTS_CHARS = 30_000+FTS_HEAD_HALF = 15_000+++def truncate_head_tail(text: str, max_chars: int = MAX_FTS_CHARS, half: int = FTS_HEAD_HALF) -> str:+ """超长正文仅保留首尾内容,沿用原有 FTS 长度与省略标记。"""+ if len(text) <= max_chars:+ return text+ return text[:half] + "\n...\n" + text[-half:]diff --git a/mineru/doclib/server.py b/mineru/doclib/server.pyindex 2a781318..7d317218 100644--- a/mineru/doclib/server.py+++ b/mineru/doclib/server.py@@ -17,8 +17,12 @@ from pydantic import ValidationErrorfrom ..config import get_configfrom ..errors import InvalidRequestError, MineruError, NotFoundError, error_response, http_status_forfrom ..filetypes import TEXT_EXTENSIONS, TEXT_FILE_TYPES+from ..model.download import verify_model_repo+from ..model.registry import ModelRepo, model_repos_for_tierfrom ..parser.page_range import (expand_page_range as _expand_page_range,+)+from ..parser.page_range import (normalize_page_range_input,normalize_result_page_range,parse_page_range_set,@@ -29,13 +33,12 @@ from ..types import (DeploymentTier,Tier,)-from ..model.download import verify_model_repo-from ..model.registry import ModelRepo, model_repos_for_tierfrom ..version import __version__from .background.parse_server_health import get_health, get_managed_parse_server_tierfrom .base import AsyncDoclibInterfacefrom .config_schema import validate_config_valuefrom .core.db import DatabaseManager+from .core.parse_artifacts import parse_batch_is_readable, parse_coveragefrom .remote_api import REMOTE_API_KEY_CONFIG, resolve_remote_api_keyfrom .rows import (ContentSearchResultRow,@@ -53,7 +56,6 @@ from .rows import (WatchStatsFileRow,WatchTargetRow,)-from .core.parse_artifacts import parse_batch_is_readable, parse_coveragefrom .services.content_read_svc import ContentReadServicefrom .telemetry.buckets import pages_bucket, results_bucketfrom .types import (@@ -146,7 +148,6 @@ from .types import ()from .utils.route_utils import get_route_info, has_route_info, route-_NO_MATCHING_DOC_SHA256 = "__mineru_no_matching_doc__"@@ -377,7 +378,7 @@ class DoclibServer(AsyncDoclibInterface):file_row: FileRow | None = Nonesha256 = doc_row["sha256"] if doc_row else Noneif sha256 is None and request.path:- file_row = await self.state.parse_svc.ensure_ingested(request.path)+ file_row = await self.state.ingest_svc.ensure_ingested(request.path)sha256 = file_row["sha256"] if file_row and file_row.get("sha256") else Noneif sha256 is None:raise NotFoundError("file_not_found", "Document not found.", "path")@@ -502,7 +503,7 @@ class DoclibServer(AsyncDoclibInterface):@route("GET", "/docs/by-path", tags=("docs",))async def get_doc_by_path(self, path: str) -> DocInfo:- await self.state.parse_svc.ensure_ingested(path)+ await self.state.ingest_svc.ensure_ingested(path)row = await self.state.db.fetchone("SELECT d.* FROM docs d JOIN files f ON f.sha256=d.sha256 WHERE f.path=? AND f.status=?",(path, FILE_STATUS_ACTIVE),@@ -652,7 +653,7 @@ class DoclibServer(AsyncDoclibInterface):ext=ext,limit=limit,offset=offset,- refresh_file=self.state.parse_svc.refresh_file,+ refresh_file=self.state.ingest_svc.refresh_file,)response = FindResponse(results=[_find_result(row) for row in results], total=total, query=query)dims = {"status": "succeeded"}@@ -668,7 +669,7 @@ class DoclibServer(AsyncDoclibInterface):@route("GET", "/files/by-path", tags=("files",))async def get_file_by_path(self, path: str) -> FileInfoResponse:- await self.state.parse_svc.ensure_ingested(path)+ await self.state.ingest_svc.ensure_ingested(path)file_row = await self.state.db.fetchone("""SELECT f.*, d.short_id AS short_iddiff --git a/mineru/doclib/services/__init__.py b/mineru/doclib/services/__init__.pyindex f44055ea..fe9b3de4 100644--- a/mineru/doclib/services/__init__.py+++ b/mineru/doclib/services/__init__.py@@ -3,13 +3,16 @@from .cleanup_svc import CleanupServicefrom .config_svc import ConfigServicefrom .content_read_svc import ContentReadService-from .parse_svc import (+from .ingest_svc import (FileRefreshResult,FileRefreshStatus,+ IngestService,+ ensure_doc_record,+)+from .parse_policy import default_parse_range+from .parse_svc import (ParseFailure,ParseService,- default_parse_range,- ensure_doc_record,expand_page_range,page_range_covered,page_range_uncovered,@@ -24,6 +27,7 @@ __all__ = ["ContentReadService","FileRefreshResult","FileRefreshStatus",+ "IngestService","ParseFailure","ParseService","ScanService",diff --git a/mineru/doclib/services/ingest_svc.py b/mineru/doclib/services/ingest_svc.pynew file mode 100644index 00000000..3c399fcc--- /dev/null+++ b/mineru/doclib/services/ingest_svc.py@@ -0,0 +1,597 @@+"""Doclib 源文件刷新、去重入库、元数据和初始解析入队。"""++from __future__ import annotations++import asyncio+import os+import time+from dataclasses import dataclass+from pathlib import Path+from typing import TYPE_CHECKING, Literal, cast++from ...errors import InvalidRequestError, MineruError+from ...filetypes import (+ CSV_EXTENSIONS,+ IMAGE_EXTENSIONS,+ INGESTIBLE_EXTENSIONS,+ PAGE_RANGE_PARSE_EXTENSIONS,+ TEXT_EXTENSIONS,+ TIERED_PARSE_EXTENSIONS,+ file_type_for_extension,+ is_office_temp_lock_file,+)+from ...parser.page_range import expand_page_range+from ...types import Tier+from ..core.db import DatabaseManager+from ..core.file_io import FileStat, MetadataExtractionError, compute_sha256, extract_metadata, get_file_stat+from ..core.fts import FTSManager, truncate_head_tail+from ..rows import FileRow, Sha256Row, ShortIdRow, WatchTargetRow+from ..types import (+ FILE_STATUS_ACTIVE,+ FILE_STATUS_DELETED,+ FILE_STATUS_UNREACHABLE,+ PARSE_STATUS_PENDING,+ RULE_TYPE_PARSING_RULE,+ WATCH_STATUS_UNREACHABLE,+ FileInfo,+)+from ..utils.path_utils import normalize_doclib_path+from . import parse_policy+from .config_svc import ConfigService++if TYPE_CHECKING:+ from ..telemetry import TelemetryService+++def _now_ms() -> int:+ """返回当前毫秒时间,保持文件状态与解析队列的时间单位。"""+ return int(time.time() * 1000)+++FileRefreshStatus = Literal["known", "new", "changed", "missing", "deleted", "unreachable", "unsupported", "error"]+++@dataclass(frozen=True)+class FileRefreshResult:+ """记录文件刷新分类和入库错误,供扫描、搜索与解析请求共同消费。"""++ file: FileInfo | None+ status: FileRefreshStatus+ error_code: str | None = None+ error_msg: str | None = None++ @property+ def needs_ingest(self) -> bool:+ """文件已登记但尚未绑定内容哈希时需要入库。"""+ return self.file is not None and self.file.sha256 is None+++def raise_refresh_error(refreshed: FileRefreshResult) -> None:+ """保持刷新失败的错误码、消息和路径参数语义。"""+ code = (+ (refreshed.file.error_code if refreshed.file and refreshed.file.error_code else None)+ or refreshed.error_code+ or "ingest_failed"+ )+ message = (+ (refreshed.file.error_msg if refreshed.file and refreshed.file.error_msg else None)+ or refreshed.error_msg+ or "File could not be ingested."+ )+ if code == "file_permission_denied":+ raise InvalidRequestError(code, message, "path")+ raise MineruError(code, message, "path")+++async def ensure_doc_record(+ db: DatabaseManager,+ *,+ sha256: str,+ size_bytes: int,+ file_type: str,+ page_count: int | None,+ title: str | None,+ author: str | None,+ subject: str | None,+ keywords: str | None,+ is_image_based: int = 0,+ language: str | None = None,+ error_code: str | None,+ error_msg: str | None,+ first_seen_at: int,+ updated_at: int,+) -> None:+ """按内容哈希只创建一次文档,短标识冲突时逐字符扩展。"""+ existing = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))+ if existing is not None:+ return++ for length in range(7, len(sha256) + 1):+ short_id = sha256[:length]+ await db.execute(+ "INSERT OR IGNORE INTO docs (sha256, short_id, size_bytes, file_type, page_count, "+ "title, author, subject, keywords, is_image_based, error_code, error_msg, first_seen_at, updated_at, language) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ (+ sha256,+ short_id,+ size_bytes,+ file_type,+ page_count,+ title,+ author,+ subject,+ keywords,+ is_image_based,+ error_code,+ error_msg,+ first_seen_at,+ updated_at,+ language,+ ),+ )+ inserted = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))+ if inserted is not None:+ return+ conflict = cast(Sha256Row | None, await db.fetchone("SELECT sha256 FROM docs WHERE short_id=?", (short_id,)))+ if conflict is not None and conflict["sha256"] != sha256:+ continue+ break++ raise RuntimeError(f"Failed to allocate unique short_id for document {sha256}")+++class IngestService:+ """独立管理源文件与文档入库,解析执行通过单独服务处理。"""++ def __init__(+ self, db: DatabaseManager, fts: FTSManager, config_svc: ConfigService, *, telemetry_svc: TelemetryService | None = None+ ) -> None:+ """保存显式数据库、索引、规则配置和可选指标记录依赖。"""+ self.db = db+ self.fts = fts+ self.config_svc = config_svc+ self.telemetry_svc = telemetry_svc++ async def refresh_file(+ self,+ path: str,+ watch_id: int | None = None,+ *,+ ensure_ingested: bool = False,+ allow_images: bool = False,+ ) -> FileRefreshResult:+ """刷新路径对应的文件状态,按需同步完成入库。"""+ path = normalize_doclib_path(path)+ ext = Path(path).suffix.lower().lstrip(".")+ existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))+ if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):+ return FileRefreshResult(file=_file_info(existing), status="unsupported")++ try:+ stat = await get_file_stat(path)+ except FileNotFoundError:+ return await self._refresh_missing_file(path, existing)+ except PermissionError as exc:+ return await self._refresh_stat_error(existing, "file_permission_denied", str(exc))+ except OSError as exc:+ return await self._refresh_stat_error(existing, "stat_failed", str(exc))++ result = await self._refresh_existing_file_with_stat(path, ext, stat, existing, watch_id)+ if ensure_ingested and result.needs_ingest:+ row = await self.ingest_file(path, watch_id=watch_id, allow_images=allow_images)+ return FileRefreshResult(file=_file_info(row), status=result.status)+ return result++ async def _refresh_existing_file_with_stat(+ self,+ path: str,+ ext: str,+ stat: FileStat,+ existing: FileRow | None,+ watch_id: int | None,+ ) -> FileRefreshResult:+ """依据大小、修改时间和历史状态登记新文件或标记变化。"""+ now = _now_ms()+ filename = Path(path).name++ if existing and existing["status"] == FILE_STATUS_ACTIVE:+ unchanged = existing["mtime_ms"] == stat.mtime_ms and existing["size_bytes"] == stat.size_bytes+ if unchanged:+ if watch_id is not None and existing["watch_id"] is None:+ await self.db.execute(+ "UPDATE files SET watch_id=?, updated_at=? WHERE id=? AND watch_id IS NULL",+ (watch_id, now, existing["id"]),+ )+ existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))+ return FileRefreshResult(file=_file_info(existing), status="known")++ await self.db.execute(+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "+ "sha256=NULL, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "+ "WHERE id=?",+ (+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ watch_id,+ now,+ existing["id"],+ ),+ )+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))+ return FileRefreshResult(file=_file_info(row), status="changed")++ if existing:+ await self.db.execute(+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "+ "sha256=NULL, watch_id=COALESCE(?, watch_id), status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, "+ "deleted_at=NULL, updated_at=? WHERE id=?",+ (+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ watch_id,+ FILE_STATUS_ACTIVE,+ now,+ existing["id"],+ ),+ )+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))+ return FileRefreshResult(file=_file_info(row), status="changed")++ file_id = await self.db.execute_insert(+ "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "+ "watch_id, first_seen_at, updated_at) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",+ (+ path,+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ watch_id,+ now,+ now,+ ),+ )+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (file_id,)))+ return FileRefreshResult(file=_file_info(row), status="new")++ async def _refresh_missing_file(self, path: str, existing: FileRow | None) -> FileRefreshResult:+ """区分已删除路径与不可达的可移动监控目录。"""+ if existing is None:+ return FileRefreshResult(file=None, status="missing")++ now = _now_ms()+ file_status = FILE_STATUS_DELETED+ deleted_at: int | None = now+ if existing["status"] != FILE_STATUS_DELETED and await self._is_missing_due_to_unreachable_watch(existing):+ file_status = FILE_STATUS_UNREACHABLE+ deleted_at = None++ await self.db.execute(+ "UPDATE files SET status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, deleted_at=?, updated_at=? WHERE id=?",+ (file_status, deleted_at, now, existing["id"]),+ )+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))+ refresh_status: FileRefreshStatus = "unreachable" if file_status == FILE_STATUS_UNREACHABLE else "deleted"+ return FileRefreshResult(file=_file_info(row), status=refresh_status)++ async def _is_missing_due_to_unreachable_watch(self, file_row: FileRow) -> bool:+ """检查缺失路径是否来自暂不可达的可移动目录。"""+ watch_id = file_row.get("watch_id")+ if watch_id is None:+ return False+ watch = cast(WatchTargetRow | None, await self.db.fetchone("SELECT * FROM watches WHERE id=?", (watch_id,)))+ if not watch or not watch.get("removable"):+ return False+ if watch["status"] == WATCH_STATUS_UNREACHABLE:+ return True+ try:+ await get_file_stat(watch["path"])+ except OSError:+ if self.config_svc is not None:+ await self.config_svc.update_watch_status(watch_id, WATCH_STATUS_UNREACHABLE)+ return True+ return False++ async def _refresh_stat_error(self, existing: FileRow | None, error_code: str, error_msg: str) -> FileRefreshResult:+ """保留原有权限和 stat 失败状态,并解除文件锁与哈希绑定。"""+ if existing is None:+ return FileRefreshResult(file=None, status="error", error_code=error_code, error_msg=error_msg)++ now = _now_ms()+ await self.db.execute(+ "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",+ (error_code, error_msg[:500], now, existing["id"]),+ )+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))+ return FileRefreshResult(file=_file_info(row), status="error", error_code=error_code, error_msg=error_msg)++ async def ensure_ingested(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:+ """确保路径已刷新并入库,保持不支持格式和错误的原有边界。"""+ path = normalize_doclib_path(path)+ refreshed = await self.refresh_file(path, watch_id=watch_id, ensure_ingested=True, allow_images=allow_images)+ if refreshed.status == "error":+ raise_refresh_error(refreshed)+ if refreshed.file is None or refreshed.status == "unsupported":+ return None+ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))++ async def ingest_file(+ self,+ path: str,+ watch_id: int | None = None,+ *,+ trigger: str = "parse",+ allow_images: bool = False,+ ) -> FileRow | None:+ """记录入库计数和耗时,将权限错误映射为稳定路径错误。"""+ path = normalize_doclib_path(path)+ start_ms = _now_ms()+ status = "succeeded"+ try:+ return await self._ingest_file(path, watch_id=watch_id, allow_images=allow_images)+ except PermissionError as exc:+ status = "failed"+ await self._mark_file_error(path, "file_permission_denied", str(exc))+ raise InvalidRequestError("file_permission_denied", str(exc), "path") from None+ except Exception:+ status = "failed"+ raise+ finally:+ dimensions = {"status": status, "trigger": trigger}+ await self._record_count("ingest.finished.count", dimensions=dimensions)+ await self._record_duration("ingest.duration_bucket.count", start_ms, dimensions=dimensions)++ async def _mark_file_error(self, path: str, error_code: str, error_msg: str) -> None:+ """保存文件错误并解除入库锁,限制持久化消息长度。"""+ path = normalize_doclib_path(path)+ now = _now_ms()+ await self.db.execute(+ "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE path=?",+ (error_code, error_msg[:500], now, path),+ )++ async def _ingest_file(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:+ """计算哈希并复用文档,提取元数据和索引,再按规则创建初始解析。"""+ ext = Path(path).suffix.lower().lstrip(".")+ if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):+ return None++ stat = await get_file_stat(path)+ sha256 = await compute_sha256(path)+ now = _now_ms()+ filename = Path(path).name++ existing_path = cast(+ FileRow | None,+ await self.db.fetchone("SELECT * FROM files WHERE path=? AND status=?", (path, FILE_STATUS_ACTIVE)),+ )+ if existing_path and existing_path["sha256"] == sha256:+ return existing_path++ # check if same content (sha256) is already tracked by another path+ existing_sha = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE sha256=?", (sha256,)))+ if existing_sha:+ # same content, possibly a new or changed path — bind this file row+ # to the existing doc without duplicating docs/parses.+ if existing_path:+ await self.db.execute(+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "+ "sha256=?, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "+ "WHERE id=?",+ (+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ sha256,+ watch_id,+ now,+ existing_path["id"],+ ),+ )+ else:+ await self.db.execute(+ "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "+ "sha256, watch_id, first_seen_at, updated_at) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",+ (+ path,+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ sha256,+ watch_id,+ now,+ now,+ ),+ )+ file_row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))+ if file_row:+ await self.fts.upsert_filename(file_row["id"], Path(path).stem, ext)+ # the source may have been moved here from another tracked path;+ # retire stale duplicates whose files no longer exist.+ await self._mark_moved_duplicate_files_deleted(sha256, path)+ return file_row++ # brand new document+ metadata_error_code = None+ metadata_error_msg = None+ try:+ metadata = await extract_metadata(path)+ metadata_error_code = metadata.get("error_code")+ metadata_error_msg = metadata.get("error_msg")+ except MetadataExtractionError as exc:+ metadata_error_code = exc.code+ metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"+ metadata = {+ "page_count": None,+ "title": None,+ "author": None,+ "subject": None,+ "keywords": None,+ "is_image_based": 0,+ }+ except Exception as exc:+ metadata_error_code = "open_failed"+ metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"+ metadata = {+ "page_count": None,+ "title": None,+ "author": None,+ "subject": None,+ "keywords": None,+ "is_image_based": 0,+ }++ # 无固有分页或元数据暂不可读的结构化格式默认按一页处理。+ page_count = metadata["page_count"]+ if page_count is None:+ if ext in ("doc", "docx", "rtf"):+ page_count = 1 # reflow docs+ elif ext in ("ppt", "pptx", "xls", "xlsx"):+ page_count = 1 # will be updated by metadata extraction+ elif ext in CSV_EXTENSIONS:+ page_count = 1++ now = _now_ms()+ await self.db.execute(+ "INSERT OR IGNORE INTO files (path, filename, ext, size_bytes, mtime_ms, "+ "watch_id, first_seen_at, updated_at) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",+ (+ path,+ filename,+ ext,+ stat.size_bytes,+ stat.mtime_ms,+ watch_id,+ now,+ now,+ ),+ )++ await ensure_doc_record(+ self.db,+ sha256=sha256,+ size_bytes=stat.size_bytes,+ file_type=file_type_for_extension(ext),+ page_count=page_count,+ title=metadata["title"],+ author=metadata["author"],+ subject=metadata["subject"],+ keywords=metadata["keywords"],+ is_image_based=int(metadata.get("is_image_based") or 0),+ language=metadata.get("language"),+ error_code=metadata_error_code,+ error_msg=metadata_error_msg,+ first_seen_at=now,+ updated_at=now,+ )++ # FTS filename index+ file_row = await self.db.fetchone("SELECT id FROM files WHERE path=?", (path,))+ if file_row:+ await self.fts.upsert_filename(+ file_row["id"],+ Path(path).stem,+ ext,+ )++ if ext in TEXT_EXTENSIONS:+ await self.db.execute(+ "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",+ (sha256, now, path),+ )+ text = await asyncio.to_thread(Path(path).read_text, encoding="utf-8", errors="replace")+ await self.fts.replace(+ sha256=sha256,+ tier=None,+ text=truncate_head_tail(text),+ title=metadata["title"] or "",+ author=metadata["author"] or "",+ )+ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))++ # determine tier and page_range for initial parse+ tier: Tier = "flash"+ privacy = "local"+ initial_page_range = (+ parse_policy.default_parse_range(page_count)+ if ext in PAGE_RANGE_PARSE_EXTENSIONS+ else expand_page_range(None, page_count or 1)+ )++ # check parsing-rules+ matched = await self.config_svc.match_rules(path, RULE_TYPE_PARSING_RULE)+ if matched:+ rule = matched[0]+ rule_tier = cast(Tier | None, rule.get("tier"))+ rule_remote = bool(rule.get("remote", False))+ if ext in TIERED_PARSE_EXTENSIONS:+ tier = rule_tier or parse_policy.resolve_parsing_rule_default_tier(remote=rule_remote)+ privacy = "remote" if rule_remote and tier != "flash" else "local"+ else:+ tier = "flash"+ privacy = "local"+ rule_page_range = rule.get("page_range")+ if rule_page_range and ext in PAGE_RANGE_PARSE_EXTENSIONS:+ initial_page_range = expand_page_range(rule_page_range, page_count or 1)++ # insert parse batch+ parse_page_range = expand_page_range(initial_page_range, page_count or 1)+ await self.db.execute(+ "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",+ (sha256, now, path),+ )+ await self.db.execute(+ "INSERT INTO parses (sha256, tier, page_range, status, privacy, priority, created_at, updated_at) "+ "VALUES (?, ?, ?, ?, ?, 0, ?, ?)",+ (sha256, tier, parse_page_range, PARSE_STATUS_PENDING, privacy, now, now),+ )+ await self._record_count("parse_task.created.count", dimensions={"tier": tier})++ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))++ async def _mark_moved_duplicate_files_deleted(self, sha256: str, keep_path: str) -> None:+ """同哈希的新路径入库后,将已不存在的旧路径标为删除。"""+ rows = cast(+ list[FileRow],+ await self.db.fetchall(+ "SELECT id, path FROM files WHERE sha256=? AND status=? AND path<>?",+ (sha256, FILE_STATUS_ACTIVE, keep_path),+ ),+ )+ now = _now_ms()+ for row in rows:+ if not os.path.exists(row["path"]):+ await self.db.execute(+ "UPDATE files SET status=?, deleted_at=?, updated_at=? WHERE id=?",+ (FILE_STATUS_DELETED, now, now, row["id"]),+ )++ async def _record_count(self, metric_name: str, *, value: int = 1, dimensions: dict[str, str] | None = None) -> None:+ """通过显式指标接口记录入库和初始解析计数。"""+ if self.telemetry_svc is not None:+ await self.telemetry_svc.record_count(metric_name, value=value, dimensions=dimensions)++ async def _record_duration(self, metric_name: str, start_ms: int, *, dimensions: dict[str, str] | None = None) -> None:+ """使用原有非负耗时和维度记录入库耗时桶。"""+ if self.telemetry_svc is not None:+ await self.telemetry_svc.record_duration_bucket(+ metric_name, duration_ms=max(0, _now_ms() - start_ms), dimensions=dimensions+ )+++def _file_info(row: FileRow | None) -> FileInfo | None:+ """将文件数据库行转换为对外文件信息。"""+ return FileInfo.model_validate(row) if row is not None else None+++__all__ = ["FileRefreshResult", "FileRefreshStatus", "IngestService", "ensure_doc_record", "raise_refresh_error"]diff --git a/mineru/doclib/services/parse_policy.py b/mineru/doclib/services/parse_policy.pynew file mode 100644index 00000000..da3dac2f--- /dev/null+++ b/mineru/doclib/services/parse_policy.py@@ -0,0 +1,25 @@+"""Doclib 入库与主动解析共用的默认页范围和后台档位策略。"""++from __future__ import annotations++from ...parser.page_range import expand_page_range+from ...types import Tier, select_parsing_rule_tier+++def default_parse_range(page_count: int | None) -> str:+ """主动读取默认选择前十页,未知页数时保留第一页请求。"""+ if page_count and page_count > 0:+ return expand_page_range("1-10", page_count)+ return "1"+++def resolve_parsing_rule_default_tier(remote: bool = False) -> Tier:+ """后台入库规则沿用质量档位降级到 Flash 的既有策略。"""+ from ..background.parse_server_health import get_health++ health = get_health()+ supported = health.remote.probe.tiers if remote else health.local.probe.tiers+ return select_parsing_rule_tier(supported)+++__all__ = ["default_parse_range", "resolve_parsing_rule_default_tier"]diff --git a/mineru/doclib/services/parse_svc.py b/mineru/doclib/services/parse_svc.pyindex e8dbe061..97c300b0 100644--- a/mineru/doclib/services/parse_svc.py+++ b/mineru/doclib/services/parse_svc.py@@ -1,4 +1,4 @@-"""Parse service: file ingestion, parse request/acquire/process."""+"""Doclib 解析请求、队列执行、产物写出和索引维护。"""from __future__ import annotations@@ -6,35 +6,37 @@ import asyncioimport loggingimport osimport time-from dataclasses import dataclassfrom pathlib import Path-from typing import Literal, cast+from typing import castfrom urllib.parse import urlparsefrom ...errors import InvalidRequestError, MineruErrorfrom ...filetypes import (- CSV_EXTENSIONS,- IMAGE_EXTENSIONS,- INGESTIBLE_EXTENSIONS,OFD_EXTENSIONS,OFFICE_EXTENSIONS,PAGE_RANGE_PARSE_EXTENSIONS,TEXT_EXTENSIONS,TIERED_PARSE_EXTENSIONS,- file_type_for_extension,is_office_temp_lock_file,)from ...parser.api_client import _APITransportError, _V1APIErrorfrom ...parser.base import ParseResultfrom ...parser.page_range import (expand_page_range,- format_page_range as _page_numbers_to_range_str,normalize_page_range_input,normalize_result_page_range,parse_page_range_set,)-from ...types import QUALITY_TIERS, TIER_ORDER, Tier, select_default_quality_tier, select_parsing_rule_tier+from ...parser.page_range import (+ format_page_range as _page_numbers_to_range_str,+)+from ...types import QUALITY_TIERS, TIER_ORDER, Tier, select_default_quality_tier+from ..core import fts as fts_utilsfrom ..core.db import DatabaseManager+from ..core.file_io import (+ metadata_to_doclib,+)+from ..core.fts import FTSManagerfrom ..core.parse_artifacts import (accessible_file_for_sha256,load_pages_from_done_batches,@@ -44,33 +46,20 @@ from ..core.parse_artifacts import (parse_coverage,write_parse_payload,)-from ..core.file_io import (- FileStat,- MetadataExtractionError,- compute_sha256,- extract_metadata,- get_file_stat,- metadata_to_doclib,-)-from ..core.fts import FTSManagerfrom ..remote_api import resolve_remote_api_key-from ..rows import DocRow, FileRow, ParseRow, Sha256Row, ShortIdRow, WatchTargetRow+from ..rows import DocRow, FileRow, ParseRowfrom ..types import (FILE_STATUS_ACTIVE,- FILE_STATUS_DELETED,- FILE_STATUS_UNREACHABLE,PARSE_STATUS_DONE,PARSE_STATUS_FAILED,PARSE_STATUS_PARSING,PARSE_STATUS_PENDING,PARSE_STATUS_SUPERSEDED,- RULE_TYPE_PARSING_RULE,- WATCH_STATUS_UNREACHABLE,- FileInfo,ParseResponse,)-from ..utils.path_utils import normalize_doclib_path+from . import parse_policyfrom .config_svc import ConfigService+from .ingest_svc import IngestService, raise_refresh_errorlogger = logging.getLogger("mineru.parse")@@ -87,41 +76,6 @@ class ParseFailure(MineruError):self.message = message-MAX_FTS_CHARS = 30_000-FTS_HEAD_HALF = 15_000---FileRefreshStatus = Literal["known", "new", "changed", "missing", "deleted", "unreachable", "unsupported", "error"]---@dataclass(frozen=True)-class FileRefreshResult:- file: FileInfo | None- status: FileRefreshStatus- error_code: str | None = None- error_msg: str | None = None-- @property- def needs_ingest(self) -> bool:- return self.file is not None and self.file.sha256 is None---def _raise_refresh_error(refreshed: FileRefreshResult) -> None:- code = (- (refreshed.file.error_code if refreshed.file and refreshed.file.error_code else None)- or refreshed.error_code- or "ingest_failed"- )- message = (- (refreshed.file.error_msg if refreshed.file and refreshed.file.error_msg else None)- or refreshed.error_msg- or "File could not be ingested."- )- if code == "file_permission_denied":- raise InvalidRequestError(code, message, "path")- raise MineruError(code, message, "path")--# ── range helpers ──────────────────────────────────────────────────@@ -146,79 +100,6 @@ def page_range_uncovered(request_page_range: str, done_batches: list[ParseRow])return needed_page_numbers - covered_page_numbers-def default_parse_range(page_count: int | None) -> str:- """Return the default page range for active reading."""- if page_count and page_count > 0:- return expand_page_range("1-10", page_count)- return "1"---async def ensure_doc_record(- db: DatabaseManager,- *,- sha256: str,- size_bytes: int,- file_type: str,- page_count: int | None,- title: str | None,- author: str | None,- subject: str | None,- keywords: str | None,- is_image_based: int = 0,- language: str | None = None,- error_code: str | None,- error_msg: str | None,- first_seen_at: int,- updated_at: int,-) -> None:- existing = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))- if existing is not None:- return-- for length in range(7, len(sha256) + 1):- short_id = sha256[:length]- await db.execute(- "INSERT OR IGNORE INTO docs (sha256, short_id, size_bytes, file_type, page_count, "- "title, author, subject, keywords, is_image_based, error_code, error_msg, first_seen_at, updated_at, language) "- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",- (- sha256,- short_id,- size_bytes,- file_type,- page_count,- title,- author,- subject,- keywords,- is_image_based,- error_code,- error_msg,- first_seen_at,- updated_at,- language,- ),- )- inserted = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))- if inserted is not None:- return- conflict = cast(Sha256Row | None, await db.fetchone("SELECT sha256 FROM docs WHERE short_id=?", (short_id,)))- if conflict is not None and conflict["sha256"] != sha256:- continue- break-- raise RuntimeError(f"Failed to allocate unique short_id for document {sha256}")---# ── text truncation ────────────────────────────────────────────────---def truncate_head_tail(text: str, max_chars: int = MAX_FTS_CHARS, half: int = FTS_HEAD_HALF) -> str:- if len(text) <= max_chars:- return text- return text[:half] + "\n...\n" + text[-half:]--# ── ParseService ───────────────────────────────────────────────────@@ -230,9 +111,12 @@ class ParseService:config_svc: ConfigService,data_dir: str,*,+ ingest_svc: IngestService,parse_lock_timeout_sec: int,telemetry_svc: object | None = None,) -> None:+ """保存解析依赖,文件刷新和入库由显式注入的服务处理。"""+ self.ingest_svc = ingest_svcself.db = dbself.fts = ftsself.config_svc = config_svc@@ -240,411 +124,6 @@ class ParseService:self.parse_lock_timeout_ms = parse_lock_timeout_sec * 1000self.telemetry_svc = telemetry_svc- # ── discovery / ingestion (shared) ──────────────────────-- async def refresh_file(- self,- path: str,- watch_id: int | None = None,- *,- ensure_ingested: bool = False,- allow_images: bool = False,- ) -> FileRefreshResult:- """Refresh one source path against the files table.-- This method owns file stat, DB row lookup, new/changed/known/missing/deleted- classification, and optional synchronous ingest.- """- path = normalize_doclib_path(path)- ext = Path(path).suffix.lower().lstrip(".")- existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))- if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):- return FileRefreshResult(file=_file_info(existing), status="unsupported")-- try:- stat = await get_file_stat(path)- except FileNotFoundError:- return await self._refresh_missing_file(path, existing)- except PermissionError as exc:- return await self._refresh_stat_error(existing, "file_permission_denied", str(exc))- except OSError as exc:- return await self._refresh_stat_error(existing, "stat_failed", str(exc))-- result = await self._refresh_existing_file_with_stat(path, ext, stat, existing, watch_id)- if ensure_ingested and result.needs_ingest:- row = await self.ingest_file(path, watch_id=watch_id, allow_images=allow_images)- return FileRefreshResult(file=_file_info(row), status=result.status)- return result-- async def _refresh_existing_file_with_stat(- self,- path: str,- ext: str,- stat: FileStat,- existing: FileRow | None,- watch_id: int | None,- ) -> FileRefreshResult:- now = _now_ms()- filename = Path(path).name-- if existing and existing["status"] == FILE_STATUS_ACTIVE:- unchanged = existing["mtime_ms"] == stat.mtime_ms and existing["size_bytes"] == stat.size_bytes- if unchanged:- if watch_id is not None and existing["watch_id"] is None:- await self.db.execute(- "UPDATE files SET watch_id=?, updated_at=? WHERE id=? AND watch_id IS NULL",- (watch_id, now, existing["id"]),- )- existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))- return FileRefreshResult(file=_file_info(existing), status="known")-- await self.db.execute(- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "- "sha256=NULL, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "- "WHERE id=?",- (- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- watch_id,- now,- existing["id"],- ),- )- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))- return FileRefreshResult(file=_file_info(row), status="changed")-- if existing:- await self.db.execute(- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "- "sha256=NULL, watch_id=COALESCE(?, watch_id), status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, "- "deleted_at=NULL, updated_at=? WHERE id=?",- (- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- watch_id,- FILE_STATUS_ACTIVE,- now,- existing["id"],- ),- )- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))- return FileRefreshResult(file=_file_info(row), status="changed")-- file_id = await self.db.execute_insert(- "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "- "watch_id, first_seen_at, updated_at) "- "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",- (- path,- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- watch_id,- now,- now,- ),- )- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (file_id,)))- return FileRefreshResult(file=_file_info(row), status="new")-- async def _refresh_missing_file(self, path: str, existing: FileRow | None) -> FileRefreshResult:- if existing is None:- return FileRefreshResult(file=None, status="missing")-- now = _now_ms()- file_status = FILE_STATUS_DELETED- deleted_at = now- if existing["status"] != FILE_STATUS_DELETED and await self._is_missing_due_to_unreachable_watch(existing):- file_status = FILE_STATUS_UNREACHABLE- deleted_at = None-- await self.db.execute(- "UPDATE files SET status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, deleted_at=?, updated_at=? WHERE id=?",- (file_status, deleted_at, now, existing["id"]),- )- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))- refresh_status: FileRefreshStatus = "unreachable" if file_status == FILE_STATUS_UNREACHABLE else "deleted"- return FileRefreshResult(file=_file_info(row), status=refresh_status)-- async def _is_missing_due_to_unreachable_watch(self, file_row: FileRow) -> bool:- watch_id = file_row.get("watch_id")- if watch_id is None:- return False- watch = cast(WatchTargetRow | None, await self.db.fetchone("SELECT * FROM watches WHERE id=?", (watch_id,)))- if not watch or not watch.get("removable"):- return False- if watch["status"] == WATCH_STATUS_UNREACHABLE:- return True- try:- await get_file_stat(watch["path"])- except OSError:- if self.config_svc is not None:- await self.config_svc.update_watch_status(watch_id, WATCH_STATUS_UNREACHABLE)- return True- return False-- async def _refresh_stat_error(self, existing: FileRow | None, error_code: str, error_msg: str) -> FileRefreshResult:- if existing is None:- return FileRefreshResult(file=None, status="error", error_code=error_code, error_msg=error_msg)-- now = _now_ms()- await self.db.execute(- "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",- (error_code, error_msg[:500], now, existing["id"]),- )- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))- return FileRefreshResult(file=_file_info(row), status="error", error_code=error_code, error_msg=error_msg)-- async def ensure_ingested(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:- """Synchronously discover and ingest a source path when needed."""- path = normalize_doclib_path(path)- refreshed = await self.refresh_file(path, watch_id=watch_id, ensure_ingested=True, allow_images=allow_images)- if refreshed.status == "error":- _raise_refresh_error(refreshed)- if refreshed.file is None or refreshed.status == "unsupported":- return None- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))-- async def ingest_file(- self,- path: str,- watch_id: int | None = None,- *,- trigger: str = "parse",- allow_images: bool = False,- ) -> FileRow | None:- """Ingest a discovered file: SHA-256 + metadata + trigger default parse."""- path = normalize_doclib_path(path)- start_ms = _now_ms()- status = "succeeded"- try:- return await self._ingest_file(path, watch_id=watch_id, allow_images=allow_images)- except PermissionError as exc:- status = "failed"- await self._mark_file_error(path, "file_permission_denied", str(exc))- raise InvalidRequestError("file_permission_denied", str(exc), "path") from None- except Exception:- status = "failed"- raise- finally:- dimensions = {"status": status, "trigger": trigger}- await self._record_count("ingest.finished.count", dimensions=dimensions)- await self._record_duration("ingest.duration_bucket.count", start_ms, dimensions=dimensions)-- async def _mark_file_error(self, path: str, error_code: str, error_msg: str) -> None:- path = normalize_doclib_path(path)- now = _now_ms()- await self.db.execute(- "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE path=?",- (error_code, error_msg[:500], now, path),- )-- async def _ingest_file(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:- """Ingest implementation without telemetry wrapper."""- ext = Path(path).suffix.lower().lstrip(".")- if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):- return None-- stat = await get_file_stat(path)- sha256 = await compute_sha256(path)- now = _now_ms()- filename = Path(path).name-- existing_path = cast(- FileRow | None,- await self.db.fetchone("SELECT * FROM files WHERE path=? AND status=?", (path, FILE_STATUS_ACTIVE)),- )- if existing_path and existing_path["sha256"] == sha256:- return existing_path-- # check if same content (sha256) is already tracked by another path- existing_sha = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE sha256=?", (sha256,)))- if existing_sha:- # same content, possibly a new or changed path — bind this file row- # to the existing doc without duplicating docs/parses.- if existing_path:- await self.db.execute(- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "- "sha256=?, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "- "WHERE id=?",- (- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- sha256,- watch_id,- now,- existing_path["id"],- ),- )- else:- await self.db.execute(- "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "- "sha256, watch_id, first_seen_at, updated_at) "- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",- (- path,- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- sha256,- watch_id,- now,- now,- ),- )- file_row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))- if file_row:- await self.fts.upsert_filename(file_row["id"], Path(path).stem, ext)- # the source may have been moved here from another tracked path;- # retire stale duplicates whose files no longer exist.- await self._mark_moved_duplicate_files_deleted(sha256, path)- return file_row-- # brand new document- metadata_error_code = None- metadata_error_msg = None- try:- metadata = await extract_metadata(path)- metadata_error_code = metadata.get("error_code")- metadata_error_msg = metadata.get("error_msg")- except MetadataExtractionError as exc:- metadata_error_code = exc.code- metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"- metadata = {- "page_count": None,- "title": None,- "author": None,- "subject": None,- "keywords": None,- "is_image_based": 0,- }- except Exception as exc:- metadata_error_code = "open_failed"- metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"- metadata = {- "page_count": None,- "title": None,- "author": None,- "subject": None,- "keywords": None,- "is_image_based": 0,- }-- # 无固有分页或元数据暂不可读的结构化格式默认按一页处理。- page_count = metadata["page_count"]- if page_count is None:- if ext in ("doc", "docx", "rtf"):- page_count = 1 # reflow docs- elif ext in ("ppt", "pptx", "xls", "xlsx"):- page_count = 1 # will be updated by metadata extraction- elif ext in CSV_EXTENSIONS:- page_count = 1-- now = _now_ms()- await self.db.execute(- "INSERT OR IGNORE INTO files (path, filename, ext, size_bytes, mtime_ms, "- "watch_id, first_seen_at, updated_at) "- "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",- (- path,- filename,- ext,- stat.size_bytes,- stat.mtime_ms,- watch_id,- now,- now,- ),- )-- await ensure_doc_record(- self.db,- sha256=sha256,- size_bytes=stat.size_bytes,- file_type=file_type_for_extension(ext),- page_count=page_count,- title=metadata["title"],- author=metadata["author"],- subject=metadata["subject"],- keywords=metadata["keywords"],- is_image_based=int(metadata.get("is_image_based") or 0),- language=metadata.get("language"),- error_code=metadata_error_code,- error_msg=metadata_error_msg,- first_seen_at=now,- updated_at=now,- )-- # FTS filename index- file_row = await self.db.fetchone("SELECT id FROM files WHERE path=?", (path,))- if file_row:- await self.fts.upsert_filename(- file_row["id"],- Path(path).stem,- ext,- )-- if ext in TEXT_EXTENSIONS:- await self.db.execute(- "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",- (sha256, now, path),- )- text = await asyncio.to_thread(Path(path).read_text, encoding="utf-8", errors="replace")- await self.fts.replace(- sha256=sha256,- tier=None,- text=truncate_head_tail(text),- title=metadata["title"] or "",- author=metadata["author"] or "",- )- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))-- # determine tier and page_range for initial parse- tier: Tier = "flash"- privacy = "local"- initial_page_range = (- default_parse_range(page_count) if ext in PAGE_RANGE_PARSE_EXTENSIONS else expand_page_range(None, page_count or 1)- )-- # check parsing-rules- matched = await self.config_svc.match_rules(path, RULE_TYPE_PARSING_RULE)- if matched:- rule = matched[0]- rule_tier = cast(Tier | None, rule.get("tier"))- rule_remote = bool(rule.get("remote", False))- if ext in TIERED_PARSE_EXTENSIONS:- tier = rule_tier or _resolve_parsing_rule_default_tier(remote=rule_remote)- privacy = "remote" if rule_remote and tier != "flash" else "local"- else:- tier = "flash"- privacy = "local"- rule_page_range = rule.get("page_range")- if rule_page_range and ext in PAGE_RANGE_PARSE_EXTENSIONS:- initial_page_range = expand_page_range(rule_page_range, page_count or 1)-- # insert parse batch- parse_page_range = expand_page_range(initial_page_range, page_count or 1)- await self.db.execute(- "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",- (sha256, now, path),- )- await self.db.execute(- "INSERT INTO parses (sha256, tier, page_range, status, privacy, priority, created_at, updated_at) "- "VALUES (?, ?, ?, ?, ?, 0, ?, ?)",- (sha256, tier, parse_page_range, PARSE_STATUS_PENDING, privacy, now, now),- )- await self._record_count("parse_task.created.count", dimensions={"tier": tier})-- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))-# ── parse request ───────────────────────────────────────────async def request_parse(@@ -659,14 +138,14 @@ class ParseService:"""Handle a parse request from CLI. Returns info for status polling."""page_range = normalize_page_range_input(page_range) or None# ensure the path is current before trusting files.sha256- refreshed = await self.refresh_file(path, ensure_ingested=True, allow_images=True)+ refreshed = await self.ingest_svc.refresh_file(path, ensure_ingested=True, allow_images=True)if refreshed.status == "unsupported":ext = Path(path).suffix.lower() or Path(path).nameraise InvalidRequestError("file_type_unsupported", _unsupported_file_type_message(ext), "path")if refreshed.status in {"missing", "deleted", "unreachable"}:raise InvalidRequestError("file_not_found", f"File {path} not found.", "path")if refreshed.status == "error":- _raise_refresh_error(refreshed)+ raise_refresh_error(refreshed)if refreshed.file is None:raise MineruError("ingest_failed", "File could not be ingested.", "path")@@ -724,7 +203,9 @@ class ParseService:# ── expand page range ──supports_page_range = ext in PAGE_RANGE_PARSE_EXTENSIONS- requested_page_range_input = page_range or (default_parse_range(page_count) if supports_page_range else None)+ requested_page_range_input = page_range or (+ parse_policy.default_parse_range(page_count) if supports_page_range else None+ )request_page_range = expand_page_range(requested_page_range_input, page_count or 1)needed_page_numbers = parse_page_range_set(request_page_range)@@ -844,22 +325,6 @@ class ParseService:await self._record_count("parse_task.started.count", dimensions={"tier": task["tier"]})return task- async def _mark_moved_duplicate_files_deleted(self, sha256: str, keep_path: str) -> None:- rows = cast(- list[FileRow],- await self.db.fetchall(- "SELECT id, path FROM files WHERE sha256=? AND status=? AND path<>?",- (sha256, FILE_STATUS_ACTIVE, keep_path),- ),- )- now = _now_ms()- for row in rows:- if not os.path.exists(row["path"]):- await self.db.execute(- "UPDATE files SET status=?, deleted_at=?, updated_at=? WHERE id=?",- (FILE_STATUS_DELETED, now, now, row["id"]),- )-async def process_doc(self, task: ParseRow) -> bool:"""Execute parse for a batch. Returns True on success."""task_start_ms = _now_ms()@@ -1348,7 +813,7 @@ class ParseService:if existing_tier and TIER_ORDER.get(existing_tier, -1) >= TIER_ORDER.get(tier, -1):return # current FTS data is from a higher or equal tier- text = truncate_head_tail(text)+ text = fts_utils.truncate_head_tail(text)await self.fts.replace(sha256=sha256,tier=tier,@@ -1426,9 +891,10 @@ class ParseService:if not pages:continue- from ...config import get_configfrom docvortex.render.markdown import render_single_block+ from ...config import get_config+delimiters = get_config().render.latex_delimitersparts: list[str] = []for page in pages:@@ -1441,7 +907,7 @@ class ParseService:)if rendered and rendered.strip():parts.append(rendered.strip())- text = truncate_head_tail("\n\n".join(parts))+ text = fts_utils.truncate_head_tail("\n\n".join(parts))await self.fts.replace(sha256=sha256,tier=tier,@@ -1489,15 +955,6 @@ def _resolve_default_tier(remote: bool = False) -> Tier:)-def _resolve_parsing_rule_default_tier(remote: bool = False) -> Tier:- """Parsing-rule 是后台批量策略,允许 standard -> basic -> flash。"""- from ..background.parse_server_health import get_health-- health = get_health()- supported = health.remote.probe.tiers if remote else health.local.probe.tiers- return select_parsing_rule_tier(supported)--def _format_action_list(actions: list[str]) -> str:if len(actions) == 1:return actions[0]@@ -1601,10 +1058,6 @@ def _is_official_remote_url(url: str) -> bool:return host in {"mineru.net", "mineru.org.cn"} or host.endswith(".mineru.net") or host.endswith(".mineru.org.cn")-def _file_info(row: FileRow | None) -> FileInfo | None:- return FileInfo.model_validate(row) if row is not None else None--def _unsupported_file_type_message(ext_or_name: str) -> str:display = ext_or_name if ext_or_name.startswith(".") else Path(ext_or_name).namereturn f"File type is not supported for parsing: {display}"diff --git a/mineru/doclib/services/scan_svc.py b/mineru/doclib/services/scan_svc.pyindex d88992c7..5edc0945 100644--- a/mineru/doclib/services/scan_svc.py+++ b/mineru/doclib/services/scan_svc.py@@ -2,25 +2,26 @@from __future__ import annotations+import loggingimport osimport time-import loggingfrom pathlib import Pathfrom typing import Any, castfrom ...errors import InvalidRequestError, NotFoundErrorfrom ...filetypes import DISCOVERABLE_EXTENSIONS, is_office_temp_lock_file+from ..core.db import DatabaseManagerfrom ..core.file_io import get_file_statfrom ..rows import PathRow, ScanRow, WatchTargetRowfrom ..types import (+ FILE_STATUS_ACTIVE,SCAN_KIND_MANUAL,SCAN_KIND_WATCH,- FILE_STATUS_ACTIVE,+ SCAN_SOURCE_UNKNOWN,SCAN_STATUS_DONE,SCAN_STATUS_FAILED,SCAN_STATUS_PENDING,SCAN_STATUS_RUNNING,- SCAN_SOURCE_UNKNOWN,WATCH_STATUS_ACTIVE,WATCH_STATUS_UNREACHABLE,ScanInfo,@@ -28,10 +29,9 @@ from ..types import (ScanSource,ScanStatus,)-from ..core.db import DatabaseManagerfrom ..utils.path_utils import normalize_doclib_pathfrom .config_svc import ConfigService-from .parse_svc import FileRefreshResult, ParseService+from .ingest_svc import FileRefreshResult, IngestServicelogger = logging.getLogger("mineru.scan")@@ -69,14 +69,15 @@ class ScanService:self,db: DatabaseManager,config_svc: ConfigService,- parse_svc: ParseService,+ ingest_svc: IngestService,*,scan_lock_timeout_sec: int,telemetry_svc: object | None = None,) -> None:+ """扫描只依赖文件刷新入库能力,不接管解析执行。"""self.db = dbself.config_svc = config_svc- self.parse_svc = parse_svc+ self.ingest_svc = ingest_svcself.scan_lock_timeout_ms = scan_lock_timeout_sec * 1000self.telemetry_svc = telemetry_svc@@ -356,7 +357,7 @@ class ScanService:await self._refresh_file(row["path"], watch_id=watch_id, counters=counters)async def _refresh_file(self, path: str, *, watch_id: int | None, counters: "_ScanCounters") -> FileRefreshResult:- result = await self.parse_svc.refresh_file(path, watch_id=watch_id)+ result = await self.ingest_svc.refresh_file(path, watch_id=watch_id)counters.add_refresh(result.status)return resultdiff --git a/mineru/doclib/services/search_svc.py b/mineru/doclib/services/search_svc.pyindex b58128da..7db9fa8b 100644--- a/mineru/doclib/services/search_svc.py+++ b/mineru/doclib/services/search_svc.py@@ -13,7 +13,7 @@ from ..rows import ContentSearchResultRow, DocRow, FilenameSearchFileRow, Filenafrom ..types import FILE_STATUS_ACTIVEif TYPE_CHECKING:- from .parse_svc import FileRefreshResult+ from .ingest_svc import FileRefreshResultFilenamePathProbe = Callable[[str], Awaitable["FileRefreshResult"]]_FILENAME_METADATA_BATCH_SIZE = 256diff --git a/tests/unittest/doclib_test_utils.py b/tests/unittest/doclib_test_utils.pynew file mode 100644index 00000000..7cf3d302--- /dev/null+++ b/tests/unittest/doclib_test_utils.py@@ -0,0 +1,33 @@+"""Doclib 测试显式组装共享入库与解析依赖。"""++from __future__ import annotations++from mineru.doclib.core.db import DatabaseManager+from mineru.doclib.core.fts import FTSManager+from mineru.doclib.services.config_svc import ConfigService+from mineru.doclib.services.ingest_svc import IngestService+from mineru.doclib.services.parse_svc import ParseService+from mineru.doclib.telemetry import TelemetryService+++def create_parse_service(+ db: DatabaseManager,+ fts: FTSManager,+ config_svc: ConfigService,+ data_dir: str,+ *,+ parse_lock_timeout_sec: int,+ telemetry_svc: TelemetryService | None = None,+ service_type: type[ParseService] = ParseService,+) -> ParseService:+ """为测试构造共享数据库、索引、规则和指标的两个真实服务。"""+ ingest_svc = IngestService(db, fts, config_svc, telemetry_svc=telemetry_svc)+ return service_type(+ db,+ fts,+ config_svc,+ data_dir,+ ingest_svc=ingest_svc,+ parse_lock_timeout_sec=parse_lock_timeout_sec,+ telemetry_svc=telemetry_svc,+ )diff --git a/tests/unittest/test_cli_parse_full_document.py b/tests/unittest/test_cli_parse_full_document.pyindex 7b1af0d1..e2af2928 100644--- a/tests/unittest/test_cli_parse_full_document.py+++ b/tests/unittest/test_cli_parse_full_document.py@@ -2,6 +2,8 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+import asyncioimport jsonimport re@@ -24,7 +26,6 @@ from mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.fts import FTSManagerfrom mineru.doclib.server import DoclibServerfrom mineru.doclib.services.config_svc import ConfigService-from mineru.doclib.services.parse_svc import ParseServicefrom mineru.doclib.types import (ContentRequestScope,DocContentExportRequest,@@ -59,14 +60,16 @@ class _DoclibHarness:self.loop = asyncio.new_event_loop()self.db = DatabaseManager(str(root / "doclib.db"))self.run(self.db.initialize())- self.service = ParseService(+ self.service = create_parse_service(self.db,FTSManager(self.db),ConfigService(self.db),str(root / "data"),parse_lock_timeout_sec=1800,)- self.server = DoclibServer(SimpleNamespace(db=self.db, data_dir=str(root / "data"), parse_svc=self.service))+ self.server = DoclibServer(+ SimpleNamespace(db=self.db, data_dir=str(root / "data"), parse_svc=self.service, ingest_svc=self.service.ingest_svc)+ )self.output_ranges: list[str | None] = []def run(self, operation: Coroutine[Any, Any, _T]) -> _T:diff --git a/tests/unittest/test_doclib_architecture.py b/tests/unittest/test_doclib_architecture.pyindex 14ffc7eb..4e7c5285 100644--- a/tests/unittest/test_doclib_architecture.py+++ b/tests/unittest/test_doclib_architecture.py@@ -49,3 +49,18 @@ def test_legacy_protocol_conversion_stays_at_cached_middle_json_boundary() -> Noif any(module.startswith("mineru.backend.postprocess.legacy_") for module in _imports(path))}assert consumers == {boundary}+++def test_ingest_service_does_not_depend_on_parse_execution_or_http() -> None:+ """文件入库拥有独立实现,不反向导入解析服务、HTTP 层或宿主应用。"""+ forbidden = ("fastapi", "starlette", "mineru.doclib.server", "mineru.doclib.app", "mineru.doclib.services.parse_svc")+ imports = _imports(_ROOT / "mineru/doclib/services/ingest_svc.py")+ assert not sorted(module for module in imports if module.startswith(forbidden))+++def test_file_discovery_workers_depend_on_ingest_service() -> None:+ """扫描和文件事件处理依赖入库边界,解析工作池独立保留解析服务依赖。"""+ for name in ("services/scan_svc.py", "background/watch.py", "background/ingest.py"):+ imports = _imports(_ROOT / "mineru/doclib" / name)+ assert "mineru.doclib.services.ingest_svc.IngestService" in imports, name+ assert not any(module.startswith("mineru.doclib.services.parse_svc") for module in imports), namediff --git a/tests/unittest/test_doclib_cache_semantics.py b/tests/unittest/test_doclib_cache_semantics.pyindex 67c1e419..f2534114 100644--- a/tests/unittest/test_doclib_cache_semantics.py+++ b/tests/unittest/test_doclib_cache_semantics.py@@ -1,4 +1,9 @@from __future__ import annotations++from doclib_test_utils import create_parse_service+from mineru.doclib.services import ingest_svc as ingest_svc_module+from mineru.doclib.services.ingest_svc import FileRefreshResult+from mineru.doclib.services.parse_policy import resolve_parsing_rule_default_tierfrom mineru.doclib.services.content_read_svc import ContentReadServicefrom docvortex.schema import Producerfrom mineru.integrations.docvortex import build_metadata@@ -47,19 +52,10 @@ from mineru.doclib.services.content_read_svc import (_ReadPlan,_locator_after,)-from mineru.doclib.services import parse_svc as parse_svc_modulefrom mineru.doclib.core import parse_artifacts as parse_artifacts_modulefrom mineru.doclib.services.cleanup_svc import CleanupServicefrom mineru.doclib.services.config_svc import ConfigService-from mineru.doclib.services.parse_svc import (- FileRefreshResult,- ParseFailure,- ParseService,- _local_parse_server_url,- _resolve_default_tier,- _resolve_parsing_rule_default_tier,- expand_page_range,-)+from mineru.doclib.services.parse_svc import ParseFailure, _local_parse_server_url, _resolve_default_tier, expand_page_rangefrom mineru.doclib.core.parse_artifacts import filter_pages_by_user_range, load_pages_from_done_batches, parse_batch_json_pathfrom mineru.doclib.services.scan_svc import ScanServicefrom mineru.doclib.services.search_svc import SearchService@@ -458,13 +454,13 @@ def test_parsing_rule_default_tier_allows_flash_fallback(monkeypatch: pytest.Monhealth = ParseServerHealth(local=ProbeState(probe=ProbeResult(tiers=["basic", "advanced"])))monkeypatch.setattr("mineru.doclib.background.parse_server_health.get_health", lambda: health)- assert _resolve_parsing_rule_default_tier() == "basic"+ assert resolve_parsing_rule_default_tier() == "basic"health.local.probe = ProbeResult(tiers=["basic", "standard", "advanced"])- assert _resolve_parsing_rule_default_tier() == "standard"+ assert resolve_parsing_rule_default_tier() == "standard"health.local.probe = ProbeResult(tiers=["flash"])- assert _resolve_parsing_rule_default_tier() == "flash"+ assert resolve_parsing_rule_default_tier() == "flash"def test_read_without_tier_uses_highest_cached_tier(tmp_path: Path) -> None:@@ -1135,7 +1131,7 @@ def test_remote_api_target_prefers_config_api_key_over_env(monkeypatch: pytest.Masync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_ConfigService(),@@ -1174,7 +1170,7 @@ def test_remote_api_target_fallback_uses_local_self_hosted_api_key(monkeypatch:async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_ConfigService(),@@ -1216,7 +1212,7 @@ def test_remote_api_target_fallback_managed_mode_sends_no_api_key(monkeypatch: pasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_ConfigService(),@@ -1255,7 +1251,7 @@ def test_remote_api_target_preserves_probe_invalid_api_key(monkeypatch: pytest.Masync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_ConfigService(),@@ -1292,7 +1288,7 @@ def test_self_hosted_api_target_preserves_probe_invalid_api_key(monkeypatch: pytasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_ConfigService(),@@ -1478,7 +1474,7 @@ def test_invalidate_deletes_fts_when_no_done_batches_remain(tmp_path: Path) -> Nparses = [{"sha256": sha256, "tier": "standard", "page_range": "1", "status": "done", "done_at": 1000}]db = _FakeDB(parses=parses, file_row={"sha256": sha256, "status": "active", "filename": "doc.pdf"})fts = _FakeFTS()- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)count = asyncio.run(service.invalidate(sha256, "standard"))@@ -1498,7 +1494,7 @@ def test_invalidate_rebuilds_fts_from_highest_remaining_done_tier(tmp_path: Path]db = _FakeDB(parses=parses, file_row={"sha256": sha256, "status": "active", "filename": "doc.pdf"})fts = _FakeFTS()- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)count = asyncio.run(service.invalidate(sha256, "standard"))@@ -1518,7 +1514,7 @@ def test_ingest_binds_file_sha_before_creating_parse_task(tmp_path: Path, monkeydb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()recording_db = _OrderRecordingDB(db)- service = ParseService(+ service = create_parse_service(db=recording_db,fts=_FilenameOnlyFTS(),config_svc=_NoRulesConfig(),@@ -1538,9 +1534,9 @@ def test_ingest_binds_file_sha_before_creating_parse_task(tmp_path: Path, monkey"is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)- row = await service.ingest_file(str(source))+ row = await service.ingest_svc.ingest_file(str(source))dangling_parses = await db.fetchall("SELECT p.id FROM parses p LEFT JOIN files f ON f.sha256=p.sha256 AND f.status='active' WHERE f.id IS NULL",)@@ -1560,7 +1556,7 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: pyasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=_FilenameOnlyFTS(),config_svc=_NoRulesConfig(),@@ -1578,7 +1574,7 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: py"is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)old_dir = tmp_path / "old"new_dir = tmp_path / "new"@@ -1586,10 +1582,10 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: pynew_dir.mkdir()source = old_dir / "doc.pdf"source.write_bytes(b"%PDF-1.4\n")- first_row = await service.ingest_file(str(source))+ first_row = await service.ingest_svc.ingest_file(str(source))source.rename(new_dir / "doc.pdf")- second_row = await service.ingest_file(str(new_dir / "doc.pdf"))+ second_row = await service.ingest_svc.ingest_file(str(new_dir / "doc.pdf"))assert first_row is not Noneassert second_row is not None@@ -1641,7 +1637,7 @@ def test_request_parse_explicit_image_ingests_and_queues_parse(tmp_path: Path, masync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -1661,7 +1657,7 @@ def test_request_parse_explicit_image_ingests_and_queues_parse(tmp_path: Path, m"is_image_based": 1,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)result = await service.request_parse(str(source), tier="flash")with pytest.raises(InvalidRequestError) as range_exc:@@ -1704,7 +1700,7 @@ def test_request_parse_rejects_quality_tiers_for_non_pdf_image_inputs(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -1724,7 +1720,7 @@ def test_request_parse_rejects_quality_tiers_for_non_pdf_image_inputs("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)with pytest.raises(InvalidRequestError) as exc_info:await service.request_parse(str(source), tier=tier)@@ -1760,7 +1756,7 @@ def test_refresh_file_applies_parsing_rule_effective_tier_and_privacy(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_RuleConfig(),@@ -1780,9 +1776,9 @@ def test_refresh_file_applies_parsing_rule_effective_tier_and_privacy("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)- refreshed = await service.refresh_file(str(source), ensure_ingested=True)+ refreshed = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)assert refreshed.file is not Noneparse_rows = await db.fetchall("SELECT tier, privacy, page_range FROM parses")@@ -1804,7 +1800,7 @@ def test_request_parse_rejects_remote_for_non_pdf_image_inputs(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -1824,7 +1820,7 @@ def test_request_parse_rejects_remote_for_non_pdf_image_inputs("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)with pytest.raises(InvalidRequestError) as exc_info:await service.request_parse(str(source), remote=True)@@ -1862,7 +1858,7 @@ def test_request_parse_reports_text_files_do_not_require_parsing(db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()fts = FTSManager(db)- service = ParseService(+ service = create_parse_service(db=db,fts=fts,config_svc=_NoRulesConfig(),@@ -1882,7 +1878,7 @@ def test_request_parse_reports_text_files_do_not_require_parsing("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)with pytest.raises(InvalidRequestError) as exc_info:await service.request_parse(str(source), tier=tier, force=force, remote=remote)@@ -1915,7 +1911,7 @@ def test_csv_doclib_parse_creates_flash_cache_and_rendered_fts(db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()fts = FTSManager(db)- service = ParseService(+ service = create_parse_service(db=db,fts=fts,config_svc=_NoRulesConfig(),@@ -1967,7 +1963,7 @@ def test_tsv_doclib_parse_creates_flash_cache_with_independent_file_type(db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()fts = FTSManager(db)- service = ParseService(+ service = create_parse_service(db=db,fts=fts,config_svc=_NoRulesConfig(),@@ -2018,7 +2014,7 @@ def test_request_parse_rejects_flash_tier_for_remote_quality_inputs(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -2038,7 +2034,7 @@ def test_request_parse_rejects_flash_tier_for_remote_quality_inputs("is_image_based": int(ext != "pdf"),}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)with pytest.raises(InvalidRequestError) as exc_info:await service.request_parse(str(source), tier="flash", remote=True)@@ -2055,7 +2051,7 @@ def test_request_parse_rejects_unsupported_file_type_before_parse_response(tmp_pasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=None,@@ -2088,7 +2084,7 @@ def test_request_parse_accepts_legacy_office_formats(tmp_path: Path, ext: str) -async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -2111,7 +2107,7 @@ def test_request_parse_raises_ingest_failed_when_file_row_has_no_sha(tmp_path: Pasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=None,@@ -2144,7 +2140,7 @@ def test_request_parse_raises_ingest_failed_when_file_row_has_no_sha(tmp_path: Pstatus="known",)- service.refresh_file = _refresh_file # type: ignore[method-assign]+ service.ingest_svc.refresh_file = _refresh_file # type: ignore[method-assign]with pytest.raises(MineruError) as exc_info:await service.request_parse(str(source), tier="flash")@@ -2198,7 +2194,7 @@ def test_force_request_reuses_active_and_creates_only_uncovered_parse(tmp_path:},doc_row={"sha256": sha256, "short_id": "eeeeeee", "page_count": 10},)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)result = asyncio.run(service.request_parse(path, tier="standard", page_range="1-10", force=True))@@ -2231,7 +2227,7 @@ def test_non_pdf_requests_use_one_full_document_batch("""依次验证初始任务、历史 partial 修复和 active 复用。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(),@@ -2252,7 +2248,7 @@ def test_non_pdf_requests_use_one_full_document_batch("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)initial = await service.request_parse(str(source))assert initial.page_range == "1-12"@@ -2306,7 +2302,7 @@ def test_list_parse_records_by_ids_returns_precise_status(tmp_path: Path) -> Non},]db = _FakeDB(parses=parses, file_row=None)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)result = asyncio.run(service.list_parse_records(ids=[2, 1]))@@ -2423,7 +2419,7 @@ def test_ensure_doc_record_extends_short_id_on_prefix_conflict(tmp_path: Path) -sha_b = ("a" * 7) + ("b" * 57)for sha256 in (sha_a, sha_b):- await parse_svc_module.ensure_doc_record(+ await ingest_svc_module.ensure_doc_record(db,sha256=sha256,size_bytes=10,@@ -2452,19 +2448,19 @@ def test_ensure_ingested_rebinds_changed_text_file_to_new_sha(tmp_path: Path) ->async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"source.write_text("old", encoding="utf-8")- first = await service.ensure_ingested(str(source))+ first = await service.ingest_svc.ensure_ingested(str(source))assert first is not Noneassert first["sha256"] == hashlib.sha256(b"old").hexdigest()source.write_text("new content", encoding="utf-8")- discovered = await service.refresh_file(str(source))+ discovered = await service.ingest_svc.refresh_file(str(source))changed_row = await db.fetchone("SELECT * FROM files WHERE path=?", (str(source),))assert discovered.status == "changed"@@ -2474,7 +2470,7 @@ def test_ensure_ingested_rebinds_changed_text_file_to_new_sha(tmp_path: Path) ->assert changed_row is not Noneassert changed_row["sha256"] is None- second = await service.ensure_ingested(str(source))+ second = await service.ingest_svc.ensure_ingested(str(source))assert second is not Noneassert second["sha256"] == hashlib.sha256(b"new content").hexdigest()@@ -2488,7 +2484,7 @@ def test_refresh_file_normalizes_lexical_path_aliases(tmp_path: Path) -> None:async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)root = tmp_path / "docs"@@ -2497,8 +2493,8 @@ def test_refresh_file_normalizes_lexical_path_aliases(tmp_path: Path) -> None:source.write_text("content", encoding="utf-8")alias = root / "." / "nested" / ".." / "note.txt"- first = await service.refresh_file(str(alias))- second = await service.refresh_file(str(source))+ first = await service.ingest_svc.refresh_file(str(alias))+ second = await service.ingest_svc.refresh_file(str(source))rows = await db.fetchall("SELECT path FROM files ORDER BY id")assert first.status == "new"@@ -2512,7 +2508,7 @@ def test_refresh_file_keeps_symlink_and_real_paths_as_distinct_file_entries(tmp_async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)real_root = tmp_path / "real-root"@@ -2522,8 +2518,8 @@ def test_refresh_file_keeps_symlink_and_real_paths_as_distinct_file_entries(tmp_source = real_root / "note.txt"source.write_text("content", encoding="utf-8")- alias_result = await service.refresh_file(str(alias_root / "note.txt"))- real_result = await service.refresh_file(str(source))+ alias_result = await service.ingest_svc.refresh_file(str(alias_root / "note.txt"))+ real_result = await service.ingest_svc.refresh_file(str(source))rows = await db.fetchall("SELECT path FROM files ORDER BY path")assert alias_result.status == "new"@@ -2537,18 +2533,18 @@ def test_refresh_file_marks_missing_known_path_deleted(tmp_path: Path) -> None:async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"source.write_text("content", encoding="utf-8")- ingested = await service.ensure_ingested(str(source))+ ingested = await service.ingest_svc.ensure_ingested(str(source))assert ingested is not Noneoriginal_sha = ingested["sha256"]source.unlink()- refreshed = await service.refresh_file(str(source))+ refreshed = await service.ingest_svc.refresh_file(str(source))row = await db.fetchone("SELECT sha256, status, deleted_at FROM files WHERE path=?", (str(source),))assert refreshed.status == "deleted"@@ -2569,7 +2565,7 @@ def test_ensure_ingested_maps_read_permission_error_to_file_permission_denied(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "locked.pdf"@@ -2578,10 +2574,10 @@ def test_ensure_ingested_maps_read_permission_error_to_file_permission_denied(async def _permission_denied(path: str) -> str:raise PermissionError("permission denied")- monkeypatch.setattr(parse_svc_module, "compute_sha256", _permission_denied)+ monkeypatch.setattr(ingest_svc_module, "compute_sha256", _permission_denied)with pytest.raises(InvalidRequestError) as exc_info:- await service.ensure_ingested(str(source))+ await service.ingest_svc.ensure_ingested(str(source))row = await db.fetchone("SELECT sha256, error_code, error_msg FROM files WHERE path=?", (str(source),))@@ -2599,21 +2595,21 @@ def test_refresh_file_records_stat_error_without_marking_deleted(tmp_path: Path,async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"source.write_text("content", encoding="utf-8")- ingested = await service.ensure_ingested(str(source))+ ingested = await service.ingest_svc.ensure_ingested(str(source))assert ingested is not Noneasync def _permission_denied(path: str) -> dict[str, Any]:raise PermissionError("permission denied")- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)- refreshed = await service.refresh_file(str(source))+ refreshed = await service.ingest_svc.refresh_file(str(source))row = await db.fetchone("SELECT status, error_code, error_msg, deleted_at FROM files WHERE path=?", (str(source),))assert refreshed.status == "error"@@ -2635,7 +2631,7 @@ def test_request_parse_maps_stat_permission_error_without_existing_file_row(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "no_exec_dir" / "inside.pdf"@@ -2643,7 +2639,7 @@ def test_request_parse_maps_stat_permission_error_without_existing_file_row(async def _permission_denied(path: str) -> dict[str, Any]:raise PermissionError(f"permission denied: {path}")- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)with pytest.raises(InvalidRequestError) as exc_info:await service.request_parse(str(source), tier="flash")@@ -2659,22 +2655,22 @@ def test_ingest_worker_skips_files_with_blocking_stat_errors(tmp_path: Path, monasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"source.write_text("content", encoding="utf-8")- assert await service.ensure_ingested(str(source)) is not None+ assert await service.ingest_svc.ensure_ingested(str(source)) is not Noneasync def _permission_denied(path: str) -> dict[str, Any]:raise PermissionError("permission denied")- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)- await service.refresh_file(str(source))+ await service.ingest_svc.refresh_file(str(source))- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)assert await worker._acquire_task() is Nonerow = await db.fetchone("SELECT error_code, error_msg, locked_at FROM files WHERE path=?", (str(source),))@@ -2690,7 +2686,7 @@ def test_ingest_worker_preserves_mineru_permission_errors(tmp_path: Path) -> Nonasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"@@ -2701,7 +2697,7 @@ def test_ingest_worker_preserves_mineru_permission_errors(tmp_path: Path) -> Non(str(source), source.name, "txt", 10, now, now, now),)- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)await worker._handle_ingest_error({"id": file_id, "path": str(source), "watch_id": None},InvalidRequestError("file_permission_denied", "permission denied", "path"),@@ -2721,7 +2717,7 @@ def test_ingest_worker_preserves_precise_file_access_errors(tmp_path: Path) -> Nasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"@@ -2732,7 +2728,7 @@ def test_ingest_worker_preserves_precise_file_access_errors(tmp_path: Path) -> N(str(source), source.name, "txt", 10, now, now, now),)- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)await worker._handle_ingest_error({"id": file_id, "path": str(source), "watch_id": None},PermissionError("permission denied"),@@ -2752,7 +2748,7 @@ def test_ingest_worker_skips_any_file_with_existing_error(tmp_path: Path) -> Nonasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "note.txt"@@ -2763,7 +2759,7 @@ def test_ingest_worker_skips_any_file_with_existing_error(tmp_path: Path) -> Non(str(source), source.name, "txt", 10, now, "ingest_failed", "boom", now, now),)- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)assert await worker._acquire_task() is Nonerow = await db.fetchone("SELECT error_code, locked_at FROM files WHERE path=?", (str(source),))@@ -2953,7 +2949,7 @@ def test_find_probes_and_filters_deleted_paths_without_rescan(tmp_path: Path) ->await db.initialize()fts = FTSManager(db)config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db,fts=fts,config_svc=config_svc,@@ -2967,11 +2963,12 @@ def test_find_probes_and_filters_deleted_paths_without_rescan(tmp_path: Path) ->parse_svc=parse_svc,search_svc=SearchService(db, fts),telemetry_svc=None,+ ingest_svc=parse_svc.ingest_svc,))source = tmp_path / "bench1.txt"source.write_text("hello", encoding="utf-8")- assert await parse_svc.ensure_ingested(str(source)) is not None+ assert await parse_svc.ingest_svc.ensure_ingested(str(source)) is not Nonesource.unlink()result = await server.find("bench1")@@ -2992,7 +2989,7 @@ def test_find_reactivates_restored_deleted_path_without_rescan(tmp_path: Path) -await db.initialize()fts = FTSManager(db)config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db,fts=fts,config_svc=config_svc,@@ -3006,11 +3003,12 @@ def test_find_reactivates_restored_deleted_path_without_rescan(tmp_path: Path) -parse_svc=parse_svc,search_svc=SearchService(db, fts),telemetry_svc=None,+ ingest_svc=parse_svc.ingest_svc,))source = tmp_path / "stale_restore.txt"source.write_text("hello", encoding="utf-8")- assert await parse_svc.ensure_ingested(str(source)) is not None+ assert await parse_svc.ingest_svc.ensure_ingested(str(source)) is not Nonesource.unlink()deleted = await server.find("stale_restore")@@ -3033,7 +3031,7 @@ def test_refresh_file_marks_only_current_file_unreachable_when_watch_root_is_misdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)root = tmp_path / "removable"@@ -3044,13 +3042,13 @@ def test_refresh_file_marks_only_current_file_unreachable_when_watch_root_is_missource.write_text("content", encoding="utf-8")other.write_text("other", encoding="utf-8")- assert await service.ensure_ingested(str(source), watch_id=watch["id"]) is not None- assert await service.ensure_ingested(str(other), watch_id=watch["id"]) is not None+ assert await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"]) is not None+ assert await service.ingest_svc.ensure_ingested(str(other), watch_id=watch["id"]) is not Nonesource.unlink()other.unlink()root.rmdir()- refreshed = await service.refresh_file(str(source), watch_id=watch["id"])+ refreshed = await service.ingest_svc.refresh_file(str(source), watch_id=watch["id"])source_row = await db.fetchone("SELECT status, error_code, error_msg, deleted_at FROM files WHERE path=?", (str(source),)@@ -3077,7 +3075,7 @@ def test_remove_watch_converts_unreachable_files_to_deleted_and_unbinds_watch(tmdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)root = tmp_path / "removable"@@ -3086,7 +3084,7 @@ def test_remove_watch_converts_unreachable_files_to_deleted_and_unbinds_watch(tmsource = root / "note.txt"source.write_text("content", encoding="utf-8")- assert await service.ensure_ingested(str(source), watch_id=watch["id"]) is not None+ assert await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"]) is not Nonenow = 123456await db.execute("INSERT INTO scans (path, kind, source, watch_id, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",@@ -3115,17 +3113,17 @@ def test_watch_scan_refreshes_known_active_paths_before_walk(tmp_path: Path) ->await db.initialize()config_svc = ConfigService(db)fts = FTSManager(db)- service = ParseService(+ service = create_parse_service(db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=service, scan_interval_sec=300)+ watch_loop = WatchLoop(db=db, config_svc=config_svc, ingest_svc=service.ingest_svc, scan_interval_sec=300)root = tmp_path / "watched"root.mkdir()watch = await config_svc.add_watch(str(root), removable=False)source = root / "note.txt"source.write_text("content", encoding="utf-8")- ingested = await service.ensure_ingested(str(source), watch_id=watch["id"])+ ingested = await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])assert ingested is not Nonesource.unlink()@@ -3145,16 +3143,16 @@ def test_watch_scan_binds_unchanged_existing_file_to_watch(tmp_path: Path) -> Noawait db.initialize()config_svc = ConfigService(db)fts = FTSManager(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)root = tmp_path / "watched"root.mkdir()source = root / "note.txt"source.write_text("content", encoding="utf-8")- initial = await parse_svc.refresh_file(str(source))+ initial = await parse_svc.ingest_svc.refresh_file(str(source))watch = await config_svc.add_watch(str(root), removable=False)scan = await scan_svc.create_scan(str(root), kind="watch", source="cli", watch_id=watch["id"])task = await scan_svc.acquire_task()@@ -3180,17 +3178,17 @@ def test_watch_scan_marks_root_unreachable_without_refreshing_all_files(tmp_pathawait db.initialize()config_svc = ConfigService(db)fts = FTSManager(db)- service = ParseService(+ service = create_parse_service(db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=service, scan_interval_sec=300)+ watch_loop = WatchLoop(db=db, config_svc=config_svc, ingest_svc=service.ingest_svc, scan_interval_sec=300)root = tmp_path / "removable"root.mkdir()watch = await config_svc.add_watch(str(root), removable=True)source = root / "note.txt"source.write_text("content", encoding="utf-8")- ingested = await service.ensure_ingested(str(source), watch_id=watch["id"])+ ingested = await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])assert ingested is not Nonesource.unlink()@@ -3213,10 +3211,10 @@ def test_device_monitor_queues_watch_scan_when_removable_watch_recovers(tmp_pathawait db.initialize()config_svc = ConfigService(db)fts = FTSManager(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)monitor = DeviceMonitor(db=db, config_svc=config_svc, interval_sec=5, scan_svc=scan_svc)root = tmp_path / "removable"root.mkdir()@@ -3224,7 +3222,7 @@ def test_device_monitor_queues_watch_scan_when_removable_watch_recovers(tmp_pathsource = root / "note.txt"source.write_text("content", encoding="utf-8")- ingested = await parse_svc.ensure_ingested(str(source), watch_id=watch["id"])+ ingested = await parse_svc.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])assert ingested is not Nonesource.unlink()@@ -3263,10 +3261,10 @@ def test_scan_service_creates_and_processes_manual_file_scan(tmp_path: Path) ->db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)source = tmp_path / "note.txt"source.write_text("content", encoding="utf-8")@@ -3296,10 +3294,10 @@ def test_scan_service_directory_scan_applies_excludes_and_counts_unsupported(tmpdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)root = tmp_path / "docs"root.mkdir()included = root / "included.txt"@@ -3332,7 +3330,7 @@ def test_refresh_file_ignores_office_temp_lock_files(tmp_path: Path) -> None:async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=ConfigService(db),@@ -3342,7 +3340,7 @@ def test_refresh_file_ignores_office_temp_lock_files(tmp_path: Path) -> None:source = tmp_path / "~$package-size.xlsx"source.write_bytes(b"\x15Microsoft Office lock")- refreshed = await service.refresh_file(str(source), ensure_ingested=True)+ refreshed = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)row = await db.fetchone("SELECT path FROM files WHERE path=?", (str(source),))assert refreshed.status == "unsupported"@@ -3356,7 +3354,7 @@ def test_ensure_ingested_ignores_existing_office_temp_lock_file_row(tmp_path: Paasync def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=ConfigService(db),@@ -3372,7 +3370,7 @@ def test_ensure_ingested_ignores_existing_office_temp_lock_file_row(tmp_path: Pa(str(source), source.name, "xlsx", source.stat().st_size, now, None, "active", now, now),)- ingested = await service.ensure_ingested(str(source))+ ingested = await service.ingest_svc.ensure_ingested(str(source))assert ingested is None@@ -3384,10 +3382,10 @@ def test_scan_service_directory_scan_ignores_office_temp_lock_files(tmp_path: Padb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)root = tmp_path / "docs"root.mkdir()regular = root / "regular.txt"@@ -3429,7 +3427,7 @@ def test_watch_event_ignores_office_temp_lock_files(tmp_path: Path) -> None:watch_loop = WatchLoop(db=SimpleNamespace(),config_svc=_ConfigService(),- parse_svc=parse_svc,+ ingest_svc=parse_svc,scan_interval_sec=300,)lock_file = tmp_path / "~$package-size.xlsx"@@ -3459,7 +3457,7 @@ def test_watch_event_ignores_image_files(tmp_path: Path) -> None:watch_loop = WatchLoop(db=SimpleNamespace(),config_svc=_ConfigService(),- parse_svc=parse_svc,+ ingest_svc=parse_svc,scan_interval_sec=300,)image_file = tmp_path / "scan.png"@@ -3498,7 +3496,7 @@ def test_watch_event_rebases_realpath_alias_to_watch_root_namespace(tmp_path: Pawatch_loop = WatchLoop(db=SimpleNamespace(),config_svc=_ConfigService(),- parse_svc=parse_svc,+ ingest_svc=parse_svc,scan_interval_sec=300,)@@ -3514,10 +3512,10 @@ def test_scan_service_reuses_pending_scan_for_same_kind_and_path(tmp_path: Path)db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)root = tmp_path / "docs"root.mkdir()@@ -3536,10 +3534,10 @@ def test_scan_service_does_not_reuse_stale_running_scan_for_same_kind_and_path(tdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1)root = tmp_path / "docs"root.mkdir()@@ -3597,11 +3595,11 @@ def test_doclib_server_list_responses_include_pagination_metadata(tmp_path: Pathdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)- server = DoclibServer(SimpleNamespace(db=db, scan_svc=scan_svc, parse_svc=_NoopParseService()))+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)+ server = DoclibServer(SimpleNamespace(db=db, scan_svc=scan_svc, ingest_svc=_NoopParseService()))for index in range(3):sha256 = str(index + 1) * 64@@ -3665,14 +3663,16 @@ def test_doclib_server_accepts_short_id_for_sha256_doc_inputs(tmp_path: Path) ->await db.initialize()config_svc = ConfigService(db)fts = FTSManager(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db,fts=fts,config_svc=config_svc,data_dir=str(tmp_path),parse_lock_timeout_sec=1800,)- server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc))+ server = DoclibServer(+ SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc)+ )now = 1000sha256 = "a" * 64short_id = "aaaaaaa"@@ -3751,14 +3751,16 @@ def test_doclib_server_reports_text_invalidation_not_required(db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()fts = FTSManager(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db,fts=fts,config_svc=ConfigService(db),data_dir=str(tmp_path / "data"),parse_lock_timeout_sec=1800,)- server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc))+ server = DoclibServer(+ SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc)+ )source = tmp_path / f"note.{ext}"source.write_text("unique searchable content", encoding="utf-8")@@ -3772,7 +3774,7 @@ def test_doclib_server_reports_text_invalidation_not_required("is_image_based": 0,}- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)with pytest.raises(InvalidRequestError) as exc_info:await server.invalidate(InvalidateRequest(path=str(source)))@@ -3884,7 +3886,7 @@ def test_get_file_by_path_missing_doclib_record_message_is_not_disk_file_not_fouasync def _run() -> None:db = _FakeDB(parses=[], file_row=None)- server = DoclibServer(SimpleNamespace(db=db, parse_svc=_ParseSvc()))+ server = DoclibServer(SimpleNamespace(db=db, ingest_svc=_ParseSvc()))with pytest.raises(NotFoundError) as exc_info:await server.get_file_by_path(str(tmp_path / "sample.png"))@@ -3903,16 +3905,16 @@ def test_get_file_by_path_maps_stat_permission_error_without_existing_file_row(async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- server = DoclibServer(SimpleNamespace(db=db, parse_svc=parse_svc))+ server = DoclibServer(SimpleNamespace(db=db, parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc))source = tmp_path / "no_exec_dir" / "inside.pdf"async def _permission_denied(path: str) -> dict[str, Any]:raise PermissionError(f"permission denied: {path}")- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)try:with pytest.raises(InvalidRequestError) as exc_info:@@ -3932,10 +3934,10 @@ def test_scan_service_cleanup_keeps_latest_terminal_scans_and_active_tasks(tmp_pdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)for index in range(1002):status = "done" if index % 2 == 0 else "failed"@@ -3970,11 +3972,13 @@ def test_watch_loop_queues_watch_scan_when_scan_service_is_available(tmp_path: Pdb = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()config_svc = ConfigService(db)- parse_svc = ParseService(+ parse_svc = create_parse_service(db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_interval_sec=300, scan_svc=scan_svc)+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)+ watch_loop = WatchLoop(+ db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_interval_sec=300, scan_svc=scan_svc+ )root = tmp_path / "watched"root.mkdir()watch = await config_svc.add_watch(str(root), removable=False)@@ -4075,7 +4079,7 @@ def test_watch_loop_starts_watcher_before_initial_scan() -> None:watch_loop = WatchLoop(db=None,config_svc=_ConfigService(),- parse_svc=None,+ ingest_svc=None,scan_interval_sec=0,)@@ -4114,7 +4118,7 @@ def test_watch_loop_wakeup_triggers_next_poll_without_interval_delay() -> None:watch_loop = WatchLoop(db=None,config_svc=_ConfigService(),- parse_svc=None,+ ingest_svc=None,scan_interval_sec=300,)task = asyncio.create_task(watch_loop.run())@@ -4509,18 +4513,18 @@ def test_ingest_records_doc_error_when_metadata_extraction_fails(tmp_path: Path,async def _run() -> None:db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=_NoRulesConfig(), data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800)source = tmp_path / "broken.pdf"source.write_bytes(b"%PDF-1.4\nnot actually a valid pdf")async def _fail_metadata(path: str) -> dict[str, Any]:- raise parse_svc_module.MetadataExtractionError("open_failed", "metadata boom")+ raise ingest_svc_module.MetadataExtractionError("open_failed", "metadata boom")- monkeypatch.setattr(parse_svc_module, "extract_metadata", _fail_metadata)+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _fail_metadata)- row = await service.ingest_file(str(source))+ row = await service.ingest_svc.ingest_file(str(source))doc = await db.fetchone("SELECT error_code, error_msg FROM docs WHERE sha256=?", (row["sha256"],))assert doc is not None@@ -4563,7 +4567,7 @@ def test_process_doc_rejects_existing_office_temp_lock_file_task(tmp_path: Path)"author": "",},)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)async def _unexpected_parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:raise AssertionError("temporary lock files should not reach parser execution")@@ -4611,7 +4615,7 @@ def test_process_doc_marks_empty_page_result_failed(tmp_path: Path) -> None:"author": "",},)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)async def _empty_parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:return ParseResult(@@ -4669,7 +4673,7 @@ def test_process_doc_preserves_remote_api_error_code(tmp_path: Path) -> None:"author": "",},)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)async def _parse_via_api(file_row: dict[str, object],@@ -4706,7 +4710,7 @@ def test_parse_via_api_maps_transport_errors_by_type(cause: httpx.TransportError,expected_code: str,) -> None:- service = ParseService(+ service = create_parse_service(db=_FakeDB(parses=[], file_row=None),fts=_FakeFTS(),config_svc=None,@@ -4752,7 +4756,7 @@ def test_parse_via_api_requests_image_cache_only_for_office(expected_page_range: str,) -> None:calls: list[bool] = []- service = ParseService(+ service = create_parse_service(db=_FakeDB(parses=[], file_row=None),fts=_FakeFTS(),config_svc=None,@@ -4844,7 +4848,7 @@ def test_process_doc_fails_when_batch_json_cannot_be_written(tmp_path: Path) ->},)fts = _FakeFTS()- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)blocked_output_dir = tmp_path / "parsed" / sha256[:2] / sha256 / "flash"blocked_output_dir.parent.mkdir(parents=True)blocked_output_dir.write_text("not a directory", encoding="utf-8")@@ -4910,7 +4914,7 @@ def test_process_doc_normalizes_full_document_range_from_actual_pages(tmp_path:},doc_row=doc_row,)- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)async def _parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:"""返回十二页整本 EPUB 结果。"""diff --git a/tests/unittest/test_doclib_find_pagination.py b/tests/unittest/test_doclib_find_pagination.pyindex de7d7107..82fd9f70 100644--- a/tests/unittest/test_doclib_find_pagination.py+++ b/tests/unittest/test_doclib_find_pagination.py@@ -2,6 +2,8 @@from __future__ import annotations+from mineru.doclib.services.ingest_svc import FileRefreshResult+import asynciofrom pathlib import Pathfrom types import SimpleNamespace@@ -17,7 +19,7 @@ from mineru.doclib.client import DoclibClientfrom mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.fts import FTSManager, tokenize_for_indexfrom mineru.doclib.server import DoclibServer-from mineru.doclib.services.parse_svc import FileRefreshResult+from mineru.doclib.services.search_svc import SearchServicefrom mineru.doclib.types import FindResponsefrom mineru.errors import InvalidRequestError@@ -166,7 +168,7 @@ def test_find_http_exposes_offset_and_validates_pagination(tmp_path: Path) -> No"""保留已有有效记录,让接口走正常刷新边界。"""return FileRefreshResult(file=None, status="known")- server = DoclibServer(SimpleNamespace(db=db, search_svc=service, parse_svc=SimpleNamespace(refresh_file=refresh)))+ server = DoclibServer(SimpleNamespace(db=db, search_svc=service, ingest_svc=SimpleNamespace(refresh_file=refresh)))try:parameters = server.app.openapi()["paths"]["/api/v1/find"]["get"]["parameters"]assert next(item for item in parameters if item["name"] == "offset")["schema"]["default"] == 0diff --git a/tests/unittest/test_doclib_ingest_service.py b/tests/unittest/test_doclib_ingest_service.pynew file mode 100644index 00000000..9cf4a4b4--- /dev/null+++ b/tests/unittest/test_doclib_ingest_service.py@@ -0,0 +1,84 @@+"""验证独立入库服务和宿主实例中的共享依赖,不执行后台解析模型。"""++from __future__ import annotations++import asyncio+from pathlib import Path++import pytest+from fastapi.testclient import TestClient++from mineru.config import PatchedConfig+from mineru.doclib import app as doclib_app+from mineru.doclib.core.db import DatabaseManager+from mineru.doclib.core.fts import FTSManager+from mineru.doclib.services.config_svc import ConfigService+from mineru.doclib.services.ingest_svc import IngestService+from mineru.doclib.services.parse_svc import ParseService+++def test_ingestion_queues_native_document_without_constructing_parse_service(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """独立入库可保存 HTML 属性和文本索引,不需要构造解析执行服务。"""++ def forbidden_parse_service(*args: object, **kwargs: object) -> None:+ """任何隐式解析服务组装都使本用例失败。"""+ raise AssertionError("IngestService constructed a ParseService")++ monkeypatch.setattr(ParseService, "__init__", forbidden_parse_service)++ async def scenario() -> None:+ """真实 SQLite 和 FTS 覆盖文件刷新、纯文本索引以及初始原生任务登记。"""+ db = DatabaseManager(str(tmp_path / "library.sqlite"))+ await db.initialize()+ try:+ fts = FTSManager(db)+ ingest = IngestService(db, fts, ConfigService(db))+ html = tmp_path / "source.html"+ html.write_text("<html><head><title>Independent ingest</title></head><body>Native source</body></html>")+ plain = tmp_path / "note.txt"+ plain.write_text("Independent indexed text", encoding="utf-8")+ document = await ingest.ensure_ingested(str(html))+ text = await ingest.ensure_ingested(str(plain))+ assert document is not None and text is not None+ row = await db.fetchone("SELECT title FROM docs WHERE sha256=?", (document["sha256"],))+ assert row["title"] == "Independent ingest"+ queue = await db.fetchall("SELECT sha256, tier, page_range, status FROM parses")+ assert queue == [{"sha256": document["sha256"], "tier": "flash", "page_range": "1", "status": "pending"}]+ indexed = await fts.search("Independent")+ assert {row["sha256"] for row in indexed} == {text["sha256"]}+ finally:+ await db.close()++ asyncio.run(scenario())+++def test_doclib_instances_share_ingest_only_with_their_own_components(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """同一应用各入口共用入库服务,两个应用的数据库和入库状态保持隔离。"""+ monkeypatch.setenv("MINERU_HOME", str(tmp_path / "home"))++ def skip_background(*args: object, **kwargs: object) -> None:+ """保留真实生命周期组装,仅关闭周期后台调度以固定检查时序。"""++ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)+ configurations = [+ PatchedConfig(doclib={"data_dir": str(tmp_path / name), "sqlite": {"path": str(tmp_path / name / "db.sqlite")}})+ for name in ("first", "second")+ ]+ with TestClient(doclib_app.create_app(configurations[0])) as first:+ with TestClient(doclib_app.create_app(configurations[1])) as second:+ left, right = first.app.state.doclib_state, second.app.state.doclib_state+ for state in (left, right):+ assert state.ingest_svc is state.parse_svc.ingest_svc+ assert state.ingest_svc is state.scan_svc.ingest_svc+ assert state.ingest_svc is state.watch.ingest_svc+ assert state.ingest_svc is state.ingest_workers.ingest_svc+ assert state.ingest_svc.db is state.db and state.ingest_svc.fts is state.fts+ assert state.ingest_svc.config_svc is state.config_svc+ assert state.ingest_svc.telemetry_svc is state.telemetry_svc+ assert left.ingest_svc is not right.ingest_svc and left.db is not right.db+ source = tmp_path / "one.txt"+ source.write_text("Only the first application stores this text", encoding="utf-8")+ assert first.portal.call(left.ingest_svc.ensure_ingested, str(source)) is not None+ assert second.portal.call(right.db.fetchall, "SELECT * FROM files") == []diff --git a/tests/unittest/test_doclib_legacy_page_ranges.py b/tests/unittest/test_doclib_legacy_page_ranges.pyindex c7830d51..fa8fed58 100644--- a/tests/unittest/test_doclib_legacy_page_ranges.py+++ b/tests/unittest/test_doclib_legacy_page_ranges.py@@ -1,4 +1,6 @@from __future__ import annotations++from doclib_test_utils import create_parse_servicefrom mineru.doclib.services.content_read_svc import ContentReadServiceimport asyncio@@ -268,10 +270,10 @@ async def _legacy_store()await _add_result(db, root, "1~5", 1000, range(1, 6), "old")_convert_fixture_family(_cache_path(root, "1~5", 1000), family)- service = ParseService(+ service = create_parse_service(db=db, fts=FTSManager(db), config_svc=ConfigService(db), data_dir=str(root), parse_lock_timeout_sec=1800)- server = DoclibServer(SimpleNamespace(db=db, parse_svc=service, data_dir=str(root)))+ server = DoclibServer(SimpleNamespace(db=db, parse_svc=service, data_dir=str(root), ingest_svc=service.ingest_svc))yield db, service, serverfinally:await db.close()diff --git a/tests/unittest/test_doclib_managed_tier_transition.py b/tests/unittest/test_doclib_managed_tier_transition.pyindex 9c0aa991..aa62ec8e 100644--- a/tests/unittest/test_doclib_managed_tier_transition.py+++ b/tests/unittest/test_doclib_managed_tier_transition.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+import asynciofrom dataclasses import dataclassfrom pathlib import Path@@ -49,7 +51,7 @@ async def _services(monkeypatch.setattr("mineru.doclib.server._ensure_managed_parse_server_tier_available", lambda tier, param: None)checker = ParseServerHealthCheck(cfg, interval_sec=30, probe_timeout_sec=1, startup_grace_sec=30, stop_timeout_sec=10)server = DoclibServer(SimpleNamespace(config_svc=cfg, health_check=checker))- parse_svc = ParseService(db, FTSManager(db), cfg, str(tmp_path), parse_lock_timeout_sec=60)+ parse_svc = create_parse_service(db, FTSManager(db), cfg, str(tmp_path), parse_lock_timeout_sec=60)return cfg, server, parse_svc, health, checkerdiff --git a/tests/unittest/test_doclib_short_id.py b/tests/unittest/test_doclib_short_id.pyindex a9df12df..2ae83eb2 100644--- a/tests/unittest/test_doclib_short_id.py+++ b/tests/unittest/test_doclib_short_id.py@@ -1,7 +1,7 @@+from mineru.doclib.services.ingest_svc import ensure_doc_recordimport asynciofrom mineru.doclib.core.db import DatabaseManager-from mineru.doclib.services.parse_svc import ensure_doc_recorddef test_ensure_doc_record_extends_short_id_on_prefix_conflict(tmp_path) -> None:diff --git a/tests/unittest/test_doclib_telemetry_core.py b/tests/unittest/test_doclib_telemetry_core.pyindex f828c680..198b5302 100644--- a/tests/unittest/test_doclib_telemetry_core.py+++ b/tests/unittest/test_doclib_telemetry_core.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+import asynciofrom types import SimpleNamespace@@ -372,13 +374,14 @@ def test_parse_service_process_doc_records_task_metrics(tmp_path) -> None:telemetry = TelemetryService(store)await telemetry.initialize()config_svc = ConfigService(db)- parse_svc = _ParseService(+ parse_svc = create_parse_service(db,_Fts(),config_svc,str(tmp_path),parse_lock_timeout_sec=30,telemetry_svc=telemetry,+ service_type=_ParseService,)now = 1_700_000_000_000(tmp_path / "a.pdf").write_bytes(b"%PDF-1.4\n")@@ -437,8 +440,7 @@ def test_scan_service_process_scan_records_task_metrics(tmp_path) -> None:)now = 1_700_000_000_000await db.execute(- "INSERT INTO scans (path, kind, source, status, started_at, created_at, updated_at) "- "VALUES (?, ?, ?, ?, ?, ?, ?)",+ "INSERT INTO scans (path, kind, source, status, started_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",(str(tmp_path), "manual", "cli", "running", now, now, now),)task = await db.fetchone("SELECT * FROM scans LIMIT 1")diff --git a/tests/unittest/test_flash_epub.py b/tests/unittest/test_flash_epub.pyindex 67031766..55fc86ab 100644--- a/tests/unittest/test_flash_epub.py+++ b/tests/unittest/test_flash_epub.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+from task_test_utils import execution_requestfrom task_test_utils import stored_source, execution_output@@ -203,7 +205,7 @@ def test_doclib_ingests_epub_as_local_flash_with_spine_page_count(tmp_path: Path"""执行隔离 SQLite 入库并检查 EPUB 文档与解析任务。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]diff --git a/tests/unittest/test_flash_odf.py b/tests/unittest/test_flash_odf.pyindex 20395188..c0c57777 100644--- a/tests/unittest/test_flash_odf.py+++ b/tests/unittest/test_flash_odf.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+from task_test_utils import execution_requestfrom task_test_utils import stored_source, execution_output@@ -19,7 +21,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyzefrom mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.file_io import extract_metadatafrom mineru.doclib.core.fts import FTSManager-from mineru.doclib.services.parse_svc import ParseServicefrom mineru.errors import InvalidRequestErrorfrom mineru.parser import api_server, parse, parse_asyncfrom mineru.parser.api_server import FileStore@@ -165,7 +166,7 @@ def test_doclib_ingests_odf_as_local_flash("""执行隔离 SQLite 入库并检查文档与解析任务。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]@@ -202,7 +203,7 @@ def test_doclib_rejects_odf_remote_parse(tmp_path: Path) -> None:"""创建隔离 doclib 并断言稳定错误码。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]diff --git a/tests/unittest/test_flash_ofd.py b/tests/unittest/test_flash_ofd.pyindex 0a6b5e78..7ed01e74 100644--- a/tests/unittest/test_flash_ofd.py+++ b/tests/unittest/test_flash_ofd.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+from task_test_utils import execution_requestfrom task_test_utils import stored_source, execution_output@@ -18,7 +20,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyzefrom mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.file_io import extract_metadatafrom mineru.doclib.core.fts import FTSManager-from mineru.doclib.services.parse_svc import ParseServicefrom mineru.errors import InvalidRequestErrorfrom mineru.parser import MinerUParser, api_serverfrom mineru.parser.api_server import FileStore@@ -109,7 +110,7 @@ def test_doclib_ingests_ofd_as_local_full_document_flash(tmp_path: Path) -> None"""执行隔离 SQLite 入库并检查 OFD 文档与解析任务。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]diff --git a/tests/unittest/test_flash_rtf.py b/tests/unittest/test_flash_rtf.pyindex 0981e133..da7528a5 100644--- a/tests/unittest/test_flash_rtf.py+++ b/tests/unittest/test_flash_rtf.py@@ -1,5 +1,7 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+import asynciofrom io import BytesIOfrom pathlib import Path@@ -12,7 +14,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyzefrom mineru.doclib.core import file_iofrom mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.fts import FTSManager-from mineru.doclib.services.parse_svc import ParseServicefrom mineru.errors import InvalidRequestErrorfrom mineru.parser import parsefrom mineru.render import RenderMode, render_docx, render_html, render_markdown, render_structured_content@@ -104,7 +105,7 @@ def test_doclib_ingests_rtf_as_one_page_flash_document(tmp_path: Path) -> None:"""执行真实 SQLite doclib 入库并检查稳定行数据。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]@@ -156,7 +157,7 @@ def test_doclib_rejects_rtf_quality_tier_and_remote("""创建隔离 doclib 并断言稳定错误契约。"""db = DatabaseManager(str(tmp_path / "doclib.db"))await db.initialize()- service = ParseService(+ service = create_parse_service(db=db,fts=FTSManager(db),config_svc=_NoRulesConfig(), # type: ignore[arg-type]diff --git a/tests/unittest/test_pdf_page_id.py b/tests/unittest/test_pdf_page_id.pyindex 2403c00e..19391aa7 100644--- a/tests/unittest/test_pdf_page_id.py+++ b/tests/unittest/test_pdf_page_id.py@@ -1,4 +1,6 @@from __future__ import annotations++from mineru.doclib.services.parse_policy import default_parse_rangefrom mineru.doclib.services.content_read_svc import ContentReadServicefrom task_test_utils import wait_task@@ -19,7 +21,7 @@ from mineru.cli.commands.parse import _validate_page_range_inputfrom mineru.cli.main import app as cli_appfrom mineru.doclib.services.content_read_svc import _normalize_content_page_rangefrom mineru.doclib.services.config_svc import ConfigService-from mineru.doclib.services.parse_svc import default_parse_range+from mineru.errors import InvalidRequestError, MineruErrorfrom mineru.kit.gradio.page_range import effective_page_range as _effective_page_rangefrom mineru.kit.gradio.artifacts import persist_parse_resultdiff --git a/tests/unittest/test_source_metadata.py b/tests/unittest/test_source_metadata.pyindex 367ae636..6a47fe24 100644--- a/tests/unittest/test_source_metadata.py+++ b/tests/unittest/test_source_metadata.py@@ -2,6 +2,9 @@from __future__ import annotations+from doclib_test_utils import create_parse_service+from mineru.doclib.services.ingest_svc import ensure_doc_record+import asynciofrom io import BytesIOfrom pathlib import Path@@ -15,7 +18,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyzefrom mineru.doclib.core.db import DatabaseManagerfrom mineru.doclib.core.fts import FTSManagerfrom mineru.doclib.services.config_svc import ConfigService-from mineru.doclib.services.parse_svc import ParseService, ensure_doc_recordfrom mineru.parser import MinerUParser, ParseResult@@ -78,7 +80,7 @@ def test_doclib_source_columns_and_fts_are_independent_of_tier(tmp_path: Path) -db = DatabaseManager(str(tmp_path / "db.sqlite"))await db.initialize()fts = FTSManager(db)- service = ParseService(db, fts, ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)+ service = create_parse_service(db, fts, ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)sha = "a" * 64await ensure_doc_record(db,@@ -126,12 +128,12 @@ def test_doclib_ingest_gets_html_metadata_without_parsing(tmp_path: Path) -> Non"""使用临时数据库与源 HTML 走入库服务。"""db = DatabaseManager(str(tmp_path / "db.sqlite"))await db.initialize()- service = ParseService(db, FTSManager(db), ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)+ service = create_parse_service(db, FTSManager(db), ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)source = tmp_path / "source.html"source.write_text('<html lang="zh-CN"><head><title>入库标题</title><meta name="author" content="作者"></head><body>正文</body></html>')- refresh = await service.refresh_file(str(source), ensure_ingested=True)+ refresh = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)assert refresh.file is not Nonerow = await db.fetchone("SELECT * FROM docs WHERE sha256=?", (refresh.file.sha256,))assert (row["title"], row["author"], row["language"]) == ("入库标题", "作者", "zh-CN")