Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

refactor: separate doclib file ingestion from parsing

myhlolif18db4a3f7a7
diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml
index 31a5a981..67b02fe8 100644
--- a/.github/workflows/cli.yml
+++ b/.github/workflows/cli.yml
@@ -117,7 +117,9 @@ jobs:
tests/unittest/test_docvortex_routing.py \
tests/unittest/test_parser_api_contract.py \
tests/unittest/test_gradio_lifecycle.py \
- tests/unittest/test_router_route_state.py
+ tests/unittest/test_router_route_state.py \
+ tests/unittest/test_doclib_ingest_service.py \
+ tests/unittest/test_doclib_architecture.py
- name: Retain test reports
if: ${{ always() }}
uses: actions/upload-artifact@v6
diff --git a/docs/next/telemetry-minimum.md b/docs/next/telemetry-minimum.md
index ab249f0a..cdc5e270 100644
--- a/docs/next/telemetry-minimum.md
+++ b/docs/next/telemetry-minimum.md
@@ -73,16 +73,18 @@ startup 初始化顺序:
DatabaseManager.initialize()
-> TelemetryStore(db)
-> TelemetryService(store, context collector)
- -> ParseService(..., telemetry_svc=state.telemetry_svc)
+ -> IngestService(..., telemetry_svc=state.telemetry_svc)
+ -> ParseService(..., ingest_svc=state.ingest_svc, telemetry_svc=state.telemetry_svc)
-> ScanService(..., telemetry_svc=state.telemetry_svc)
- -> SearchService(..., telemetry_svc=state.telemetry_svc)
+ -> SearchService(db, fts)
-> background workers
-> TelemetryFlushWorker(...)
```
传递规则:
-- `ParseService`、`ScanService`、`SearchService` 构造函数显式接收 `telemetry_svc`。
+- `IngestService`、`ParseService`、`ScanService` 构造函数显式接收 `telemetry_svc`。
+- 文件入库及初始解析任务创建指标由 `IngestService` 记录;扫描、watch 和入库工作池共享应用的同一入库服务。
- `DoclibServer` route 通过 `self.state.telemetry_svc` 记录 route-level metric。
- background worker 不创建 telemetry service,只使用已注入到 service 的 telemetry。
- telemetry service 所有 record 方法内部吞掉异常,最多记录 debug / warning log,不影响主流程。
diff --git a/mineru/doclib/app.py b/mineru/doclib/app.py
index 5565146d..0cda4e0a 100644
--- a/mineru/doclib/app.py
+++ b/mineru/doclib/app.py
@@ -30,6 +30,8 @@ from .types import PARSE_STATUS_FAILED, PARSE_STATUS_PARSING, SCAN_STATUS_FAILED
if TYPE_CHECKING:
from loguru import Message as LoguruMessage
+
+ from .services.ingest_svc import IngestService
else:
LoguruMessage = Any
@@ -65,6 +67,7 @@ def create_app(cfg: Config | None = None) -> FastAPI:
from .core.fts import FTSManager
from .services.cleanup_svc import CleanupService
from .services.config_svc import ConfigService
+ from .services.ingest_svc import IngestService
from .services.parse_svc import ParseService
from .services.scan_svc import ScanService
from .services.search_svc import SearchService
@@ -85,18 +88,20 @@ def create_app(cfg: Config | None = None) -> FastAPI:
state.config_svc = ConfigService(state.db)
state.telemetry_svc = TelemetryService(TelemetryStore(state.db))
await state.telemetry_svc.initialize()
+ state.ingest_svc = IngestService(state.db, state.fts, state.config_svc, telemetry_svc=state.telemetry_svc)
state.parse_svc = ParseService(
state.db,
state.fts,
state.config_svc,
data_dir,
+ ingest_svc=state.ingest_svc,
parse_lock_timeout_sec=cfg.doclib.parse_lock_timeout_sec,
telemetry_svc=state.telemetry_svc,
)
state.scan_svc = ScanService(
state.db,
state.config_svc,
- state.parse_svc,
+ state.ingest_svc,
scan_lock_timeout_sec=cfg.doclib.scan_lock_timeout_sec,
telemetry_svc=state.telemetry_svc,
)
@@ -149,13 +154,13 @@ def create_app(cfg: Config | None = None) -> FastAPI:
state.watch = WatchLoop(
state.db,
state.config_svc,
- state.parse_svc,
+ state.ingest_svc,
scan_interval_sec=cfg.doclib.scan_interval_sec,
scan_svc=state.scan_svc,
)
state.scan_workers = ScanWorkerPool(state.scan_svc, num_workers=1)
state.ingest_workers = IngestWorkerPool(
- state.parse_svc,
+ state.ingest_svc,
num_workers=cfg.doclib.ingest_workers,
lock_timeout_sec=cfg.doclib.ingest_lock_timeout_sec,
)
@@ -295,6 +300,7 @@ class AppState:
self.db: Any = None
self.fts: Any = None
self.config_svc: Any = None
+ self.ingest_svc: IngestService | None = None
self.parse_svc: Any = None
self.scan_svc: Any = None
self.search_svc: Any = None
diff --git a/mineru/doclib/background/ingest.py b/mineru/doclib/background/ingest.py
index bed7e9fd..7bad3810 100644
--- a/mineru/doclib/background/ingest.py
+++ b/mineru/doclib/background/ingest.py
@@ -9,15 +9,16 @@ from typing import cast
from ...errors import MineruError
from ..rows import CountRow, FileRow, IngestTaskRow
-from ..services.parse_svc import ParseService
+from ..services.ingest_svc import IngestService
from ..types import FILE_STATUS_ACTIVE
logger = logging.getLogger("mineru.ingest")
class IngestWorkerPool:
- def __init__(self, parse_svc: ParseService, *, num_workers: int, lock_timeout_sec: int) -> None:
- self.parse_svc = parse_svc
+ def __init__(self, ingest_svc: IngestService, *, num_workers: int, lock_timeout_sec: int) -> None:
+ """保存入库服务与原有工作池数量、抢占超时设置。"""
+ self.ingest_svc = ingest_svc
self.num_workers = num_workers
self.lock_timeout_ms = lock_timeout_sec * 1000
self.running = False
@@ -42,7 +43,7 @@ class IngestWorkerPool:
if no_task_count == 1 or no_task_count % 20 == 0:
q = cast(
CountRow | None,
- await self.parse_svc.db.fetchone(
+ await self.ingest_svc.db.fetchone(
"SELECT COUNT(*) as cnt FROM files WHERE sha256 IS NULL AND status=? AND error_code IS NULL",
(FILE_STATUS_ACTIVE,),
),
@@ -63,7 +64,7 @@ class IngestWorkerPool:
no_task_count = 0
try:
- await self.parse_svc.ingest_file(task["path"], watch_id=task.get("watch_id"), trigger="background")
+ await self.ingest_svc.ingest_file(task["path"], watch_id=task.get("watch_id"), trigger="background")
processed += 1
if processed % 100 == 0:
logger.info(f"Ingest worker {worker_id} ingested {processed} files")
@@ -84,7 +85,7 @@ class IngestWorkerPool:
timeout = now - self.lock_timeout_ms
return cast(
FileRow | None,
- await self.parse_svc.db.fetchone_write(
+ await self.ingest_svc.db.fetchone_write(
"UPDATE files SET locked_at=? "
"WHERE id = ("
" SELECT id FROM files "
@@ -101,7 +102,7 @@ class IngestWorkerPool:
try:
if isinstance(exc, FileNotFoundError):
watch_id = task.get("watch_id")
- await self.parse_svc.refresh_file(
+ await self.ingest_svc.refresh_file(
str(task["path"]),
watch_id=watch_id if isinstance(watch_id, int) else None,
)
@@ -118,7 +119,7 @@ class IngestWorkerPool:
error_code = "stat_failed"
now = int(time.time() * 1000)
- await self.parse_svc.db.execute(
+ await self.ingest_svc.db.execute(
"UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",
(error_code, error_msg, now, task["id"]),
)
diff --git a/mineru/doclib/background/watch.py b/mineru/doclib/background/watch.py
index fc2f01fb..8ab62df3 100644
--- a/mineru/doclib/background/watch.py
+++ b/mineru/doclib/background/watch.py
@@ -9,10 +9,10 @@ from pathlib import Path
from watchfiles import awatch
-from ..core.db import DatabaseManager
from ...filetypes import DISCOVERABLE_EXTENSIONS, is_office_temp_lock_file
+from ..core.db import DatabaseManager
from ..services.config_svc import ConfigService
-from ..services.parse_svc import ParseService
+from ..services.ingest_svc import IngestService
from ..services.scan_svc import ScanService
from ..types import FILE_STATUS_ACTIVE, WATCH_STATUS_ACTIVE, WATCH_STATUS_UNREACHABLE
from ..utils.path_utils import normalize_doclib_path, rebase_watch_event_path
@@ -25,14 +25,15 @@ class WatchLoop:
self,
db: DatabaseManager,
config_svc: ConfigService,
- parse_svc: ParseService,
+ ingest_svc: IngestService,
*,
scan_interval_sec: int,
scan_svc: ScanService | None = None,
) -> None:
+ """保存目录监控和入库依赖,文件事件交给同一刷新入口。"""
self.db = db
self.config_svc = config_svc
- self.parse_svc = parse_svc
+ self.ingest_svc = ingest_svc
self.scan_svc = scan_svc
self.scan_interval_sec = scan_interval_sec
self.running = False
@@ -131,11 +132,7 @@ class WatchLoop:
)
async def _handle_event(self, filepath: str, watch_id: int, *, watch_root: str | None = None) -> None:
- filepath = (
- rebase_watch_event_path(filepath, watch_root)
- if watch_root is not None
- else normalize_doclib_path(filepath)
- )
+ filepath = rebase_watch_event_path(filepath, watch_root) if watch_root is not None else normalize_doclib_path(filepath)
ext = Path(filepath).suffix.lstrip(".").lower()
if ext not in DISCOVERABLE_EXTENSIONS or is_office_temp_lock_file(filepath):
return
@@ -144,7 +141,7 @@ class WatchLoop:
await self._refresh_file(filepath, watch_id)
async def _refresh_file(self, filepath: str, watch_id: int) -> None:
- await self.parse_svc.refresh_file(filepath, watch_id=watch_id)
+ await self.ingest_svc.refresh_file(filepath, watch_id=watch_id)
async def stop(self) -> None:
self.running = False
diff --git a/mineru/doclib/core/fts.py b/mineru/doclib/core/fts.py
index 35e2c201..1e9ec098 100644
--- a/mineru/doclib/core/fts.py
+++ b/mineru/doclib/core/fts.py
@@ -148,3 +148,14 @@ def _sanitize_query_tokens(tokenized: str) -> list[str]:
if t and t.upper() not in ("AND", "OR", "NOT", "NEAR"):
safe.append(t)
return safe
+
+
+MAX_FTS_CHARS = 30_000
+FTS_HEAD_HALF = 15_000
+
+
+def truncate_head_tail(text: str, max_chars: int = MAX_FTS_CHARS, half: int = FTS_HEAD_HALF) -> str:
+ """超长正文仅保留首尾内容,沿用原有 FTS 长度与省略标记。"""
+ if len(text) <= max_chars:
+ return text
+ return text[:half] + "\n...\n" + text[-half:]
diff --git a/mineru/doclib/server.py b/mineru/doclib/server.py
index 2a781318..7d317218 100644
--- a/mineru/doclib/server.py
+++ b/mineru/doclib/server.py
@@ -17,8 +17,12 @@ from pydantic import ValidationError
from ..config import get_config
from ..errors import InvalidRequestError, MineruError, NotFoundError, error_response, http_status_for
from ..filetypes import TEXT_EXTENSIONS, TEXT_FILE_TYPES
+from ..model.download import verify_model_repo
+from ..model.registry import ModelRepo, model_repos_for_tier
from ..parser.page_range import (
expand_page_range as _expand_page_range,
+)
+from ..parser.page_range import (
normalize_page_range_input,
normalize_result_page_range,
parse_page_range_set,
@@ -29,13 +33,12 @@ from ..types import (
DeploymentTier,
Tier,
)
-from ..model.download import verify_model_repo
-from ..model.registry import ModelRepo, model_repos_for_tier
from ..version import __version__
from .background.parse_server_health import get_health, get_managed_parse_server_tier
from .base import AsyncDoclibInterface
from .config_schema import validate_config_value
from .core.db import DatabaseManager
+from .core.parse_artifacts import parse_batch_is_readable, parse_coverage
from .remote_api import REMOTE_API_KEY_CONFIG, resolve_remote_api_key
from .rows import (
ContentSearchResultRow,
@@ -53,7 +56,6 @@ from .rows import (
WatchStatsFileRow,
WatchTargetRow,
)
-from .core.parse_artifacts import parse_batch_is_readable, parse_coverage
from .services.content_read_svc import ContentReadService
from .telemetry.buckets import pages_bucket, results_bucket
from .types import (
@@ -146,7 +148,6 @@ from .types import (
)
from .utils.route_utils import get_route_info, has_route_info, route
-
_NO_MATCHING_DOC_SHA256 = "__mineru_no_matching_doc__"
@@ -377,7 +378,7 @@ class DoclibServer(AsyncDoclibInterface):
file_row: FileRow | None = None
sha256 = doc_row["sha256"] if doc_row else None
if sha256 is None and request.path:
- file_row = await self.state.parse_svc.ensure_ingested(request.path)
+ file_row = await self.state.ingest_svc.ensure_ingested(request.path)
sha256 = file_row["sha256"] if file_row and file_row.get("sha256") else None
if sha256 is None:
raise NotFoundError("file_not_found", "Document not found.", "path")
@@ -502,7 +503,7 @@ class DoclibServer(AsyncDoclibInterface):
@route("GET", "/docs/by-path", tags=("docs",))
async def get_doc_by_path(self, path: str) -> DocInfo:
- await self.state.parse_svc.ensure_ingested(path)
+ await self.state.ingest_svc.ensure_ingested(path)
row = await self.state.db.fetchone(
"SELECT d.* FROM docs d JOIN files f ON f.sha256=d.sha256 WHERE f.path=? AND f.status=?",
(path, FILE_STATUS_ACTIVE),
@@ -652,7 +653,7 @@ class DoclibServer(AsyncDoclibInterface):
ext=ext,
limit=limit,
offset=offset,
- refresh_file=self.state.parse_svc.refresh_file,
+ refresh_file=self.state.ingest_svc.refresh_file,
)
response = FindResponse(results=[_find_result(row) for row in results], total=total, query=query)
dims = {"status": "succeeded"}
@@ -668,7 +669,7 @@ class DoclibServer(AsyncDoclibInterface):
@route("GET", "/files/by-path", tags=("files",))
async def get_file_by_path(self, path: str) -> FileInfoResponse:
- await self.state.parse_svc.ensure_ingested(path)
+ await self.state.ingest_svc.ensure_ingested(path)
file_row = await self.state.db.fetchone(
"""
SELECT f.*, d.short_id AS short_id
diff --git a/mineru/doclib/services/__init__.py b/mineru/doclib/services/__init__.py
index f44055ea..fe9b3de4 100644
--- a/mineru/doclib/services/__init__.py
+++ b/mineru/doclib/services/__init__.py
@@ -3,13 +3,16 @@
from .cleanup_svc import CleanupService
from .config_svc import ConfigService
from .content_read_svc import ContentReadService
-from .parse_svc import (
+from .ingest_svc import (
FileRefreshResult,
FileRefreshStatus,
+ IngestService,
+ ensure_doc_record,
+)
+from .parse_policy import default_parse_range
+from .parse_svc import (
ParseFailure,
ParseService,
- default_parse_range,
- ensure_doc_record,
expand_page_range,
page_range_covered,
page_range_uncovered,
@@ -24,6 +27,7 @@ __all__ = [
"ContentReadService",
"FileRefreshResult",
"FileRefreshStatus",
+ "IngestService",
"ParseFailure",
"ParseService",
"ScanService",
diff --git a/mineru/doclib/services/ingest_svc.py b/mineru/doclib/services/ingest_svc.py
new file mode 100644
index 00000000..3c399fcc
--- /dev/null
+++ b/mineru/doclib/services/ingest_svc.py
@@ -0,0 +1,597 @@
+"""Doclib 源文件刷新、去重入库、元数据和初始解析入队。"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal, cast
+
+from ...errors import InvalidRequestError, MineruError
+from ...filetypes import (
+ CSV_EXTENSIONS,
+ IMAGE_EXTENSIONS,
+ INGESTIBLE_EXTENSIONS,
+ PAGE_RANGE_PARSE_EXTENSIONS,
+ TEXT_EXTENSIONS,
+ TIERED_PARSE_EXTENSIONS,
+ file_type_for_extension,
+ is_office_temp_lock_file,
+)
+from ...parser.page_range import expand_page_range
+from ...types import Tier
+from ..core.db import DatabaseManager
+from ..core.file_io import FileStat, MetadataExtractionError, compute_sha256, extract_metadata, get_file_stat
+from ..core.fts import FTSManager, truncate_head_tail
+from ..rows import FileRow, Sha256Row, ShortIdRow, WatchTargetRow
+from ..types import (
+ FILE_STATUS_ACTIVE,
+ FILE_STATUS_DELETED,
+ FILE_STATUS_UNREACHABLE,
+ PARSE_STATUS_PENDING,
+ RULE_TYPE_PARSING_RULE,
+ WATCH_STATUS_UNREACHABLE,
+ FileInfo,
+)
+from ..utils.path_utils import normalize_doclib_path
+from . import parse_policy
+from .config_svc import ConfigService
+
+if TYPE_CHECKING:
+ from ..telemetry import TelemetryService
+
+
+def _now_ms() -> int:
+ """返回当前毫秒时间,保持文件状态与解析队列的时间单位。"""
+ return int(time.time() * 1000)
+
+
+FileRefreshStatus = Literal["known", "new", "changed", "missing", "deleted", "unreachable", "unsupported", "error"]
+
+
+@dataclass(frozen=True)
+class FileRefreshResult:
+ """记录文件刷新分类和入库错误,供扫描、搜索与解析请求共同消费。"""
+
+ file: FileInfo | None
+ status: FileRefreshStatus
+ error_code: str | None = None
+ error_msg: str | None = None
+
+ @property
+ def needs_ingest(self) -> bool:
+ """文件已登记但尚未绑定内容哈希时需要入库。"""
+ return self.file is not None and self.file.sha256 is None
+
+
+def raise_refresh_error(refreshed: FileRefreshResult) -> None:
+ """保持刷新失败的错误码、消息和路径参数语义。"""
+ code = (
+ (refreshed.file.error_code if refreshed.file and refreshed.file.error_code else None)
+ or refreshed.error_code
+ or "ingest_failed"
+ )
+ message = (
+ (refreshed.file.error_msg if refreshed.file and refreshed.file.error_msg else None)
+ or refreshed.error_msg
+ or "File could not be ingested."
+ )
+ if code == "file_permission_denied":
+ raise InvalidRequestError(code, message, "path")
+ raise MineruError(code, message, "path")
+
+
+async def ensure_doc_record(
+ db: DatabaseManager,
+ *,
+ sha256: str,
+ size_bytes: int,
+ file_type: str,
+ page_count: int | None,
+ title: str | None,
+ author: str | None,
+ subject: str | None,
+ keywords: str | None,
+ is_image_based: int = 0,
+ language: str | None = None,
+ error_code: str | None,
+ error_msg: str | None,
+ first_seen_at: int,
+ updated_at: int,
+) -> None:
+ """按内容哈希只创建一次文档,短标识冲突时逐字符扩展。"""
+ existing = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))
+ if existing is not None:
+ return
+
+ for length in range(7, len(sha256) + 1):
+ short_id = sha256[:length]
+ await db.execute(
+ "INSERT OR IGNORE INTO docs (sha256, short_id, size_bytes, file_type, page_count, "
+ "title, author, subject, keywords, is_image_based, error_code, error_msg, first_seen_at, updated_at, language) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ (
+ sha256,
+ short_id,
+ size_bytes,
+ file_type,
+ page_count,
+ title,
+ author,
+ subject,
+ keywords,
+ is_image_based,
+ error_code,
+ error_msg,
+ first_seen_at,
+ updated_at,
+ language,
+ ),
+ )
+ inserted = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))
+ if inserted is not None:
+ return
+ conflict = cast(Sha256Row | None, await db.fetchone("SELECT sha256 FROM docs WHERE short_id=?", (short_id,)))
+ if conflict is not None and conflict["sha256"] != sha256:
+ continue
+ break
+
+ raise RuntimeError(f"Failed to allocate unique short_id for document {sha256}")
+
+
+class IngestService:
+ """独立管理源文件与文档入库,解析执行通过单独服务处理。"""
+
+ def __init__(
+ self, db: DatabaseManager, fts: FTSManager, config_svc: ConfigService, *, telemetry_svc: TelemetryService | None = None
+ ) -> None:
+ """保存显式数据库、索引、规则配置和可选指标记录依赖。"""
+ self.db = db
+ self.fts = fts
+ self.config_svc = config_svc
+ self.telemetry_svc = telemetry_svc
+
+ async def refresh_file(
+ self,
+ path: str,
+ watch_id: int | None = None,
+ *,
+ ensure_ingested: bool = False,
+ allow_images: bool = False,
+ ) -> FileRefreshResult:
+ """刷新路径对应的文件状态,按需同步完成入库。"""
+ path = normalize_doclib_path(path)
+ ext = Path(path).suffix.lower().lstrip(".")
+ existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
+ if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):
+ return FileRefreshResult(file=_file_info(existing), status="unsupported")
+
+ try:
+ stat = await get_file_stat(path)
+ except FileNotFoundError:
+ return await self._refresh_missing_file(path, existing)
+ except PermissionError as exc:
+ return await self._refresh_stat_error(existing, "file_permission_denied", str(exc))
+ except OSError as exc:
+ return await self._refresh_stat_error(existing, "stat_failed", str(exc))
+
+ result = await self._refresh_existing_file_with_stat(path, ext, stat, existing, watch_id)
+ if ensure_ingested and result.needs_ingest:
+ row = await self.ingest_file(path, watch_id=watch_id, allow_images=allow_images)
+ return FileRefreshResult(file=_file_info(row), status=result.status)
+ return result
+
+ async def _refresh_existing_file_with_stat(
+ self,
+ path: str,
+ ext: str,
+ stat: FileStat,
+ existing: FileRow | None,
+ watch_id: int | None,
+ ) -> FileRefreshResult:
+ """依据大小、修改时间和历史状态登记新文件或标记变化。"""
+ now = _now_ms()
+ filename = Path(path).name
+
+ if existing and existing["status"] == FILE_STATUS_ACTIVE:
+ unchanged = existing["mtime_ms"] == stat.mtime_ms and existing["size_bytes"] == stat.size_bytes
+ if unchanged:
+ if watch_id is not None and existing["watch_id"] is None:
+ await self.db.execute(
+ "UPDATE files SET watch_id=?, updated_at=? WHERE id=? AND watch_id IS NULL",
+ (watch_id, now, existing["id"]),
+ )
+ existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
+ return FileRefreshResult(file=_file_info(existing), status="known")
+
+ await self.db.execute(
+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
+ "sha256=NULL, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "
+ "WHERE id=?",
+ (
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ watch_id,
+ now,
+ existing["id"],
+ ),
+ )
+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
+ return FileRefreshResult(file=_file_info(row), status="changed")
+
+ if existing:
+ await self.db.execute(
+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
+ "sha256=NULL, watch_id=COALESCE(?, watch_id), status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, "
+ "deleted_at=NULL, updated_at=? WHERE id=?",
+ (
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ watch_id,
+ FILE_STATUS_ACTIVE,
+ now,
+ existing["id"],
+ ),
+ )
+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
+ return FileRefreshResult(file=_file_info(row), status="changed")
+
+ file_id = await self.db.execute_insert(
+ "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "
+ "watch_id, first_seen_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+ (
+ path,
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ watch_id,
+ now,
+ now,
+ ),
+ )
+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (file_id,)))
+ return FileRefreshResult(file=_file_info(row), status="new")
+
+ async def _refresh_missing_file(self, path: str, existing: FileRow | None) -> FileRefreshResult:
+ """区分已删除路径与不可达的可移动监控目录。"""
+ if existing is None:
+ return FileRefreshResult(file=None, status="missing")
+
+ now = _now_ms()
+ file_status = FILE_STATUS_DELETED
+ deleted_at: int | None = now
+ if existing["status"] != FILE_STATUS_DELETED and await self._is_missing_due_to_unreachable_watch(existing):
+ file_status = FILE_STATUS_UNREACHABLE
+ deleted_at = None
+
+ await self.db.execute(
+ "UPDATE files SET status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, deleted_at=?, updated_at=? WHERE id=?",
+ (file_status, deleted_at, now, existing["id"]),
+ )
+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
+ refresh_status: FileRefreshStatus = "unreachable" if file_status == FILE_STATUS_UNREACHABLE else "deleted"
+ return FileRefreshResult(file=_file_info(row), status=refresh_status)
+
+ async def _is_missing_due_to_unreachable_watch(self, file_row: FileRow) -> bool:
+ """检查缺失路径是否来自暂不可达的可移动目录。"""
+ watch_id = file_row.get("watch_id")
+ if watch_id is None:
+ return False
+ watch = cast(WatchTargetRow | None, await self.db.fetchone("SELECT * FROM watches WHERE id=?", (watch_id,)))
+ if not watch or not watch.get("removable"):
+ return False
+ if watch["status"] == WATCH_STATUS_UNREACHABLE:
+ return True
+ try:
+ await get_file_stat(watch["path"])
+ except OSError:
+ if self.config_svc is not None:
+ await self.config_svc.update_watch_status(watch_id, WATCH_STATUS_UNREACHABLE)
+ return True
+ return False
+
+ async def _refresh_stat_error(self, existing: FileRow | None, error_code: str, error_msg: str) -> FileRefreshResult:
+ """保留原有权限和 stat 失败状态,并解除文件锁与哈希绑定。"""
+ if existing is None:
+ return FileRefreshResult(file=None, status="error", error_code=error_code, error_msg=error_msg)
+
+ now = _now_ms()
+ await self.db.execute(
+ "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",
+ (error_code, error_msg[:500], now, existing["id"]),
+ )
+ row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
+ return FileRefreshResult(file=_file_info(row), status="error", error_code=error_code, error_msg=error_msg)
+
+ async def ensure_ingested(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:
+ """确保路径已刷新并入库,保持不支持格式和错误的原有边界。"""
+ path = normalize_doclib_path(path)
+ refreshed = await self.refresh_file(path, watch_id=watch_id, ensure_ingested=True, allow_images=allow_images)
+ if refreshed.status == "error":
+ raise_refresh_error(refreshed)
+ if refreshed.file is None or refreshed.status == "unsupported":
+ return None
+ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
+
+ async def ingest_file(
+ self,
+ path: str,
+ watch_id: int | None = None,
+ *,
+ trigger: str = "parse",
+ allow_images: bool = False,
+ ) -> FileRow | None:
+ """记录入库计数和耗时,将权限错误映射为稳定路径错误。"""
+ path = normalize_doclib_path(path)
+ start_ms = _now_ms()
+ status = "succeeded"
+ try:
+ return await self._ingest_file(path, watch_id=watch_id, allow_images=allow_images)
+ except PermissionError as exc:
+ status = "failed"
+ await self._mark_file_error(path, "file_permission_denied", str(exc))
+ raise InvalidRequestError("file_permission_denied", str(exc), "path") from None
+ except Exception:
+ status = "failed"
+ raise
+ finally:
+ dimensions = {"status": status, "trigger": trigger}
+ await self._record_count("ingest.finished.count", dimensions=dimensions)
+ await self._record_duration("ingest.duration_bucket.count", start_ms, dimensions=dimensions)
+
+ async def _mark_file_error(self, path: str, error_code: str, error_msg: str) -> None:
+ """保存文件错误并解除入库锁,限制持久化消息长度。"""
+ path = normalize_doclib_path(path)
+ now = _now_ms()
+ await self.db.execute(
+ "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE path=?",
+ (error_code, error_msg[:500], now, path),
+ )
+
+ async def _ingest_file(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:
+ """计算哈希并复用文档,提取元数据和索引,再按规则创建初始解析。"""
+ ext = Path(path).suffix.lower().lstrip(".")
+ if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):
+ return None
+
+ stat = await get_file_stat(path)
+ sha256 = await compute_sha256(path)
+ now = _now_ms()
+ filename = Path(path).name
+
+ existing_path = cast(
+ FileRow | None,
+ await self.db.fetchone("SELECT * FROM files WHERE path=? AND status=?", (path, FILE_STATUS_ACTIVE)),
+ )
+ if existing_path and existing_path["sha256"] == sha256:
+ return existing_path
+
+ # check if same content (sha256) is already tracked by another path
+ existing_sha = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE sha256=?", (sha256,)))
+ if existing_sha:
+ # same content, possibly a new or changed path — bind this file row
+ # to the existing doc without duplicating docs/parses.
+ if existing_path:
+ await self.db.execute(
+ "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
+ "sha256=?, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "
+ "WHERE id=?",
+ (
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ sha256,
+ watch_id,
+ now,
+ existing_path["id"],
+ ),
+ )
+ else:
+ await self.db.execute(
+ "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "
+ "sha256, watch_id, first_seen_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ (
+ path,
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ sha256,
+ watch_id,
+ now,
+ now,
+ ),
+ )
+ file_row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
+ if file_row:
+ await self.fts.upsert_filename(file_row["id"], Path(path).stem, ext)
+ # the source may have been moved here from another tracked path;
+ # retire stale duplicates whose files no longer exist.
+ await self._mark_moved_duplicate_files_deleted(sha256, path)
+ return file_row
+
+ # brand new document
+ metadata_error_code = None
+ metadata_error_msg = None
+ try:
+ metadata = await extract_metadata(path)
+ metadata_error_code = metadata.get("error_code")
+ metadata_error_msg = metadata.get("error_msg")
+ except MetadataExtractionError as exc:
+ metadata_error_code = exc.code
+ metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"
+ metadata = {
+ "page_count": None,
+ "title": None,
+ "author": None,
+ "subject": None,
+ "keywords": None,
+ "is_image_based": 0,
+ }
+ except Exception as exc:
+ metadata_error_code = "open_failed"
+ metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"
+ metadata = {
+ "page_count": None,
+ "title": None,
+ "author": None,
+ "subject": None,
+ "keywords": None,
+ "is_image_based": 0,
+ }
+
+ # 无固有分页或元数据暂不可读的结构化格式默认按一页处理。
+ page_count = metadata["page_count"]
+ if page_count is None:
+ if ext in ("doc", "docx", "rtf"):
+ page_count = 1 # reflow docs
+ elif ext in ("ppt", "pptx", "xls", "xlsx"):
+ page_count = 1 # will be updated by metadata extraction
+ elif ext in CSV_EXTENSIONS:
+ page_count = 1
+
+ now = _now_ms()
+ await self.db.execute(
+ "INSERT OR IGNORE INTO files (path, filename, ext, size_bytes, mtime_ms, "
+ "watch_id, first_seen_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+ (
+ path,
+ filename,
+ ext,
+ stat.size_bytes,
+ stat.mtime_ms,
+ watch_id,
+ now,
+ now,
+ ),
+ )
+
+ await ensure_doc_record(
+ self.db,
+ sha256=sha256,
+ size_bytes=stat.size_bytes,
+ file_type=file_type_for_extension(ext),
+ page_count=page_count,
+ title=metadata["title"],
+ author=metadata["author"],
+ subject=metadata["subject"],
+ keywords=metadata["keywords"],
+ is_image_based=int(metadata.get("is_image_based") or 0),
+ language=metadata.get("language"),
+ error_code=metadata_error_code,
+ error_msg=metadata_error_msg,
+ first_seen_at=now,
+ updated_at=now,
+ )
+
+ # FTS filename index
+ file_row = await self.db.fetchone("SELECT id FROM files WHERE path=?", (path,))
+ if file_row:
+ await self.fts.upsert_filename(
+ file_row["id"],
+ Path(path).stem,
+ ext,
+ )
+
+ if ext in TEXT_EXTENSIONS:
+ await self.db.execute(
+ "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",
+ (sha256, now, path),
+ )
+ text = await asyncio.to_thread(Path(path).read_text, encoding="utf-8", errors="replace")
+ await self.fts.replace(
+ sha256=sha256,
+ tier=None,
+ text=truncate_head_tail(text),
+ title=metadata["title"] or "",
+ author=metadata["author"] or "",
+ )
+ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
+
+ # determine tier and page_range for initial parse
+ tier: Tier = "flash"
+ privacy = "local"
+ initial_page_range = (
+ parse_policy.default_parse_range(page_count)
+ if ext in PAGE_RANGE_PARSE_EXTENSIONS
+ else expand_page_range(None, page_count or 1)
+ )
+
+ # check parsing-rules
+ matched = await self.config_svc.match_rules(path, RULE_TYPE_PARSING_RULE)
+ if matched:
+ rule = matched[0]
+ rule_tier = cast(Tier | None, rule.get("tier"))
+ rule_remote = bool(rule.get("remote", False))
+ if ext in TIERED_PARSE_EXTENSIONS:
+ tier = rule_tier or parse_policy.resolve_parsing_rule_default_tier(remote=rule_remote)
+ privacy = "remote" if rule_remote and tier != "flash" else "local"
+ else:
+ tier = "flash"
+ privacy = "local"
+ rule_page_range = rule.get("page_range")
+ if rule_page_range and ext in PAGE_RANGE_PARSE_EXTENSIONS:
+ initial_page_range = expand_page_range(rule_page_range, page_count or 1)
+
+ # insert parse batch
+ parse_page_range = expand_page_range(initial_page_range, page_count or 1)
+ await self.db.execute(
+ "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",
+ (sha256, now, path),
+ )
+ await self.db.execute(
+ "INSERT INTO parses (sha256, tier, page_range, status, privacy, priority, created_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, 0, ?, ?)",
+ (sha256, tier, parse_page_range, PARSE_STATUS_PENDING, privacy, now, now),
+ )
+ await self._record_count("parse_task.created.count", dimensions={"tier": tier})
+
+ return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
+
+ async def _mark_moved_duplicate_files_deleted(self, sha256: str, keep_path: str) -> None:
+ """同哈希的新路径入库后,将已不存在的旧路径标为删除。"""
+ rows = cast(
+ list[FileRow],
+ await self.db.fetchall(
+ "SELECT id, path FROM files WHERE sha256=? AND status=? AND path<>?",
+ (sha256, FILE_STATUS_ACTIVE, keep_path),
+ ),
+ )
+ now = _now_ms()
+ for row in rows:
+ if not os.path.exists(row["path"]):
+ await self.db.execute(
+ "UPDATE files SET status=?, deleted_at=?, updated_at=? WHERE id=?",
+ (FILE_STATUS_DELETED, now, now, row["id"]),
+ )
+
+ async def _record_count(self, metric_name: str, *, value: int = 1, dimensions: dict[str, str] | None = None) -> None:
+ """通过显式指标接口记录入库和初始解析计数。"""
+ if self.telemetry_svc is not None:
+ await self.telemetry_svc.record_count(metric_name, value=value, dimensions=dimensions)
+
+ async def _record_duration(self, metric_name: str, start_ms: int, *, dimensions: dict[str, str] | None = None) -> None:
+ """使用原有非负耗时和维度记录入库耗时桶。"""
+ if self.telemetry_svc is not None:
+ await self.telemetry_svc.record_duration_bucket(
+ metric_name, duration_ms=max(0, _now_ms() - start_ms), dimensions=dimensions
+ )
+
+
+def _file_info(row: FileRow | None) -> FileInfo | None:
+ """将文件数据库行转换为对外文件信息。"""
+ return FileInfo.model_validate(row) if row is not None else None
+
+
+__all__ = ["FileRefreshResult", "FileRefreshStatus", "IngestService", "ensure_doc_record", "raise_refresh_error"]
diff --git a/mineru/doclib/services/parse_policy.py b/mineru/doclib/services/parse_policy.py
new file mode 100644
index 00000000..da3dac2f
--- /dev/null
+++ b/mineru/doclib/services/parse_policy.py
@@ -0,0 +1,25 @@
+"""Doclib 入库与主动解析共用的默认页范围和后台档位策略。"""
+
+from __future__ import annotations
+
+from ...parser.page_range import expand_page_range
+from ...types import Tier, select_parsing_rule_tier
+
+
+def default_parse_range(page_count: int | None) -> str:
+ """主动读取默认选择前十页,未知页数时保留第一页请求。"""
+ if page_count and page_count > 0:
+ return expand_page_range("1-10", page_count)
+ return "1"
+
+
+def resolve_parsing_rule_default_tier(remote: bool = False) -> Tier:
+ """后台入库规则沿用质量档位降级到 Flash 的既有策略。"""
+ from ..background.parse_server_health import get_health
+
+ health = get_health()
+ supported = health.remote.probe.tiers if remote else health.local.probe.tiers
+ return select_parsing_rule_tier(supported)
+
+
+__all__ = ["default_parse_range", "resolve_parsing_rule_default_tier"]
diff --git a/mineru/doclib/services/parse_svc.py b/mineru/doclib/services/parse_svc.py
index e8dbe061..97c300b0 100644
--- a/mineru/doclib/services/parse_svc.py
+++ b/mineru/doclib/services/parse_svc.py
@@ -1,4 +1,4 @@
-"""Parse service: file ingestion, parse request/acquire/process."""
+"""Doclib 解析请求、队列执行、产物写出和索引维护。"""
from __future__ import annotations
@@ -6,35 +6,37 @@ import asyncio
import logging
import os
import time
-from dataclasses import dataclass
from pathlib import Path
-from typing import Literal, cast
+from typing import cast
from urllib.parse import urlparse
from ...errors import InvalidRequestError, MineruError
from ...filetypes import (
- CSV_EXTENSIONS,
- IMAGE_EXTENSIONS,
- INGESTIBLE_EXTENSIONS,
OFD_EXTENSIONS,
OFFICE_EXTENSIONS,
PAGE_RANGE_PARSE_EXTENSIONS,
TEXT_EXTENSIONS,
TIERED_PARSE_EXTENSIONS,
- file_type_for_extension,
is_office_temp_lock_file,
)
from ...parser.api_client import _APITransportError, _V1APIError
from ...parser.base import ParseResult
from ...parser.page_range import (
expand_page_range,
- format_page_range as _page_numbers_to_range_str,
normalize_page_range_input,
normalize_result_page_range,
parse_page_range_set,
)
-from ...types import QUALITY_TIERS, TIER_ORDER, Tier, select_default_quality_tier, select_parsing_rule_tier
+from ...parser.page_range import (
+ format_page_range as _page_numbers_to_range_str,
+)
+from ...types import QUALITY_TIERS, TIER_ORDER, Tier, select_default_quality_tier
+from ..core import fts as fts_utils
from ..core.db import DatabaseManager
+from ..core.file_io import (
+ metadata_to_doclib,
+)
+from ..core.fts import FTSManager
from ..core.parse_artifacts import (
accessible_file_for_sha256,
load_pages_from_done_batches,
@@ -44,33 +46,20 @@ from ..core.parse_artifacts import (
parse_coverage,
write_parse_payload,
)
-from ..core.file_io import (
- FileStat,
- MetadataExtractionError,
- compute_sha256,
- extract_metadata,
- get_file_stat,
- metadata_to_doclib,
-)
-from ..core.fts import FTSManager
from ..remote_api import resolve_remote_api_key
-from ..rows import DocRow, FileRow, ParseRow, Sha256Row, ShortIdRow, WatchTargetRow
+from ..rows import DocRow, FileRow, ParseRow
from ..types import (
FILE_STATUS_ACTIVE,
- FILE_STATUS_DELETED,
- FILE_STATUS_UNREACHABLE,
PARSE_STATUS_DONE,
PARSE_STATUS_FAILED,
PARSE_STATUS_PARSING,
PARSE_STATUS_PENDING,
PARSE_STATUS_SUPERSEDED,
- RULE_TYPE_PARSING_RULE,
- WATCH_STATUS_UNREACHABLE,
- FileInfo,
ParseResponse,
)
-from ..utils.path_utils import normalize_doclib_path
+from . import parse_policy
from .config_svc import ConfigService
+from .ingest_svc import IngestService, raise_refresh_error
logger = logging.getLogger("mineru.parse")
@@ -87,41 +76,6 @@ class ParseFailure(MineruError):
self.message = message
-MAX_FTS_CHARS = 30_000
-FTS_HEAD_HALF = 15_000
-
-
-FileRefreshStatus = Literal["known", "new", "changed", "missing", "deleted", "unreachable", "unsupported", "error"]
-
-
-@dataclass(frozen=True)
-class FileRefreshResult:
- file: FileInfo | None
- status: FileRefreshStatus
- error_code: str | None = None
- error_msg: str | None = None
-
- @property
- def needs_ingest(self) -> bool:
- return self.file is not None and self.file.sha256 is None
-
-
-def _raise_refresh_error(refreshed: FileRefreshResult) -> None:
- code = (
- (refreshed.file.error_code if refreshed.file and refreshed.file.error_code else None)
- or refreshed.error_code
- or "ingest_failed"
- )
- message = (
- (refreshed.file.error_msg if refreshed.file and refreshed.file.error_msg else None)
- or refreshed.error_msg
- or "File could not be ingested."
- )
- if code == "file_permission_denied":
- raise InvalidRequestError(code, message, "path")
- raise MineruError(code, message, "path")
-
-
# ── range helpers ──────────────────────────────────────────────────
@@ -146,79 +100,6 @@ def page_range_uncovered(request_page_range: str, done_batches: list[ParseRow])
return needed_page_numbers - covered_page_numbers
-def default_parse_range(page_count: int | None) -> str:
- """Return the default page range for active reading."""
- if page_count and page_count > 0:
- return expand_page_range("1-10", page_count)
- return "1"
-
-
-async def ensure_doc_record(
- db: DatabaseManager,
- *,
- sha256: str,
- size_bytes: int,
- file_type: str,
- page_count: int | None,
- title: str | None,
- author: str | None,
- subject: str | None,
- keywords: str | None,
- is_image_based: int = 0,
- language: str | None = None,
- error_code: str | None,
- error_msg: str | None,
- first_seen_at: int,
- updated_at: int,
-) -> None:
- existing = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))
- if existing is not None:
- return
-
- for length in range(7, len(sha256) + 1):
- short_id = sha256[:length]
- await db.execute(
- "INSERT OR IGNORE INTO docs (sha256, short_id, size_bytes, file_type, page_count, "
- "title, author, subject, keywords, is_image_based, error_code, error_msg, first_seen_at, updated_at, language) "
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
- (
- sha256,
- short_id,
- size_bytes,
- file_type,
- page_count,
- title,
- author,
- subject,
- keywords,
- is_image_based,
- error_code,
- error_msg,
- first_seen_at,
- updated_at,
- language,
- ),
- )
- inserted = cast(ShortIdRow | None, await db.fetchone("SELECT short_id FROM docs WHERE sha256=?", (sha256,)))
- if inserted is not None:
- return
- conflict = cast(Sha256Row | None, await db.fetchone("SELECT sha256 FROM docs WHERE short_id=?", (short_id,)))
- if conflict is not None and conflict["sha256"] != sha256:
- continue
- break
-
- raise RuntimeError(f"Failed to allocate unique short_id for document {sha256}")
-
-
-# ── text truncation ────────────────────────────────────────────────
-
-
-def truncate_head_tail(text: str, max_chars: int = MAX_FTS_CHARS, half: int = FTS_HEAD_HALF) -> str:
- if len(text) <= max_chars:
- return text
- return text[:half] + "\n...\n" + text[-half:]
-
-
# ── ParseService ───────────────────────────────────────────────────
@@ -230,9 +111,12 @@ class ParseService:
config_svc: ConfigService,
data_dir: str,
*,
+ ingest_svc: IngestService,
parse_lock_timeout_sec: int,
telemetry_svc: object | None = None,
) -> None:
+ """保存解析依赖,文件刷新和入库由显式注入的服务处理。"""
+ self.ingest_svc = ingest_svc
self.db = db
self.fts = fts
self.config_svc = config_svc
@@ -240,411 +124,6 @@ class ParseService:
self.parse_lock_timeout_ms = parse_lock_timeout_sec * 1000
self.telemetry_svc = telemetry_svc
- # ── discovery / ingestion (shared) ──────────────────────
-
- async def refresh_file(
- self,
- path: str,
- watch_id: int | None = None,
- *,
- ensure_ingested: bool = False,
- allow_images: bool = False,
- ) -> FileRefreshResult:
- """Refresh one source path against the files table.
-
- This method owns file stat, DB row lookup, new/changed/known/missing/deleted
- classification, and optional synchronous ingest.
- """
- path = normalize_doclib_path(path)
- ext = Path(path).suffix.lower().lstrip(".")
- existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
- if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):
- return FileRefreshResult(file=_file_info(existing), status="unsupported")
-
- try:
- stat = await get_file_stat(path)
- except FileNotFoundError:
- return await self._refresh_missing_file(path, existing)
- except PermissionError as exc:
- return await self._refresh_stat_error(existing, "file_permission_denied", str(exc))
- except OSError as exc:
- return await self._refresh_stat_error(existing, "stat_failed", str(exc))
-
- result = await self._refresh_existing_file_with_stat(path, ext, stat, existing, watch_id)
- if ensure_ingested and result.needs_ingest:
- row = await self.ingest_file(path, watch_id=watch_id, allow_images=allow_images)
- return FileRefreshResult(file=_file_info(row), status=result.status)
- return result
-
- async def _refresh_existing_file_with_stat(
- self,
- path: str,
- ext: str,
- stat: FileStat,
- existing: FileRow | None,
- watch_id: int | None,
- ) -> FileRefreshResult:
- now = _now_ms()
- filename = Path(path).name
-
- if existing and existing["status"] == FILE_STATUS_ACTIVE:
- unchanged = existing["mtime_ms"] == stat.mtime_ms and existing["size_bytes"] == stat.size_bytes
- if unchanged:
- if watch_id is not None and existing["watch_id"] is None:
- await self.db.execute(
- "UPDATE files SET watch_id=?, updated_at=? WHERE id=? AND watch_id IS NULL",
- (watch_id, now, existing["id"]),
- )
- existing = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
- return FileRefreshResult(file=_file_info(existing), status="known")
-
- await self.db.execute(
- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
- "sha256=NULL, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "
- "WHERE id=?",
- (
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- watch_id,
- now,
- existing["id"],
- ),
- )
- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
- return FileRefreshResult(file=_file_info(row), status="changed")
-
- if existing:
- await self.db.execute(
- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
- "sha256=NULL, watch_id=COALESCE(?, watch_id), status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, "
- "deleted_at=NULL, updated_at=? WHERE id=?",
- (
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- watch_id,
- FILE_STATUS_ACTIVE,
- now,
- existing["id"],
- ),
- )
- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
- return FileRefreshResult(file=_file_info(row), status="changed")
-
- file_id = await self.db.execute_insert(
- "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "
- "watch_id, first_seen_at, updated_at) "
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
- (
- path,
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- watch_id,
- now,
- now,
- ),
- )
- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (file_id,)))
- return FileRefreshResult(file=_file_info(row), status="new")
-
- async def _refresh_missing_file(self, path: str, existing: FileRow | None) -> FileRefreshResult:
- if existing is None:
- return FileRefreshResult(file=None, status="missing")
-
- now = _now_ms()
- file_status = FILE_STATUS_DELETED
- deleted_at = now
- if existing["status"] != FILE_STATUS_DELETED and await self._is_missing_due_to_unreachable_watch(existing):
- file_status = FILE_STATUS_UNREACHABLE
- deleted_at = None
-
- await self.db.execute(
- "UPDATE files SET status=?, locked_at=NULL, error_code=NULL, error_msg=NULL, deleted_at=?, updated_at=? WHERE id=?",
- (file_status, deleted_at, now, existing["id"]),
- )
- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
- refresh_status: FileRefreshStatus = "unreachable" if file_status == FILE_STATUS_UNREACHABLE else "deleted"
- return FileRefreshResult(file=_file_info(row), status=refresh_status)
-
- async def _is_missing_due_to_unreachable_watch(self, file_row: FileRow) -> bool:
- watch_id = file_row.get("watch_id")
- if watch_id is None:
- return False
- watch = cast(WatchTargetRow | None, await self.db.fetchone("SELECT * FROM watches WHERE id=?", (watch_id,)))
- if not watch or not watch.get("removable"):
- return False
- if watch["status"] == WATCH_STATUS_UNREACHABLE:
- return True
- try:
- await get_file_stat(watch["path"])
- except OSError:
- if self.config_svc is not None:
- await self.config_svc.update_watch_status(watch_id, WATCH_STATUS_UNREACHABLE)
- return True
- return False
-
- async def _refresh_stat_error(self, existing: FileRow | None, error_code: str, error_msg: str) -> FileRefreshResult:
- if existing is None:
- return FileRefreshResult(file=None, status="error", error_code=error_code, error_msg=error_msg)
-
- now = _now_ms()
- await self.db.execute(
- "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE id=?",
- (error_code, error_msg[:500], now, existing["id"]),
- )
- row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE id=?", (existing["id"],)))
- return FileRefreshResult(file=_file_info(row), status="error", error_code=error_code, error_msg=error_msg)
-
- async def ensure_ingested(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:
- """Synchronously discover and ingest a source path when needed."""
- path = normalize_doclib_path(path)
- refreshed = await self.refresh_file(path, watch_id=watch_id, ensure_ingested=True, allow_images=allow_images)
- if refreshed.status == "error":
- _raise_refresh_error(refreshed)
- if refreshed.file is None or refreshed.status == "unsupported":
- return None
- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
-
- async def ingest_file(
- self,
- path: str,
- watch_id: int | None = None,
- *,
- trigger: str = "parse",
- allow_images: bool = False,
- ) -> FileRow | None:
- """Ingest a discovered file: SHA-256 + metadata + trigger default parse."""
- path = normalize_doclib_path(path)
- start_ms = _now_ms()
- status = "succeeded"
- try:
- return await self._ingest_file(path, watch_id=watch_id, allow_images=allow_images)
- except PermissionError as exc:
- status = "failed"
- await self._mark_file_error(path, "file_permission_denied", str(exc))
- raise InvalidRequestError("file_permission_denied", str(exc), "path") from None
- except Exception:
- status = "failed"
- raise
- finally:
- dimensions = {"status": status, "trigger": trigger}
- await self._record_count("ingest.finished.count", dimensions=dimensions)
- await self._record_duration("ingest.duration_bucket.count", start_ms, dimensions=dimensions)
-
- async def _mark_file_error(self, path: str, error_code: str, error_msg: str) -> None:
- path = normalize_doclib_path(path)
- now = _now_ms()
- await self.db.execute(
- "UPDATE files SET sha256=NULL, locked_at=NULL, error_code=?, error_msg=?, updated_at=? WHERE path=?",
- (error_code, error_msg[:500], now, path),
- )
-
- async def _ingest_file(self, path: str, watch_id: int | None = None, *, allow_images: bool = False) -> FileRow | None:
- """Ingest implementation without telemetry wrapper."""
- ext = Path(path).suffix.lower().lstrip(".")
- if ext not in INGESTIBLE_EXTENSIONS or (ext in IMAGE_EXTENSIONS and not allow_images) or is_office_temp_lock_file(path):
- return None
-
- stat = await get_file_stat(path)
- sha256 = await compute_sha256(path)
- now = _now_ms()
- filename = Path(path).name
-
- existing_path = cast(
- FileRow | None,
- await self.db.fetchone("SELECT * FROM files WHERE path=? AND status=?", (path, FILE_STATUS_ACTIVE)),
- )
- if existing_path and existing_path["sha256"] == sha256:
- return existing_path
-
- # check if same content (sha256) is already tracked by another path
- existing_sha = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE sha256=?", (sha256,)))
- if existing_sha:
- # same content, possibly a new or changed path — bind this file row
- # to the existing doc without duplicating docs/parses.
- if existing_path:
- await self.db.execute(
- "UPDATE files SET filename=?, ext=?, size_bytes=?, mtime_ms=?, "
- "sha256=?, watch_id=COALESCE(?, watch_id), locked_at=NULL, error_code=NULL, error_msg=NULL, updated_at=? "
- "WHERE id=?",
- (
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- sha256,
- watch_id,
- now,
- existing_path["id"],
- ),
- )
- else:
- await self.db.execute(
- "INSERT INTO files (path, filename, ext, size_bytes, mtime_ms, "
- "sha256, watch_id, first_seen_at, updated_at) "
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
- (
- path,
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- sha256,
- watch_id,
- now,
- now,
- ),
- )
- file_row = cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
- if file_row:
- await self.fts.upsert_filename(file_row["id"], Path(path).stem, ext)
- # the source may have been moved here from another tracked path;
- # retire stale duplicates whose files no longer exist.
- await self._mark_moved_duplicate_files_deleted(sha256, path)
- return file_row
-
- # brand new document
- metadata_error_code = None
- metadata_error_msg = None
- try:
- metadata = await extract_metadata(path)
- metadata_error_code = metadata.get("error_code")
- metadata_error_msg = metadata.get("error_msg")
- except MetadataExtractionError as exc:
- metadata_error_code = exc.code
- metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"
- metadata = {
- "page_count": None,
- "title": None,
- "author": None,
- "subject": None,
- "keywords": None,
- "is_image_based": 0,
- }
- except Exception as exc:
- metadata_error_code = "open_failed"
- metadata_error_msg = str(exc)[:500] or "Failed to extract document metadata"
- metadata = {
- "page_count": None,
- "title": None,
- "author": None,
- "subject": None,
- "keywords": None,
- "is_image_based": 0,
- }
-
- # 无固有分页或元数据暂不可读的结构化格式默认按一页处理。
- page_count = metadata["page_count"]
- if page_count is None:
- if ext in ("doc", "docx", "rtf"):
- page_count = 1 # reflow docs
- elif ext in ("ppt", "pptx", "xls", "xlsx"):
- page_count = 1 # will be updated by metadata extraction
- elif ext in CSV_EXTENSIONS:
- page_count = 1
-
- now = _now_ms()
- await self.db.execute(
- "INSERT OR IGNORE INTO files (path, filename, ext, size_bytes, mtime_ms, "
- "watch_id, first_seen_at, updated_at) "
- "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
- (
- path,
- filename,
- ext,
- stat.size_bytes,
- stat.mtime_ms,
- watch_id,
- now,
- now,
- ),
- )
-
- await ensure_doc_record(
- self.db,
- sha256=sha256,
- size_bytes=stat.size_bytes,
- file_type=file_type_for_extension(ext),
- page_count=page_count,
- title=metadata["title"],
- author=metadata["author"],
- subject=metadata["subject"],
- keywords=metadata["keywords"],
- is_image_based=int(metadata.get("is_image_based") or 0),
- language=metadata.get("language"),
- error_code=metadata_error_code,
- error_msg=metadata_error_msg,
- first_seen_at=now,
- updated_at=now,
- )
-
- # FTS filename index
- file_row = await self.db.fetchone("SELECT id FROM files WHERE path=?", (path,))
- if file_row:
- await self.fts.upsert_filename(
- file_row["id"],
- Path(path).stem,
- ext,
- )
-
- if ext in TEXT_EXTENSIONS:
- await self.db.execute(
- "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",
- (sha256, now, path),
- )
- text = await asyncio.to_thread(Path(path).read_text, encoding="utf-8", errors="replace")
- await self.fts.replace(
- sha256=sha256,
- tier=None,
- text=truncate_head_tail(text),
- title=metadata["title"] or "",
- author=metadata["author"] or "",
- )
- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
-
- # determine tier and page_range for initial parse
- tier: Tier = "flash"
- privacy = "local"
- initial_page_range = (
- default_parse_range(page_count) if ext in PAGE_RANGE_PARSE_EXTENSIONS else expand_page_range(None, page_count or 1)
- )
-
- # check parsing-rules
- matched = await self.config_svc.match_rules(path, RULE_TYPE_PARSING_RULE)
- if matched:
- rule = matched[0]
- rule_tier = cast(Tier | None, rule.get("tier"))
- rule_remote = bool(rule.get("remote", False))
- if ext in TIERED_PARSE_EXTENSIONS:
- tier = rule_tier or _resolve_parsing_rule_default_tier(remote=rule_remote)
- privacy = "remote" if rule_remote and tier != "flash" else "local"
- else:
- tier = "flash"
- privacy = "local"
- rule_page_range = rule.get("page_range")
- if rule_page_range and ext in PAGE_RANGE_PARSE_EXTENSIONS:
- initial_page_range = expand_page_range(rule_page_range, page_count or 1)
-
- # insert parse batch
- parse_page_range = expand_page_range(initial_page_range, page_count or 1)
- await self.db.execute(
- "UPDATE files SET sha256=?, locked_at=NULL, updated_at=? WHERE path=?",
- (sha256, now, path),
- )
- await self.db.execute(
- "INSERT INTO parses (sha256, tier, page_range, status, privacy, priority, created_at, updated_at) "
- "VALUES (?, ?, ?, ?, ?, 0, ?, ?)",
- (sha256, tier, parse_page_range, PARSE_STATUS_PENDING, privacy, now, now),
- )
- await self._record_count("parse_task.created.count", dimensions={"tier": tier})
-
- return cast(FileRow | None, await self.db.fetchone("SELECT * FROM files WHERE path=?", (path,)))
-
# ── parse request ───────────────────────────────────────────
async def request_parse(
@@ -659,14 +138,14 @@ class ParseService:
"""Handle a parse request from CLI. Returns info for status polling."""
page_range = normalize_page_range_input(page_range) or None
# ensure the path is current before trusting files.sha256
- refreshed = await self.refresh_file(path, ensure_ingested=True, allow_images=True)
+ refreshed = await self.ingest_svc.refresh_file(path, ensure_ingested=True, allow_images=True)
if refreshed.status == "unsupported":
ext = Path(path).suffix.lower() or Path(path).name
raise InvalidRequestError("file_type_unsupported", _unsupported_file_type_message(ext), "path")
if refreshed.status in {"missing", "deleted", "unreachable"}:
raise InvalidRequestError("file_not_found", f"File {path} not found.", "path")
if refreshed.status == "error":
- _raise_refresh_error(refreshed)
+ raise_refresh_error(refreshed)
if refreshed.file is None:
raise MineruError("ingest_failed", "File could not be ingested.", "path")
@@ -724,7 +203,9 @@ class ParseService:
# ── expand page range ──
supports_page_range = ext in PAGE_RANGE_PARSE_EXTENSIONS
- requested_page_range_input = page_range or (default_parse_range(page_count) if supports_page_range else None)
+ requested_page_range_input = page_range or (
+ parse_policy.default_parse_range(page_count) if supports_page_range else None
+ )
request_page_range = expand_page_range(requested_page_range_input, page_count or 1)
needed_page_numbers = parse_page_range_set(request_page_range)
@@ -844,22 +325,6 @@ class ParseService:
await self._record_count("parse_task.started.count", dimensions={"tier": task["tier"]})
return task
- async def _mark_moved_duplicate_files_deleted(self, sha256: str, keep_path: str) -> None:
- rows = cast(
- list[FileRow],
- await self.db.fetchall(
- "SELECT id, path FROM files WHERE sha256=? AND status=? AND path<>?",
- (sha256, FILE_STATUS_ACTIVE, keep_path),
- ),
- )
- now = _now_ms()
- for row in rows:
- if not os.path.exists(row["path"]):
- await self.db.execute(
- "UPDATE files SET status=?, deleted_at=?, updated_at=? WHERE id=?",
- (FILE_STATUS_DELETED, now, now, row["id"]),
- )
-
async def process_doc(self, task: ParseRow) -> bool:
"""Execute parse for a batch. Returns True on success."""
task_start_ms = _now_ms()
@@ -1348,7 +813,7 @@ class ParseService:
if existing_tier and TIER_ORDER.get(existing_tier, -1) >= TIER_ORDER.get(tier, -1):
return # current FTS data is from a higher or equal tier
- text = truncate_head_tail(text)
+ text = fts_utils.truncate_head_tail(text)
await self.fts.replace(
sha256=sha256,
tier=tier,
@@ -1426,9 +891,10 @@ class ParseService:
if not pages:
continue
- from ...config import get_config
from docvortex.render.markdown import render_single_block
+ from ...config import get_config
+
delimiters = get_config().render.latex_delimiters
parts: list[str] = []
for page in pages:
@@ -1441,7 +907,7 @@ class ParseService:
)
if rendered and rendered.strip():
parts.append(rendered.strip())
- text = truncate_head_tail("\n\n".join(parts))
+ text = fts_utils.truncate_head_tail("\n\n".join(parts))
await self.fts.replace(
sha256=sha256,
tier=tier,
@@ -1489,15 +955,6 @@ def _resolve_default_tier(remote: bool = False) -> Tier:
)
-def _resolve_parsing_rule_default_tier(remote: bool = False) -> Tier:
- """Parsing-rule 是后台批量策略,允许 standard -> basic -> flash。"""
- from ..background.parse_server_health import get_health
-
- health = get_health()
- supported = health.remote.probe.tiers if remote else health.local.probe.tiers
- return select_parsing_rule_tier(supported)
-
-
def _format_action_list(actions: list[str]) -> str:
if len(actions) == 1:
return actions[0]
@@ -1601,10 +1058,6 @@ def _is_official_remote_url(url: str) -> bool:
return host in {"mineru.net", "mineru.org.cn"} or host.endswith(".mineru.net") or host.endswith(".mineru.org.cn")
-def _file_info(row: FileRow | None) -> FileInfo | None:
- return FileInfo.model_validate(row) if row is not None else None
-
-
def _unsupported_file_type_message(ext_or_name: str) -> str:
display = ext_or_name if ext_or_name.startswith(".") else Path(ext_or_name).name
return f"File type is not supported for parsing: {display}"
diff --git a/mineru/doclib/services/scan_svc.py b/mineru/doclib/services/scan_svc.py
index d88992c7..5edc0945 100644
--- a/mineru/doclib/services/scan_svc.py
+++ b/mineru/doclib/services/scan_svc.py
@@ -2,25 +2,26 @@
from __future__ import annotations
+import logging
import os
import time
-import logging
from pathlib import Path
from typing import Any, cast
from ...errors import InvalidRequestError, NotFoundError
from ...filetypes import DISCOVERABLE_EXTENSIONS, is_office_temp_lock_file
+from ..core.db import DatabaseManager
from ..core.file_io import get_file_stat
from ..rows import PathRow, ScanRow, WatchTargetRow
from ..types import (
+ FILE_STATUS_ACTIVE,
SCAN_KIND_MANUAL,
SCAN_KIND_WATCH,
- FILE_STATUS_ACTIVE,
+ SCAN_SOURCE_UNKNOWN,
SCAN_STATUS_DONE,
SCAN_STATUS_FAILED,
SCAN_STATUS_PENDING,
SCAN_STATUS_RUNNING,
- SCAN_SOURCE_UNKNOWN,
WATCH_STATUS_ACTIVE,
WATCH_STATUS_UNREACHABLE,
ScanInfo,
@@ -28,10 +29,9 @@ from ..types import (
ScanSource,
ScanStatus,
)
-from ..core.db import DatabaseManager
from ..utils.path_utils import normalize_doclib_path
from .config_svc import ConfigService
-from .parse_svc import FileRefreshResult, ParseService
+from .ingest_svc import FileRefreshResult, IngestService
logger = logging.getLogger("mineru.scan")
@@ -69,14 +69,15 @@ class ScanService:
self,
db: DatabaseManager,
config_svc: ConfigService,
- parse_svc: ParseService,
+ ingest_svc: IngestService,
*,
scan_lock_timeout_sec: int,
telemetry_svc: object | None = None,
) -> None:
+ """扫描只依赖文件刷新入库能力,不接管解析执行。"""
self.db = db
self.config_svc = config_svc
- self.parse_svc = parse_svc
+ self.ingest_svc = ingest_svc
self.scan_lock_timeout_ms = scan_lock_timeout_sec * 1000
self.telemetry_svc = telemetry_svc
@@ -356,7 +357,7 @@ class ScanService:
await self._refresh_file(row["path"], watch_id=watch_id, counters=counters)
async def _refresh_file(self, path: str, *, watch_id: int | None, counters: "_ScanCounters") -> FileRefreshResult:
- result = await self.parse_svc.refresh_file(path, watch_id=watch_id)
+ result = await self.ingest_svc.refresh_file(path, watch_id=watch_id)
counters.add_refresh(result.status)
return result
diff --git a/mineru/doclib/services/search_svc.py b/mineru/doclib/services/search_svc.py
index b58128da..7db9fa8b 100644
--- a/mineru/doclib/services/search_svc.py
+++ b/mineru/doclib/services/search_svc.py
@@ -13,7 +13,7 @@ from ..rows import ContentSearchResultRow, DocRow, FilenameSearchFileRow, Filena
from ..types import FILE_STATUS_ACTIVE
if TYPE_CHECKING:
- from .parse_svc import FileRefreshResult
+ from .ingest_svc import FileRefreshResult
FilenamePathProbe = Callable[[str], Awaitable["FileRefreshResult"]]
_FILENAME_METADATA_BATCH_SIZE = 256
diff --git a/tests/unittest/doclib_test_utils.py b/tests/unittest/doclib_test_utils.py
new file mode 100644
index 00000000..7cf3d302
--- /dev/null
+++ b/tests/unittest/doclib_test_utils.py
@@ -0,0 +1,33 @@
+"""Doclib 测试显式组装共享入库与解析依赖。"""
+
+from __future__ import annotations
+
+from mineru.doclib.core.db import DatabaseManager
+from mineru.doclib.core.fts import FTSManager
+from mineru.doclib.services.config_svc import ConfigService
+from mineru.doclib.services.ingest_svc import IngestService
+from mineru.doclib.services.parse_svc import ParseService
+from mineru.doclib.telemetry import TelemetryService
+
+
+def create_parse_service(
+ db: DatabaseManager,
+ fts: FTSManager,
+ config_svc: ConfigService,
+ data_dir: str,
+ *,
+ parse_lock_timeout_sec: int,
+ telemetry_svc: TelemetryService | None = None,
+ service_type: type[ParseService] = ParseService,
+) -> ParseService:
+ """为测试构造共享数据库、索引、规则和指标的两个真实服务。"""
+ ingest_svc = IngestService(db, fts, config_svc, telemetry_svc=telemetry_svc)
+ return service_type(
+ db,
+ fts,
+ config_svc,
+ data_dir,
+ ingest_svc=ingest_svc,
+ parse_lock_timeout_sec=parse_lock_timeout_sec,
+ telemetry_svc=telemetry_svc,
+ )
diff --git a/tests/unittest/test_cli_parse_full_document.py b/tests/unittest/test_cli_parse_full_document.py
index 7b1af0d1..e2af2928 100644
--- a/tests/unittest/test_cli_parse_full_document.py
+++ b/tests/unittest/test_cli_parse_full_document.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
import asyncio
import json
import re
@@ -24,7 +26,6 @@ from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.fts import FTSManager
from mineru.doclib.server import DoclibServer
from mineru.doclib.services.config_svc import ConfigService
-from mineru.doclib.services.parse_svc import ParseService
from mineru.doclib.types import (
ContentRequestScope,
DocContentExportRequest,
@@ -59,14 +60,16 @@ class _DoclibHarness:
self.loop = asyncio.new_event_loop()
self.db = DatabaseManager(str(root / "doclib.db"))
self.run(self.db.initialize())
- self.service = ParseService(
+ self.service = create_parse_service(
self.db,
FTSManager(self.db),
ConfigService(self.db),
str(root / "data"),
parse_lock_timeout_sec=1800,
)
- self.server = DoclibServer(SimpleNamespace(db=self.db, data_dir=str(root / "data"), parse_svc=self.service))
+ self.server = DoclibServer(
+ SimpleNamespace(db=self.db, data_dir=str(root / "data"), parse_svc=self.service, ingest_svc=self.service.ingest_svc)
+ )
self.output_ranges: list[str | None] = []
def run(self, operation: Coroutine[Any, Any, _T]) -> _T:
diff --git a/tests/unittest/test_doclib_architecture.py b/tests/unittest/test_doclib_architecture.py
index 14ffc7eb..4e7c5285 100644
--- a/tests/unittest/test_doclib_architecture.py
+++ b/tests/unittest/test_doclib_architecture.py
@@ -49,3 +49,18 @@ def test_legacy_protocol_conversion_stays_at_cached_middle_json_boundary() -> No
if any(module.startswith("mineru.backend.postprocess.legacy_") for module in _imports(path))
}
assert consumers == {boundary}
+
+
+def test_ingest_service_does_not_depend_on_parse_execution_or_http() -> None:
+ """文件入库拥有独立实现,不反向导入解析服务、HTTP 层或宿主应用。"""
+ forbidden = ("fastapi", "starlette", "mineru.doclib.server", "mineru.doclib.app", "mineru.doclib.services.parse_svc")
+ imports = _imports(_ROOT / "mineru/doclib/services/ingest_svc.py")
+ assert not sorted(module for module in imports if module.startswith(forbidden))
+
+
+def test_file_discovery_workers_depend_on_ingest_service() -> None:
+ """扫描和文件事件处理依赖入库边界,解析工作池独立保留解析服务依赖。"""
+ for name in ("services/scan_svc.py", "background/watch.py", "background/ingest.py"):
+ imports = _imports(_ROOT / "mineru/doclib" / name)
+ assert "mineru.doclib.services.ingest_svc.IngestService" in imports, name
+ assert not any(module.startswith("mineru.doclib.services.parse_svc") for module in imports), name
diff --git a/tests/unittest/test_doclib_cache_semantics.py b/tests/unittest/test_doclib_cache_semantics.py
index 67c1e419..f2534114 100644
--- a/tests/unittest/test_doclib_cache_semantics.py
+++ b/tests/unittest/test_doclib_cache_semantics.py
@@ -1,4 +1,9 @@
from __future__ import annotations
+
+from doclib_test_utils import create_parse_service
+from mineru.doclib.services import ingest_svc as ingest_svc_module
+from mineru.doclib.services.ingest_svc import FileRefreshResult
+from mineru.doclib.services.parse_policy import resolve_parsing_rule_default_tier
from mineru.doclib.services.content_read_svc import ContentReadService
from docvortex.schema import Producer
from mineru.integrations.docvortex import build_metadata
@@ -47,19 +52,10 @@ from mineru.doclib.services.content_read_svc import (
_ReadPlan,
_locator_after,
)
-from mineru.doclib.services import parse_svc as parse_svc_module
from mineru.doclib.core import parse_artifacts as parse_artifacts_module
from mineru.doclib.services.cleanup_svc import CleanupService
from mineru.doclib.services.config_svc import ConfigService
-from mineru.doclib.services.parse_svc import (
- FileRefreshResult,
- ParseFailure,
- ParseService,
- _local_parse_server_url,
- _resolve_default_tier,
- _resolve_parsing_rule_default_tier,
- expand_page_range,
-)
+from mineru.doclib.services.parse_svc import ParseFailure, _local_parse_server_url, _resolve_default_tier, expand_page_range
from mineru.doclib.core.parse_artifacts import filter_pages_by_user_range, load_pages_from_done_batches, parse_batch_json_path
from mineru.doclib.services.scan_svc import ScanService
from mineru.doclib.services.search_svc import SearchService
@@ -458,13 +454,13 @@ def test_parsing_rule_default_tier_allows_flash_fallback(monkeypatch: pytest.Mon
health = ParseServerHealth(local=ProbeState(probe=ProbeResult(tiers=["basic", "advanced"])))
monkeypatch.setattr("mineru.doclib.background.parse_server_health.get_health", lambda: health)
- assert _resolve_parsing_rule_default_tier() == "basic"
+ assert resolve_parsing_rule_default_tier() == "basic"
health.local.probe = ProbeResult(tiers=["basic", "standard", "advanced"])
- assert _resolve_parsing_rule_default_tier() == "standard"
+ assert resolve_parsing_rule_default_tier() == "standard"
health.local.probe = ProbeResult(tiers=["flash"])
- assert _resolve_parsing_rule_default_tier() == "flash"
+ assert resolve_parsing_rule_default_tier() == "flash"
def test_read_without_tier_uses_highest_cached_tier(tmp_path: Path) -> None:
@@ -1135,7 +1131,7 @@ def test_remote_api_target_prefers_config_api_key_over_env(monkeypatch: pytest.M
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_ConfigService(),
@@ -1174,7 +1170,7 @@ def test_remote_api_target_fallback_uses_local_self_hosted_api_key(monkeypatch:
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_ConfigService(),
@@ -1216,7 +1212,7 @@ def test_remote_api_target_fallback_managed_mode_sends_no_api_key(monkeypatch: p
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_ConfigService(),
@@ -1255,7 +1251,7 @@ def test_remote_api_target_preserves_probe_invalid_api_key(monkeypatch: pytest.M
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_ConfigService(),
@@ -1292,7 +1288,7 @@ def test_self_hosted_api_target_preserves_probe_invalid_api_key(monkeypatch: pyt
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_ConfigService(),
@@ -1478,7 +1474,7 @@ def test_invalidate_deletes_fts_when_no_done_batches_remain(tmp_path: Path) -> N
parses = [{"sha256": sha256, "tier": "standard", "page_range": "1", "status": "done", "done_at": 1000}]
db = _FakeDB(parses=parses, file_row={"sha256": sha256, "status": "active", "filename": "doc.pdf"})
fts = _FakeFTS()
- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
count = asyncio.run(service.invalidate(sha256, "standard"))
@@ -1498,7 +1494,7 @@ def test_invalidate_rebuilds_fts_from_highest_remaining_done_tier(tmp_path: Path
]
db = _FakeDB(parses=parses, file_row={"sha256": sha256, "status": "active", "filename": "doc.pdf"})
fts = _FakeFTS()
- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
count = asyncio.run(service.invalidate(sha256, "standard"))
@@ -1518,7 +1514,7 @@ def test_ingest_binds_file_sha_before_creating_parse_task(tmp_path: Path, monkey
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
recording_db = _OrderRecordingDB(db)
- service = ParseService(
+ service = create_parse_service(
db=recording_db,
fts=_FilenameOnlyFTS(),
config_svc=_NoRulesConfig(),
@@ -1538,9 +1534,9 @@ def test_ingest_binds_file_sha_before_creating_parse_task(tmp_path: Path, monkey
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
- row = await service.ingest_file(str(source))
+ row = await service.ingest_svc.ingest_file(str(source))
dangling_parses = await db.fetchall(
"SELECT p.id FROM parses p LEFT JOIN files f ON f.sha256=p.sha256 AND f.status='active' WHERE f.id IS NULL",
)
@@ -1560,7 +1556,7 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: py
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=_FilenameOnlyFTS(),
config_svc=_NoRulesConfig(),
@@ -1578,7 +1574,7 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: py
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
old_dir = tmp_path / "old"
new_dir = tmp_path / "new"
@@ -1586,10 +1582,10 @@ def test_ingest_moved_file_retargets_to_new_path(tmp_path: Path, monkeypatch: py
new_dir.mkdir()
source = old_dir / "doc.pdf"
source.write_bytes(b"%PDF-1.4\n")
- first_row = await service.ingest_file(str(source))
+ first_row = await service.ingest_svc.ingest_file(str(source))
source.rename(new_dir / "doc.pdf")
- second_row = await service.ingest_file(str(new_dir / "doc.pdf"))
+ second_row = await service.ingest_svc.ingest_file(str(new_dir / "doc.pdf"))
assert first_row is not None
assert second_row is not None
@@ -1641,7 +1637,7 @@ def test_request_parse_explicit_image_ingests_and_queues_parse(tmp_path: Path, m
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -1661,7 +1657,7 @@ def test_request_parse_explicit_image_ingests_and_queues_parse(tmp_path: Path, m
"is_image_based": 1,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
result = await service.request_parse(str(source), tier="flash")
with pytest.raises(InvalidRequestError) as range_exc:
@@ -1704,7 +1700,7 @@ def test_request_parse_rejects_quality_tiers_for_non_pdf_image_inputs(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -1724,7 +1720,7 @@ def test_request_parse_rejects_quality_tiers_for_non_pdf_image_inputs(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
with pytest.raises(InvalidRequestError) as exc_info:
await service.request_parse(str(source), tier=tier)
@@ -1760,7 +1756,7 @@ def test_refresh_file_applies_parsing_rule_effective_tier_and_privacy(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_RuleConfig(),
@@ -1780,9 +1776,9 @@ def test_refresh_file_applies_parsing_rule_effective_tier_and_privacy(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
- refreshed = await service.refresh_file(str(source), ensure_ingested=True)
+ refreshed = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)
assert refreshed.file is not None
parse_rows = await db.fetchall("SELECT tier, privacy, page_range FROM parses")
@@ -1804,7 +1800,7 @@ def test_request_parse_rejects_remote_for_non_pdf_image_inputs(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -1824,7 +1820,7 @@ def test_request_parse_rejects_remote_for_non_pdf_image_inputs(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
with pytest.raises(InvalidRequestError) as exc_info:
await service.request_parse(str(source), remote=True)
@@ -1862,7 +1858,7 @@ def test_request_parse_reports_text_files_do_not_require_parsing(
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
fts = FTSManager(db)
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=fts,
config_svc=_NoRulesConfig(),
@@ -1882,7 +1878,7 @@ def test_request_parse_reports_text_files_do_not_require_parsing(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
with pytest.raises(InvalidRequestError) as exc_info:
await service.request_parse(str(source), tier=tier, force=force, remote=remote)
@@ -1915,7 +1911,7 @@ def test_csv_doclib_parse_creates_flash_cache_and_rendered_fts(
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
fts = FTSManager(db)
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=fts,
config_svc=_NoRulesConfig(),
@@ -1967,7 +1963,7 @@ def test_tsv_doclib_parse_creates_flash_cache_with_independent_file_type(
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
fts = FTSManager(db)
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=fts,
config_svc=_NoRulesConfig(),
@@ -2018,7 +2014,7 @@ def test_request_parse_rejects_flash_tier_for_remote_quality_inputs(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -2038,7 +2034,7 @@ def test_request_parse_rejects_flash_tier_for_remote_quality_inputs(
"is_image_based": int(ext != "pdf"),
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
with pytest.raises(InvalidRequestError) as exc_info:
await service.request_parse(str(source), tier="flash", remote=True)
@@ -2055,7 +2051,7 @@ def test_request_parse_rejects_unsupported_file_type_before_parse_response(tmp_p
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=None,
@@ -2088,7 +2084,7 @@ def test_request_parse_accepts_legacy_office_formats(tmp_path: Path, ext: str) -
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -2111,7 +2107,7 @@ def test_request_parse_raises_ingest_failed_when_file_row_has_no_sha(tmp_path: P
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=None,
@@ -2144,7 +2140,7 @@ def test_request_parse_raises_ingest_failed_when_file_row_has_no_sha(tmp_path: P
status="known",
)
- service.refresh_file = _refresh_file # type: ignore[method-assign]
+ service.ingest_svc.refresh_file = _refresh_file # type: ignore[method-assign]
with pytest.raises(MineruError) as exc_info:
await service.request_parse(str(source), tier="flash")
@@ -2198,7 +2194,7 @@ def test_force_request_reuses_active_and_creates_only_uncovered_parse(tmp_path:
},
doc_row={"sha256": sha256, "short_id": "eeeeeee", "page_count": 10},
)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
result = asyncio.run(service.request_parse(path, tier="standard", page_range="1-10", force=True))
@@ -2231,7 +2227,7 @@ def test_non_pdf_requests_use_one_full_document_batch(
"""依次验证初始任务、历史 partial 修复和 active 复用。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(),
@@ -2252,7 +2248,7 @@ def test_non_pdf_requests_use_one_full_document_batch(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
initial = await service.request_parse(str(source))
assert initial.page_range == "1-12"
@@ -2306,7 +2302,7 @@ def test_list_parse_records_by_ids_returns_precise_status(tmp_path: Path) -> Non
},
]
db = _FakeDB(parses=parses, file_row=None)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
result = asyncio.run(service.list_parse_records(ids=[2, 1]))
@@ -2423,7 +2419,7 @@ def test_ensure_doc_record_extends_short_id_on_prefix_conflict(tmp_path: Path) -
sha_b = ("a" * 7) + ("b" * 57)
for sha256 in (sha_a, sha_b):
- await parse_svc_module.ensure_doc_record(
+ await ingest_svc_module.ensure_doc_record(
db,
sha256=sha256,
size_bytes=10,
@@ -2452,19 +2448,19 @@ def test_ensure_ingested_rebinds_changed_text_file_to_new_sha(tmp_path: Path) ->
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
source.write_text("old", encoding="utf-8")
- first = await service.ensure_ingested(str(source))
+ first = await service.ingest_svc.ensure_ingested(str(source))
assert first is not None
assert first["sha256"] == hashlib.sha256(b"old").hexdigest()
source.write_text("new content", encoding="utf-8")
- discovered = await service.refresh_file(str(source))
+ discovered = await service.ingest_svc.refresh_file(str(source))
changed_row = await db.fetchone("SELECT * FROM files WHERE path=?", (str(source),))
assert discovered.status == "changed"
@@ -2474,7 +2470,7 @@ def test_ensure_ingested_rebinds_changed_text_file_to_new_sha(tmp_path: Path) ->
assert changed_row is not None
assert changed_row["sha256"] is None
- second = await service.ensure_ingested(str(source))
+ second = await service.ingest_svc.ensure_ingested(str(source))
assert second is not None
assert second["sha256"] == hashlib.sha256(b"new content").hexdigest()
@@ -2488,7 +2484,7 @@ def test_refresh_file_normalizes_lexical_path_aliases(tmp_path: Path) -> None:
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
root = tmp_path / "docs"
@@ -2497,8 +2493,8 @@ def test_refresh_file_normalizes_lexical_path_aliases(tmp_path: Path) -> None:
source.write_text("content", encoding="utf-8")
alias = root / "." / "nested" / ".." / "note.txt"
- first = await service.refresh_file(str(alias))
- second = await service.refresh_file(str(source))
+ first = await service.ingest_svc.refresh_file(str(alias))
+ second = await service.ingest_svc.refresh_file(str(source))
rows = await db.fetchall("SELECT path FROM files ORDER BY id")
assert first.status == "new"
@@ -2512,7 +2508,7 @@ def test_refresh_file_keeps_symlink_and_real_paths_as_distinct_file_entries(tmp_
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
real_root = tmp_path / "real-root"
@@ -2522,8 +2518,8 @@ def test_refresh_file_keeps_symlink_and_real_paths_as_distinct_file_entries(tmp_
source = real_root / "note.txt"
source.write_text("content", encoding="utf-8")
- alias_result = await service.refresh_file(str(alias_root / "note.txt"))
- real_result = await service.refresh_file(str(source))
+ alias_result = await service.ingest_svc.refresh_file(str(alias_root / "note.txt"))
+ real_result = await service.ingest_svc.refresh_file(str(source))
rows = await db.fetchall("SELECT path FROM files ORDER BY path")
assert alias_result.status == "new"
@@ -2537,18 +2533,18 @@ def test_refresh_file_marks_missing_known_path_deleted(tmp_path: Path) -> None:
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
source.write_text("content", encoding="utf-8")
- ingested = await service.ensure_ingested(str(source))
+ ingested = await service.ingest_svc.ensure_ingested(str(source))
assert ingested is not None
original_sha = ingested["sha256"]
source.unlink()
- refreshed = await service.refresh_file(str(source))
+ refreshed = await service.ingest_svc.refresh_file(str(source))
row = await db.fetchone("SELECT sha256, status, deleted_at FROM files WHERE path=?", (str(source),))
assert refreshed.status == "deleted"
@@ -2569,7 +2565,7 @@ def test_ensure_ingested_maps_read_permission_error_to_file_permission_denied(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "locked.pdf"
@@ -2578,10 +2574,10 @@ def test_ensure_ingested_maps_read_permission_error_to_file_permission_denied(
async def _permission_denied(path: str) -> str:
raise PermissionError("permission denied")
- monkeypatch.setattr(parse_svc_module, "compute_sha256", _permission_denied)
+ monkeypatch.setattr(ingest_svc_module, "compute_sha256", _permission_denied)
with pytest.raises(InvalidRequestError) as exc_info:
- await service.ensure_ingested(str(source))
+ await service.ingest_svc.ensure_ingested(str(source))
row = await db.fetchone("SELECT sha256, error_code, error_msg FROM files WHERE path=?", (str(source),))
@@ -2599,21 +2595,21 @@ def test_refresh_file_records_stat_error_without_marking_deleted(tmp_path: Path,
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
source.write_text("content", encoding="utf-8")
- ingested = await service.ensure_ingested(str(source))
+ ingested = await service.ingest_svc.ensure_ingested(str(source))
assert ingested is not None
async def _permission_denied(path: str) -> dict[str, Any]:
raise PermissionError("permission denied")
- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)
+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)
- refreshed = await service.refresh_file(str(source))
+ refreshed = await service.ingest_svc.refresh_file(str(source))
row = await db.fetchone("SELECT status, error_code, error_msg, deleted_at FROM files WHERE path=?", (str(source),))
assert refreshed.status == "error"
@@ -2635,7 +2631,7 @@ def test_request_parse_maps_stat_permission_error_without_existing_file_row(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "no_exec_dir" / "inside.pdf"
@@ -2643,7 +2639,7 @@ def test_request_parse_maps_stat_permission_error_without_existing_file_row(
async def _permission_denied(path: str) -> dict[str, Any]:
raise PermissionError(f"permission denied: {path}")
- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)
+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)
with pytest.raises(InvalidRequestError) as exc_info:
await service.request_parse(str(source), tier="flash")
@@ -2659,22 +2655,22 @@ def test_ingest_worker_skips_files_with_blocking_stat_errors(tmp_path: Path, mon
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
source.write_text("content", encoding="utf-8")
- assert await service.ensure_ingested(str(source)) is not None
+ assert await service.ingest_svc.ensure_ingested(str(source)) is not None
async def _permission_denied(path: str) -> dict[str, Any]:
raise PermissionError("permission denied")
- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)
+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)
- await service.refresh_file(str(source))
+ await service.ingest_svc.refresh_file(str(source))
- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)
+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)
assert await worker._acquire_task() is None
row = await db.fetchone("SELECT error_code, error_msg, locked_at FROM files WHERE path=?", (str(source),))
@@ -2690,7 +2686,7 @@ def test_ingest_worker_preserves_mineru_permission_errors(tmp_path: Path) -> Non
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
@@ -2701,7 +2697,7 @@ def test_ingest_worker_preserves_mineru_permission_errors(tmp_path: Path) -> Non
(str(source), source.name, "txt", 10, now, now, now),
)
- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)
+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)
await worker._handle_ingest_error(
{"id": file_id, "path": str(source), "watch_id": None},
InvalidRequestError("file_permission_denied", "permission denied", "path"),
@@ -2721,7 +2717,7 @@ def test_ingest_worker_preserves_precise_file_access_errors(tmp_path: Path) -> N
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
@@ -2732,7 +2728,7 @@ def test_ingest_worker_preserves_precise_file_access_errors(tmp_path: Path) -> N
(str(source), source.name, "txt", 10, now, now, now),
)
- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)
+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)
await worker._handle_ingest_error(
{"id": file_id, "path": str(source), "watch_id": None},
PermissionError("permission denied"),
@@ -2752,7 +2748,7 @@ def test_ingest_worker_skips_any_file_with_existing_error(tmp_path: Path) -> Non
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "note.txt"
@@ -2763,7 +2759,7 @@ def test_ingest_worker_skips_any_file_with_existing_error(tmp_path: Path) -> Non
(str(source), source.name, "txt", 10, now, "ingest_failed", "boom", now, now),
)
- worker = IngestWorkerPool(service, num_workers=1, lock_timeout_sec=60)
+ worker = IngestWorkerPool(service.ingest_svc, num_workers=1, lock_timeout_sec=60)
assert await worker._acquire_task() is None
row = await db.fetchone("SELECT error_code, locked_at FROM files WHERE path=?", (str(source),))
@@ -2953,7 +2949,7 @@ def test_find_probes_and_filters_deleted_paths_without_rescan(tmp_path: Path) ->
await db.initialize()
fts = FTSManager(db)
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db,
fts=fts,
config_svc=config_svc,
@@ -2967,11 +2963,12 @@ def test_find_probes_and_filters_deleted_paths_without_rescan(tmp_path: Path) ->
parse_svc=parse_svc,
search_svc=SearchService(db, fts),
telemetry_svc=None,
+ ingest_svc=parse_svc.ingest_svc,
)
)
source = tmp_path / "bench1.txt"
source.write_text("hello", encoding="utf-8")
- assert await parse_svc.ensure_ingested(str(source)) is not None
+ assert await parse_svc.ingest_svc.ensure_ingested(str(source)) is not None
source.unlink()
result = await server.find("bench1")
@@ -2992,7 +2989,7 @@ def test_find_reactivates_restored_deleted_path_without_rescan(tmp_path: Path) -
await db.initialize()
fts = FTSManager(db)
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db,
fts=fts,
config_svc=config_svc,
@@ -3006,11 +3003,12 @@ def test_find_reactivates_restored_deleted_path_without_rescan(tmp_path: Path) -
parse_svc=parse_svc,
search_svc=SearchService(db, fts),
telemetry_svc=None,
+ ingest_svc=parse_svc.ingest_svc,
)
)
source = tmp_path / "stale_restore.txt"
source.write_text("hello", encoding="utf-8")
- assert await parse_svc.ensure_ingested(str(source)) is not None
+ assert await parse_svc.ingest_svc.ensure_ingested(str(source)) is not None
source.unlink()
deleted = await server.find("stale_restore")
@@ -3033,7 +3031,7 @@ def test_refresh_file_marks_only_current_file_unreachable_when_watch_root_is_mis
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
root = tmp_path / "removable"
@@ -3044,13 +3042,13 @@ def test_refresh_file_marks_only_current_file_unreachable_when_watch_root_is_mis
source.write_text("content", encoding="utf-8")
other.write_text("other", encoding="utf-8")
- assert await service.ensure_ingested(str(source), watch_id=watch["id"]) is not None
- assert await service.ensure_ingested(str(other), watch_id=watch["id"]) is not None
+ assert await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"]) is not None
+ assert await service.ingest_svc.ensure_ingested(str(other), watch_id=watch["id"]) is not None
source.unlink()
other.unlink()
root.rmdir()
- refreshed = await service.refresh_file(str(source), watch_id=watch["id"])
+ refreshed = await service.ingest_svc.refresh_file(str(source), watch_id=watch["id"])
source_row = await db.fetchone(
"SELECT status, error_code, error_msg, deleted_at FROM files WHERE path=?", (str(source),)
@@ -3077,7 +3075,7 @@ def test_remove_watch_converts_unreachable_files_to_deleted_and_unbinds_watch(tm
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
root = tmp_path / "removable"
@@ -3086,7 +3084,7 @@ def test_remove_watch_converts_unreachable_files_to_deleted_and_unbinds_watch(tm
source = root / "note.txt"
source.write_text("content", encoding="utf-8")
- assert await service.ensure_ingested(str(source), watch_id=watch["id"]) is not None
+ assert await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"]) is not None
now = 123456
await db.execute(
"INSERT INTO scans (path, kind, source, watch_id, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
@@ -3115,17 +3113,17 @@ def test_watch_scan_refreshes_known_active_paths_before_walk(tmp_path: Path) ->
await db.initialize()
config_svc = ConfigService(db)
fts = FTSManager(db)
- service = ParseService(
+ service = create_parse_service(
db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=service, scan_interval_sec=300)
+ watch_loop = WatchLoop(db=db, config_svc=config_svc, ingest_svc=service.ingest_svc, scan_interval_sec=300)
root = tmp_path / "watched"
root.mkdir()
watch = await config_svc.add_watch(str(root), removable=False)
source = root / "note.txt"
source.write_text("content", encoding="utf-8")
- ingested = await service.ensure_ingested(str(source), watch_id=watch["id"])
+ ingested = await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])
assert ingested is not None
source.unlink()
@@ -3145,16 +3143,16 @@ def test_watch_scan_binds_unchanged_existing_file_to_watch(tmp_path: Path) -> No
await db.initialize()
config_svc = ConfigService(db)
fts = FTSManager(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
root = tmp_path / "watched"
root.mkdir()
source = root / "note.txt"
source.write_text("content", encoding="utf-8")
- initial = await parse_svc.refresh_file(str(source))
+ initial = await parse_svc.ingest_svc.refresh_file(str(source))
watch = await config_svc.add_watch(str(root), removable=False)
scan = await scan_svc.create_scan(str(root), kind="watch", source="cli", watch_id=watch["id"])
task = await scan_svc.acquire_task()
@@ -3180,17 +3178,17 @@ def test_watch_scan_marks_root_unreachable_without_refreshing_all_files(tmp_path
await db.initialize()
config_svc = ConfigService(db)
fts = FTSManager(db)
- service = ParseService(
+ service = create_parse_service(
db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=service, scan_interval_sec=300)
+ watch_loop = WatchLoop(db=db, config_svc=config_svc, ingest_svc=service.ingest_svc, scan_interval_sec=300)
root = tmp_path / "removable"
root.mkdir()
watch = await config_svc.add_watch(str(root), removable=True)
source = root / "note.txt"
source.write_text("content", encoding="utf-8")
- ingested = await service.ensure_ingested(str(source), watch_id=watch["id"])
+ ingested = await service.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])
assert ingested is not None
source.unlink()
@@ -3213,10 +3211,10 @@ def test_device_monitor_queues_watch_scan_when_removable_watch_recovers(tmp_path
await db.initialize()
config_svc = ConfigService(db)
fts = FTSManager(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=fts, config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
monitor = DeviceMonitor(db=db, config_svc=config_svc, interval_sec=5, scan_svc=scan_svc)
root = tmp_path / "removable"
root.mkdir()
@@ -3224,7 +3222,7 @@ def test_device_monitor_queues_watch_scan_when_removable_watch_recovers(tmp_path
source = root / "note.txt"
source.write_text("content", encoding="utf-8")
- ingested = await parse_svc.ensure_ingested(str(source), watch_id=watch["id"])
+ ingested = await parse_svc.ingest_svc.ensure_ingested(str(source), watch_id=watch["id"])
assert ingested is not None
source.unlink()
@@ -3263,10 +3261,10 @@ def test_scan_service_creates_and_processes_manual_file_scan(tmp_path: Path) ->
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
source = tmp_path / "note.txt"
source.write_text("content", encoding="utf-8")
@@ -3296,10 +3294,10 @@ def test_scan_service_directory_scan_applies_excludes_and_counts_unsupported(tmp
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
root = tmp_path / "docs"
root.mkdir()
included = root / "included.txt"
@@ -3332,7 +3330,7 @@ def test_refresh_file_ignores_office_temp_lock_files(tmp_path: Path) -> None:
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=ConfigService(db),
@@ -3342,7 +3340,7 @@ def test_refresh_file_ignores_office_temp_lock_files(tmp_path: Path) -> None:
source = tmp_path / "~$package-size.xlsx"
source.write_bytes(b"\x15Microsoft Office lock")
- refreshed = await service.refresh_file(str(source), ensure_ingested=True)
+ refreshed = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)
row = await db.fetchone("SELECT path FROM files WHERE path=?", (str(source),))
assert refreshed.status == "unsupported"
@@ -3356,7 +3354,7 @@ def test_ensure_ingested_ignores_existing_office_temp_lock_file_row(tmp_path: Pa
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=ConfigService(db),
@@ -3372,7 +3370,7 @@ def test_ensure_ingested_ignores_existing_office_temp_lock_file_row(tmp_path: Pa
(str(source), source.name, "xlsx", source.stat().st_size, now, None, "active", now, now),
)
- ingested = await service.ensure_ingested(str(source))
+ ingested = await service.ingest_svc.ensure_ingested(str(source))
assert ingested is None
@@ -3384,10 +3382,10 @@ def test_scan_service_directory_scan_ignores_office_temp_lock_files(tmp_path: Pa
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
root = tmp_path / "docs"
root.mkdir()
regular = root / "regular.txt"
@@ -3429,7 +3427,7 @@ def test_watch_event_ignores_office_temp_lock_files(tmp_path: Path) -> None:
watch_loop = WatchLoop(
db=SimpleNamespace(),
config_svc=_ConfigService(),
- parse_svc=parse_svc,
+ ingest_svc=parse_svc,
scan_interval_sec=300,
)
lock_file = tmp_path / "~$package-size.xlsx"
@@ -3459,7 +3457,7 @@ def test_watch_event_ignores_image_files(tmp_path: Path) -> None:
watch_loop = WatchLoop(
db=SimpleNamespace(),
config_svc=_ConfigService(),
- parse_svc=parse_svc,
+ ingest_svc=parse_svc,
scan_interval_sec=300,
)
image_file = tmp_path / "scan.png"
@@ -3498,7 +3496,7 @@ def test_watch_event_rebases_realpath_alias_to_watch_root_namespace(tmp_path: Pa
watch_loop = WatchLoop(
db=SimpleNamespace(),
config_svc=_ConfigService(),
- parse_svc=parse_svc,
+ ingest_svc=parse_svc,
scan_interval_sec=300,
)
@@ -3514,10 +3512,10 @@ def test_scan_service_reuses_pending_scan_for_same_kind_and_path(tmp_path: Path)
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
root = tmp_path / "docs"
root.mkdir()
@@ -3536,10 +3534,10 @@ def test_scan_service_does_not_reuse_stale_running_scan_for_same_kind_and_path(t
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1)
root = tmp_path / "docs"
root.mkdir()
@@ -3597,11 +3595,11 @@ def test_doclib_server_list_responses_include_pagination_metadata(tmp_path: Path
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
- server = DoclibServer(SimpleNamespace(db=db, scan_svc=scan_svc, parse_svc=_NoopParseService()))
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
+ server = DoclibServer(SimpleNamespace(db=db, scan_svc=scan_svc, ingest_svc=_NoopParseService()))
for index in range(3):
sha256 = str(index + 1) * 64
@@ -3665,14 +3663,16 @@ def test_doclib_server_accepts_short_id_for_sha256_doc_inputs(tmp_path: Path) ->
await db.initialize()
config_svc = ConfigService(db)
fts = FTSManager(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db,
fts=fts,
config_svc=config_svc,
data_dir=str(tmp_path),
parse_lock_timeout_sec=1800,
)
- server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc))
+ server = DoclibServer(
+ SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc)
+ )
now = 1000
sha256 = "a" * 64
short_id = "aaaaaaa"
@@ -3751,14 +3751,16 @@ def test_doclib_server_reports_text_invalidation_not_required(
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
fts = FTSManager(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db,
fts=fts,
config_svc=ConfigService(db),
data_dir=str(tmp_path / "data"),
parse_lock_timeout_sec=1800,
)
- server = DoclibServer(SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc))
+ server = DoclibServer(
+ SimpleNamespace(db=db, data_dir=str(tmp_path), parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc)
+ )
source = tmp_path / f"note.{ext}"
source.write_text("unique searchable content", encoding="utf-8")
@@ -3772,7 +3774,7 @@ def test_doclib_server_reports_text_invalidation_not_required(
"is_image_based": 0,
}
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _metadata)
with pytest.raises(InvalidRequestError) as exc_info:
await server.invalidate(InvalidateRequest(path=str(source)))
@@ -3884,7 +3886,7 @@ def test_get_file_by_path_missing_doclib_record_message_is_not_disk_file_not_fou
async def _run() -> None:
db = _FakeDB(parses=[], file_row=None)
- server = DoclibServer(SimpleNamespace(db=db, parse_svc=_ParseSvc()))
+ server = DoclibServer(SimpleNamespace(db=db, ingest_svc=_ParseSvc()))
with pytest.raises(NotFoundError) as exc_info:
await server.get_file_by_path(str(tmp_path / "sample.png"))
@@ -3903,16 +3905,16 @@ def test_get_file_by_path_maps_stat_permission_error_without_existing_file_row(
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=None, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- server = DoclibServer(SimpleNamespace(db=db, parse_svc=parse_svc))
+ server = DoclibServer(SimpleNamespace(db=db, parse_svc=parse_svc, ingest_svc=parse_svc.ingest_svc))
source = tmp_path / "no_exec_dir" / "inside.pdf"
async def _permission_denied(path: str) -> dict[str, Any]:
raise PermissionError(f"permission denied: {path}")
- monkeypatch.setattr(parse_svc_module, "get_file_stat", _permission_denied)
+ monkeypatch.setattr(ingest_svc_module, "get_file_stat", _permission_denied)
try:
with pytest.raises(InvalidRequestError) as exc_info:
@@ -3932,10 +3934,10 @@ def test_scan_service_cleanup_keeps_latest_terminal_scans_and_active_tasks(tmp_p
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
for index in range(1002):
status = "done" if index % 2 == 0 else "failed"
@@ -3970,11 +3972,13 @@ def test_watch_loop_queues_watch_scan_when_scan_service_is_available(tmp_path: P
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
config_svc = ConfigService(db)
- parse_svc = ParseService(
+ parse_svc = create_parse_service(
db=db, fts=FTSManager(db), config_svc=config_svc, data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
- scan_svc = ScanService(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_lock_timeout_sec=1800)
- watch_loop = WatchLoop(db=db, config_svc=config_svc, parse_svc=parse_svc, scan_interval_sec=300, scan_svc=scan_svc)
+ scan_svc = ScanService(db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_lock_timeout_sec=1800)
+ watch_loop = WatchLoop(
+ db=db, config_svc=config_svc, ingest_svc=parse_svc.ingest_svc, scan_interval_sec=300, scan_svc=scan_svc
+ )
root = tmp_path / "watched"
root.mkdir()
watch = await config_svc.add_watch(str(root), removable=False)
@@ -4075,7 +4079,7 @@ def test_watch_loop_starts_watcher_before_initial_scan() -> None:
watch_loop = WatchLoop(
db=None,
config_svc=_ConfigService(),
- parse_svc=None,
+ ingest_svc=None,
scan_interval_sec=0,
)
@@ -4114,7 +4118,7 @@ def test_watch_loop_wakeup_triggers_next_poll_without_interval_delay() -> None:
watch_loop = WatchLoop(
db=None,
config_svc=_ConfigService(),
- parse_svc=None,
+ ingest_svc=None,
scan_interval_sec=300,
)
task = asyncio.create_task(watch_loop.run())
@@ -4509,18 +4513,18 @@ def test_ingest_records_doc_error_when_metadata_extraction_fails(tmp_path: Path,
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=_NoRulesConfig(), data_dir=str(tmp_path / "data"), parse_lock_timeout_sec=1800
)
source = tmp_path / "broken.pdf"
source.write_bytes(b"%PDF-1.4\nnot actually a valid pdf")
async def _fail_metadata(path: str) -> dict[str, Any]:
- raise parse_svc_module.MetadataExtractionError("open_failed", "metadata boom")
+ raise ingest_svc_module.MetadataExtractionError("open_failed", "metadata boom")
- monkeypatch.setattr(parse_svc_module, "extract_metadata", _fail_metadata)
+ monkeypatch.setattr(ingest_svc_module, "extract_metadata", _fail_metadata)
- row = await service.ingest_file(str(source))
+ row = await service.ingest_svc.ingest_file(str(source))
doc = await db.fetchone("SELECT error_code, error_msg FROM docs WHERE sha256=?", (row["sha256"],))
assert doc is not None
@@ -4563,7 +4567,7 @@ def test_process_doc_rejects_existing_office_temp_lock_file_task(tmp_path: Path)
"author": "",
},
)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
async def _unexpected_parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:
raise AssertionError("temporary lock files should not reach parser execution")
@@ -4611,7 +4615,7 @@ def test_process_doc_marks_empty_page_result_failed(tmp_path: Path) -> None:
"author": "",
},
)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
async def _empty_parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:
return ParseResult(
@@ -4669,7 +4673,7 @@ def test_process_doc_preserves_remote_api_error_code(tmp_path: Path) -> None:
"author": "",
},
)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
async def _parse_via_api(
file_row: dict[str, object],
@@ -4706,7 +4710,7 @@ def test_parse_via_api_maps_transport_errors_by_type(
cause: httpx.TransportError,
expected_code: str,
) -> None:
- service = ParseService(
+ service = create_parse_service(
db=_FakeDB(parses=[], file_row=None),
fts=_FakeFTS(),
config_svc=None,
@@ -4752,7 +4756,7 @@ def test_parse_via_api_requests_image_cache_only_for_office(
expected_page_range: str,
) -> None:
calls: list[bool] = []
- service = ParseService(
+ service = create_parse_service(
db=_FakeDB(parses=[], file_row=None),
fts=_FakeFTS(),
config_svc=None,
@@ -4844,7 +4848,7 @@ def test_process_doc_fails_when_batch_json_cannot_be_written(tmp_path: Path) ->
},
)
fts = _FakeFTS()
- service = ParseService(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=fts, config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
blocked_output_dir = tmp_path / "parsed" / sha256[:2] / sha256 / "flash"
blocked_output_dir.parent.mkdir(parents=True)
blocked_output_dir.write_text("not a directory", encoding="utf-8")
@@ -4910,7 +4914,7 @@ def test_process_doc_normalizes_full_document_range_from_actual_pages(tmp_path:
},
doc_row=doc_row,
)
- service = ParseService(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
+ service = create_parse_service(db=db, fts=_FakeFTS(), config_svc=None, data_dir=str(tmp_path), parse_lock_timeout_sec=1800)
async def _parse(file_row: dict, tier: Tier, page_range: str) -> ParseResult:
"""返回十二页整本 EPUB 结果。"""
diff --git a/tests/unittest/test_doclib_find_pagination.py b/tests/unittest/test_doclib_find_pagination.py
index de7d7107..82fd9f70 100644
--- a/tests/unittest/test_doclib_find_pagination.py
+++ b/tests/unittest/test_doclib_find_pagination.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+from mineru.doclib.services.ingest_svc import FileRefreshResult
+
import asyncio
from pathlib import Path
from types import SimpleNamespace
@@ -17,7 +19,7 @@ from mineru.doclib.client import DoclibClient
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.fts import FTSManager, tokenize_for_index
from mineru.doclib.server import DoclibServer
-from mineru.doclib.services.parse_svc import FileRefreshResult
+
from mineru.doclib.services.search_svc import SearchService
from mineru.doclib.types import FindResponse
from mineru.errors import InvalidRequestError
@@ -166,7 +168,7 @@ def test_find_http_exposes_offset_and_validates_pagination(tmp_path: Path) -> No
"""保留已有有效记录,让接口走正常刷新边界。"""
return FileRefreshResult(file=None, status="known")
- server = DoclibServer(SimpleNamespace(db=db, search_svc=service, parse_svc=SimpleNamespace(refresh_file=refresh)))
+ server = DoclibServer(SimpleNamespace(db=db, search_svc=service, ingest_svc=SimpleNamespace(refresh_file=refresh)))
try:
parameters = server.app.openapi()["paths"]["/api/v1/find"]["get"]["parameters"]
assert next(item for item in parameters if item["name"] == "offset")["schema"]["default"] == 0
diff --git a/tests/unittest/test_doclib_ingest_service.py b/tests/unittest/test_doclib_ingest_service.py
new file mode 100644
index 00000000..9cf4a4b4
--- /dev/null
+++ b/tests/unittest/test_doclib_ingest_service.py
@@ -0,0 +1,84 @@
+"""验证独立入库服务和宿主实例中的共享依赖,不执行后台解析模型。"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from mineru.config import PatchedConfig
+from mineru.doclib import app as doclib_app
+from mineru.doclib.core.db import DatabaseManager
+from mineru.doclib.core.fts import FTSManager
+from mineru.doclib.services.config_svc import ConfigService
+from mineru.doclib.services.ingest_svc import IngestService
+from mineru.doclib.services.parse_svc import ParseService
+
+
+def test_ingestion_queues_native_document_without_constructing_parse_service(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """独立入库可保存 HTML 属性和文本索引,不需要构造解析执行服务。"""
+
+ def forbidden_parse_service(*args: object, **kwargs: object) -> None:
+ """任何隐式解析服务组装都使本用例失败。"""
+ raise AssertionError("IngestService constructed a ParseService")
+
+ monkeypatch.setattr(ParseService, "__init__", forbidden_parse_service)
+
+ async def scenario() -> None:
+ """真实 SQLite 和 FTS 覆盖文件刷新、纯文本索引以及初始原生任务登记。"""
+ db = DatabaseManager(str(tmp_path / "library.sqlite"))
+ await db.initialize()
+ try:
+ fts = FTSManager(db)
+ ingest = IngestService(db, fts, ConfigService(db))
+ html = tmp_path / "source.html"
+ html.write_text("<html><head><title>Independent ingest</title></head><body>Native source</body></html>")
+ plain = tmp_path / "note.txt"
+ plain.write_text("Independent indexed text", encoding="utf-8")
+ document = await ingest.ensure_ingested(str(html))
+ text = await ingest.ensure_ingested(str(plain))
+ assert document is not None and text is not None
+ row = await db.fetchone("SELECT title FROM docs WHERE sha256=?", (document["sha256"],))
+ assert row["title"] == "Independent ingest"
+ queue = await db.fetchall("SELECT sha256, tier, page_range, status FROM parses")
+ assert queue == [{"sha256": document["sha256"], "tier": "flash", "page_range": "1", "status": "pending"}]
+ indexed = await fts.search("Independent")
+ assert {row["sha256"] for row in indexed} == {text["sha256"]}
+ finally:
+ await db.close()
+
+ asyncio.run(scenario())
+
+
+def test_doclib_instances_share_ingest_only_with_their_own_components(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """同一应用各入口共用入库服务,两个应用的数据库和入库状态保持隔离。"""
+ monkeypatch.setenv("MINERU_HOME", str(tmp_path / "home"))
+
+ def skip_background(*args: object, **kwargs: object) -> None:
+ """保留真实生命周期组装,仅关闭周期后台调度以固定检查时序。"""
+
+ monkeypatch.setattr(doclib_app, "_create_background_task", skip_background)
+ configurations = [
+ PatchedConfig(doclib={"data_dir": str(tmp_path / name), "sqlite": {"path": str(tmp_path / name / "db.sqlite")}})
+ for name in ("first", "second")
+ ]
+ with TestClient(doclib_app.create_app(configurations[0])) as first:
+ with TestClient(doclib_app.create_app(configurations[1])) as second:
+ left, right = first.app.state.doclib_state, second.app.state.doclib_state
+ for state in (left, right):
+ assert state.ingest_svc is state.parse_svc.ingest_svc
+ assert state.ingest_svc is state.scan_svc.ingest_svc
+ assert state.ingest_svc is state.watch.ingest_svc
+ assert state.ingest_svc is state.ingest_workers.ingest_svc
+ assert state.ingest_svc.db is state.db and state.ingest_svc.fts is state.fts
+ assert state.ingest_svc.config_svc is state.config_svc
+ assert state.ingest_svc.telemetry_svc is state.telemetry_svc
+ assert left.ingest_svc is not right.ingest_svc and left.db is not right.db
+ source = tmp_path / "one.txt"
+ source.write_text("Only the first application stores this text", encoding="utf-8")
+ assert first.portal.call(left.ingest_svc.ensure_ingested, str(source)) is not None
+ assert second.portal.call(right.db.fetchall, "SELECT * FROM files") == []
diff --git a/tests/unittest/test_doclib_legacy_page_ranges.py b/tests/unittest/test_doclib_legacy_page_ranges.py
index c7830d51..fa8fed58 100644
--- a/tests/unittest/test_doclib_legacy_page_ranges.py
+++ b/tests/unittest/test_doclib_legacy_page_ranges.py
@@ -1,4 +1,6 @@
from __future__ import annotations
+
+from doclib_test_utils import create_parse_service
from mineru.doclib.services.content_read_svc import ContentReadService
import asyncio
@@ -268,10 +270,10 @@ async def _legacy_store(
)
await _add_result(db, root, "1~5", 1000, range(1, 6), "old")
_convert_fixture_family(_cache_path(root, "1~5", 1000), family)
- service = ParseService(
+ service = create_parse_service(
db=db, fts=FTSManager(db), config_svc=ConfigService(db), data_dir=str(root), parse_lock_timeout_sec=1800
)
- server = DoclibServer(SimpleNamespace(db=db, parse_svc=service, data_dir=str(root)))
+ server = DoclibServer(SimpleNamespace(db=db, parse_svc=service, data_dir=str(root), ingest_svc=service.ingest_svc))
yield db, service, server
finally:
await db.close()
diff --git a/tests/unittest/test_doclib_managed_tier_transition.py b/tests/unittest/test_doclib_managed_tier_transition.py
index 9c0aa991..aa62ec8e 100644
--- a/tests/unittest/test_doclib_managed_tier_transition.py
+++ b/tests/unittest/test_doclib_managed_tier_transition.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
import asyncio
from dataclasses import dataclass
from pathlib import Path
@@ -49,7 +51,7 @@ async def _services(
monkeypatch.setattr("mineru.doclib.server._ensure_managed_parse_server_tier_available", lambda tier, param: None)
checker = ParseServerHealthCheck(cfg, interval_sec=30, probe_timeout_sec=1, startup_grace_sec=30, stop_timeout_sec=10)
server = DoclibServer(SimpleNamespace(config_svc=cfg, health_check=checker))
- parse_svc = ParseService(db, FTSManager(db), cfg, str(tmp_path), parse_lock_timeout_sec=60)
+ parse_svc = create_parse_service(db, FTSManager(db), cfg, str(tmp_path), parse_lock_timeout_sec=60)
return cfg, server, parse_svc, health, checker
diff --git a/tests/unittest/test_doclib_short_id.py b/tests/unittest/test_doclib_short_id.py
index a9df12df..2ae83eb2 100644
--- a/tests/unittest/test_doclib_short_id.py
+++ b/tests/unittest/test_doclib_short_id.py
@@ -1,7 +1,7 @@
+from mineru.doclib.services.ingest_svc import ensure_doc_record
import asyncio
from mineru.doclib.core.db import DatabaseManager
-from mineru.doclib.services.parse_svc import ensure_doc_record
def test_ensure_doc_record_extends_short_id_on_prefix_conflict(tmp_path) -> None:
diff --git a/tests/unittest/test_doclib_telemetry_core.py b/tests/unittest/test_doclib_telemetry_core.py
index f828c680..198b5302 100644
--- a/tests/unittest/test_doclib_telemetry_core.py
+++ b/tests/unittest/test_doclib_telemetry_core.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
import asyncio
from types import SimpleNamespace
@@ -372,13 +374,14 @@ def test_parse_service_process_doc_records_task_metrics(tmp_path) -> None:
telemetry = TelemetryService(store)
await telemetry.initialize()
config_svc = ConfigService(db)
- parse_svc = _ParseService(
+ parse_svc = create_parse_service(
db,
_Fts(),
config_svc,
str(tmp_path),
parse_lock_timeout_sec=30,
telemetry_svc=telemetry,
+ service_type=_ParseService,
)
now = 1_700_000_000_000
(tmp_path / "a.pdf").write_bytes(b"%PDF-1.4\n")
@@ -437,8 +440,7 @@ def test_scan_service_process_scan_records_task_metrics(tmp_path) -> None:
)
now = 1_700_000_000_000
await db.execute(
- "INSERT INTO scans (path, kind, source, status, started_at, created_at, updated_at) "
- "VALUES (?, ?, ?, ?, ?, ?, ?)",
+ "INSERT INTO scans (path, kind, source, status, started_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
(str(tmp_path), "manual", "cli", "running", now, now, now),
)
task = await db.fetchone("SELECT * FROM scans LIMIT 1")
diff --git a/tests/unittest/test_flash_epub.py b/tests/unittest/test_flash_epub.py
index 67031766..55fc86ab 100644
--- a/tests/unittest/test_flash_epub.py
+++ b/tests/unittest/test_flash_epub.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output
@@ -203,7 +205,7 @@ def test_doclib_ingests_epub_as_local_flash_with_spine_page_count(tmp_path: Path
"""执行隔离 SQLite 入库并检查 EPUB 文档与解析任务。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
diff --git a/tests/unittest/test_flash_odf.py b/tests/unittest/test_flash_odf.py
index 20395188..c0c57777 100644
--- a/tests/unittest/test_flash_odf.py
+++ b/tests/unittest/test_flash_odf.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output
@@ -19,7 +21,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.file_io import extract_metadata
from mineru.doclib.core.fts import FTSManager
-from mineru.doclib.services.parse_svc import ParseService
from mineru.errors import InvalidRequestError
from mineru.parser import api_server, parse, parse_async
from mineru.parser.api_server import FileStore
@@ -165,7 +166,7 @@ def test_doclib_ingests_odf_as_local_flash(
"""执行隔离 SQLite 入库并检查文档与解析任务。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
@@ -202,7 +203,7 @@ def test_doclib_rejects_odf_remote_parse(tmp_path: Path) -> None:
"""创建隔离 doclib 并断言稳定错误码。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
diff --git a/tests/unittest/test_flash_ofd.py b/tests/unittest/test_flash_ofd.py
index 0a6b5e78..7ed01e74 100644
--- a/tests/unittest/test_flash_ofd.py
+++ b/tests/unittest/test_flash_ofd.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output
@@ -18,7 +20,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.file_io import extract_metadata
from mineru.doclib.core.fts import FTSManager
-from mineru.doclib.services.parse_svc import ParseService
from mineru.errors import InvalidRequestError
from mineru.parser import MinerUParser, api_server
from mineru.parser.api_server import FileStore
@@ -109,7 +110,7 @@ def test_doclib_ingests_ofd_as_local_full_document_flash(tmp_path: Path) -> None
"""执行隔离 SQLite 入库并检查 OFD 文档与解析任务。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
diff --git a/tests/unittest/test_flash_rtf.py b/tests/unittest/test_flash_rtf.py
index 0981e133..da7528a5 100644
--- a/tests/unittest/test_flash_rtf.py
+++ b/tests/unittest/test_flash_rtf.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+
import asyncio
from io import BytesIO
from pathlib import Path
@@ -12,7 +14,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.doclib.core import file_io
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.fts import FTSManager
-from mineru.doclib.services.parse_svc import ParseService
from mineru.errors import InvalidRequestError
from mineru.parser import parse
from mineru.render import RenderMode, render_docx, render_html, render_markdown, render_structured_content
@@ -104,7 +105,7 @@ def test_doclib_ingests_rtf_as_one_page_flash_document(tmp_path: Path) -> None:
"""执行真实 SQLite doclib 入库并检查稳定行数据。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
@@ -156,7 +157,7 @@ def test_doclib_rejects_rtf_quality_tier_and_remote(
"""创建隔离 doclib 并断言稳定错误契约。"""
db = DatabaseManager(str(tmp_path / "doclib.db"))
await db.initialize()
- service = ParseService(
+ service = create_parse_service(
db=db,
fts=FTSManager(db),
config_svc=_NoRulesConfig(), # type: ignore[arg-type]
diff --git a/tests/unittest/test_pdf_page_id.py b/tests/unittest/test_pdf_page_id.py
index 2403c00e..19391aa7 100644
--- a/tests/unittest/test_pdf_page_id.py
+++ b/tests/unittest/test_pdf_page_id.py
@@ -1,4 +1,6 @@
from __future__ import annotations
+
+from mineru.doclib.services.parse_policy import default_parse_range
from mineru.doclib.services.content_read_svc import ContentReadService
from task_test_utils import wait_task
@@ -19,7 +21,7 @@ from mineru.cli.commands.parse import _validate_page_range_input
from mineru.cli.main import app as cli_app
from mineru.doclib.services.content_read_svc import _normalize_content_page_range
from mineru.doclib.services.config_svc import ConfigService
-from mineru.doclib.services.parse_svc import default_parse_range
+
from mineru.errors import InvalidRequestError, MineruError
from mineru.kit.gradio.page_range import effective_page_range as _effective_page_range
from mineru.kit.gradio.artifacts import persist_parse_result
diff --git a/tests/unittest/test_source_metadata.py b/tests/unittest/test_source_metadata.py
index 367ae636..6a47fe24 100644
--- a/tests/unittest/test_source_metadata.py
+++ b/tests/unittest/test_source_metadata.py
@@ -2,6 +2,9 @@
from __future__ import annotations
+from doclib_test_utils import create_parse_service
+from mineru.doclib.services.ingest_svc import ensure_doc_record
+
import asyncio
from io import BytesIO
from pathlib import Path
@@ -15,7 +18,6 @@ from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.fts import FTSManager
from mineru.doclib.services.config_svc import ConfigService
-from mineru.doclib.services.parse_svc import ParseService, ensure_doc_record
from mineru.parser import MinerUParser, ParseResult
@@ -78,7 +80,7 @@ def test_doclib_source_columns_and_fts_are_independent_of_tier(tmp_path: Path) -
db = DatabaseManager(str(tmp_path / "db.sqlite"))
await db.initialize()
fts = FTSManager(db)
- service = ParseService(db, fts, ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)
+ service = create_parse_service(db, fts, ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)
sha = "a" * 64
await ensure_doc_record(
db,
@@ -126,12 +128,12 @@ def test_doclib_ingest_gets_html_metadata_without_parsing(tmp_path: Path) -> Non
"""使用临时数据库与源 HTML 走入库服务。"""
db = DatabaseManager(str(tmp_path / "db.sqlite"))
await db.initialize()
- service = ParseService(db, FTSManager(db), ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)
+ service = create_parse_service(db, FTSManager(db), ConfigService(db), str(tmp_path), parse_lock_timeout_sec=30)
source = tmp_path / "source.html"
source.write_text(
'<html lang="zh-CN"><head><title>入库标题</title><meta name="author" content="作者"></head><body>正文</body></html>'
)
- refresh = await service.refresh_file(str(source), ensure_ingested=True)
+ refresh = await service.ingest_svc.refresh_file(str(source), ensure_ingested=True)
assert refresh.file is not None
row = await db.fetchone("SELECT * FROM docs WHERE sha256=?", (refresh.file.sha256,))
assert (row["title"], row["author"], row["language"]) == ("入库标题", "作者", "zh-CN")