Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
from doclib_test_utils import DOCLIB_AUTH_HEADERS, authenticated_doclib_config
import asyncio
import logging
import subprocess
import tomllib
import uuid
from logging.handlers import RotatingFileHandler
from pathlib import Path
from unittest.mock import Mock
import pytest
from fastapi.testclient import TestClient
from loguru import logger as loguru_logger
from packaging.requirements import Requirement
from mineru.config import Config, LogConfig, PatchedConfig, get_config
from mineru.doclib import app as doclib_app
from mineru.doclib.app import _assert_required_schema
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.server import _tail_log
from mineru.doclib.services.content_read_svc import _write_temp_asset
from mineru.model import download as model_download
from mineru.parser import tier as parser_tier
from mineru.version import __version__
def _clear_test_loggers() -> None:
for name in ("mineru", "uvicorn", "uvicorn.error", "uvicorn.access", "py.warnings"):
logger = logging.getLogger(name)
for handler in list(logger.handlers):
logger.removeHandler(handler)
handler.close()
logger.propagate = True
def test_required_schema_check_fails_before_migration_and_passes_after(tmp_path) -> None:
async def _run() -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
with pytest.raises(RuntimeError, match="missing tables"):
await _assert_required_schema(db)
await db.initialize()
await _assert_required_schema(db)
row = await db.fetchone("SELECT name FROM sqlite_master WHERE type='table' AND name='parses'")
assert row == {"name": "parses"}
asyncio.run(_run())
def test_doclib_runtime_dependencies_are_in_base_install() -> None:
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
dependencies = pyproject["project"]["dependencies"]
dependency_names = {dependency.split(">", 1)[0].split("=", 1)[0].lower() for dependency in dependencies}
assert "aiosqlite" in dependency_names
assert "packaging" in dependency_names
assert "watchfiles" in dependency_names
def test_huggingface_hub_base_dependency_enables_xet() -> None:
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
dependencies = (Requirement(raw_dependency) for raw_dependency in pyproject["project"]["dependencies"])
huggingface_hub = next(dependency for dependency in dependencies if dependency.name == "huggingface-hub")
assert huggingface_hub.extras == {"hf_xet"}
def test_torch_extra_includes_preflight_runtime_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
"""完整模型栈的预检依赖由 torch extra 提供,不依赖已删除的 basic extra。"""
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
torch_dependencies = pyproject["project"]["optional-dependencies"]["torch"]
dependency_names = {Requirement(dependency).name for dependency in torch_dependencies}
dependency_names.update(Requirement(dependency).name for dependency in pyproject["project"]["dependencies"])
module_to_distribution = {
"torch": "torch",
"torchvision": "torchvision",
"transformers": "transformers",
}
monkeypatch.setattr(get_config().model, "small_backend", "torch")
missing = [
module_name
for module_name in parser_tier.required_modules_for_tier("basic")
if module_to_distribution.get(module_name, module_name) not in dependency_names
]
assert missing == []
def test_full_extra_composes_torch_and_platform_engines() -> None:
"""full extra 组合 Torch 和平台引擎,普通 test extra 保持独立。"""
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
extras = pyproject["project"]["optional-dependencies"]
assert "advanced" not in extras
assert "mineru[torch]" in extras["full"]
assert {"vllm", "lmdeploy"} <= {Requirement(item).name for item in extras["full"]}
assert "mlx-vlm" not in {Requirement(item).name for item in extras["full"]}
assert not any(Requirement(item).name == "mineru" for item in extras["test"])
@pytest.mark.parametrize(
("key", "value"),
[
("parse_server.local.mode", "totally-invalid-mode"),
("parse_server.local.managed_tier", "ultra"),
("parse_server.remote.url", "ftp://example.com/api"),
("parse_server.local.self_hosted_url", "not-a-url"),
],
)
def test_config_set_rejects_invalid_known_config_values(key: str, value: str, monkeypatch, tmp_path) -> None:
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.put(f"/api/v1/configs/{key}", json={"value": value})
config_response = client.get(f"/api/v1/configs/{key}")
assert response.status_code == 400
payload = response.json()
assert payload["error"]["code"] == "invalid_config_value"
assert payload["error"]["param"] == "value"
assert config_response.json()["source"] == "default"
@pytest.mark.parametrize(("tier", "model_tier"), [("basic", "basic"), ("standard", "standard"), ("advanced", "standard")])
def test_config_set_managed_tier_rejects_missing_models(
tier: str, model_tier: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""托管档位缺少模型时拒绝保存,Advanced 提示下载共享的 Standard 模型。"""
def _skip_background_task(*args, **kwargs):
"""跳过后台任务,隔离配置接口与模型就绪检查。"""
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None)
monkeypatch.setattr(model_download.get_config().model, "base_dir", str(tmp_path / "models"))
# 无 torch 环境会把小模型后端解析为 onnx,显式固定以保持仓库名断言确定。
monkeypatch.setattr(model_download.get_config().model, "small_backend", "torch")
monkeypatch.setattr(model_download.get_config().model.vlm, "engine", "llama-cpp")
monkeypatch.setattr(model_download.get_config().model.vlm, "server_url", "")
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.put("/api/v1/configs/parse_server.local.managed_tier", json={"value": tier})
config_response = client.get("/api/v1/configs/parse_server.local.managed_tier")
assert response.status_code == 400
payload = response.json()
assert payload["error"]["code"] == "parse_server_model_not_ready"
assert payload["error"]["param"] == "parse_server.local.managed_tier"
assert "MinerU-4_models_torch" in payload["error"]["message"]
assert f"mineru-kit models download --tier {model_tier}" in payload["error"]["message"]
assert config_response.json()["value"] == "standard"
assert config_response.json()["source"] == "default"
def test_config_set_managed_advanced_accepts_ready_standard_models(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Advanced 使用 Standard 依赖和模型,通过检查后保存为显式托管上限。"""
check_dependencies = Mock(return_value=None)
verify_models = Mock(return_value=model_download.ModelReadyResult(ready=True, root=tmp_path / "models"))
monkeypatch.setattr(doclib_app, "_create_background_task", Mock(return_value=None))
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", check_dependencies)
monkeypatch.setattr("mineru.doclib.server.verify_model_repo", verify_models)
monkeypatch.setattr(model_download.get_config().model, "small_backend", "torch")
monkeypatch.setattr(model_download.get_config().model.vlm, "engine", "llama-cpp")
monkeypatch.setattr(model_download.get_config().model.vlm, "server_url", "")
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.put("/api/v1/configs/parse_server.local.managed_tier", json={"value": "advanced"})
config_response = client.get("/api/v1/configs/parse_server.local.managed_tier")
assert response.status_code == 200
assert response.json()["value"] == "advanced"
assert config_response.json()["value"] == "advanced"
assert config_response.json()["source"] == "override"
check_dependencies.assert_called_once_with("standard")
assert [call.args[0].name for call in verify_models.call_args_list] == [
"MinerU-4_models_torch",
"MinerU2.5-Pro-2605-1.2B-GGUF",
]
def test_config_set_managed_mode_rejects_missing_models_for_current_tier(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None)
monkeypatch.setattr(model_download.get_config().model, "base_dir", str(tmp_path / "models"))
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.put("/api/v1/configs/parse_server.local.mode", json={"value": "managed"})
config_response = client.get("/api/v1/configs/parse_server.local.mode")
assert response.status_code == 400
payload = response.json()
assert payload["error"]["code"] == "parse_server_model_not_ready"
assert payload["error"]["param"] == "parse_server.local.mode"
assert "mineru-kit models download --tier standard" in payload["error"]["message"]
assert config_response.json()["value"] == "disabled"
assert config_response.json()["source"] == "default"
def test_background_task_crash_is_logged(caplog: pytest.LogCaptureFixture) -> None:
async def _boom() -> None:
raise RuntimeError("boom")
async def _run() -> None:
task = asyncio.create_task(_boom())
await asyncio.sleep(0)
doclib_app._log_background_task_result("test-task", task)
with caplog.at_level(logging.ERROR):
asyncio.run(_run())
assert "Background task test-task crashed: boom" in caplog.text
def test_background_tasks_are_cancelled_and_awaited_on_shutdown() -> None:
async def _run() -> None:
state = doclib_app.AppState()
state.db_ready = asyncio.Event()
async def _blocked() -> None:
await asyncio.Event().wait()
task = doclib_app._create_background_task(state, "blocked", _blocked)
await asyncio.sleep(0)
await doclib_app._cancel_background_tasks(state)
assert task.cancelled()
assert task.done()
assert state.background_tasks == []
asyncio.run(_run())
def test_pending_loop_tasks_are_cancelled_before_loop_close() -> None:
loop = asyncio.new_event_loop()
try:
task = loop.create_task(asyncio.Event().wait())
doclib_app._cancel_pending_loop_tasks(loop)
assert task.cancelled()
assert task.done()
finally:
loop.close()
def test_bind_tcp_socket_tries_configured_port_range(monkeypatch: pytest.MonkeyPatch) -> None:
bind_calls: list[tuple[str, int]] = []
class _Socket:
def __init__(self) -> None:
self.bound_port: int | None = None
def setsockopt(self, *args: object) -> None:
return None
def bind(self, address: tuple[str, int]) -> None:
bind_calls.append(address)
host, port = address
if port in (15980, 15981):
raise OSError(doclib_app.errno.EADDRINUSE, "in use")
self.bound_port = port
def getsockname(self) -> tuple[str, int]:
assert self.bound_port is not None
return ("127.0.0.1", self.bound_port)
monkeypatch.setattr(doclib_app.socket, "socket", lambda *args, **kwargs: _Socket())
tcp_sock, port = doclib_app._bind_tcp_socket("127.0.0.1", 15980, strict_port=False, port_probe_count=3)
assert isinstance(tcp_sock, _Socket)
assert port == 15982
assert bind_calls == [("127.0.0.1", 15980), ("127.0.0.1", 15981), ("127.0.0.1", 15982)]
def test_bind_tcp_socket_strict_port_does_not_probe(monkeypatch: pytest.MonkeyPatch) -> None:
bind_calls: list[tuple[str, int]] = []
class _Socket:
def setsockopt(self, *args: object) -> None:
return None
def bind(self, address: tuple[str, int]) -> None:
bind_calls.append(address)
raise OSError(doclib_app.errno.EADDRINUSE, "in use")
monkeypatch.setattr(doclib_app.socket, "socket", lambda *args, **kwargs: _Socket())
with pytest.raises(OSError):
doclib_app._bind_tcp_socket("127.0.0.1", 15980, strict_port=True, port_probe_count=3)
assert bind_calls == [("127.0.0.1", 15980)]
def test_doclib_log_level_inherits_global_and_accepts_local_override() -> None:
"""验证 Doclib 未设置级别时继承当前应用的全局配置,显式设置时局部覆盖。"""
assert doclib_app._resolve_log_level(LogConfig(), global_level="debug") == logging.DEBUG
assert doclib_app._resolve_log_level(LogConfig(level="warning"), global_level="debug") == logging.WARNING
def test_explicit_app_configuration_is_copied_without_loading_defaults(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""显式启动配置独立于默认配置,创建后修改调用方对象不会影响应用快照。"""
cfg = Config(doclib={"log": {"dir": str(tmp_path)}}, log={"level": "debug"})
def reject_default_configuration() -> Config:
"""显式配置路径不得再次装载进程默认配置。"""
pytest.fail("explicit app configuration must not load defaults")
monkeypatch.setattr(doclib_app, "load_doclib_server_config", reject_default_configuration)
try:
app = doclib_app.create_app(authenticated_doclib_config(cfg))
snapshot = app.state.doclib_state.config
assert snapshot is not cfg
assert logging.getLogger("mineru").level == logging.DEBUG
cfg.doclib.parse_workers = 7
cfg.log.level = "error"
assert snapshot.doclib.parse_workers == 2
assert snapshot.log.level == "debug"
finally:
_clear_test_loggers()
def test_setup_logging_routes_application_logs_to_rotating_file_without_stderr_duplication(tmp_path: Path) -> None:
_clear_test_loggers()
log_path = tmp_path / "doclib.log"
access_log_path = tmp_path / "doclib.access.log"
try:
doclib_app._setup_logging(LogConfig(app_path=str(log_path), access_path=str(access_log_path)), global_level="info")
mineru_logger = logging.getLogger("mineru")
uvicorn_error_logger = logging.getLogger("uvicorn.error")
uvicorn_access_logger = logging.getLogger("uvicorn.access")
warnings_logger = logging.getLogger("py.warnings")
mineru_rotating_handlers = [handler for handler in mineru_logger.handlers if isinstance(handler, RotatingFileHandler)]
mineru_plain_stream_handlers = [
handler
for handler in mineru_logger.handlers
if isinstance(handler, logging.StreamHandler) and not isinstance(handler, RotatingFileHandler)
]
assert len(mineru_rotating_handlers) == 1
assert mineru_rotating_handlers[0].baseFilename == str(log_path)
assert mineru_rotating_handlers[0].encoding == "utf-8"
assert mineru_rotating_handlers[0].errors == "backslashreplace"
assert mineru_plain_stream_handlers == []
assert mineru_logger.propagate is False
uvicorn_error_log_paths = [
handler.baseFilename for handler in uvicorn_error_logger.handlers if isinstance(handler, RotatingFileHandler)
]
assert uvicorn_error_log_paths == [str(log_path)]
assert [handler.baseFilename for handler in warnings_logger.handlers if isinstance(handler, RotatingFileHandler)] == [
str(log_path)
]
uvicorn_access_log_paths = [
handler.baseFilename for handler in uvicorn_access_logger.handlers if isinstance(handler, RotatingFileHandler)
]
assert uvicorn_access_log_paths == [str(access_log_path)]
mineru_logger.info("Unicode log: \u223c \u4e2d\u6587 \u2713")
mineru_rotating_handlers[0].flush()
assert "Unicode log: \u223c \u4e2d\u6587 \u2713" in log_path.read_text(encoding="utf-8")
finally:
_clear_test_loggers()
def test_setup_logging_routes_loguru_records_to_application_log(tmp_path: Path) -> None:
_clear_test_loggers()
log_path = tmp_path / "doclib.log"
access_log_path = tmp_path / "doclib.access.log"
try:
doclib_app._setup_logging(LogConfig(app_path=str(log_path), access_path=str(access_log_path)), global_level="info")
patched_logger = loguru_logger.patch(lambda record: record.update(name="mineru.test_loguru"))
patched_logger.error("loguru bridged message")
for handler in logging.getLogger("mineru").handlers:
handler.flush()
content = log_path.read_text(encoding="utf-8")
assert "mineru.test_loguru" in content
assert "loguru bridged message" in content
finally:
_clear_test_loggers()
def test_shutdown_removes_only_endpoint_owned_by_current_server(tmp_path: Path) -> None:
endpoint_path = tmp_path / "doclib.endpoint.json"
doclib_app.write_endpoint_file(endpoint_path, pid=123, server_id="current-server", transports=[])
doclib_app._remove_owned_endpoint_file(str(endpoint_path), "old-server")
assert endpoint_path.exists()
doclib_app._remove_owned_endpoint_file(str(endpoint_path), "current-server")
assert not endpoint_path.exists()
def test_shutdown_removes_only_uds_path_with_bound_identity(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
uds_path = tmp_path / "doclib.sock"
uds_path.write_text("original", encoding="utf-8")
# Linux 会复用刚释放的 inode,真实 stat 无法稳定构造"同路径不同身份";用固定身份值验证清理契约。
identities = iter([(0, 2), (0, 2)])
monkeypatch.setattr(doclib_app, "_path_identity", lambda _path: next(identities))
doclib_app._remove_owned_uds_path(str(uds_path), (0, 1))
assert uds_path.read_text(encoding="utf-8") == "original"
doclib_app._remove_owned_uds_path(str(uds_path), (0, 2))
assert not uds_path.exists()
def test_discovery_cleanup_runs_only_once() -> None:
state = doclib_app.AppState()
events: list[str] = []
state.discovery_cleanup = lambda: events.append("cleanup")
doclib_app._run_discovery_cleanup(state)
doclib_app._run_discovery_cleanup(state)
assert events == ["cleanup"]
def test_server_status_reports_configured_socket_path(monkeypatch, tmp_path) -> None:
app_log_path = tmp_path / "doclib.log"
access_log_path = tmp_path / "doclib.access.log"
stdout_log_path = tmp_path / "doclib.stdout.log"
stderr_log_path = tmp_path / "doclib.stderr.log"
parse_server_stdout_log_path = tmp_path / "doclib.parse-server.stdout.log"
parse_server_stderr_log_path = tmp_path / "doclib.parse-server.stderr.log"
for prefix, path, count in (
("app", app_log_path, 30),
("access", access_log_path, 12),
("stdout", stdout_log_path, 12),
("stderr", stderr_log_path, 12),
("parse-stdout", parse_server_stdout_log_path, 12),
("parse-stderr", parse_server_stderr_log_path, 12),
):
path.write_text("".join(f"{prefix}-{idx}\n" for idx in range(count)), encoding="utf-8")
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(app_log_path),
"access_path": str(access_log_path),
"stdout_path": str(stdout_log_path),
"stderr_path": str(stderr_log_path),
"parse_server_stdout_path": str(parse_server_stdout_log_path),
"parse_server_stderr_path": str(parse_server_stderr_log_path),
},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.get("/api/v1/server/status")
assert response.status_code == 200
payload = response.json()
assert uuid.UUID(payload["server_id"])
assert payload["mineru_home"]
assert payload["version"] == __version__
assert isinstance(payload["python_version"], str)
assert payload["socket_path"] == str(tmp_path / "doclib.sock")
assert payload["sqlite_path"] == str(tmp_path / "doclib.db")
assert payload["log_path"] == str(tmp_path / "doclib.log")
assert payload["access_log_path"] == str(access_log_path)
assert payload["stdout_log_path"] == str(stdout_log_path)
assert payload["stderr_log_path"] == str(stderr_log_path)
assert payload["tcp"] == {"enabled": cfg.doclib.resolved_tcp_enabled, "host": cfg.doclib.tcp.host, "port": None}
assert payload["app_logs"] == [f"app-{idx}\n" for idx in range(5, 30)]
assert payload["access_logs"] == [f"access-{idx}\n" for idx in range(2, 12)]
assert payload["stdout_logs"] == [f"stdout-{idx}\n" for idx in range(2, 12)]
assert payload["stderr_logs"] == [f"stderr-{idx}\n" for idx in range(2, 12)]
assert payload["parse_server_stdout_logs"] == [f"parse-stdout-{idx}\n" for idx in range(2, 12)]
assert payload["parse_server_stderr_logs"] == [f"parse-stderr-{idx}\n" for idx in range(2, 12)]
def test_doclib_app_uses_lifespan_instead_of_deprecated_event_handlers(monkeypatch, tmp_path) -> None:
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
app = doclib_app.create_app(authenticated_doclib_config(cfg))
assert app.router.on_startup == []
assert app.router.on_shutdown == []
with TestClient(app, headers=DOCLIB_AUTH_HEADERS) as client:
response = client.get("/api/v1/server/status")
assert response.status_code == 200
def test_managed_parse_server_startup_writes_stdout_and_stderr_logs(monkeypatch, tmp_path) -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
asyncio.run(db.initialize())
asyncio.run(db.execute("INSERT INTO config (key, value) VALUES (?, ?)", ("parse_server.local.mode", "managed")))
parse_stdout_log_path = tmp_path / "doclib.parse-server.stdout.log"
parse_stderr_log_path = tmp_path / "doclib.parse-server.stderr.log"
popen_calls: list[dict[str, object]] = []
class _Proc:
pid = 12345
def poll(self) -> None:
return None
def terminate(self) -> None:
return None
def wait(self, timeout: int) -> None:
return None
def _popen(*args: object, **kwargs: object) -> _Proc:
popen_calls.append({"args": args, "kwargs": kwargs})
cmd = args[0]
assert "--host" in cmd
assert cmd[cmd.index("--host") + 1] == "127.0.0.2"
assert "--port" in cmd
assert cmd[cmd.index("--port") + 1] == "16582"
assert kwargs["stdout"] is not subprocess.DEVNULL
assert kwargs["stderr"] is not subprocess.DEVNULL
assert kwargs["stdout"] is not kwargs["stderr"]
assert kwargs["stdin"] is subprocess.DEVNULL
assert kwargs["env"]["MINERU_MANAGED_CONTROL"]
kwargs["stdout"].write("parse stdout\n")
kwargs["stdout"].flush()
kwargs["stderr"].write("parse stderr\n")
kwargs["stderr"].flush()
return _Proc()
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
"parse_server_stdout_path": str(parse_stdout_log_path),
"parse_server_stderr_path": str(parse_stderr_log_path),
},
"managed_parse_server": {
"host": "127.0.0.2",
"port": 16580,
"port_probe_count": 3,
},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr(
"mineru.doclib.background.parse_server_health.select_available_managed_port", lambda *args, **kwargs: 16582
)
monkeypatch.setattr("mineru.doclib.background.parse_server_health.subprocess.Popen", _popen)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.get("/api/v1/server/status")
assert response.status_code == 200
assert popen_calls
assert parse_stdout_log_path.read_text(encoding="utf-8").endswith("parse stdout\n")
assert parse_stderr_log_path.read_text(encoding="utf-8").endswith("parse stderr\n")
def test_managed_parse_server_startup_ignores_invalid_tier_override(monkeypatch, tmp_path) -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
asyncio.run(db.initialize())
asyncio.run(db.execute("INSERT INTO config (key, value) VALUES (?, ?)", ("parse_server.local.mode", "managed")))
asyncio.run(db.execute("INSERT INTO config (key, value) VALUES (?, ?)", ("parse_server.local.managed_tier", "ultra")))
class _Proc:
pid = 12345
def poll(self) -> None:
return None
def terminate(self) -> None:
"""模拟正常结束,使启动配置测试使用完整的进程生命周期替身。"""
return None
def wait(self, timeout: int) -> None:
return None
def _popen(*args: object, **kwargs: object) -> _Proc:
cmd = args[0]
assert "--tier" in cmd
assert cmd[cmd.index("--tier") + 1] == "standard"
return _Proc()
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr(
"mineru.doclib.background.parse_server_health.select_available_managed_port", lambda *args, **kwargs: 16582
)
monkeypatch.setattr("mineru.doclib.background.parse_server_health.subprocess.Popen", _popen)
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
"parse_server_stdout_path": str(tmp_path / "parse.stdout.log"),
"parse_server_stderr_path": str(tmp_path / "parse.stderr.log"),
},
"managed_parse_server": {"port": 16580},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
config_response = client.get("/api/v1/configs/parse_server.local.managed_tier")
status_response = client.get("/api/v1/server/status")
assert config_response.json()["value"] == "ultra"
assert status_response.json()["parse_server"]["local"]["managed_tier"] == "standard"
def test_managed_parse_server_shutdown_uses_health_proc(monkeypatch, tmp_path) -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
asyncio.run(db.initialize())
asyncio.run(db.execute("INSERT INTO config (key, value) VALUES (?, ?)", ("parse_server.local.mode", "managed")))
events: list[str] = []
class _Stdin:
closed = False
def close(self) -> None:
events.append("stdin.close")
self.closed = True
class _Proc:
pid = 12345
stdin = _Stdin()
def poll(self) -> None:
return None
def wait(self, timeout: int) -> None:
events.append(f"wait:{timeout}")
def terminate(self) -> None:
events.append("terminate")
def kill(self) -> None:
events.append("kill")
def _popen(*args: object, **kwargs: object) -> _Proc:
return _Proc()
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
"parse_server_stdout_path": str(tmp_path / "parse.stdout.log"),
"parse_server_stderr_path": str(tmp_path / "parse.stderr.log"),
},
"managed_parse_server": {"port": 16580},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr(
"mineru.doclib.background.parse_server_health.select_available_managed_port", lambda *args, **kwargs: 16582
)
monkeypatch.setattr("mineru.doclib.background.parse_server_health.subprocess.Popen", _popen)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
state = client.app.state.doclib_state
assert not hasattr(state, "parse_server_proc")
assert client.get("/api/v1/server/status").status_code == 200
assert events == ["terminate", "wait:3.0"]
def test_startup_resets_running_scans_to_failed(monkeypatch, tmp_path) -> None:
db = DatabaseManager(str(tmp_path / "doclib.db"))
asyncio.run(db.initialize())
asyncio.run(
db.execute(
"INSERT INTO scans (path, kind, source, status, locked_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
(str(tmp_path / "watch-root"), "watch", "watch", "running", 123, 1000, 1000),
)
)
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.get("/api/v1/server/status")
assert response.status_code == 200
row = asyncio.run(db.fetchone("SELECT status, locked_at, error_code FROM scans"))
assert row == {"status": "failed", "locked_at": None, "error_code": "scan_interrupted"}
def test_telemetry_observation_route_uses_context_headers(monkeypatch, tmp_path) -> None:
cfg = PatchedConfig(
doclib={
"data_dir": str(tmp_path),
"sqlite": {"path": str(tmp_path / "doclib.db")},
"log": {
"app_path": str(tmp_path / "doclib.log"),
"access_path": str(tmp_path / "doclib.access.log"),
},
"uds": {"path": str(tmp_path / "doclib.sock")},
}
)
def _skip_background_task(*args, **kwargs):
return None
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
with TestClient(doclib_app.create_app(authenticated_doclib_config(cfg)), headers=DOCLIB_AUTH_HEADERS) as client:
response = client.post(
"/api/v1/observations",
headers={"X-MinerU-Telemetry-Source": "cli", "X-MinerU-Telemetry-Caller": "agent"},
json={"observations": [{"metric_name": "parse.request.count"}]},
)
preview = client.get("/api/v1/telemetry/preview")
assert response.status_code == 200
assert response.json() == {"accepted": 1}
assert preview.status_code == 200
metrics = preview.json()["body"]["metrics"]
assert metrics == [{"name": "parse.request.count", "value": 1, "dimensions": {"caller": "agent", "source": "cli"}}]
def test_write_temp_asset_uses_temp_read_assets_directory(tmp_path: Path) -> None:
asset = _write_temp_asset(
str(tmp_path),
"abc1234",
"png",
b"image-bytes",
mime_type="image/png",
width=10,
height=10,
)
asset_path = Path(asset.path)
assert asset_path.parent == tmp_path / "temp" / "read-assets"
def test_tail_log_uses_explicit_path_without_process_default(tmp_path: Path) -> None:
log_path = tmp_path / "doclib.log"
log_path.write_text("line-1\nline-2\n", encoding="utf-8")
assert _tail_log(str(log_path)) == ["line-1\n", "line-2\n"]
assert _tail_log("") == []